From 8c7edef55f20e45418120404e65f3a6dfe539025 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Thu, 5 Nov 2026 18:08:39 +0000 Subject: [PATCH 01/20] [RF][HS3] Reject circular dependencies on import Objects that depend on each other made the import recurse until the stack overflowed. Keep track of the objects that are being imported and report the cycle instead. Assisted-by: claude-code:claude-opus-5-5 --- roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h | 4 ++++ roofit/hs3/src/RooJSONFactoryWSTool.cxx | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h index b0bd15cf2208f..669da2ba3abb8 100644 --- a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h +++ b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h @@ -24,6 +24,7 @@ #include #include #include +#include namespace RooFit { namespace JSONIO { @@ -274,5 +275,8 @@ class RooJSONFactoryWSTool { // every sibling node. std::unordered_map _functionsByName; std::unordered_map _distributionsByName; + + // The objects that are currently being imported, innermost last, to detect circular dependencies. + std::vector _importStack; }; #endif diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index e951197158419..e680bac8ab83a 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -1279,6 +1279,19 @@ void RooJSONFactoryWSTool::importFunction(const JSONNode &p, bool importAllDepen if (_workspace.arg(name)) { return; } + // Objects that depend on each other would otherwise recurse until the stack overflows. + if (std::find(_importStack.begin(), _importStack.end(), name) != _importStack.end()) { + std::string chain; + for (std::string const &ancestor : _importStack) { + chain += ancestor + " -> "; + } + RooJSONFactoryWSTool::error("circular dependency in the document: " + chain + name); + } + _importStack.push_back(name); + struct PopOnExit { + std::vector &stack; + ~PopOnExit() { stack.pop_back(); } + } popOnExit{_importStack}; // if the key we found is not a map, it's an error if (!p.is_map()) { RooJSONFactoryWSTool::error("RooJSONFactoryWSTool() function node " + name + " is not a map!"); From d04c8535fffb4792eba9652334506a951551ff41 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Thu, 5 Nov 2026 18:08:41 +0000 Subject: [PATCH 02/20] [RF][HS3] Reject unknown parameters of interest An analysis that listed an unknown parameter of interest dereferenced a null pointer. Assisted-by: claude-code:claude-opus-5-5 --- roofit/hs3/src/RooJSONFactoryWSTool.cxx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index e680bac8ab83a..a6eadcabd230e 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -634,7 +634,12 @@ void importAnalysis(const JSONNode &rootnode, const JSONNode &analysisNode, cons RooArgSet pois; if (poiNode) { for (auto const &child : poiNode->children()) { - pois.add(*workspace.arg(child.val())); + RooAbsArg *arg = workspace.arg(child.val()); + if (!arg) { + RooJSONFactoryWSTool::error("the object '" + child.val() + "' in \"" + poiNode->key() + + "\" of analysis '" + analysisName + "' does not exist"); + } + pois.add(*arg); } } mc->SetParametersOfInterest(pois); From 781ac09758b7a39f9b8cdd65adbdce3c05545ca3 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Thu, 5 Nov 2026 18:33:54 +0000 Subject: [PATCH 03/20] [RF][HS3] Add a typed representation of HS3 documents Parsing and building RooFit objects are intertwined, so that malformed documents are only noticed halfway through an import, and every importer deals with JSON details on its own. Add RooFit::HS3Document, which follows the structure of the standard and doesn't depend on RooFit, and a parser that reports all problems with their location in the document. The header stays private for now. Distributions and functions are generic objects for the time being, which are still interpreted by the registered importers. Assisted-by: claude-code:claude-opus-5-5 --- roofit/hs3/CMakeLists.txt | 3 + roofit/hs3/src/HS3Document.h | 299 ++++++++++++++ roofit/hs3/src/HS3Parser.cxx | 732 +++++++++++++++++++++++++++++++++++ roofit/hs3/src/HS3Schema.h | 184 +++++++++ 4 files changed, 1218 insertions(+) create mode 100644 roofit/hs3/src/HS3Document.h create mode 100644 roofit/hs3/src/HS3Parser.cxx create mode 100644 roofit/hs3/src/HS3Schema.h diff --git a/roofit/hs3/CMakeLists.txt b/roofit/hs3/CMakeLists.txt index ffeea5e753c31..0601ea4296d75 100644 --- a/roofit/hs3/CMakeLists.txt +++ b/roofit/hs3/CMakeLists.txt @@ -20,6 +20,7 @@ ROOT_STANDARD_LIBRARY_PACKAGE(RooFitHS3 ${ROOFITHS3_HEADERS} SOURCES src/Domains.cxx + src/HS3Parser.cxx src/JSONInterface.cxx src/JSONIO.cxx src/RooJSONFactoryWSTool.cxx @@ -49,6 +50,8 @@ target_sources( ${RELATIVE_INC_HEADERS} # LinkDef.h # seemingly not being used by ROOT_STANDARD_LIBRARY_PACKAGE call src/Domains.h + src/HS3Document.h + src/HS3Schema.h src/JSONIOUtils.h src/static_execute.h ) diff --git a/roofit/hs3/src/HS3Document.h b/roofit/hs3/src/HS3Document.h new file mode 100644 index 0000000000000..b20611d9face2 --- /dev/null +++ b/roofit/hs3/src/HS3Document.h @@ -0,0 +1,299 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef RooFit_HS3Document_h +#define RooFit_HS3Document_h + +#include +#include +#include +#include +#include +#include + +namespace RooFit { + +namespace Detail { +class JSONNode; +class JSONTree; +} // namespace Detail + +/// Types that represent an HS3 document. They follow the structure of the HS3 standard, and this header must not depend +/// on RooFit: creating RooFit objects from a document is a separate step. +namespace HS3 { + +struct Version { + int major = 0; + int minor = 0; + int patch = 0; +}; + +inline bool operator<(Version const &l, Version const &r) +{ + if (l.major != r.major) { + return l.major < r.major; + } + if (l.minor != r.minor) { + return l.minor < r.minor; + } + return l.patch < r.patch; +} + +/// A name that says what it refers to, so that for example a DataRef can't be used as a DomainRef. +template +struct Ref { + std::string name; +}; + +using DistributionRef = Ref; +using DataRef = Ref; +using DomainRef = Ref; +using LikelihoodRef = Ref; +using ParameterPointRef = Ref; +using ParameterRef = Ref; + +struct Package { + std::string name; + std::string version; +}; + +struct Metadata { + std::string hs3Version; ///< As it was written in the document, empty for fragments of a document. + Version version; ///< The parsed version. Versions that can't be parsed count as the oldest one. + std::vector packages; + std::vector authors; + std::vector publications; + std::optional description; +}; + +struct RegularBinning { + int nbins = 0; +}; +struct EdgesBinning { + std::vector edges; +}; +using Binning = std::variant; + +/// The axis of a domain. Infinite bounds are written as null, and bounds can be missing in older documents. The binning +/// is a RooFit extension. +struct DomainAxis { + std::string name; + std::optional min; + std::optional max; + std::optional binning; +}; + +/// The axis of point data or unbinned data, which can't have a binning. +struct ContinuousAxis { + std::string name; + std::optional min; + std::optional max; +}; + +/// The axis of binned data. +struct BinnedAxis { + std::string name; + double min = 0.0; + double max = 0.0; + Binning binning; +}; + +struct CategoricalAxis { + std::string name; + std::vector categories; +}; + +struct Domain { + std::string name; + std::vector> axes; +}; + +struct ParameterValue { + std::string name; + std::optional> value; ///< A string selects the state of a category. + bool isConst = false; + // RooFit extensions that were written by older versions: + std::optional error; + std::optional relError; + std::optional nbins; +}; + +struct ParameterPoint { + std::string name; + std::vector parameters; +}; + +struct GaussianUncertainty { + std::vector sigma; + std::vector> correlation; ///< Empty if there is no correlation. +}; + +struct PointData { + double value = 0.0; + std::optional uncertainty; + std::vector axes; +}; + +struct UnbinnedData { + std::vector> axes; + /// The coordinates, row by row. A value on a categorical axis is the index in the list of categories of that axis. + std::vector entries; + std::optional> weights; + bool hasEntriesUncertainties = false; + + std::size_t numEntries() const { return axes.empty() ? 0 : entries.size() / axes.size(); } +}; + +struct BinnedData { + std::vector> axes; + std::vector contents; + std::optional uncertainty; +}; + +struct Data { + std::string name; + std::variant body; +}; + +struct LikelihoodTerm { + DistributionRef distribution; + std::variant data; +}; + +struct Likelihood { + std::string name; + std::vector terms; + std::vector auxDistributions; +}; + +struct Analysis { + std::string name; + LikelihoodRef likelihood; + std::vector parametersOfInterest; + std::vector domains; + std::optional init; + std::optional prior; +}; + +/// A distribution or function that is not interpreted by the document, but by the importers that are registered for +/// its type in RooFit::JSONIO::importers(), or by the factory expressions. +struct GenericObject { + std::string type; + Detail::JSONNode const *node = nullptr; +}; + +struct Distribution { + std::string name; + GenericObject body; +}; + +struct Function { + std::string name; + GenericObject body; +}; + +// The following types are stored in the "misc" section, for which the HS3 standard doesn't prescribe a format. + +struct Attributes { + std::map dict; + std::vector tags; +}; + +struct CombinedDistribution { + std::string indexCat; + std::vector labels; + std::vector indices; + std::vector distributions; +}; + +struct CombinedDataset { + std::string indexCat; + std::vector labels; + std::vector indices; +}; + +struct ModelConfigInfo { + std::optional mcName; + std::optional pdfName; + std::optional combinedDataName; +}; + +struct StepWidth { + std::string name; + double width = 0.0; +}; + +/// The content of misc.ROOT_internal and misc.minimization. +struct RooFitInternal { + std::map attributes; + std::map combinedDistributions; + std::map combinedDatasets; + std::map modelConfigs; + std::vector stepWidths; +}; + +struct Diagnostic { + enum class Severity { + Warning, + Error + }; + Severity severity = Severity::Warning; + std::string path; ///< Where in the document, for example "distributions[3](pdf).x". + std::string message; +}; + +using Diagnostics = std::vector; + +/// Whether there is a diagnostic with error severity. +bool hasErrors(Diagnostics const &diagnostics); + +} // namespace HS3 + +/// An HS3 document. It is move-only, because generic objects refer to the JSON tree that the document was read from. +class HS3Document { +public: + HS3::Metadata metadata; + std::vector distributions; + std::vector functions; + std::vector data; + std::vector likelihoods; + std::vector domains; + std::vector parameterPoints; + std::vector analyses; + HS3::RooFitInternal rooFit; + + HS3Document() = default; + HS3Document(HS3Document &&) = default; + HS3Document &operator=(HS3Document &&) = default; + HS3Document(HS3Document const &) = delete; + HS3Document &operator=(HS3Document const &) = delete; +}; + +namespace HS3 { + +/// Options for parseDocument(). +struct ParseOptions { + /// Fragments of a document can be imported without metadata. + bool requireMetadata = true; +}; + +/// Reads a document. The generic objects in the document refer to nodes of the JSON tree, so the tree must outlive the +/// document. Problems are reported in `diagnostics`, and if there are errors the document must not be used. +HS3Document parseDocument(Detail::JSONNode const &root, Diagnostics &diagnostics, ParseOptions const &options = {}); + +/// Reads the leading "MAJOR.MINOR" and the optional ".PATCH" of an HS3 version string. +bool parseVersion(std::string const &version, Version &out); + +} // namespace HS3 +} // namespace RooFit + +#endif diff --git a/roofit/hs3/src/HS3Parser.cxx b/roofit/hs3/src/HS3Parser.cxx new file mode 100644 index 0000000000000..8e5bfba06ea4b --- /dev/null +++ b/roofit/hs3/src/HS3Parser.cxx @@ -0,0 +1,732 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "HS3Schema.h" + +#include +#include +#include +#include +#include +#include + +using RooFit::Detail::JSONNode; + +namespace RooFit { +namespace HS3 { + +constexpr auto supportedVersion = "0.3"; + +bool hasErrors(Diagnostics const &diagnostics) +{ + return std::any_of(diagnostics.begin(), diagnostics.end(), + [](Diagnostic const &d) { return d.severity == Diagnostic::Severity::Error; }); +} + +std::string Cursor::path() const +{ + if (!_parent) { + return ""; + } + std::string out = _parent->path(); + if (_key) { + if (!out.empty()) { + out += "."; + } + out += _key; + return out; + } + out += "[" + std::to_string(_index) + "]"; + if (_node.is_map() && _node.has_child("name") && !_node["name"].is_container() && !_node["name"].is_null()) { + out += "(" + _node["name"].val() + ")"; + } + return out; +} + +void Cursor::error(std::string const &message) const +{ + _diagnostics.push_back({Diagnostic::Severity::Error, path(), message}); +} + +void Cursor::warning(std::string const &message) const +{ + _diagnostics.push_back({Diagnostic::Severity::Warning, path(), message}); +} + +void readValue(Cursor const &c, std::string &out) +{ + if (c.node().is_container() || c.node().is_null()) { + c.error("must be a string"); + return; + } + out = c.node().val(); +} + +void readValue(Cursor const &c, double &out) +{ + if (!c.node().is_number()) { + c.error("must be a number"); + return; + } + out = c.node().val_double(); +} + +void readValue(Cursor const &c, int &out) +{ + // Read through double, such that an integer that is encoded as a JSON float like 1e6 is accepted. + const double value = c.node().is_number() ? c.node().val_double() : 0.5; + if (value != std::floor(value) || std::abs(value) > static_cast(std::numeric_limits::max())) { + c.error("must be an integer"); + return; + } + out = static_cast(value); +} + +void readValue(Cursor const &c, bool &out) +{ + if (c.node().is_container() || c.node().is_null()) { + c.error("must be a boolean"); + return; + } + try { + out = c.node().val_bool(); + } catch (std::exception const &) { + c.error("must be a boolean"); + } +} + +bool parseVersion(std::string const &version, Version &out) +{ + std::size_t pos = 0; + auto readNumber = [&](int &number) { + const std::size_t begin = pos; + while (pos < version.size() && std::isdigit(static_cast(version[pos]))) { + ++pos; + } + // Reject empty and absurdly long numbers, which std::stoi could not represent. + if (pos == begin || pos - begin > 9) { + return false; + } + number = std::stoi(version.substr(begin, pos - begin)); + return true; + }; + Version parsed; + if (!readNumber(parsed.major) || pos >= version.size() || version[pos] != '.') { + return false; + } + ++pos; + if (!readNumber(parsed.minor)) { + return false; + } + if (pos < version.size() && version[pos] == '.') { + ++pos; + if (!readNumber(parsed.patch)) { + parsed.patch = 0; + } + } + out = parsed; + return true; +} + +namespace { + +// Read a bound of an axis. Infinite bounds are written as null. +void readBound(Reader &ar, const char *key, std::optional &out, double valueIfNull) +{ + if (!ar.has(key)) { + return; + } + Cursor c{ar.cursor().node()[key], ar.cursor(), key}; + if (c.node().is_null()) { + out = valueIfNull; + return; + } + out.emplace(); + readValue(c, *out); +} + +constexpr double infinity = std::numeric_limits::infinity(); + +// Read bin edges, which must be at least two strictly ascending numbers. +void readEdges(Reader &ar, std::string const &axisName, std::vector &edges) +{ + Cursor c{ar.cursor().node()["edges"], ar.cursor(), "edges"}; + const std::string context = "\"edges\" of axis '" + axisName + "' must "; + if (!c.node().is_seq()) { + c.error(context + "be a list"); + return; + } + readValue(c, edges); + for (std::size_t i = 1; i < edges.size(); ++i) { + if (!(edges[i] > edges[i - 1])) { + c.error(context + "be strictly ascending"); + return; + } + } + if (edges.size() < 2) { + c.error(context + "contain at least two values"); + } +} + +} // namespace + +void describe(Reader &ar, Package &p) +{ + ar("name", p.name); + ar("version", p.version); +} + +void describe(Reader &ar, DomainAxis &axis) +{ + ar("name", axis.name); + readBound(ar, "min", axis.min, -infinity); + readBound(ar, "max", axis.max, infinity); + if (ar.has("edges")) { + // Not necessarily ascending, because RooFit also writes parameterized ranges as edges. + EdgesBinning binning; + ar("edges", binning.edges); + if (!binning.edges.empty()) { + if (!axis.min) { + axis.min = binning.edges.front(); + } + if (!axis.max) { + axis.max = binning.edges.back(); + } + } + axis.binning = std::move(binning); + } else if (ar.has("nbins")) { + RegularBinning binning; + ar("nbins", binning.nbins); + if (binning.nbins != 0) { + axis.binning = binning; + } + } +} + +void describe(Reader &ar, ContinuousAxis &axis) +{ + ar("name", axis.name); + readBound(ar, "min", axis.min, -infinity); + readBound(ar, "max", axis.max, infinity); +} + +void describe(Reader &ar, BinnedAxis &axis) +{ + ar("name", axis.name); + if (ar.has("edges")) { + EdgesBinning binning; + readEdges(ar, axis.name, binning.edges); + if (binning.edges.size() >= 2) { + axis.min = binning.edges.front(); + axis.max = binning.edges.back(); + } + axis.binning = std::move(binning); + return; + } + ar("min", axis.min); + ar("max", axis.max); + RegularBinning binning; + ar("nbins", binning.nbins); + axis.binning = binning; +} + +void describe(Reader &ar, CategoricalAxis &axis) +{ + ar("name", axis.name); + ar("categories", axis.categories); +} + +namespace { + +// The kind of an axis is determined by its components. +template +void readAxis(Cursor const &c, std::variant &out) +{ + if (!c.node().is_map()) { + c.error("must be a struct"); + return; + } + Reader reader{c}; + if (reader.has("categories")) { + CategoricalAxis axis; + describe(reader, axis); + out = std::move(axis); + } else { + ContinuousAxis_t axis; + describe(reader, axis); + out = std::move(axis); + } +} + +} // namespace + +void readValue(Cursor const &c, std::variant &out) +{ + readAxis(c, out); +} + +void readValue(Cursor const &c, std::variant &out) +{ + readAxis(c, out); +} + +void readValue(Cursor const &c, std::variant &out) +{ + readAxis(c, out); +} + +void describe(Reader &ar, Domain &domain) +{ + ar("name", domain.name); + std::string type; + if (ar.has("type")) { + ar("type", type); + } + if (type != "product_domain") { + ar.cursor().error("only domains of type \"product_domain\" are currently supported!"); + return; + } + ar.list("axes", domain.axes); +} + +void readValue(Cursor const &c, ParameterValue &p) +{ + if (!c.node().is_map()) { + c.error("must be a struct"); + return; + } + Reader ar{c}; + ar("name", p.name); + if (ar.has("value")) { + Cursor crsr{ar.cursor().node()["value"], ar.cursor(), "value"}; + if (crsr.node().is_number()) { + p.value = crsr.node().val_double(); + } else { + std::string label; + readValue(crsr, label); + p.value = std::move(label); + } + } + std::optional isConst; + ar("const", isConst); + p.isConst = isConst.value_or(false); + ar("err", p.error); + ar("relErr", p.relError); + ar("nbins", p.nbins); +} + +void describe(Reader &ar, ParameterPoint &point) +{ + ar("name", point.name); + ar.list("parameters", point.parameters); +} + +void describe(Reader &ar, PointData &data) +{ + ar("value", data.value); + ar("uncertainty", data.uncertainty); + ar.list("axes", data.axes); +} + +void describe(Reader &ar, UnbinnedData &data) +{ + ar.list("axes", data.axes); + ar("weights", data.weights); + data.hasEntriesUncertainties = ar.has("entries_uncertainties"); + + if (!ar.has("entries")) { + ar.cursor().error("does not define the required component \"entries\""); + return; + } + Cursor entries{ar.cursor().node()["entries"], ar.cursor(), "entries"}; + if (!entries.node().is_seq()) { + entries.error("key 'entries' is not a list!"); + return; + } + if (data.weights && data.weights->size() != entries.node().num_children()) { + ar.cursor().error("inconsistent number of entries and weights!"); + return; + } + data.entries.reserve(entries.node().num_children() * data.axes.size()); + std::size_t iRow = 0; + for (JSONNode const &row : entries.node().children()) { + Cursor rowCursor{row, entries, iRow}; + if (!row.is_seq()) { + rowCursor.error("coordinate point '" + std::to_string(iRow) + "' is not a list!"); + return; + } + if (row.num_children() != data.axes.size()) { + rowCursor.error("inconsistent number of entries and observables!"); + return; + } + std::size_t iAxis = 0; + for (JSONNode const &value : row.children()) { + Cursor valueCursor{value, rowCursor, iAxis}; + if (auto const *category = std::get_if(&data.axes[iAxis])) { + std::string label; + readValue(valueCursor, label); + auto found = std::find(category->categories.begin(), category->categories.end(), label); + if (found == category->categories.end()) { + valueCursor.error("'" + label + "' is not a category of the axis '" + category->name + "'"); + return; + } + data.entries.push_back(static_cast(std::distance(category->categories.begin(), found))); + } else { + double coordinate = 0.0; + readValue(valueCursor, coordinate); + data.entries.push_back(coordinate); + } + ++iAxis; + } + ++iRow; + } +} + +void describe(Reader &ar, GaussianUncertainty &uncertainty) +{ + if (ar.has("type")) { + std::string type; + ar("type", type); + if (type != "gaussian_uncertainty") { + ar.cursor().error("unsupported type of uncertainty '" + type + "'"); + } + } + ar("sigma", uncertainty.sigma); + if (ar.has("correlation")) { + Cursor c{ar.cursor().node()["correlation"], ar.cursor(), "correlation"}; + // Besides a matrix, it can be the number 0 to indicate that there is no correlation. + if (c.node().is_seq()) { + readValue(c, uncertainty.correlation); + } else if (!c.node().is_number() || c.node().val_double() != 0.0) { + c.error("must be a matrix or 0"); + } + } +} + +void describe(Reader &ar, BinnedData &data) +{ + ar.list("axes", data.axes); + ar("contents", data.contents); + ar("uncertainty", data.uncertainty); + // RooFit extension: older versions read the errors like in the samples of a HistFactory distribution. + if (ar.has("errors")) { + Cursor errors{ar.cursor().node()["errors"], ar.cursor(), "errors"}; + if (!errors.node().is_seq()) { + errors.error("errors are not in list form"); + } else if (!data.uncertainty) { + data.uncertainty.emplace(); + readValue(errors, data.uncertainty->sigma); + } + } +} + +void readValue(Cursor const &c, Data &out) +{ + if (!c.node().is_map()) { + c.error("must be a struct"); + return; + } + Reader ar{c}; + ar("name", out.name); + std::string type; + ar("type", type); + if (type == "binned") { + BinnedData body; + describe(ar, body); + out.body = std::move(body); + } else if (type == "unbinned") { + UnbinnedData body; + describe(ar, body); + out.body = std::move(body); + } else if (type == "point") { + PointData body; + describe(ar, body); + out.body = std::move(body); + } else if (ar.has("type")) { + c.error("unknown type of dataset '" + type + "'"); + } +} + +void describe(Reader &ar, Likelihood &likelihood) +{ + ar("name", likelihood.name); + ar.list("aux_distributions", likelihood.auxDistributions); + + std::vector distributions; + ar("distributions", distributions); + if (!ar.has("data")) { + ar.cursor().error("does not define the required component \"data\""); + return; + } + Cursor data{ar.cursor().node()["data"], ar.cursor(), "data"}; + if (!data.node().is_seq()) { + data.error("must be an array"); + return; + } + if (data.node().num_children() != distributions.size()) { + ar.cursor().error("\"distributions\" and \"data\" must have the same length"); + return; + } + likelihood.terms.reserve(distributions.size()); + std::size_t i = 0; + for (JSONNode const &node : data.node().children()) { + LikelihoodTerm term; + term.distribution = std::move(distributions[i]); + // Older versions of RooFit used the string "0" to say that there is no dataset. + if (node.is_number() || (!node.is_container() && !node.is_null() && node.val() == "0")) { + term.data = node.is_number() ? node.val_double() : 0.0; + } else { + DataRef ref; + readValue(Cursor{node, data, i}, ref); + term.data = std::move(ref); + } + likelihood.terms.push_back(std::move(term)); + ++i; + } +} + +void describe(Reader &ar, Analysis &analysis) +{ + ar("name", analysis.name); + ar("likelihood", analysis.likelihood); + // HS3 v0.2.9 used the singular keys "parameter_of_interest" (a list) and "domain" (a single name). They are still + // accepted, but the plural keys take precedence. + auto isLegacy = [&](const char *key, const char *legacyKey) { + if (ar.has(key) || !ar.has(legacyKey)) { + return false; + } + ar.cursor().warning(std::string("the key \"") + legacyKey + "\" of analysis '" + analysis.name + + "' is deprecated, please use \"" + key + "\" instead."); + return true; + }; + ar.list(isLegacy("parameters_of_interest", "parameter_of_interest") ? "parameter_of_interest" + : "parameters_of_interest", + analysis.parametersOfInterest); + if (!isLegacy("domains", "domain")) { + ar.list("domains", analysis.domains); + } else if (ar.cursor().node()["domain"].is_seq()) { + ar.list("domain", analysis.domains); + } else { + analysis.domains.emplace_back(); + ar("domain", analysis.domains.back()); + } + ar("init", analysis.init); + ar("prior", analysis.prior); +} + +namespace { + +template +void readGenericObject(Cursor const &c, Object_t &out) +{ + if (!c.node().is_map()) { + c.error("must be a struct"); + return; + } + Reader ar{c}; + ar("name", out.name); + ar("type", out.body.type); + out.body.node = &c.node(); +} + +} // namespace + +void readValue(Cursor const &c, Distribution &out) +{ + readGenericObject(c, out); +} + +void readValue(Cursor const &c, Function &out) +{ + readGenericObject(c, out); +} + +void describe(Reader &ar, Attributes &attributes) +{ + if (ar.has("dict")) { + Cursor dict{ar.cursor().node()["dict"], ar.cursor(), "dict"}; + if (!dict.node().is_map()) { + dict.error("must be a struct"); + return; + } + for (JSONNode const &item : dict.node().children()) { + readValue(Cursor{item, dict, item.key().c_str()}, attributes.dict[item.key()]); + } + } + ar.list("tags", attributes.tags); +} + +void describe(Reader &ar, CombinedDistribution &info) +{ + ar("index_cat", info.indexCat); + ar("labels", info.labels); + ar("indices", info.indices); + ar("distributions", info.distributions); + if (info.labels.size() != info.indices.size() || info.labels.size() != info.distributions.size()) { + ar.cursor().error("mismatch in number of labels, indices and distributions!"); + } +} + +void describe(Reader &ar, CombinedDataset &info) +{ + ar("index_cat", info.indexCat); + ar("labels", info.labels); + ar("indices", info.indices); + if (info.labels.size() != info.indices.size()) { + ar.cursor().error("mismatch in number of indices and labels!"); + } +} + +void describe(Reader &ar, ModelConfigInfo &info) +{ + ar("mcName", info.mcName); + ar("pdfName", info.pdfName); + ar("combined_data_name", info.combinedDataName); +} + +namespace { + +template +void readMap(Reader &ar, const char *key, Map_t &out) +{ + if (!ar.has(key)) { + return; + } + Cursor c{ar.cursor().node()[key], ar.cursor(), key}; + if (c.node().is_null()) { + return; + } + if (!c.node().is_map()) { + c.error("must be a struct"); + return; + } + for (JSONNode const &item : c.node().children()) { + const std::string itemKey = item.key(); + readValue(Cursor{item, c, itemKey.c_str()}, out[itemKey]); + } +} + +// The content of "misc" is not supposed to change the result of evaluating the model, so problems with the step widths +// are only warnings. +void readStepWidths(Cursor const &misc, std::vector &out) +{ + JSONNode const *node = misc.node().find("minimization", "parameter_stepwidths"); + if (!node) { + return; + } + if (!node->is_seq()) { + misc.warning("minimization.parameter_stepwidths is not a sequence, skipping."); + return; + } + for (JSONNode const &item : node->children()) { + if (!item.is_map() || !item.has_child("name") || !item.has_child("step_width") || + !item["step_width"].is_number() || item["name"].is_container() || item["name"].is_null()) { + misc.warning("skipping malformed parameter_stepwidths entry."); + continue; + } + out.push_back({item["name"].val(), item["step_width"].val_double()}); + } +} + +void readMisc(Cursor const &root, RooFitInternal &out) +{ + if (!root.node().has_child("misc")) { + return; + } + Cursor misc{root.node()["misc"], root, "misc"}; + if (!misc.node().is_map()) { + return; + } + readStepWidths(misc, out.stepWidths); + if (!misc.node().has_child("ROOT_internal")) { + return; + } + Cursor internal{misc.node()["ROOT_internal"], misc, "ROOT_internal"}; + if (!internal.node().is_map()) { + return; + } + Reader ar{internal}; + readMap(ar, "attributes", out.attributes); + readMap(ar, "combined_distributions", out.combinedDistributions); + readMap(ar, "combined_datasets", out.combinedDatasets); + readMap(ar, "ModelConfigs", out.modelConfigs); +} + +void readMetadata(Cursor const &root, Metadata &out, ParseOptions const &options) +{ + JSONNode const *metadata = root.node().find("metadata"); + JSONNode const *versionNode = metadata ? metadata->find("hs3_version") : nullptr; + if (!versionNode) { + if (options.requireMetadata) { + std::stringstream ss; + ss << "The HS3 version is missing in the JSON!\n" + << "Please include the HS3 version in the metadata field, e.g.:\n" + << " \"metadata\" :\n" + << " {\n" + << " \"hs3_version\" : \"" << supportedVersion << "\"\n" + << " }"; + root.error(ss.str()); + } + return; + } + Cursor metadataCursor{*metadata, root, "metadata"}; + Cursor versionCursor{*versionNode, metadataCursor, "hs3_version"}; + if (versionNode->is_container() || versionNode->is_null() || versionNode->is_number()) { + versionCursor.error(std::string("The HS3 version in the metadata must be a string, e.g. \"") + supportedVersion + + "\""); + return; + } + out.hs3Version = versionNode->val(); + + Version supported; + parseVersion(supportedVersion, supported); + if (!parseVersion(out.hs3Version, out.version)) { + versionCursor.warning("cannot parse hs3_version '" + out.hs3Version + + "', expected a string of the form \"MAJOR.MINOR\"."); + } else if (supported < Version{out.version.major, out.version.minor}) { + versionCursor.warning("the document has hs3_version '" + out.hs3Version + + "', which is newer than the version supported by this RooFit ('" + supportedVersion + + "'). Some content might not be imported correctly."); + } + + Reader ar{metadataCursor}; + ar.list("packages", out.packages); + ar.list("authors", out.authors); + ar.list("publications", out.publications); + ar("description", out.description); +} + +} // namespace + +HS3Document parseDocument(JSONNode const &root, Diagnostics &diagnostics, ParseOptions const &options) +{ + HS3Document doc; + Cursor cursor{root, diagnostics}; + readMetadata(cursor, doc.metadata, options); + if (hasErrors(diagnostics)) { + return doc; + } + + Reader ar{cursor}; + ar.list("distributions", doc.distributions); + ar.list("functions", doc.functions); + ar.list("data", doc.data); + ar.list("likelihoods", doc.likelihoods); + ar.list("domains", doc.domains); + ar.list("parameter_points", doc.parameterPoints); + ar.list("analyses", doc.analyses); + readMisc(cursor, doc.rooFit); + return doc; +} + +} // namespace HS3 +} // namespace RooFit diff --git a/roofit/hs3/src/HS3Schema.h b/roofit/hs3/src/HS3Schema.h new file mode 100644 index 0000000000000..006406f4ed1ae --- /dev/null +++ b/roofit/hs3/src/HS3Schema.h @@ -0,0 +1,184 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef RooFit_HS3Schema_h +#define RooFit_HS3Schema_h + +#include "HS3Document.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace RooFit { +namespace HS3 { + +/// A position in the JSON tree. Keeps track of how it got there, in order to create the paths in error messages only +/// when they are needed. +class Cursor { +public: + Cursor(Detail::JSONNode const &node, Diagnostics &diagnostics) : _node{node}, _diagnostics{diagnostics} {} + Cursor(Detail::JSONNode const &node, Cursor const &parent, const char *key) + : _node{node}, _diagnostics{parent._diagnostics}, _parent{&parent}, _key{key} + { + } + Cursor(Detail::JSONNode const &node, Cursor const &parent, std::size_t index) + : _node{node}, _diagnostics{parent._diagnostics}, _parent{&parent}, _index{index} + { + } + + Detail::JSONNode const &node() const { return _node; } + std::string path() const; + void error(std::string const &message) const; + void warning(std::string const &message) const; + +private: + Detail::JSONNode const &_node; + Diagnostics &_diagnostics; + Cursor const *_parent = nullptr; + const char *_key = nullptr; + std::size_t _index = 0; +}; + +/// Reads the components of a struct. The functions take the same arguments as the ones of the Writer, such that a +/// single function can describe both directions. +class Reader { +public: + explicit Reader(Cursor const &cursor) : _cursor{cursor} {} + + /// Read a required component, or an optional one if the field is a std::optional. + template + void operator()(const char *key, T &field); + template + void operator()(const char *key, std::optional &field); + + /// Read an array that can be omitted, which is equivalent to an empty one. + template + void list(const char *key, std::vector &field); + + /// Read the components of a struct that is nested in a component. + template + void nested(const char *key, Fn &&fn); + + bool has(const char *key) const { return _cursor.node().has_child(key); } + + Cursor const &cursor() const { return _cursor; } + +private: + Cursor const &_cursor; +}; + +void readValue(Cursor const &c, std::string &out); +void readValue(Cursor const &c, double &out); +void readValue(Cursor const &c, int &out); +void readValue(Cursor const &c, bool &out); +void readValue(Cursor const &c, ParameterValue &out); +void readValue(Cursor const &c, Data &out); + +template +void readValue(Cursor const &c, Ref &out) +{ + readValue(c, out.name); +} + +template +void readValue(Cursor const &c, std::vector &out) +{ + Detail::JSONNode const &node = c.node(); + if (!node.is_seq()) { + c.error("must be an array"); + return; + } + out.clear(); + out.reserve(node.num_children()); + std::size_t i = 0; + for (Detail::JSONNode const &child : node.children()) { + out.emplace_back(); + readValue(Cursor{child, c, i}, out.back()); + ++i; + } +} + +/// Everything that has a describe() function is read from a struct. +template +auto readValue(Cursor const &c, T &out) -> std::void_t(), out))> +{ + if (!c.node().is_map()) { + c.error("must be a struct"); + return; + } + Reader reader{c}; + describe(reader, out); +} + +template +void Reader::operator()(const char *key, T &field) +{ + if (!has(key)) { + _cursor.error(std::string("does not define the required component \"") + key + "\""); + return; + } + readValue(Cursor{_cursor.node()[key], _cursor, key}, field); +} + +template +void Reader::operator()(const char *key, std::optional &field) +{ + if (!has(key)) { + return; + } + Detail::JSONNode const &node = _cursor.node()[key]; + if (node.is_null()) { + return; + } + field.emplace(); + readValue(Cursor{node, _cursor, key}, *field); +} + +template +void Reader::list(const char *key, std::vector &field) +{ + if (!has(key)) { + return; + } + Detail::JSONNode const &node = _cursor.node()[key]; + if (node.is_null()) { + return; + } + readValue(Cursor{node, _cursor, key}, field); +} + +template +void Reader::nested(const char *key, Fn &&fn) +{ + if (!has(key)) { + _cursor.error(std::string("does not define the required component \"") + key + "\""); + return; + } + Cursor child{_cursor.node()[key], _cursor, key}; + if (!child.node().is_map()) { + child.error("must be a struct"); + return; + } + Reader reader{child}; + fn(reader); +} + +} // namespace HS3 +} // namespace RooFit + +#endif From 482002b35c52fe23618f9c4267f4b64ef78fc0af Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Thu, 5 Nov 2026 18:33:56 +0000 Subject: [PATCH 04/20] [RF][HS3] Import documents through their typed representation Parse the whole document before anything is created in the workspace, and use the typed representation for all top-level sections. Most of the import state moves out of the public header. The exported JSON doesn't change. Assisted-by: claude-code:claude-opus-5-5 --- .../hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h | 19 +- roofit/hs3/src/Domains.cxx | 77 +- roofit/hs3/src/Domains.h | 7 +- roofit/hs3/src/HS3Document.h | 4 +- roofit/hs3/src/HS3Parser.cxx | 21 +- roofit/hs3/src/JSONIOUtils.cxx | 4 - roofit/hs3/src/RooJSONFactoryWSTool.cxx | 990 ++++++++---------- roofit/hs3/test/testRooFitHS3.cxx | 1 + 8 files changed, 487 insertions(+), 636 deletions(-) diff --git a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h index 669da2ba3abb8..ac834fe5ba559 100644 --- a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h +++ b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h @@ -23,7 +23,6 @@ #include #include #include -#include #include namespace RooFit { @@ -32,6 +31,10 @@ namespace Detail { class Domains; } } // namespace JSONIO +namespace HS3 { +struct ParameterValue; +struct ToolImportState; +} // namespace HS3 } // namespace RooFit namespace RooStats { class ModelConfig; @@ -238,8 +241,8 @@ class RooJSONFactoryWSTool { void importAllNodes(const RooFit::Detail::JSONNode &n); - void importVariable(const RooFit::Detail::JSONNode &p); - void checkFixedValue(std::string const &name, const RooFit::Detail::JSONNode &p, std::string const &pointName); + void importVariable(RooFit::HS3::ParameterValue const &p); + void checkFixedValue(RooFit::HS3::ParameterValue const &p, std::string const &pointName); void warnIfFreeWithoutDomain(RooRealVar const &var); void importDependants(const RooFit::Detail::JSONNode &n); @@ -257,8 +260,6 @@ class RooJSONFactoryWSTool { std::map const *dataComponents); // member variables - const RooFit::Detail::JSONNode *_rootnodeInput = nullptr; - const RooFit::Detail::JSONNode *_attributesNode = nullptr; RooFit::Detail::JSONNode *_rootnodeOutput = nullptr; RooFit::Detail::JSONNode *_varsNode = nullptr; std::string _hs3VersionInput; @@ -266,16 +267,10 @@ class RooJSONFactoryWSTool { // objects to represent intermediate information std::unique_ptr _domains; + std::unique_ptr _import; std::vector _serversToExport; std::vector _serversToDelete; - // Name-keyed indices over the top-level "functions" and "distributions" - // sequences of the input JSON. Built once at the start of importAllNodes() - // so that requestImpl() lookups become O(1) instead of an O(N) scan over - // every sibling node. - std::unordered_map _functionsByName; - std::unordered_map _distributionsByName; - // The objects that are currently being imported, innermost last, to detect circular dependencies. std::vector _importStack; }; diff --git a/roofit/hs3/src/Domains.cxx b/roofit/hs3/src/Domains.cxx index b02197eb38471..ac401887d0561 100644 --- a/roofit/hs3/src/Domains.cxx +++ b/roofit/hs3/src/Domains.cxx @@ -11,6 +11,7 @@ */ #include "Domains.h" +#include "HS3Document.h" #include "JSONIOUtils.h" #include @@ -93,15 +94,11 @@ void Domains::writeVariable(RooRealVar &var) const } } -void Domains::readJSON(RooFit::Detail::JSONNode const &node) +void Domains::read(std::vector const &domains) { // The "default_domain" is written by RooFit, but it is not required by the HS3 standard. - for (auto &domain : node.children()) { - if (!domain.has_child("name")) { - RooJSONFactoryWSTool::error("encountered domain without \"name\""); - } - auto &name = domain["name"]; - _map[name.val()].readJSON(domain); + for (auto const &domain : domains) { + _map[domain.name].read(domain); } // A parameter with a single value has no other domain: every domain that lists it must give it the same value. for (auto const &[domainName, domain] : _map) { @@ -269,61 +266,35 @@ void Domains::ProductDomain::readFixedValue(const char *name, double value) elem.fixedValue = value; } -void Domains::ProductDomain::readJSON(RooFit::Detail::JSONNode const &node) +void Domains::ProductDomain::read(RooFit::HS3::Domain const &domain) { - if (!node.has_child("type") || node["type"].val() != "product_domain") { - RooJSONFactoryWSTool::error("only domains of type \"product_domain\" are currently supported!"); - } - for (auto const &varNode : node["axes"].children()) { - const std::string varName = RooJSONFactoryWSTool::name(varNode); - auto &elem = _map[varName]; - - if (varNode.has_child(fixedValueKey)) { - // The domain of the parameter is the set with this single value: no range, binning or categories. - for (const char *key : {"min", "max", "nbins", "edges", "categories"}) { - if (varNode.has_child(key)) { - RooJSONFactoryWSTool::error("the axis '" + varName + "' of the domain '" + - RooJSONFactoryWSTool::name(node) + "' has \"" + fixedValueKey + "\" and \"" + - key + "\", which exclude each other"); - } - } - auto const &valueNode = varNode[fixedValueKey]; - if (!valueNode.is_number()) { - RooJSONFactoryWSTool::error("the \"" + std::string(fixedValueKey) + "\" of the axis '" + varName + - "' must be a number"); - } - elem = ProductDomainElement{}; - elem.hasFixedValue = true; - elem.fixedValue = valueNode.val_double(); + for (auto const &axisVariant : domain.axes) { + auto const *axis = std::get_if(&axisVariant); + if (!axis) { + continue; + } + if (axis->fixedValue) { + readFixedValue(axis->name.c_str(), *axis->fixedValue); continue; } + auto &elem = _map[axis->name]; - if (varNode.has_child("min")) { - elem.min = readBound(varNode, "min", -RooNumber::infinity()); + if (axis->min) { + elem.min = *axis->min; elem.hasMin = true; } - if (varNode.has_child("max")) { - elem.max = readBound(varNode, "max", RooNumber::infinity()); + if (axis->max) { + elem.max = *axis->max; elem.hasMax = true; } - if (varNode.has_child("edges")) { + if (!axis->binning) { + continue; + } + if (auto const *edges = std::get_if(&*axis->binning)) { elem.hasNBins = false; - elem.edges.clear(); - for (auto const &edge : varNode["edges"].children()) { - elem.edges.push_back(edge.val_double()); - } - if (!elem.edges.empty()) { - if (!elem.hasMin) { - elem.min = elem.edges.front(); - elem.hasMin = true; - } - if (!elem.hasMax) { - elem.max = elem.edges.back(); - elem.hasMax = true; - } - } - } else if (varNode.has_child("nbins")) { - elem.nBins = varNode["nbins"].val_int(); + elem.edges = edges->edges; + } else { + elem.nBins = std::get(*axis->binning).nbins; elem.hasNBins = elem.nBins != 0; elem.edges.clear(); } diff --git a/roofit/hs3/src/Domains.h b/roofit/hs3/src/Domains.h index b0825c0d8198c..96f5b7dee6781 100644 --- a/roofit/hs3/src/Domains.h +++ b/roofit/hs3/src/Domains.h @@ -26,6 +26,9 @@ namespace Detail { class JSONNode; class JSONTree; } // namespace Detail +namespace HS3 { +struct Domain; +} } // namespace RooFit namespace RooFit { @@ -41,7 +44,7 @@ class Domains { // The single value of the parameter `name` if a domain has one for it, otherwise nullptr. double const *fixedValue(const char *name) const; - void readJSON(RooFit::Detail::JSONNode const &); + void read(std::vector const &); void writeJSON(RooFit::Detail::JSONNode &) const; bool hasVariable(const char *name) const; @@ -56,7 +59,7 @@ class Domains { void readVariable(const char *name, RooAbsBinning const &binning); void writeVariable(RooRealVar &) const; - void readJSON(RooFit::Detail::JSONNode const &); + void read(RooFit::HS3::Domain const &); void writeJSON(RooFit::Detail::JSONNode &) const; bool hasVariable(const char *name) const; diff --git a/roofit/hs3/src/HS3Document.h b/roofit/hs3/src/HS3Document.h index b20611d9face2..8c8e5962e8c47 100644 --- a/roofit/hs3/src/HS3Document.h +++ b/roofit/hs3/src/HS3Document.h @@ -84,9 +84,10 @@ struct EdgesBinning { using Binning = std::variant; /// The axis of a domain. Infinite bounds are written as null, and bounds can be missing in older documents. The binning -/// is a RooFit extension. +/// is a RooFit extension. A fixed axis has a single value instead, which excludes bounds and a binning. struct DomainAxis { std::string name; + std::optional fixedValue; std::optional min; std::optional max; std::optional binning; @@ -206,6 +207,7 @@ struct Function { struct Attributes { std::map dict; std::vector tags; + bool isConstVar = false; ///< How ROOT versions before HS3 0.3 marked named RooConstVars. }; struct CombinedDistribution { diff --git a/roofit/hs3/src/HS3Parser.cxx b/roofit/hs3/src/HS3Parser.cxx index 8e5bfba06ea4b..8e79cd5469107 100644 --- a/roofit/hs3/src/HS3Parser.cxx +++ b/roofit/hs3/src/HS3Parser.cxx @@ -11,6 +11,7 @@ */ #include "HS3Schema.h" +#include "JSONIOUtils.h" #include #include @@ -188,6 +189,18 @@ void describe(Reader &ar, Package &p) void describe(Reader &ar, DomainAxis &axis) { ar("name", axis.name); + if (ar.has(fixedValueKey)) { + // The domain of the parameter is the set with this single value: no range, binning or categories. + for (const char *key : {"min", "max", "nbins", "edges", "categories"}) { + if (ar.has(key)) { + ar.cursor().error("the axis '" + axis.name + "' has \"" + fixedValueKey + "\" and \"" + key + + "\", which exclude each other"); + } + } + // Not read as an optional component, for which null would be the same as leaving it out. + ar(fixedValueKey, axis.fixedValue.emplace()); + return; + } readBound(ar, "min", axis.min, -infinity); readBound(ar, "max", axis.max, infinity); if (ar.has("edges")) { @@ -256,7 +269,8 @@ void readAxis(Cursor const &c, std::variant & return; } Reader reader{c}; - if (reader.has("categories")) { + // A fixed axis is not categorical. That it has no categories is checked when it is read. + if (reader.has("categories") && !(std::is_same_v && reader.has(fixedValueKey))) { CategoricalAxis axis; describe(reader, axis); out = std::move(axis); @@ -563,6 +577,11 @@ void describe(Reader &ar, Attributes &attributes) } } ar.list("tags", attributes.tags); + if (ar.has("is_const_var")) { + int isConstVar = 0; + ar("is_const_var", isConstVar); + attributes.isConstVar = isConstVar == 1; + } } void describe(Reader &ar, CombinedDistribution &info) diff --git a/roofit/hs3/src/JSONIOUtils.cxx b/roofit/hs3/src/JSONIOUtils.cxx index dd634f469444e..757c9a9ba02fc 100644 --- a/roofit/hs3/src/JSONIOUtils.cxx +++ b/roofit/hs3/src/JSONIOUtils.cxx @@ -79,10 +79,6 @@ std::unique_ptr varJSONString(const JSONNode &treeRoot if (isConstant && !isRange) { _parameters["const"] << true; - JSONNode &_misc = n["misc"].set_map(); - JSONNode &rootInternal = _misc["ROOT_internal"].set_map(); - JSONNode &_var = rootInternal[varName].set_map(); - _var["tags"] << "Constant"; } return jsonDict; diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index a6eadcabd230e..2de5d34341ffe 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -30,6 +30,7 @@ #include "JSONIOUtils.h" #include "Domains.h" +#include "HS3Schema.h" #include @@ -108,19 +109,61 @@ tool.writedoc("hs3.tex") constexpr auto hs3VersionTag = "0.3"; +using RooFit::HS3Document; using RooFit::Detail::JSONNode; using RooFit::Detail::JSONTree; +namespace HS3 = RooFit::HS3; + +/// What the tool needs to know about the document that it is importing. +struct RooFit::HS3::ToolImportState { + explicit ToolImportState(HS3Document const &doc) : document{doc} + { + // Without these indices, every cross-reference resolved during import (e.g. dependencies of a + // PiecewiseInterpolation, or factory-expression arguments) would need a linear scan over all sibling objects, + // which becomes O(N^2) on workspaces with thousands of entries. + for (auto const &function : doc.functions) { + functions.emplace(function.name, &function); + } + for (auto const &distribution : doc.distributions) { + distributions.emplace(distribution.name, &distribution); + } + for (auto const &point : doc.parameterPoints) { + if (point.name == "default_values") { + defaultValues = &point; + for (auto const &p : point.parameters) { + defaultValuesByName.emplace(p.name, &p); + } + break; + } + } + } + + HS3Document const &document; + std::unordered_map functions; + std::unordered_map distributions; + HS3::ParameterPoint const *defaultValues = nullptr; + std::unordered_map defaultValuesByName; +}; + namespace { -std::vector valsToStringVec(JSONNode const &node) +// Forward the warnings to the message service, and throw if there are errors. +void report(HS3::Diagnostics const &diagnostics) { - std::vector out; - out.reserve(node.num_children()); - for (JSONNode const &elem : node.children()) { - out.push_back(elem.val()); + std::string errors; + for (HS3::Diagnostic const &diagnostic : diagnostics) { + const std::string text = + diagnostic.path.empty() ? diagnostic.message : diagnostic.path + ": " + diagnostic.message; + if (diagnostic.severity == HS3::Diagnostic::Severity::Error) { + errors += (errors.empty() ? "" : "\n") + text; + } else { + RooJSONFactoryWSTool::warning("RooFitHS3: " + text); + } + } + if (!errors.empty()) { + RooJSONFactoryWSTool::error(errors); } - return out; } // True if the number of components in `data` matches the number of categories in `pdf`. @@ -140,37 +183,22 @@ bool isNumber(const std::string &str) return (stream >> value) && stream.eof(); } -// Configure `v` (value, error, binning, constness) from the JSON node `p`. -void configureVariable(RooFit::JSONIO::Detail::Domains &domains, const JSONNode &p, RooRealVar &v) +// Configure `v` (value, error, binning, constness) from the parameter `p`. +void configureVariable(RooFit::JSONIO::Detail::Domains &domains, HS3::ParameterValue const &p, RooRealVar &v) { - if (!p.has_child("name")) { - RooJSONFactoryWSTool::error("cannot instantiate variable without \"name\"!"); + if (p.value) { + if (auto const *value = std::get_if(&*p.value)) { + v.setVal(*value); + } } - if (auto n = p.find("value")) - v.setVal(n->val_double()); domains.writeVariable(v); - if (auto n = p.find("nbins")) - v.setBins(n->val_int()); - if (auto n = p.find("relErr")) - v.setError(v.getVal() * n->val_double()); - if (auto n = p.find("err")) - v.setError(n->val_double()); - if (auto n = p.find("const")) { - v.setConstant(n->val_bool()); - } else { - v.setConstant(false); - } -} - -JSONNode const *getVariablesNode(JSONNode const &rootNode) -{ - auto paramPointsNode = rootNode.find("parameter_points"); - if (!paramPointsNode) - return nullptr; - auto out = RooJSONFactoryWSTool::findNamedChild(*paramPointsNode, "default_values"); - if (out == nullptr) - return nullptr; - return &((*out)["parameters"]); + if (p.nbins) + v.setBins(*p.nbins); + if (p.relError) + v.setError(v.getVal() * *p.relError); + if (p.error) + v.setError(*p.error); + v.setConstant(p.isConst); } std::string genPrefix(const JSONNode &p, bool trailing_underscore) @@ -205,18 +233,14 @@ void genIndicesHelper(std::vector> &combinations, std::vectorchildren()) { - arg->setStringAttribute(attr.key().c_str(), attr.val().c_str()); - } + for (auto const &attr : attributes.dict) { + arg->setStringAttribute(attr.first.c_str(), attr.second.c_str()); } - if (auto seq = node.find("tags")) { - for (const auto &attr : seq->children()) { - arg->setAttribute(attr.val().c_str()); - } + for (auto const &tag : attributes.tags) { + arg->setAttribute(tag.c_str()); } } @@ -298,33 +322,17 @@ void exportParameterStepWidths(RooWorkspace const &workspace, std::vectoris_seq()) { - RooJSONFactoryWSTool::warning("RooFitHS3: misc.minimization.parameter_stepwidths is not a sequence, skipping."); - return; - } - - for (JSONNode const &stepWidthNode : parameterStepWidthsNode->children()) { - if (!stepWidthNode.is_map() || !stepWidthNode.has_child("name") || !stepWidthNode.has_child("step_width")) { - RooJSONFactoryWSTool::warning("RooFitHS3: skipping malformed parameter_stepwidths entry."); - continue; - } - - const std::string name = RooJSONFactoryWSTool::name(stepWidthNode); - RooAbsArg *arg = workspace.arg(name); - auto *var = dynamic_cast(arg); + for (HS3::StepWidth const &stepWidth : doc.rooFit.stepWidths) { + auto *var = dynamic_cast(workspace.arg(stepWidth.name)); if (!var) { - RooJSONFactoryWSTool::warning( - "RooFitHS3: skipping parameter_stepwidths entry for unknown or non-real variable '" + name + "'."); + RooJSONFactoryWSTool::warning("RooFitHS3: skipping parameter_stepwidths entry for unknown or non-real " + "variable '" + + stepWidth.name + "'."); continue; } - - var->setError(stepWidthNode.find("step_width")->val_double()); + var->setError(stepWidth.width); } } @@ -430,117 +438,128 @@ void exportAttributes(const RooAbsArg *arg, JSONNode &rootnode) } } -// Collect the observables named in the "axes" field of `node` from the workspace into `out`. -void getObservables(RooWorkspace const &ws, const JSONNode &node, RooAbsCollection &out) +// Create the variables for the axes of binned data. +RooArgSet createBinnedObservables(std::vector> const &axes) { - for (const auto &p : node["axes"].children()) { - std::string name(RooJSONFactoryWSTool::name(p)); - if (ws.var(name)) { - out.add(*ws.var(name)); + RooArgSet vars; + for (auto const &axisVariant : axes) { + auto const *axis = std::get_if(&axisVariant); + if (!axis) { + RooJSONFactoryWSTool::error("categorical axes of binned data are not supported yet"); + } + auto obs = std::make_unique(axis->name.c_str(), axis->name.c_str(), axis->min, axis->max); + if (auto const *edges = std::get_if(&axis->binning)) { + RooBinning bins(obs->getMin(), obs->getMax()); + for (double edge : edges->edges) { + bins.addBoundary(edge); + } + obs->setBinning(bins); } else { - RooJSONFactoryWSTool::error("The observable \"" + name + "\" could not be found in the workspace!"); + obs->setBins(std::get(axis->binning).nbins); } + vars.addOwned(std::move(obs)); + } + return vars; +} + +// Create the RooDataHist with the contents and errors. Negative errors stand for no error. +std::unique_ptr createDataHist(std::string const &name, RooArgSet const &vars, + std::vector const &contents, std::vector const *errors) +{ + if (errors && errors->size() != contents.size()) { + RooJSONFactoryWSTool::error("inconsistent number of errors and contents in '" + name + "': errors=" + + std::to_string(errors->size()) + ", contents=" + std::to_string(contents.size())); + } + auto bins = generateBinIndices(vars); + if (contents.size() != bins.size()) { + RooJSONFactoryWSTool::error("inconsistent bin numbers: contents=" + std::to_string(contents.size()) + + ", bins=" + std::to_string(bins.size())); + } + auto dh = std::make_unique(name, name, vars); + // RooDataHist only starts to store squared weights when it sees a positive error, and then initializes them from + // the weights so far. Trigger that before setting any weight, so that errors of zero stay zero. + if (errors && !errors->empty()) { + dh->set(0, 0., 1.); + } + for (std::size_t ibin = 0; ibin < bins.size(); ++ibin) { + dh->set(ibin, contents[ibin], errors ? (*errors)[ibin] : -1); } + return dh; } -// Make the ranges of the observables of unbinned data consistent with the "min" and "max" of the axes. Variables that +// Make the range of an observable of unbinned data consistent with the "min" and "max" of the axis. Variables that // didn't get their range from a domain take it from the axis. If `domains` is null, all ranges are considered explicit. -void applyUnbinnedAxisRanges(const JSONNode &p, RooWorkspace &workspace, RooFit::JSONIO::Detail::Domains const *domains) +void applyUnbinnedAxisRange(HS3::ContinuousAxis const &axis, std::string const &dataName, RooRealVar &var, + RooFit::JSONIO::Detail::Domains const *domains) { - for (const auto &axis : p["axes"].children()) { - if (!axis.has_child("min") || !axis.has_child("max")) { - continue; + if (!axis.min || !axis.max) { + return; + } + if (domains && !domains->hasVariable(axis.name.c_str())) { + var.setRange(*axis.min, *axis.max); + } else if (var.getMin() != *axis.min || var.getMax() != *axis.max) { + std::stringstream ss; + ss << "RooFitHS3: the range [" << *axis.min << ", " << *axis.max << "] of the axis '" << axis.name + << "' of the dataset '" << dataName << "' differs from the range [" << var.getMin() << ", " << var.getMax() + << "] of the variable."; + RooJSONFactoryWSTool::warning(ss.str()); + } +} + +std::unique_ptr createUnbinnedData(std::string const &name, HS3::UnbinnedData const &input, + RooWorkspace &workspace, RooFit::JSONIO::Detail::Domains const *domains) +{ + RooArgList varlist; + for (auto const &axisVariant : input.axes) { + auto const *axis = std::get_if(&axisVariant); + if (!axis) { + RooJSONFactoryWSTool::error("categorical axes of unbinned data are not supported yet"); } - const std::string name = RooJSONFactoryWSTool::name(axis); - RooRealVar *var = workspace.var(name); + RooRealVar *var = workspace.var(axis->name); if (!var) { - continue; // reported by getObservables() + RooJSONFactoryWSTool::error("The observable \"" + axis->name + "\" could not be found in the workspace!"); } - const double min = readBound(axis, "min", -RooNumber::infinity()); - const double max = readBound(axis, "max", RooNumber::infinity()); - if (domains && !domains->hasVariable(name.c_str())) { - var->setRange(min, max); - } else if (var->getMin() != min || var->getMax() != max) { - std::stringstream ss; - ss << "RooFitHS3: the range [" << min << ", " << max << "] of the axis '" << name << "' of the dataset '" - << RooJSONFactoryWSTool::name(p) << "' differs from the range [" << var->getMin() << ", " << var->getMax() - << "] of the variable."; - RooJSONFactoryWSTool::warning(ss.str()); + varlist.add(*var); + applyUnbinnedAxisRange(*axis, name, *var, domains); + } + RooArgSet vars(varlist); + auto data = std::make_unique(name, name, vars, RooFit::WeightVar()); + const std::size_t nAxes = input.axes.size(); + for (std::size_t i = 0; i < input.numEntries(); ++i) { + for (std::size_t j = 0; j < nAxes; ++j) { + static_cast(varlist.at(j))->setVal(input.entries[i * nAxes + j]); } + data->add(vars, input.weights ? (*input.weights)[i] : 1.); } + return data; } -// Create a RooAbsData (binned or unbinned) from the JSON node `p`. +// Create a RooAbsData (binned or unbinned) from its description. std::unique_ptr -loadData(const JSONNode &p, RooWorkspace &workspace, RooFit::JSONIO::Detail::Domains const *domains) +loadData(HS3::Data const &input, RooWorkspace &workspace, RooFit::JSONIO::Detail::Domains const *domains) { - std::string name(RooJSONFactoryWSTool::name(p)); + RooJSONFactoryWSTool::testValidName(input.name, true); - RooJSONFactoryWSTool::testValidName(name, true); - - std::string const &type = p["type"].val(); - if (type == "binned") { - // binned - return RooJSONFactoryWSTool::readBinnedData(p, name, RooJSONFactoryWSTool::readAxes(p)); - } else if (type == "unbinned") { - // unbinned - RooArgList varlist; - getObservables(workspace, p, varlist); - applyUnbinnedAxisRanges(p, workspace, domains); - RooArgSet vars(varlist); - auto data = std::make_unique(name, name, vars, RooFit::WeightVar()); - auto &coords = p["entries"]; - if (!coords.is_seq()) { - RooJSONFactoryWSTool::error("key 'entries' is not a list!"); - } - std::vector weightVals; - if (p.has_child("weights")) { - auto &weights = p["weights"]; - if (coords.num_children() != weights.num_children()) { - RooJSONFactoryWSTool::error("inconsistent number of entries and weights!"); - } - for (auto const &weight : weights.children()) { - weightVals.push_back(weight.val_double()); - } - } - std::size_t i = 0; - for (auto const &point : coords.children()) { - if (!point.is_seq()) { - RooJSONFactoryWSTool::error("coordinate point '" + std::to_string(i) + "' is not a list!"); - } - if (point.num_children() != varlist.size()) { - RooJSONFactoryWSTool::error("inconsistent number of entries and observables!"); - } - std::size_t j = 0; - for (auto const &pointj : point.children()) { - auto *v = static_cast(varlist.at(j)); - v->setVal(pointj.val_double()); - ++j; - } - if (weightVals.size() > 0) { - data->add(vars, weightVals[i]); - } else { - data->add(vars, 1.); - } - ++i; - } - return data; + if (auto const *binned = std::get_if(&input.body)) { + return createDataHist(input.name, createBinnedObservables(binned->axes), binned->contents, + binned->uncertainty ? &binned->uncertainty->sigma : nullptr); } - - RooJSONFactoryWSTool::error("RooJSONFactoryWSTool() failed to create dataset " + name); + if (auto const *unbinned = std::get_if(&input.body)) { + return createUnbinnedData(input.name, *unbinned, workspace, domains); + } + RooJSONFactoryWSTool::error("RooJSONFactoryWSTool() failed to create dataset " + input.name); } // Import an analysis (likelihood + domains) as one or more ModelConfig objects into the workspace. -void importAnalysis(const JSONNode &rootnode, const JSONNode &analysisNode, const JSONNode &likelihoodsNode, - const JSONNode &domainsNode, RooWorkspace &workspace, +void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWorkspace &workspace, const std::vector> &datasets) { // if this is a toplevel pdf, also create a modelConfig for it - std::string const &analysisName = RooJSONFactoryWSTool::name(analysisNode); - JSONNode const *mcAuxNode = findRooFitInternal(rootnode, "ModelConfigs", analysisName); + std::string const &analysisName = analysis.name; + auto foundInfo = doc.rooFit.modelConfigs.find(analysisName); + HS3::ModelConfigInfo const *mcInfo = foundInfo != doc.rooFit.modelConfigs.end() ? &foundInfo->second : nullptr; - JSONNode const *mcNameNode = mcAuxNode ? mcAuxNode->find("mcName") : nullptr; - std::string mcname = mcNameNode ? mcNameNode->val() : analysisName; + std::string mcname = mcInfo && mcInfo->mcName ? *mcInfo->mcName : analysisName; if (workspace.obj(mcname)) return; @@ -548,38 +567,42 @@ void importAnalysis(const JSONNode &rootnode, const JSONNode &analysisNode, cons auto *mc = static_cast(workspace.obj(mcname)); mc->SetWS(workspace); - auto *nllNode = RooJSONFactoryWSTool::findNamedChild(likelihoodsNode, analysisNode["likelihood"].val()); - if (!nllNode) { + auto foundLikelihood = std::find_if(doc.likelihoods.begin(), doc.likelihoods.end(), + [&](HS3::Likelihood const &l) { return l.name == analysis.likelihood.name; }); + if (foundLikelihood == doc.likelihoods.end()) { throw std::runtime_error("likelihood node not found!"); } - if (!nllNode->has_child("distributions")) { - throw std::runtime_error("likelihood node has no distributions attached!"); - } - if (!nllNode->has_child("data")) { - throw std::runtime_error("likelihood node has no data attached!"); + HS3::Likelihood const &likelihood = *foundLikelihood; + + std::vector nllDistNames; + nllDistNames.reserve(likelihood.terms.size()); + for (auto const &term : likelihood.terms) { + nllDistNames.push_back(term.distribution.name); } - std::vector nllDistNames = valsToStringVec((*nllNode)["distributions"]); RooArgSet extConstraints; - for (auto &nameNode : (*nllNode)["aux_distributions"].children()) { - if (RooAbsArg *extConstraint = workspace.arg(nameNode.val())) { + for (auto const &aux : likelihood.auxDistributions) { + if (RooAbsArg *extConstraint = workspace.arg(aux.name)) { extConstraints.add(*extConstraint); } } RooArgSet observables; - for (auto &nameNode : (*nllNode)["data"].children()) { + for (auto const &term : likelihood.terms) { + auto const *dataRef = std::get_if(&term.data); + if (!dataRef) { + continue; + } bool found = false; for (const auto &d : datasets) { - if (d->GetName() == nameNode.val()) { + if (d->GetName() == dataRef->name) { found = true; observables.add(*d->get(), true); } } - if (nameNode.val() != "0" && !found) - throw std::runtime_error("dataset '" + nameNode.val() + "' cannot be found!"); + if (!found) + throw std::runtime_error("dataset '" + dataRef->name + "' cannot be found!"); } - JSONNode const *pdfNameNode = mcAuxNode ? mcAuxNode->find("pdfName") : nullptr; - std::string const pdfName = pdfNameNode ? pdfNameNode->val() : "simPdf"; + std::string const pdfName = mcInfo && mcInfo->pdfName ? *mcInfo->pdfName : "simPdf"; RooAbsPdf *pdf = workspace.pdf(pdfName); @@ -609,40 +632,17 @@ void importAnalysis(const JSONNode &rootnode, const JSONNode &analysisNode, cons if (!extConstraints.empty()) mc->SetExternalConstraints(extConstraints); - // HS3 v0.2.9 used the singular keys "parameter_of_interest" (a list) and "domain" (a single name). They are still - // accepted, but the plural keys take precedence. - auto findAnalysisKey = [&](const char *key, const char *legacyKey) -> JSONNode const * { - if (auto node = analysisNode.find(key)) { - return node; - } - if (auto node = analysisNode.find(legacyKey)) { - RooJSONFactoryWSTool::warning(std::string("RooFitHS3: the key \"") + legacyKey + "\" of analysis '" + - analysisName + "' is deprecated, please use \"" + key + "\" instead."); - return node; - } - return nullptr; - }; - JSONNode const *poiNode = findAnalysisKey("parameters_of_interest", "parameter_of_interest"); - JSONNode const *domainsNodeOfAnalysis = findAnalysisKey("domains", "domain"); - std::vector analysisDomains; - if (domainsNodeOfAnalysis && domainsNodeOfAnalysis->is_seq()) { - analysisDomains = valsToStringVec(*domainsNodeOfAnalysis); - } else if (domainsNodeOfAnalysis) { - analysisDomains.push_back(domainsNodeOfAnalysis->val()); - } - - RooArgSet pois; - if (poiNode) { - for (auto const &child : poiNode->children()) { - RooAbsArg *arg = workspace.arg(child.val()); - if (!arg) { - RooJSONFactoryWSTool::error("the object '" + child.val() + "' in \"" + poiNode->key() + - "\" of analysis '" + analysisName + "' does not exist"); - } - pois.add(*arg); + RooArgSet parametersOfInterest; + for (auto const &poi : analysis.parametersOfInterest) { + RooAbsArg *arg = workspace.arg(poi.name); + if (!arg) { + RooJSONFactoryWSTool::error("the object '" + poi.name + "' in \"parameters_of_interest\" of analysis '" + + analysisName + "' does not exist"); } + parametersOfInterest.add(*arg); } - mc->SetParametersOfInterest(pois); + + mc->SetParametersOfInterest(parametersOfInterest); mc->SetObservables(observables); RooArgSet pars; pdf->getParameters(&observables, pars); @@ -654,12 +654,13 @@ void importAnalysis(const JSONNode &rootnode, const JSONNode &analysisNode, cons pdf->getAllConstraints(observables, mainPars, /*stripDisconnected*/ true); RooArgSet domainPars; - for (std::string const &domain : analysisDomains) { - const auto &thisDomain = RooJSONFactoryWSTool::findNamedChild(domainsNode, domain); - if (!thisDomain || !thisDomain->has_child("axes")) + for (auto const &domainRef : analysis.domains) { + auto thisDomain = std::find_if(doc.domains.begin(), doc.domains.end(), + [&](HS3::Domain const &d) { return d.name == domainRef.name; }); + if (thisDomain == doc.domains.end()) continue; - for (auto &var : (*thisDomain)["axes"].children()) { - auto *wsvar = workspace.var(RooJSONFactoryWSTool::name(var)); + for (auto const &axis : thisDomain->axes) { + auto *wsvar = workspace.var(std::visit([](auto const &a) -> std::string const & { return a.name; }, axis)); if (wsvar) domainPars.add(*wsvar); } @@ -680,14 +681,12 @@ void importAnalysis(const JSONNode &rootnode, const JSONNode &analysisNode, cons mc->SetGlobalObservables(globs); mc->SetNuisanceParameters(nps); - if (mcAuxNode) { - if (auto found = mcAuxNode->find("combined_data_name")) { - pdf->setStringAttribute("combined_data_name", found->val().c_str()); - } + if (mcInfo && mcInfo->combinedDataName) { + pdf->setStringAttribute("combined_data_name", mcInfo->combinedDataName->c_str()); } - if (analysisNode.has_child("init")) { - const std::string initName = analysisNode["init"].val(); + if (analysis.init) { + const std::string &initName = analysis.init->name; RooArgSet const *init = workspace.getSnapshot(initName.c_str()); if (!init) { RooJSONFactoryWSTool::warning("RooFitHS3: the parameter point '" + initName + @@ -711,33 +710,18 @@ void importAnalysis(const JSONNode &rootnode, const JSONNode &analysisNode, cons } } -void combinePdfs(const JSONNode &rootnode, RooWorkspace &ws) +void combinePdfs(HS3Document const &doc, RooWorkspace &ws) { - auto *combinedPdfInfoNode = findRooFitInternal(rootnode, "combined_distributions"); - - // If there is no info on combining pdfs - if (combinedPdfInfoNode == nullptr) { - return; - } + for (auto const &item : doc.rooFit.combinedDistributions) { + std::string const &combinedName = item.first; + HS3::CombinedDistribution const &info = item.second; - for (auto &info : combinedPdfInfoNode->children()) { - - // parse the information - std::string combinedName = info.key(); - std::string indexCatName = info["index_cat"].val(); - std::vector labels = valsToStringVec(info["labels"]); - std::vector indices; - std::vector pdfNames = valsToStringVec(info["distributions"]); - for (auto &n : info["indices"].children()) { - indices.push_back(n.val_int()); - } - - RooCategory indexCat{indexCatName.c_str(), indexCatName.c_str()}; + RooCategory indexCat{info.indexCat.c_str(), info.indexCat.c_str()}; std::map pdfMap; - for (std::size_t iChannel = 0; iChannel < labels.size(); ++iChannel) { - indexCat.defineType(labels[iChannel], indices[iChannel]); - pdfMap[labels[iChannel]] = ws.pdf(pdfNames[iChannel]); + for (std::size_t iChannel = 0; iChannel < info.labels.size(); ++iChannel) { + indexCat.defineType(info.labels[iChannel], info.indices[iChannel]); + pdfMap[info.labels[iChannel]] = ws.pdf(info.distributions[iChannel].name); } RooSimultaneous simPdf{combinedName.c_str(), combinedName.c_str(), pdfMap, indexCat}; @@ -745,44 +729,28 @@ void combinePdfs(const JSONNode &rootnode, RooWorkspace &ws) } } -void combineDatasets(const JSONNode &rootnode, std::vector> &datasets) +void combineDatasets(HS3Document const &doc, std::vector> &datasets) { - auto *combinedDataInfoNode = findRooFitInternal(rootnode, "combined_datasets"); - - // If there is no info on combining datasets - if (combinedDataInfoNode == nullptr) { - return; - } - - for (auto &info : combinedDataInfoNode->children()) { - - // parse the information - std::string combinedName = info.key(); - std::string indexCatName = info["index_cat"].val(); - std::vector labels = valsToStringVec(info["labels"]); - std::vector indices; - for (auto &n : info["indices"].children()) { - indices.push_back(n.val_int()); - } - if (indices.size() != labels.size()) { - RooJSONFactoryWSTool::error("mismatch in number of indices and labels!"); - } + for (auto const &item : doc.rooFit.combinedDatasets) { + std::string const &combinedName = item.first; + HS3::CombinedDataset const &info = item.second; // Create the combined dataset for RooFit std::map> dsMap; - RooCategory indexCat{indexCatName.c_str(), indexCatName.c_str()}; + RooCategory indexCat{info.indexCat.c_str(), info.indexCat.c_str()}; RooArgSet allVars{indexCat}; - for (std::size_t iChannel = 0; iChannel < labels.size(); ++iChannel) { - auto componentName = combinedName + "_" + labels[iChannel]; + for (std::size_t iChannel = 0; iChannel < info.labels.size(); ++iChannel) { + auto componentName = combinedName + "_" + info.labels[iChannel]; // We move the found channel data out of the "datasets" vector, such that // the data components don't get imported anymore. - std::unique_ptr &component = *std::find_if( - datasets.begin(), datasets.end(), [&](auto &d) { return d && d->GetName() == componentName; }); - if (!component) + auto found = + std::find_if(datasets.begin(), datasets.end(), [&](auto &d) { return d && d->GetName() == componentName; }); + if (found == datasets.end()) { RooJSONFactoryWSTool::error("unable to obtain component matching component name '" + componentName + "'"); - allVars.add(*component->get(), true); - dsMap.insert({labels[iChannel], std::move(component)}); - indexCat.defineType(labels[iChannel], indices[iChannel]); + } + allVars.add(*(*found)->get(), true); + dsMap.insert({info.labels[iChannel], std::move(*found)}); + indexCat.defineType(info.labels[iChannel], info.indices[iChannel]); } auto combined = std::make_unique(combinedName, combinedName, allVars, RooFit::Import(dsMap), @@ -830,73 +798,54 @@ RooAbsCategory *findCategoryObservable(RooAbsData const &data) return cat; } -/// Warn if the HS3 version of an imported document is not understood or newer than what RooFit writes. -void checkHS3Version(std::string const &version) -{ - int major = 0; - int minor = 0; - int supportedMajor = 0; - int supportedMinor = 0; - parseHS3Version(hs3VersionTag, supportedMajor, supportedMinor); - if (!parseHS3Version(version, major, minor)) { - RooJSONFactoryWSTool::warning("RooFitHS3: cannot parse hs3_version '" + version + - "', expected a string of the form \"MAJOR.MINOR\"."); - return; - } - if (std::make_pair(major, minor) > std::make_pair(supportedMajor, supportedMinor)) { - RooJSONFactoryWSTool::warning("RooFitHS3: the document has hs3_version '" + version + - "', which is newer than the version supported by this RooFit ('" + hs3VersionTag + - "'). Some content might not be imported correctly."); - } -} - /// HS3 requires the names of all top-level objects to be unique within a document. If `strict`, throw if a name is /// used twice within one section or by a distribution and a function. Other clashes, and all clashes if not /// `strict`, only warn: RooFit's own export still produces them, e.g. for a range and a pdf of one name, and it /// must be able to read its own output. -void checkUniqueNames(JSONNode const &rootnode, bool strict) +void checkUniqueNames(HS3Document const &doc, bool strict) { std::map sectionOfName; - for (const char *section : - {"distributions", "functions", "data", "domains", "parameter_points", "likelihoods", "analyses"}) { - auto seq = rootnode.find(section); - if (!seq || !seq->is_seq()) { - continue; + auto check = [&](const char *section, std::string const &name) { + auto inserted = sectionOfName.emplace(name, section); + if (inserted.second) { + return; } - for (JSONNode const &child : seq->children()) { - if (!child.is_map() || !child.has_child("name")) { - continue; // malformed entries are reported when they are imported - } - std::string const name = child["name"].val(); - auto inserted = sectionOfName.emplace(name, section); - if (inserted.second) { - continue; - } - std::string const &firstSection = inserted.first->second; - // RooFit used to give analyses and likelihoods the name of the pdf or the analysis. Accept this for backwards - // compatibility: analyses are not referenced by name, and likelihoods only from analyses, so there is no - // ambiguity. - auto isLegacyNamed = [](std::string const &s) { return s == "likelihoods" || s == "analyses"; }; - // Within one section, and between distributions and functions (both can be referenced by name), a clash is - // ambiguous. Other clashes are produced by RooFit itself (a range, snapshot or data set named like a pdf). - auto isReferable = [](std::string const &s) { return s == "distributions" || s == "functions"; }; - const bool ambiguous = firstSection == section || (isReferable(firstSection) && isReferable(section)); - if (!strict || !ambiguous || isLegacyNamed(firstSection) || isLegacyNamed(section)) { - RooJSONFactoryWSTool::warning("RooFitHS3: the name '" + name + "' is used in \"" + firstSection + - "\" and in \"" + section + "\", which is not allowed by the HS3 standard."); - continue; - } - RooJSONFactoryWSTool::error("duplicate name '" + name + "': it is defined in \"" + firstSection + - "\" and in \"" + section + "\", but names must be unique within the document"); + std::string const &firstSection = inserted.first->second; + // RooFit used to give analyses and likelihoods the name of the pdf or the analysis. Accept this for backwards + // compatibility: analyses are not referenced by name, and likelihoods only from analyses, so there is no + // ambiguity. + auto isLegacyNamed = [](std::string const &s) { return s == "likelihoods" || s == "analyses"; }; + // Within one section, and between distributions and functions (both can be referenced by name), a clash is + // ambiguous. Other clashes are produced by RooFit itself (a range, snapshot or data set named like a pdf). + auto isReferable = [](std::string const &s) { return s == "distributions" || s == "functions"; }; + const bool ambiguous = firstSection == section || (isReferable(firstSection) && isReferable(section)); + if (!strict || !ambiguous || isLegacyNamed(firstSection) || isLegacyNamed(section)) { + RooJSONFactoryWSTool::warning("RooFitHS3: the name '" + name + "' is used in \"" + firstSection + + "\" and in \"" + section + "\", which is not allowed by the HS3 standard."); + return; } - } + RooJSONFactoryWSTool::error("duplicate name '" + name + "': it is defined in \"" + firstSection + "\" and in \"" + + section + "\", but names must be unique within the document"); + }; + auto checkSection = [&](const char *section, auto const &objects) { + for (auto const &object : objects) { + check(section, object.name); + } + }; + checkSection("distributions", doc.distributions); + checkSection("functions", doc.functions); + checkSection("data", doc.data); + checkSection("domains", doc.domains); + checkSection("parameter_points", doc.parameterPoints); + checkSection("likelihoods", doc.likelihoods); + checkSection("analyses", doc.analyses); } } // namespace RooJSONFactoryWSTool::RooJSONFactoryWSTool(RooWorkspace &ws) : _workspace{ws} {} -RooJSONFactoryWSTool::~RooJSONFactoryWSTool() {} +RooJSONFactoryWSTool::~RooJSONFactoryWSTool() = default; void RooJSONFactoryWSTool::fillSeq(JSONNode &node, RooAbsCollection const &coll, size_t nMax) { @@ -1004,9 +953,10 @@ RooRealVar *RooJSONFactoryWSTool::requestImpl(const std::string &obj { if (RooRealVar *retval = _workspace.var(objname)) return retval; - if (const auto *vars = getVariablesNode(*_rootnodeInput)) { - if (const auto &node = vars->find(objname)) { - this->importVariable(*node); + if (_import) { + auto it = _import->defaultValuesByName.find(objname); + if (it != _import->defaultValuesByName.end()) { + this->importVariable(*it->second); if (RooRealVar *retval = _workspace.var(objname)) return retval; } @@ -1019,11 +969,13 @@ RooAbsPdf *RooJSONFactoryWSTool::requestImpl(const std::string &objna { if (RooAbsPdf *retval = _workspace.pdf(objname)) return retval; - auto it = _distributionsByName.find(objname); - if (it != _distributionsByName.end()) { - this->importFunction(*it->second, true); - if (RooAbsPdf *retval = _workspace.pdf(objname)) - return retval; + if (_import) { + auto it = _import->distributions.find(objname); + if (it != _import->distributions.end()) { + this->importFunction(*it->second->body.node, true); + if (RooAbsPdf *retval = _workspace.pdf(objname)) + return retval; + } } return nullptr; } @@ -1041,11 +993,13 @@ RooAbsReal *RooJSONFactoryWSTool::requestImpl(const std::string &obj return var; if (RooAbsReal *retval = _workspace.function(objname)) return retval; - auto it = _functionsByName.find(objname); - if (it != _functionsByName.end()) { - this->importFunction(*it->second, true); - if (RooAbsReal *retval = _workspace.function(objname)) - return retval; + if (_import) { + auto it = _import->functions.find(objname); + if (it != _import->functions.end()) { + this->importFunction(*it->second->body.node, true); + if (RooAbsReal *retval = _workspace.function(objname)) + return retval; + } } return nullptr; } @@ -1687,87 +1641,56 @@ RooJSONFactoryWSTool::readBinnedData(const JSONNode &n, const std::string &name, if (!contents.is_seq()) RooJSONFactoryWSTool::error("contents are not in list form"); - JSONNode const *errors = nullptr; - if (n.has_child("errors")) { - errors = &n["errors"]; - if (!errors->is_seq()) - RooJSONFactoryWSTool::error("errors are not in list form"); - if (errors->num_children() != contents.num_children()) { - RooJSONFactoryWSTool::error("inconsistent number of errors and contents in '" + name + - "': errors=" + std::to_string(errors->num_children()) + - ", contents=" + std::to_string(contents.num_children())); - } - } - - auto bins = generateBinIndices(vars); - if (contents.num_children() != bins.size()) { - RooJSONFactoryWSTool::error("inconsistent bin numbers: contents=" + std::to_string(contents.num_children()) + - ", bins=" + std::to_string(bins.size())); - } - auto dh = std::make_unique(name, name, vars); std::vector contentVals; - contentVals.reserve(contents.num_children()); - for (auto const &cont : contents.children()) { - contentVals.push_back(cont.val_double()); + contentVals << contents; + + if (!n.has_child("errors")) { + return createDataHist(name, vars, contentVals, nullptr); } + if (!n["errors"].is_seq()) + RooJSONFactoryWSTool::error("errors are not in list form"); std::vector errorVals; - if (errors) { - errorVals.reserve(errors->num_children()); - for (auto const &err : errors->children()) { - errorVals.push_back(err.val_double()); - } - } - // RooDataHist only starts to store squared weights when it sees a positive error, and then initializes them from - // the weights so far. Trigger that before setting any weight, so that errors of zero stay zero. - if (!errorVals.empty()) { - dh->set(0, 0., 1.); - } - for (size_t ibin = 0; ibin < bins.size(); ++ibin) { - const double err = errors ? errorVals[ibin] : -1; - dh->set(ibin, contentVals[ibin], err); - } - return dh; + errorVals << n["errors"]; + return createDataHist(name, vars, contentVals, &errorVals); } -// Import a single variable (RooRealVar or RooConstVar) from the JSON node `p` into the workspace. -void RooJSONFactoryWSTool::importVariable(const JSONNode &p) +// Import a single variable (RooRealVar or RooConstVar) from the parameter `p` into the workspace. +void RooJSONFactoryWSTool::importVariable(HS3::ParameterValue const &p) { // import a RooRealVar object - std::string name(RooJSONFactoryWSTool::name(p)); - RooJSONFactoryWSTool::testValidName(name, true); + RooJSONFactoryWSTool::testValidName(p.name, true); - if (!p.is_map()) { - oocoutE(nullptr, InputArguments) << "RooJSONFactoryWSTool() node '" << name << "' is not a map, skipping." - << std::endl; + checkFixedValue(p, "default_values"); + if (_workspace.arg(p.name)) return; - } - checkFixedValue(name, p, "default_values"); - if (_workspace.arg(name)) + // A string selects the state of a category, which is done together with the other parameter points. + if (p.value && std::holds_alternative(*p.value)) return; - if (!_domains->hasVariable(name.c_str())) { + if (!_domains->hasVariable(p.name.c_str())) { // ROOT versions before HS3 0.3 marked named RooConstVars with an attribute. - JSONNode const *attrNode = _attributesNode ? _attributesNode->find(name) : nullptr; - if (attrNode && attrNode->has_child("is_const_var") && attrNode->find("is_const_var")->val_int() == 1) { - if (!p.has_child("value")) { - RooJSONFactoryWSTool::error("cannot instantiate RooConstVar '" + name + "' without \"value\"!"); + auto const &attributes = _import->document.rooFit.attributes; + auto found = attributes.find(p.name); + if (found != attributes.end() && found->second.isConstVar) { + if (!p.value) { + RooJSONFactoryWSTool::error("cannot instantiate RooConstVar '" + p.name + "' without \"value\"!"); } - wsEmplace(name, p["value"].val_double()); + wsEmplace(p.name, std::get(*p.value)); return; } } - RooRealVar &var = wsEmplace(name, 1.); + RooRealVar &var = wsEmplace(p.name, 1.); configureVariable(*_domains, p, var); warnIfFreeWithoutDomain(var); } // A parameter with a single value in its domain only takes that value: a parameter point must not give it another. -void RooJSONFactoryWSTool::checkFixedValue(std::string const &name, JSONNode const &p, std::string const &pointName) +void RooJSONFactoryWSTool::checkFixedValue(HS3::ParameterValue const &p, std::string const &pointName) { - double const *fixed = _domains ? _domains->fixedValue(name.c_str()) : nullptr; - JSONNode const *valueNode = p.find("value"); - if (fixed && valueNode && valueNode->is_number() && valueNode->val_double() != *fixed) { - RooJSONFactoryWSTool::error("the parameter point '" + pointName + "' gives the parameter '" + name + - "' the value " + valueNode->val() + ", but its domain only has the value " + + double const *fixed = _domains ? _domains->fixedValue(p.name.c_str()) : nullptr; + double const *value = p.value ? std::get_if(&*p.value) : nullptr; + if (fixed && value && *value != *fixed) { + RooJSONFactoryWSTool::error("the parameter point '" + pointName + "' gives the parameter '" + p.name + + "' the value " + std::to_string(*value) + ", but its domain only has the value " + std::to_string(*fixed)); } } @@ -1782,23 +1705,14 @@ void RooJSONFactoryWSTool::warnIfFreeWithoutDomain(RooRealVar const &var) } } -// Import all dependants (variables, functions and distributions) of node `n` into the workspace. +// Import the objects of a fragment of a document, for which the node `n` plays the role of the document. void RooJSONFactoryWSTool::importDependants(const JSONNode &n) { - // import all the dependants of an object - if (JSONNode const *varsNode = getVariablesNode(n)) { - for (const auto &p : varsNode->children()) { - importVariable(p); - } - } - if (auto seq = n.find("functions")) { - for (const auto &p : seq->children()) { - this->importFunction(p, true); - } - } - if (auto seq = n.find("distributions")) { - for (const auto &p : seq->children()) { - this->importFunction(p, true); + for (const char *section : {"functions", "distributions"}) { + if (auto seq = n.find(section)) { + for (const auto &p : seq->children()) { + this->importFunction(p, true); + } } } } @@ -2104,17 +2018,14 @@ bool RooJSONFactoryWSTool::exportJSON(std::string const &filename) bool RooJSONFactoryWSTool::hasAttribute(const std::string &obj, const std::string &attrib) { - if (!_attributesNode) + if (!_import) return false; - if (auto attrNode = _attributesNode->find(obj)) { - if (auto seq = attrNode->find("tags")) { - for (auto &a : seq->children()) { - if (a.val() == attrib) - return true; - } - } - } - return false; + auto const &attributes = _import->document.rooFit.attributes; + auto found = attributes.find(obj); + if (found == attributes.end()) + return false; + auto const &tags = found->second.tags; + return std::find(tags.begin(), tags.end(), attrib) != tags.end(); } void RooJSONFactoryWSTool::setAttribute(const std::string &obj, const std::string &attrib) { @@ -2126,16 +2037,14 @@ void RooJSONFactoryWSTool::setAttribute(const std::string &obj, const std::strin std::string RooJSONFactoryWSTool::getStringAttribute(const std::string &obj, const std::string &attrib) { - if (!_attributesNode) + if (!_import) return ""; - if (auto attrNode = _attributesNode->find(obj)) { - if (auto dict = attrNode->find("dict")) { - if (auto *a = dict->find(attrib)) { - return a->val(); - } - } - } - return ""; + auto const &attributes = _import->document.rooFit.attributes; + auto found = attributes.find(obj); + if (found == attributes.end()) + return ""; + auto value = found->second.dict.find(attrib); + return value != found->second.dict.end() ? value->second : ""; } void RooJSONFactoryWSTool::setStringAttribute(const std::string &obj, const std::string &attrib, const std::string &value) @@ -2146,139 +2055,109 @@ void RooJSONFactoryWSTool::setStringAttribute(const std::string &obj, const std: dict[attrib] << value; } -// Import all nodes of the JSON document rooted at `n` into the workspace. +// Import the JSON document rooted at `n` into the workspace. void RooJSONFactoryWSTool::importAllNodes(const JSONNode &n) { - // Per HS3 standard, the hs3_version in the metadata is required. So we - // error out if it is missing or not a string, and warn if the document is - // newer than what this version of RooFit was written for. - auto metadata = n.find("metadata"); - auto versionNode = metadata ? metadata->find("hs3_version") : nullptr; - if (!versionNode) { - std::stringstream ss; - ss << "The HS3 version is missing in the JSON!\n" - << "Please include the HS3 version in the metadata field, e.g.:\n" - << " \"metadata\" :\n" - << " {\n" - << " \"hs3_version\" : \"" << hs3VersionTag << "\"\n" - << " }"; - error(ss.str()); - } - if (versionNode->is_container() || versionNode->is_null() || versionNode->is_number()) { - error(std::string("The HS3 version in the metadata must be a string, e.g. \"") + hs3VersionTag + "\""); - } - _hs3VersionInput = versionNode->val(); - checkHS3Version(_hs3VersionInput); - checkUniqueNames(n, !isHS3VersionBefore(_hs3VersionInput, 0, 3)); + HS3::Diagnostics diagnostics; + HS3Document doc = HS3::parseDocument(n, diagnostics); + report(diagnostics); + + // The state of the import refers to the document, so it must not survive it, not even if there is an exception. + struct ImportScope { + RooJSONFactoryWSTool &tool; + ~ImportScope() + { + tool._import.reset(); + tool._domains.reset(); + tool._hs3VersionInput.clear(); + } + } importScope{*this}; - _rootnodeInput = &n; + _hs3VersionInput = doc.metadata.hs3Version; + checkUniqueNames(doc, !isHS3VersionBefore(_hs3VersionInput, 0, 3)); - _attributesNode = findRooFitInternal(*_rootnodeInput, "attributes"); + _import = std::make_unique(doc); _domains = std::make_unique(); - if (auto domains = n.find("domains")) { - _domains->readJSON(*domains); - } + _domains->read(doc.domains); _domains->populate(_workspace); - // Build name-keyed indices over the "functions" and "distributions" - // sequences. Without these, every cross-reference resolved during import - // (e.g. dependencies of a PiecewiseInterpolation, or factory-expression - // arguments) triggers a linear scan over all sibling nodes via - // findNamedChild(), which becomes O(N^2) on workspaces with thousands of - // entries. Populating the maps up-front turns each lookup into O(1). - auto buildIndex = [&n](const char *key, auto &index) { - index.clear(); - auto seq = n.find(key); - if (!seq || !seq->is_seq()) - return; - index.reserve(seq->num_children()); - for (const auto &p : seq->children()) { - index.emplace(RooJSONFactoryWSTool::name(p), &p); + if (_import->defaultValues) { + for (auto const &p : _import->defaultValues->parameters) { + importVariable(p); } - }; - buildIndex("functions", _functionsByName); - buildIndex("distributions", _distributionsByName); - - this->importDependants(n); + } + for (auto const &function : doc.functions) { + this->importFunction(*function.body.node, true); + } + for (auto const &distribution : doc.distributions) { + this->importFunction(*distribution.body.node, true); + } // This creates the index categories, which the parameter points can refer to. - combinePdfs(*_rootnodeInput, _workspace); - - if (auto paramPointsNode = n.find("parameter_points")) { - for (const auto &snsh : paramPointsNode->children()) { - std::string name = RooJSONFactoryWSTool::name(snsh); - RooJSONFactoryWSTool::testValidName(name, true); - - RooArgSet vars; - for (const auto &var : snsh["parameters"].children()) { - const std::string varName = RooJSONFactoryWSTool::name(var); - JSONNode const *valueNode = var.find("value"); - const bool hasStringValue = valueNode && !valueNode->is_number(); - if (RooCategory *cat = _workspace.cat(varName)) { - if (hasStringValue) { - cat->setLabel(valueNode->val().c_str()); - } else if (valueNode) { - cat->setIndex(valueNode->val_int()); - } - vars.add(*cat); - continue; - } - if (hasStringValue) { - warning("RooFitHS3: ignoring the string value of parameter '" + varName + "' in parameter point '" + - name + "', because there is no category with that name."); - continue; - } - checkFixedValue(varName, var, name); - RooRealVar *rrv = _workspace.var(varName); - if (!rrv && !_workspace.arg(varName)) { - // The parameter is not used by any other component: create it. "const": false is the same as no - // "const" at all. - testValidName(varName, true); - rrv = &wsEmplace(varName, 1.); - configureVariable(*_domains, var, *rrv); - warnIfFreeWithoutDomain(*rrv); - } else if (rrv) { - configureVariable(*_domains, var, *rrv); - } - if (rrv) { - vars.add(*rrv); + combinePdfs(doc, _workspace); + + for (auto const &point : doc.parameterPoints) { + RooJSONFactoryWSTool::testValidName(point.name, true); + + RooArgSet vars; + for (auto const &var : point.parameters) { + std::string const *label = var.value ? std::get_if(&*var.value) : nullptr; + if (RooCategory *cat = _workspace.cat(var.name)) { + if (label) { + cat->setLabel(label->c_str()); + } else if (var.value) { + cat->setIndex(static_cast(std::get(*var.value))); } + vars.add(*cat); + continue; + } + if (label) { + warning("RooFitHS3: ignoring the string value of parameter '" + var.name + "' in parameter point '" + + point.name + "', because there is no category with that name."); + continue; + } + checkFixedValue(var, point.name); + RooRealVar *rrv = _workspace.var(var.name); + if (!rrv && !_workspace.arg(var.name)) { + // The parameter is not used by any other component: create it. "const": false is the same as no + // "const" at all. + testValidName(var.name, true); + rrv = &wsEmplace(var.name, 1.); + configureVariable(*_domains, var, *rrv); + warnIfFreeWithoutDomain(*rrv); + } else if (rrv) { + configureVariable(*_domains, var, *rrv); + } + if (rrv) { + vars.add(*rrv); } - _workspace.saveSnapshot(name, vars); } + _workspace.saveSnapshot(point.name, vars); } // Import attributes - if (_attributesNode) { - for (const auto &elem : _attributesNode->children()) { - if (RooAbsArg *arg = _workspace.arg(elem.key())) - importAttributes(arg, elem); - } + for (auto const &item : doc.rooFit.attributes) { + if (RooAbsArg *arg = _workspace.arg(item.first)) + importAttributes(arg, item.second); } - _attributesNode = nullptr; - // We delay the import of the data to after combineDatasets(), because it // might be that some datasets are merged to combined datasets there. In // that case, we will remove the components from the "datasets" vector so they // don't get imported. std::vector> datasets; - if (auto dataNode = n.find("data")) { - for (const auto &p : dataNode->children()) { - datasets.push_back(loadData(p, _workspace, _domains.get())); - } + for (auto const &data : doc.data) { + datasets.push_back(loadData(data, _workspace, _domains.get())); } // Now, read in analyses and likelihoods if there are any - if (auto analysesNode = n.find("analyses")) { - for (JSONNode const &analysisNode : analysesNode->children()) { - importAnalysis(*_rootnodeInput, analysisNode, n["likelihoods"], n["domains"], _workspace, datasets); - } + for (auto const &analysis : doc.analyses) { + importAnalysis(doc, analysis, _workspace, datasets); } - combineDatasets(*_rootnodeInput, datasets); + combineDatasets(doc, datasets); for (auto const &d : datasets) { if (d) { @@ -2291,11 +2170,10 @@ void RooJSONFactoryWSTool::importAllNodes(const JSONNode &n) } } - _rootnodeInput = nullptr; - _hs3VersionInput.clear(); - _domains.reset(); - _functionsByName.clear(); - _distributionsByName.clear(); + if (_workspace.getSnapshot("default_values")) { + _workspace.loadSnapshot("default_values"); + } + importParameterStepWidths(_workspace, doc); } /** @@ -2308,12 +2186,7 @@ bool RooJSONFactoryWSTool::importJSON(std::istream &is) { // import a JSON file to the workspace std::unique_ptr tree = JSONTree::create(is); - JSONNode const &rootnode = tree->rootnode(); - this->importAllNodes(rootnode); - if (this->workspace()->getSnapshot("default_values")) { - this->workspace()->loadSnapshot("default_values"); - } - importParameterStepWidths(*this->workspace(), rootnode); + this->importAllNodes(tree->rootnode()); return true; } @@ -2369,8 +2242,11 @@ void RooJSONFactoryWSTool::importJSONElement(const std::string &name, const std: if (isVariable) { this->importVariableElement(n); } else if (isData) { - auto absData = loadData(n, _workspace, nullptr); - _workspace.import(*absData, RooFit::Silence()); + HS3::Diagnostics diagnostics; + HS3::Data data; + readValue(HS3::Cursor{n, diagnostics}, data); + report(diagnostics); + _workspace.import(*loadData(data, _workspace, nullptr), RooFit::Silence()); } else { this->importFunction(n, false); } @@ -2379,34 +2255,22 @@ void RooJSONFactoryWSTool::importJSONElement(const std::string &name, const std: void RooJSONFactoryWSTool::importVariableElement(const JSONNode &elementNode) { std::unique_ptr tree = varJSONString(elementNode); - JSONNode &n = tree->rootnode(); - _domains = std::make_unique(); - if (auto domains = n.find("domains")) - _domains->readJSON(*domains); - - _rootnodeInput = &n; - _attributesNode = findRooFitInternal(*_rootnodeInput, "attributes"); - JSONNode const *varsNode = getVariablesNode(n); - const auto &p = varsNode->child(0); - importVariable(p); + HS3::Diagnostics diagnostics; + HS3::ParseOptions options; + options.requireMetadata = false; + HS3Document doc = HS3::parseDocument(tree->rootnode(), diagnostics, options); + report(diagnostics); - auto paramPointsNode = n.find("parameter_points"); - const auto &var = paramPointsNode->child(0)["parameters"].child(0); - if (RooRealVar *rrv = _workspace.var(RooJSONFactoryWSTool::name(var))) { - configureVariable(*_domains, var, *rrv); - } + _domains = std::make_unique(); + _domains->read(doc.domains); - // Import attributes - if (_attributesNode) { - for (const auto &elem : _attributesNode->children()) { - if (RooAbsArg *arg = _workspace.arg(elem.key())) - importAttributes(arg, elem); - } + HS3::ParameterValue const ¶meter = doc.parameterPoints.front().parameters.front(); + importVariable(parameter); + if (RooRealVar *rrv = _workspace.var(parameter.name)) { + configureVariable(*_domains, parameter, *rrv); } - _attributesNode = nullptr; - _rootnodeInput = nullptr; _domains.reset(); } diff --git a/roofit/hs3/test/testRooFitHS3.cxx b/roofit/hs3/test/testRooFitHS3.cxx index 07da2618ccf5c..699daff1ea97b 100644 --- a/roofit/hs3/test/testRooFitHS3.cxx +++ b/roofit/hs3/test/testRooFitHS3.cxx @@ -5048,6 +5048,7 @@ TEST(RooFitHS3, FixedDomainAxis) "has \"value\" and"); } expectImportThrowsWithError(document(R"({"name": "mZ", "value": "heavy"})"), "must be a number"); + expectImportThrowsWithError(document(R"({"name": "mZ", "value": null})"), "must be a number"); // All domains that list the parameter give it the same single value. expectImportThrowsWithError( From 72fea96d9992b2295d1a24574e912164facfb6e9 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Thu, 5 Nov 2026 18:57:16 +0000 Subject: [PATCH 05/20] [RF][HS3] Build HistFactory distributions from typed modifiers Parse HistFactory distributions into typed modifiers with a variant for the constraint, and create all constraints in one place. What older RooFit versions wrote differently, like integer interpolation codes, "constraint_type", or implied Gaussian constraints, is rewritten into the current form before parsing, for which JSONNode gets erase(). The builder has no version checks anymore. This fixes some imports that crashed or built the wrong model: - a "Gauss" constraint of a normfactor, whose nominal value is one - a sample without "modifiers", which lost its histogram - a shapesys or shapefactor with a single "parameter", as in the example of the standard, and a shapefactor without data - modifiers without name in different samples, which shared objects because they got the same generated name - a custom modifier that refers to an object later in the document Objects of different samples or modifiers that would get the same name are reported instead of being shared. A function as parameter of a modifier is reported instead of corrupting memory, and a constant without domain is no longer used as a variable, which was undefined behaviour. Importers that users register for "histfactory_dist" still have priority. The exported JSON doesn't change. Assisted-by: claude-code:claude-opus-5-5 --- roofit/hs3/CMakeLists.txt | 2 + roofit/hs3/inc/RooFit/Detail/JSONInterface.h | 2 + .../hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h | 4 +- roofit/hs3/src/HS3Builder.h | 53 + roofit/hs3/src/HS3Document.h | 196 ++- roofit/hs3/src/HS3Parser.cxx | 446 ++++++- roofit/hs3/src/HS3Schema.h | 1 + roofit/hs3/src/HS3Upgrade.cxx | 386 ++++++ roofit/hs3/src/JSONFactories_HistFactory.cxx | 1160 +++++++---------- roofit/hs3/src/JSONInterface.cxx | 5 + roofit/hs3/src/RooJSONFactoryWSTool.cxx | 182 ++- roofit/hs3/test/testRooFitHS3.cxx | 2 +- 12 files changed, 1655 insertions(+), 784 deletions(-) create mode 100644 roofit/hs3/src/HS3Builder.h create mode 100644 roofit/hs3/src/HS3Upgrade.cxx diff --git a/roofit/hs3/CMakeLists.txt b/roofit/hs3/CMakeLists.txt index 0601ea4296d75..314fec0a5d802 100644 --- a/roofit/hs3/CMakeLists.txt +++ b/roofit/hs3/CMakeLists.txt @@ -21,6 +21,7 @@ ROOT_STANDARD_LIBRARY_PACKAGE(RooFitHS3 SOURCES src/Domains.cxx src/HS3Parser.cxx + src/HS3Upgrade.cxx src/JSONInterface.cxx src/JSONIO.cxx src/RooJSONFactoryWSTool.cxx @@ -50,6 +51,7 @@ target_sources( ${RELATIVE_INC_HEADERS} # LinkDef.h # seemingly not being used by ROOT_STANDARD_LIBRARY_PACKAGE call src/Domains.h + src/HS3Builder.h src/HS3Document.h src/HS3Schema.h src/JSONIOUtils.h diff --git a/roofit/hs3/inc/RooFit/Detail/JSONInterface.h b/roofit/hs3/inc/RooFit/Detail/JSONInterface.h index 10abfbd941ec3..e8fc73679160f 100644 --- a/roofit/hs3/inc/RooFit/Detail/JSONInterface.h +++ b/roofit/hs3/inc/RooFit/Detail/JSONInterface.h @@ -92,6 +92,8 @@ class JSONNode { JSONNode &set_seq(); JSONNode &set_null(); void clear(); + /// Remove a child of a map, if it exists. + void erase(std::string const &key); std::string key() const; std::string val() const; diff --git a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h index ac834fe5ba559..5c310ab29c80e 100644 --- a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h +++ b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h @@ -32,6 +32,7 @@ class Domains; } } // namespace JSONIO namespace HS3 { +struct Distribution; struct ParameterValue; struct ToolImportState; } // namespace HS3 @@ -239,7 +240,8 @@ class RooJSONFactoryWSTool { void exportData(RooAbsData const &data); RooJSONFactoryWSTool::CombinedData exportCombinedData(RooAbsData const &data); - void importAllNodes(const RooFit::Detail::JSONNode &n); + void importAllNodes(RooFit::Detail::JSONNode &n); + void importDistribution(RooFit::HS3::Distribution const &distribution); void importVariable(RooFit::HS3::ParameterValue const &p); void checkFixedValue(RooFit::HS3::ParameterValue const &p, std::string const &pointName); diff --git a/roofit/hs3/src/HS3Builder.h b/roofit/hs3/src/HS3Builder.h new file mode 100644 index 0000000000000..355365decce63 --- /dev/null +++ b/roofit/hs3/src/HS3Builder.h @@ -0,0 +1,53 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef RooFit_HS3Builder_h +#define RooFit_HS3Builder_h + +#include "HS3Document.h" + +#include + +#include +#include +#include + +class RooDataHist; +class RooJSONFactoryWSTool; + +namespace RooFit { +namespace JSONIO { +class Importer; +} + +namespace HS3 { + +/// Forward the warnings to the message service, and throw if there are errors. +void report(Diagnostics const &diagnostics); + +/// Create the variables for the axes of binned data. +RooArgSet createObservables(std::vector const &axes); + +/// Create a RooDataHist with the contents and errors. If there are no errors, it doesn't store squared weights. +std::unique_ptr createDataHist(std::string const &name, RooArgSet const &vars, + std::vector const &contents, std::vector const *errors); + +/// Create the RooFit objects of a HistFactory distribution in the workspace of the tool. +void buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string const &name, HistFactoryDist const &dist); + +/// Whether this is the importer that RooFit registers for HistFactory distributions, as opposed to the one of a user. +bool isBuiltinHistFactoryImporter(JSONIO::Importer const &importer); + +} // namespace HS3 +} // namespace RooFit + +#endif diff --git a/roofit/hs3/src/HS3Document.h b/roofit/hs3/src/HS3Document.h index 8c8e5962e8c47..a0bea7a56c97a 100644 --- a/roofit/hs3/src/HS3Document.h +++ b/roofit/hs3/src/HS3Document.h @@ -48,6 +48,14 @@ inline bool operator<(Version const &l, Version const &r) return l.patch < r.patch; } +/// Combines lambdas into one visitor for std::visit(). The compiler complains about the alternatives without lambda. +template +struct overloaded : Ts... { + using Ts::operator()...; +}; +template +overloaded(Ts...) -> overloaded; + /// A name that says what it refers to, so that for example a DataRef can't be used as a DomainRef. template struct Ref { @@ -192,9 +200,177 @@ struct GenericObject { Detail::JSONNode const *node = nullptr; }; +enum class InterpolationType { + Add, + Mult +}; + +enum class InterpolationFunction { + Poly1, + Poly2, + Poly6, + Exp +}; + +/// How a modifier of a HistFactory distribution interpolates and extrapolates its effect. +struct Interpolation { + InterpolationType type = InterpolationType::Add; + InterpolationFunction in = InterpolationFunction::Poly1; + std::optional out; ///< If not set, the function inside the bounds is used outside as well. +}; + +inline bool operator==(Interpolation const &l, Interpolation const &r) +{ + return l.type == r.type && l.in == r.in && l.out == r.out; +} +inline bool operator!=(Interpolation const &l, Interpolation const &r) +{ + return !(l == r); +} +inline bool operator<(Interpolation const &l, Interpolation const &r) +{ + if (l.type != r.type) { + return l.type < r.type; + } + if (l.in != r.in) { + return l.in < r.in; + } + return l.out < r.out; +} + +inline constexpr Interpolation additivePiecewiseLinear{InterpolationType::Add, InterpolationFunction::Poly1, + std::nullopt}; +inline constexpr Interpolation multiplicativePiecewiseExponential{InterpolationType::Mult, InterpolationFunction::Exp, + std::nullopt}; +inline constexpr Interpolation additiveQuadraticLinear{InterpolationType::Add, InterpolationFunction::Poly2, + InterpolationFunction::Poly1}; +inline constexpr Interpolation additivePolynomialLinear{InterpolationType::Add, InterpolationFunction::Poly6, + InterpolationFunction::Poly1}; +inline constexpr Interpolation multiplicativePolynomialExponential{ + InterpolationType::Mult, InterpolationFunction::Poly6, InterpolationFunction::Exp}; +inline constexpr Interpolation multiplicativePolynomialLinear{InterpolationType::Mult, InterpolationFunction::Poly6, + InterpolationFunction::Poly1}; + +char const *toString(InterpolationType type); +char const *toString(InterpolationFunction function); + +/// The RooFit classes that implement the interpolations, which encode them with integer codes. +enum class InterpolationClass { + Piecewise, + Flexible +}; + +/// The interpolation for an integer code, or nothing if the class doesn't support the code. +std::optional interpolationFromCode(int code, InterpolationClass interpolationClass); +/// The integer code of an interpolation, or nothing if the class can't represent the interpolation. +std::optional codeFromInterpolation(Interpolation const &interpolation, InterpolationClass interpolationClass); + +/// The constraint types that older documents named, for which RooFit creates the constraint terms. +enum class ConstraintType { + Gauss, + Poisson, + LogNormal +}; + +char const *toString(ConstraintType type); + +/// The parameter of the modifier is not constrained. +struct Unconstrained {}; + +/// The constraint term is a distribution of the document, named in "constraints". +struct ConstraintPdf { + DistributionRef pdf; +}; + +using Constraint = std::variant; + +struct NormFactor { + std::string name; + ParameterRef parameter; + Constraint constraint; +}; + +struct NormSys { + std::string name; + ParameterRef parameter; + double hi = 1.0; + double lo = 1.0; + Constraint constraint; + std::optional interpolation; +}; + +struct HistoSysVariation { + std::vector contents; + std::optional> errors; ///< Not in the standard, but older versions of RooFit read them. +}; + +struct HistoSys { + std::string name; + ParameterRef parameter; + HistoSysVariation hi; + HistoSysVariation lo; + Constraint constraint; + std::optional interpolation; +}; + +/// The gamma parameters are constant, which is a RooFit extension. +struct ConstantGammas {}; + +/// Every gamma parameter has its own constraint term, or none, as named in "constraints". +struct GammaConstraintPdfs { + std::vector> pdfs; +}; + +using GammaConstraint = std::variant; + +/// Scales each bin by its own parameter. The modifier type "shapesys" is a deprecated alias of "shapefactor". +struct ShapeSys { + std::string name; + std::vector parameters; ///< If empty, the parameters are named after the modifier. + std::vector vals; + GammaConstraint constraint; +}; + +/// Scales each bin by a parameter, that is constrained by the statistical uncertainty of all samples that have this +/// modifier. +struct StatError { + std::string name; + std::vector parameters; ///< If empty, the parameters are named after the channel. + GammaConstraint constraint; +}; + +/// Multiplies the sample with a function of the document. +struct CustomModifier { + std::string name; +}; + +using Modifier = std::variant; + +struct Sample { + std::string name; + std::vector contents; + std::optional> errors; + std::vector modifiers; +}; + +/// Settings of the staterror modifiers that apply to the whole channel. This is a RooFit extension. +struct StatErrorConfig { + double relThreshold = 0.0; + GammaConstraint constraint; + /// Samples without "errors" have Poisson uncertainties, as in documents before HS3 0.3. + bool impliedPoissonErrors = false; +}; + +struct HistFactoryDist { + std::vector axes; + std::optional defaultInterpolation; + std::vector samples; + StatErrorConfig statError; +}; + struct Distribution { std::string name; - GenericObject body; + std::variant body; }; struct Function { @@ -282,16 +458,32 @@ class HS3Document { namespace HS3 { -/// Options for parseDocument(). +/// Options for upgradeDocument() and parseDocument(). struct ParseOptions { /// Fragments of a document can be imported without metadata. bool requireMetadata = true; + /// If false, HistFactory distributions stay generic objects, because a user registered their own importer. + bool typedHistFactory = true; }; +/// Rewrites everything that older versions of RooFit wrote in a way that is not part of the standard into the form that +/// parseDocument() understands. +void upgradeDocument(Detail::JSONNode &root, Diagnostics &diagnostics, ParseOptions const &options = {}); + +/// Same as upgradeDocument(), but for a single HistFactory distribution that doesn't come with a document. Names that +/// can't be resolved in the document are assumed to refer to distributions. +void upgradeHistFactoryDist(Detail::JSONNode &distribution, Version const &version, Diagnostics &diagnostics); + /// Reads a document. The generic objects in the document refer to nodes of the JSON tree, so the tree must outlive the /// document. Problems are reported in `diagnostics`, and if there are errors the document must not be used. HS3Document parseDocument(Detail::JSONNode const &root, Diagnostics &diagnostics, ParseOptions const &options = {}); +/// Reads a HistFactory distribution that doesn't come with a document. +HistFactoryDist parseHistFactoryDist(Detail::JSONNode const &distribution, Diagnostics &diagnostics); + +/// Reads an interpolation that doesn't come with a document. +Interpolation parseInterpolation(Detail::JSONNode const &interpolation, Diagnostics &diagnostics); + /// Reads the leading "MAJOR.MINOR" and the optional ".PATCH" of an HS3 version string. bool parseVersion(std::string const &version, Version &out); diff --git a/roofit/hs3/src/HS3Parser.cxx b/roofit/hs3/src/HS3Parser.cxx index 8e79cd5469107..a4129b3401e6c 100644 --- a/roofit/hs3/src/HS3Parser.cxx +++ b/roofit/hs3/src/HS3Parser.cxx @@ -27,6 +27,69 @@ namespace HS3 { constexpr auto supportedVersion = "0.3"; +namespace { + +// Single source of truth for the mapping between the structured HS3 interpolation +// descriptors and the RooFit integer codes of PiecewiseInterpolation and +// FlexibleInterpVar. A code of `kUnrepresentable` means the descriptor cannot be +// expressed by that class: FlexibleInterpVar internally remaps code 4 to code 5, +// so it has no additive-linear or multiplicative-linear poly6 variant. +struct InterpolationCodes { + Interpolation descriptor; + int piecewise; + int flexible; +}; + +constexpr int kUnrepresentable = -1; + +// clang-format off +constexpr InterpolationCodes interpolationTable[]{ + {additivePiecewiseLinear, 0, 0}, + {multiplicativePiecewiseExponential, 1, 1}, + {additiveQuadraticLinear, 2, 2}, + {additivePolynomialLinear, 4, kUnrepresentable}, + {multiplicativePolynomialExponential, 5, 4}, + {multiplicativePolynomialLinear, 6, kUnrepresentable}, +}; +// clang-format on + +int codeForClass(InterpolationCodes const &row, InterpolationClass interpolationClass) +{ + return interpolationClass == InterpolationClass::Piecewise ? row.piecewise : row.flexible; +} + +} // namespace + +std::optional interpolationFromCode(int code, InterpolationClass interpolationClass) +{ + // Code 3 was historically an unimplemented alias of code 2 for both classes, + // and FlexibleInterpVar treats code 5 identically to its canonical code 4. + if (code == 3) { + code = 2; + } + if (interpolationClass == InterpolationClass::Flexible && code == 5) { + code = 4; + } + for (auto const &row : interpolationTable) { + const int rowCode = codeForClass(row, interpolationClass); + if (rowCode != kUnrepresentable && rowCode == code) { + return row.descriptor; + } + } + return std::nullopt; +} + +std::optional codeFromInterpolation(Interpolation const &interpolation, InterpolationClass interpolationClass) +{ + for (auto const &row : interpolationTable) { + const int rowCode = codeForClass(row, interpolationClass); + if (rowCode != kUnrepresentable && row.descriptor == interpolation) { + return rowCode; + } + } + return std::nullopt; +} + bool hasErrors(Diagnostics const &diagnostics) { return std::any_of(diagnostics.begin(), diagnostics.end(), @@ -539,29 +602,365 @@ void describe(Reader &ar, Analysis &analysis) namespace { -template -void readGenericObject(Cursor const &c, Object_t &out) +// The strings of the interpolation types and functions in the standard. +std::optional interpolationTypeFromString(std::string const &str) +{ + for (auto type : {InterpolationType::Add, InterpolationType::Mult}) { + if (str == toString(type)) { + return type; + } + } + return std::nullopt; +} + +std::optional interpolationFunctionFromString(std::string const &str) +{ + for (auto function : {InterpolationFunction::Poly1, InterpolationFunction::Poly2, InterpolationFunction::Poly6, + InterpolationFunction::Exp}) { + if (str == toString(function)) { + return function; + } + } + return std::nullopt; +} + +} // namespace + +char const *toString(InterpolationType type) +{ + return type == InterpolationType::Add ? "add" : "mult"; +} + +char const *toString(InterpolationFunction function) +{ + switch (function) { + case InterpolationFunction::Poly1: return "poly1"; + case InterpolationFunction::Poly2: return "poly2"; + case InterpolationFunction::Poly6: return "poly6"; + case InterpolationFunction::Exp: return "exp"; + } + return ""; +} + +char const *toString(ConstraintType type) +{ + switch (type) { + case ConstraintType::Gauss: return "Gauss"; + case ConstraintType::Poisson: return "Poisson"; + case ConstraintType::LogNormal: return "LogNormal"; + } + return ""; +} + +void readValue(Cursor const &c, Interpolation &out) +{ + JSONNode const &node = c.node(); + if (!node.is_map()) { + c.error("must be a struct with components 'type', 'in', and 'out'"); + return; + } + for (const char *component : {"type", "in", "out"}) { + if (!node.has_child(component)) { + c.error(std::string("does not define the required '") + component + "' component"); + return; + } + } + JSONNode const &typeNode = node["type"]; + JSONNode const &inNode = node["in"]; + JSONNode const &outNode = node["out"]; + if (typeNode.is_container() || typeNode.is_null() || inNode.is_container() || inNode.is_null()) { + c.error("components 'type' and 'in' must be strings"); + return; + } + + auto type = interpolationTypeFromString(typeNode.val()); + if (!type) { + c.error("has unknown interpolation type '" + typeNode.val() + "'"); + return; + } + auto in = interpolationFunctionFromString(inNode.val()); + if (!in) { + c.error("has unknown interpolation function '" + inNode.val() + "'"); + return; + } + out.type = *type; + out.in = *in; + out.out.reset(); + + if (!outNode.is_null()) { + if (outNode.is_container()) { + c.error("component 'out' must be a string or null"); + return; + } + out.out = interpolationFunctionFromString(outNode.val()); + if (!out.out) { + c.error("has unknown extrapolation function '" + outNode.val() + "'"); + } + } +} + +namespace { + +// Constraints: "constraints" names the constraint distribution of each parameter, or is null for an unconstrained one. +// Older documents give a type in "constraint", or a distribution in "constraint_name", to which upgradeDocument() +// translated the even older forms. +std::optional readConstraintType(Cursor const &c) +{ + std::string value; + readValue(c, value); + for (auto type : {ConstraintType::Gauss, ConstraintType::Poisson, ConstraintType::LogNormal}) { + if (value == toString(type)) { + return type; + } + } + c.error("unknown constraint type '" + value + "', supported are \"Gauss\", \"Poisson\" and \"LogNormal\""); + return std::nullopt; +} + +void readConstraint(Reader &ar, Constraint &out) +{ + if (ar.has("constraints")) { + Cursor c{ar.cursor().node()["constraints"], ar.cursor(), "constraints"}; + if (!c.node().is_seq() || c.node().num_children() != 1) { + c.error("must be an array with one entry"); + return; + } + JSONNode const &item = c.node().child(0); + if (item.is_null()) { + out = Unconstrained{}; + return; + } + ConstraintPdf pdf; + readValue(Cursor{item, c, std::size_t{0}}, pdf.pdf); + out = std::move(pdf); + } else if (ar.has("constraint_name")) { + ConstraintPdf pdf; + ar("constraint_name", pdf.pdf); + out = std::move(pdf); + } else if (ar.has("constraint")) { + if (auto type = readConstraintType(Cursor{ar.cursor().node()["constraint"], ar.cursor(), "constraint"})) { + out = *type; + } + } +} + +void readGammaConstraint(Reader &ar, GammaConstraint &out) +{ + if (ar.has("constraints")) { + GammaConstraintPdfs constraints; + Cursor c{ar.cursor().node()["constraints"], ar.cursor(), "constraints"}; + if (!c.node().is_seq()) { + c.error("must be an array"); + return; + } + std::size_t i = 0; + for (JSONNode const &item : c.node().children()) { + constraints.pdfs.emplace_back(); + if (!item.is_null()) { + constraints.pdfs.back().emplace(); + readValue(Cursor{item, c, i}, *constraints.pdfs.back()); + } + ++i; + } + out = std::move(constraints); + } else if (ar.has("constraint")) { + Cursor c{ar.cursor().node()["constraint"], ar.cursor(), "constraint"}; + // "Const" is a RooFit extension. + if (!c.node().is_container() && !c.node().is_null() && c.node().val() == "Const") { + out = ConstantGammas{}; + } else if (auto type = readConstraintType(c)) { + out = *type; + } + } +} + +} // namespace + +// The standard doesn't require a name for modifiers, but RooFit uses it to name the objects it creates. +void readModifierName(Reader &ar, std::string &name) +{ + std::optional optionalName; + ar("name", optionalName); + name = optionalName.value_or(""); +} + +// The parameter of a modifier with a single parameter is given in "parameters", or in "parameter" in older documents. +void readParameter(Reader &ar, ParameterRef ¶meter) +{ + if (!ar.has("parameters")) { + ar("parameter", parameter); + return; + } + Cursor c{ar.cursor().node()["parameters"], ar.cursor(), "parameters"}; + if (!c.node().is_seq() || c.node().num_children() != 1) { + c.error("must be an array with one entry"); + return; + } + readValue(Cursor{c.node().child(0), c, std::size_t{0}}, parameter); +} + +void describe(Reader &ar, NormFactor &m) +{ + readModifierName(ar, m.name); + readParameter(ar, m.parameter); + readConstraint(ar, m.constraint); +} + +void describe(Reader &ar, NormSys &m) +{ + readModifierName(ar, m.name); + readParameter(ar, m.parameter); + ar.nested("data", [&](Reader &data) { + data("hi", m.hi); + data("lo", m.lo); + }); + readConstraint(ar, m.constraint); + ar("interpolation", m.interpolation); +} + +void describe(Reader &ar, HistoSysVariation &variation) +{ + ar("contents", variation.contents); + ar("errors", variation.errors); +} + +void describe(Reader &ar, HistoSys &m) +{ + readModifierName(ar, m.name); + readParameter(ar, m.parameter); + ar.nested("data", [&](Reader &data) { + data("hi", m.hi); + data("lo", m.lo); + }); + readConstraint(ar, m.constraint); + ar("interpolation", m.interpolation); +} + +void describe(Reader &ar, ShapeSys &m) +{ + readModifierName(ar, m.name); + if (ar.has("parameter")) { + // A single name stands for one parameter per bin, which are named after it. + ar("parameter", m.name); + } + ar.list("parameters", m.parameters); + if (ar.has("data")) { + ar.nested("data", [&](Reader &data) { data.list("vals", m.vals); }); + } + readGammaConstraint(ar, m.constraint); +} + +void describe(Reader &ar, StatError &m) +{ + readModifierName(ar, m.name); + ar.list("parameters", m.parameters); + readGammaConstraint(ar, m.constraint); +} + +void describe(Reader &ar, CustomModifier &m) +{ + ar("name", m.name); +} + +void readValue(Cursor const &c, Modifier &out) { if (!c.node().is_map()) { c.error("must be a struct"); return; } Reader ar{c}; - ar("name", out.name); - ar("type", out.body.type); - out.body.node = &c.node(); + std::string type; + ar("type", type); + auto read = [&](auto modifier) { + describe(ar, modifier); + out = std::move(modifier); + }; + if (type == "normfactor") { + read(NormFactor{}); + } else if (type == "normsys") { + read(NormSys{}); + } else if (type == "histosys") { + read(HistoSys{}); + } else if (type == "shapesys" || type == "shapefactor") { + read(ShapeSys{}); + } else if (type == "staterror") { + read(StatError{}); + } else if (type == "custom") { + read(CustomModifier{}); + } else if (ar.has("type")) { + std::string name; + if (ar.has("name")) { + ar("name", name); + } + c.error("modifier '" + name + "' of unknown type '" + type + "'"); + } } -} // namespace +void describe(Reader &ar, Sample &sample) +{ + ar("name", sample.name); + ar.nested("data", [&](Reader &data) { + data("contents", sample.contents); + data("errors", sample.errors); + }); + ar.list("modifiers", sample.modifiers); +} + +void describe(Reader &ar, StatErrorConfig &config) +{ + std::optional relThreshold; + ar("relThreshold", relThreshold); + config.relThreshold = relThreshold.value_or(0.0); + readGammaConstraint(ar, config.constraint); + // Written by upgradeDocument() for older documents. + std::optional impliedPoissonErrors; + ar("implied_poisson_errors", impliedPoissonErrors); + config.impliedPoissonErrors = impliedPoissonErrors.value_or(false); +} + +void describe(Reader &ar, HistFactoryDist &dist) +{ + ar("axes", dist.axes); + ar("default_interpolation", dist.defaultInterpolation); + ar("samples", dist.samples); + if (ar.has("staterror")) { + readValue(Cursor{ar.cursor().node()["staterror"], ar.cursor(), "staterror"}, dist.statError); + } +} -void readValue(Cursor const &c, Distribution &out) +namespace { + +void readDistribution(Cursor const &c, Distribution &out, ParseOptions const &options) { - readGenericObject(c, out); + if (!c.node().is_map()) { + c.error("must be a struct"); + return; + } + Reader ar{c}; + ar("name", out.name); + std::string type; + ar("type", type); + if (options.typedHistFactory && type == "histfactory_dist") { + HistFactoryDist body; + describe(ar, body); + out.body = std::move(body); + } else { + out.body = GenericObject{type, &c.node()}; + } } +} // namespace + void readValue(Cursor const &c, Function &out) { - readGenericObject(c, out); + if (!c.node().is_map()) { + c.error("must be a struct"); + return; + } + Reader ar{c}; + ar("name", out.name); + ar("type", out.body.type); + out.body.node = &c.node(); } void describe(Reader &ar, Attributes &attributes) @@ -735,8 +1134,21 @@ HS3Document parseDocument(JSONNode const &root, Diagnostics &diagnostics, ParseO return doc; } + if (root.has_child("distributions") && !root["distributions"].is_null()) { + Cursor distributions{root["distributions"], cursor, "distributions"}; + if (!distributions.node().is_seq()) { + distributions.error("must be an array"); + } else { + std::size_t i = 0; + doc.distributions.reserve(distributions.node().num_children()); + for (JSONNode const &item : distributions.node().children()) { + doc.distributions.emplace_back(); + readDistribution(Cursor{item, distributions, i}, doc.distributions.back(), options); + ++i; + } + } + } Reader ar{cursor}; - ar.list("distributions", doc.distributions); ar.list("functions", doc.functions); ar.list("data", doc.data); ar.list("likelihoods", doc.likelihoods); @@ -747,5 +1159,19 @@ HS3Document parseDocument(JSONNode const &root, Diagnostics &diagnostics, ParseO return doc; } +HistFactoryDist parseHistFactoryDist(JSONNode const &distribution, Diagnostics &diagnostics) +{ + HistFactoryDist out; + readValue(Cursor{distribution, diagnostics}, out); + return out; +} + +Interpolation parseInterpolation(JSONNode const &interpolation, Diagnostics &diagnostics) +{ + Interpolation out; + readValue(Cursor{interpolation, diagnostics}, out); + return out; +} + } // namespace HS3 } // namespace RooFit diff --git a/roofit/hs3/src/HS3Schema.h b/roofit/hs3/src/HS3Schema.h index 006406f4ed1ae..adc815881b108 100644 --- a/roofit/hs3/src/HS3Schema.h +++ b/roofit/hs3/src/HS3Schema.h @@ -88,6 +88,7 @@ void readValue(Cursor const &c, int &out); void readValue(Cursor const &c, bool &out); void readValue(Cursor const &c, ParameterValue &out); void readValue(Cursor const &c, Data &out); +void readValue(Cursor const &c, Interpolation &out); template void readValue(Cursor const &c, Ref &out) diff --git a/roofit/hs3/src/HS3Upgrade.cxx b/roofit/hs3/src/HS3Upgrade.cxx new file mode 100644 index 0000000000000..512b62b8202f1 --- /dev/null +++ b/roofit/hs3/src/HS3Upgrade.cxx @@ -0,0 +1,386 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +// Everything that older versions of RooFit wrote in a way that is not part of the HS3 standard is translated here into +// the form that the parser understands. The rewrites go by the shape of the JSON: the same version number was written +// by ROOT releases with different conventions. Only what a missing component means depends on the version. + +#include "HS3Schema.h" + +#include +#include +#include +#include + +using RooFit::Detail::JSONNode; + +namespace RooFit { +namespace HS3 { +namespace { + +// The version that the standard requires the interpolation of modifiers from on. +constexpr Version explicitInterpolationVersion{0, 3, 0}; +// Before this version, normsys and histosys modifiers without constraint had an implicit Gaussian constraint. +constexpr Version explicitConstraintVersion{0, 2, 0}; +// From this version on, the gamma parameters of modifiers are free unless "constraints" names their constraint terms, +// "shapesys" is deprecated, and a sample without "errors" has no statistical uncertainty. +constexpr Version namedConstraintVersion{0, 3, 0}; + +using IsDistribution = std::function; + +bool isScalar(JSONNode const &node) +{ + return !node.is_container() && !node.is_null(); +} + +bool isConstraintType(std::string const &value) +{ + return value == "Gauss" || value == "Poisson" || value == "LogNormal"; +} + +// Older versions of RooFit spelled "LogNormal" differently. +std::string canonicalConstraintType(std::string const &value) +{ + return value == "Lognormal" ? "LogNormal" : value; +} + +// The "constraints" of the standard take precedence over the keys that older documents used. +bool eraseLegacyConstraints(JSONNode &mod) +{ + if (!mod.has_child("constraints")) { + return false; + } + mod.erase("constraint"); + mod.erase("constraint_name"); + mod.erase("constraint_type"); + return true; +} + +// Write a legacy constraint of a normsys, histosys or normfactor modifier as "constraint" if it is a type, or +// "constraint_name" if it refers to a distribution. +void upgradeConstraint(JSONNode &mod, IsDistribution const &isDistribution) +{ + if (eraseLegacyConstraints(mod)) { + return; + } + if (mod.has_child("constraint_name")) { + mod.erase("constraint"); + mod.erase("constraint_type"); + return; + } + if (mod.has_child("constraint")) { + mod.erase("constraint_type"); + JSONNode &constraint = mod["constraint"]; + if (!isScalar(constraint)) { + return; // reported by the parser + } + const std::string value = constraint.val(); + // In older documents, the "constraint" can be the name of a distribution, which has priority. + if (isDistribution(value)) { + mod["constraint_name"] << value; + mod.erase("constraint"); + } else { + constraint << canonicalConstraintType(value); + } + return; + } + if (mod.has_child("constraint_type")) { + JSONNode &constraintType = mod["constraint_type"]; + if (isScalar(constraintType)) { + mod["constraint"] << canonicalConstraintType(constraintType.val()); + } + mod.erase("constraint_type"); + } +} + +// The constraint of the gamma parameters of a shapesys modifier. In contrast to the other modifiers, it is not possible +// to name a distribution in this way, and "constraint_type" has priority. +void upgradeGammaConstraint(JSONNode &mod) +{ + if (eraseLegacyConstraints(mod)) { + return; + } + if (mod.has_child("constraint_type")) { + JSONNode &constraintType = mod["constraint_type"]; + if (isScalar(constraintType)) { + mod["constraint"] << canonicalConstraintType(constraintType.val()); + } + mod.erase("constraint_type"); + } else if (mod.has_child("constraint") && isScalar(mod["constraint"])) { + mod["constraint"] << canonicalConstraintType(mod["constraint"].val()); + } +} + +// An integer code that was used before the structured interpolations. +void upgradeInterpolationCode(JSONNode &mod, InterpolationClass interpolationClass, Cursor const &c) +{ + if (!mod.has_child("interpolation") || mod["interpolation"].is_map()) { + return; + } + JSONNode &node = mod["interpolation"]; + Cursor nodeCursor{node, c, "interpolation"}; + if (!isScalar(node)) { + nodeCursor.error("must be a structured interpolation or a legacy integer code"); + return; + } + const std::string value = node.val(); + int code = 0; + const auto result = std::from_chars(value.data(), value.data() + value.size(), code); + if (result.ec != std::errc{} || result.ptr != value.data() + value.size()) { + nodeCursor.error("has invalid legacy interpolation code '" + value + "'"); + return; + } + auto interpolation = interpolationFromCode(code, interpolationClass); + if (!interpolation) { + nodeCursor.error( + std::string("has unsupported ") + + (interpolationClass == InterpolationClass::Piecewise ? "PiecewiseInterpolation" : "FlexibleInterpVar") + + " code " + value); + return; + } + node.set_null(); + node.set_map(); + node["type"] << toString(interpolation->type); + node["in"] << toString(interpolation->in); + if (interpolation->out) { + node["out"] << toString(*interpolation->out); + } else { + node["out"].set_null(); + } +} + +struct ChannelState { + std::string channelName; + bool warnedInterpolation = false; + bool warnedUnconstrained = false; + bool warnedShapeSys = false; +}; + +// Before structured interpolations were introduced, both modifier classes used the integer code 4 as their implicit +// default. The meaning of the code is class-dependent. HS3 forbids implementation-specific defaults. +void upgradeMissingInterpolation(JSONNode &mod, InterpolationClass interpolationClass, Version const &version, + bool hasDefault, std::string const &context, ChannelState &state, Cursor const &c) +{ + if (mod.has_child("interpolation") || hasDefault || !(version < explicitInterpolationVersion)) { + return; + } + if (!state.warnedInterpolation) { + c.warning(context + " is not given, and the channel has no \"default_interpolation\". Using the implicit default " + "of older RooFit versions (code 4), which is deprecated."); + state.warnedInterpolation = true; + } + mod["interpolation"] << 4; + upgradeInterpolationCode(mod, interpolationClass, c); +} + +// Whether "constraints" leaves the single parameter of a modifier unconstrained. +bool isExplicitlyUnconstrained(JSONNode const &constraints) +{ + return constraints.is_seq() && constraints.num_children() == 1 && constraints.child(0).is_null(); +} + +void upgradeModifier(JSONNode &mod, std::size_t index, std::string const &sampleName, Version const &version, + bool hasDefaultInterpolation, IsDistribution const &isDistribution, ChannelState &state, + Cursor const &c) +{ + if (!mod.has_child("type") || !isScalar(mod["type"])) { + return; // reported by the parser + } + const std::string type = mod["type"].val(); + const bool hasSingleParameter = type == "normfactor" || type == "normsys" || type == "histosys"; + + // The "parameters" of the standard take precedence over the "parameter" of older documents. + std::string parameter; + if (hasSingleParameter && mod.has_child("parameters")) { + mod.erase("parameter"); + JSONNode const ¶meters = mod["parameters"]; + if (parameters.is_seq() && parameters.num_children() == 1 && isScalar(parameters.child(0))) { + parameter = parameters.child(0).val(); + } + } else if (mod.has_child("parameter") && isScalar(mod["parameter"])) { + parameter = mod["parameter"].val(); + } + + // The name of a modifier is not part of the standard. RooFit names objects after it, so a generated name must be + // unique, unlike the name of a parameter, which is shared on purpose. + if (!mod.has_child("name")) { + mod["name"] << (!parameter.empty() ? parameter + : state.channelName + "_" + sampleName + "_syst_" + std::to_string(index)); + } + const std::string name = isScalar(mod["name"]) ? mod["name"].val() : ""; + + if (hasSingleParameter && !mod.has_child("parameter") && !mod.has_child("parameters")) { + parameter = type == "normfactor" ? name : "alpha_" + name; + mod["parameter"] << parameter; + } + + if (type == "normfactor") { + upgradeConstraint(mod, isDistribution); + } else if (type == "normsys" || type == "histosys") { + upgradeConstraint(mod, isDistribution); + const bool constrained = mod.has_child("constraint") || mod.has_child("constraint_name") || + (mod.has_child("constraints") && !isExplicitlyUnconstrained(mod["constraints"])); + if (!constrained) { + if (version < explicitConstraintVersion) { + mod.erase("constraints"); + mod["constraint"] << "Gauss"; + } else if (!state.warnedUnconstrained) { + c.warning("modifier '" + name + "' of sample '" + sampleName + "' has no constraint, so its parameter '" + + parameter + + "' is unconstrained. Older RooFit versions implied a unit Gaussian. Other modifiers of this " + "channel are not reported."); + state.warnedUnconstrained = true; + } + } + const auto interpolationClass = type == "normsys" ? InterpolationClass::Flexible : InterpolationClass::Piecewise; + const std::string context = "interpolation for " + type + " modifier '" + name + "' in sample '" + sampleName + + "' of channel '" + state.channelName + "'"; + upgradeInterpolationCode(mod, interpolationClass, c); + upgradeMissingInterpolation(mod, interpolationClass, version, hasDefaultInterpolation, context, state, c); + } else if (type == "shapesys" || type == "shapefactor") { + if (type == "shapesys" && !(version < namedConstraintVersion) && !state.warnedShapeSys) { + c.warning("the modifier type \"shapesys\" is deprecated, please use \"shapefactor\" instead."); + state.warnedShapeSys = true; + } + upgradeGammaConstraint(mod); + if (!mod.has_child("constraint") && !mod.has_child("constraints")) { + if (version < namedConstraintVersion) { + // Without any information about the constraint, older versions created Gaussian constraints. + mod["constraint"] << "Gauss"; + } else if (mod.has_child("data") && mod["data"].is_map() && mod["data"].has_child("vals")) { + if (mod["data"]["vals"].is_seq() && mod["data"]["vals"].num_children() > 0) { + c.warning("the \"data.vals\" of modifier '" + name + "' of sample '" + sampleName + + "' are ignored, because it names no constraints, so its parameters are free"); + } + mod["data"].erase("vals"); + } + } + } else if (type == "staterror") { + if (eraseLegacyConstraints(mod)) { + return; + } + // ROOT 6.38 and 6.40.00 wrote "constraint_type". + if (mod.has_child("constraint_type")) { + if (!mod.has_child("constraint") && isScalar(mod["constraint_type"])) { + mod["constraint"] << mod["constraint_type"].val(); + } + mod.erase("constraint_type"); + } + } +} + +void upgradeHistFactory(JSONNode &dist, Version const &version, IsDistribution const &isDistribution, Cursor const &c) +{ + ChannelState state; + if (dist.has_child("name") && isScalar(dist["name"])) { + state.channelName = dist["name"].val(); + if (state.channelName.rfind("model_", 0) == 0) { + state.channelName.erase(0, 6); + } + } + + // Before HS3 0.3, the gamma parameters of the staterror modifiers had Poisson constraints unless the document said + // otherwise, which RooFit also creates if the document has the channel-level settings of RooFit. Samples without + // "errors" had Poisson uncertainties. + const bool legacyStatError = version < namedConstraintVersion; + if (legacyStatError && !dist.has_child("staterror")) { + dist["staterror"].set_map(); + } + if (dist.has_child("staterror") && dist["staterror"].is_map()) { + JSONNode &config = dist["staterror"]; + upgradeGammaConstraint(config); + if (!config.has_child("constraint") && !config.has_child("constraints")) { + config["constraint"] << "Poisson"; + } + if (legacyStatError) { + config["implied_poisson_errors"] << true; + } + } + + if (!dist.has_child("samples") || !dist["samples"].is_seq()) { + return; + } + const bool hasDefaultInterpolation = dist.has_child("default_interpolation"); + + Cursor samples{dist["samples"], c, "samples"}; + std::size_t iSample = 0; + for (JSONNode &sample : dist["samples"].children()) { + Cursor sampleCursor{sample, samples, iSample++}; + if (!sample.is_map() || !sample.has_child("modifiers") || !sample["modifiers"].is_seq()) { + continue; + } + const std::string sampleName = sample.has_child("name") && isScalar(sample["name"]) ? sample["name"].val() : ""; + Cursor modifiers{sample["modifiers"], sampleCursor, "modifiers"}; + std::size_t iModifier = 0; + for (JSONNode &mod : sample["modifiers"].children()) { + Cursor modCursor{mod, modifiers, iModifier}; + if (mod.is_map()) { + upgradeModifier(mod, iModifier, sampleName, version, hasDefaultInterpolation, isDistribution, state, + modCursor); + } + ++iModifier; + } + } +} + +} // namespace + +void upgradeDocument(JSONNode &root, Diagnostics &diagnostics, ParseOptions const &options) +{ + if (!root.is_map() || !root.has_child("distributions") || !root["distributions"].is_seq()) { + return; + } + if (!options.typedHistFactory) { + return; + } + + Version version; + if (JSONNode const *versionNode = root.find("metadata", "hs3_version")) { + if (isScalar(*versionNode)) { + parseVersion(versionNode->val(), version); + } + } + + std::set distributionNames; + for (JSONNode const &dist : root["distributions"].children()) { + if (dist.is_map() && dist.has_child("name") && isScalar(dist["name"])) { + distributionNames.insert(dist["name"].val()); + } + } + // The version from which on "constraint" is only a type of the standard. + const bool constraintCanBeName = version < explicitInterpolationVersion; + IsDistribution isDistribution = [&](std::string const &name) { + return constraintCanBeName && distributionNames.count(name) > 0; + }; + + Cursor rootCursor{root, diagnostics}; + Cursor distributions{root["distributions"], rootCursor, "distributions"}; + std::size_t i = 0; + for (JSONNode &dist : root["distributions"].children()) { + Cursor distCursor{dist, distributions, i++}; + if (dist.is_map() && dist.has_child("type") && isScalar(dist["type"]) && + dist["type"].val() == "histfactory_dist") { + upgradeHistFactory(dist, version, isDistribution, distCursor); + } + } +} + +void upgradeHistFactoryDist(JSONNode &distribution, Version const &version, Diagnostics &diagnostics) +{ + IsDistribution isDistribution = [](std::string const &name) { + return !isConstraintType(name) && name != "Lognormal" && name != "Const"; + }; + upgradeHistFactory(distribution, version, isDistribution, Cursor{distribution, diagnostics}); +} + +} // namespace HS3 +} // namespace RooFit diff --git a/roofit/hs3/src/JSONFactories_HistFactory.cxx b/roofit/hs3/src/JSONFactories_HistFactory.cxx index 25e4c9ce0cf94..d68553a4abfcd 100644 --- a/roofit/hs3/src/JSONFactories_HistFactory.cxx +++ b/roofit/hs3/src/JSONFactories_HistFactory.cxx @@ -33,19 +33,31 @@ #include #include +#include #include #include +#include #include +#include #include #include #include #include -#include +#include +#include #include "static_execute.h" +#include "HS3Builder.h" #include "JSONIOUtils.h" using RooFit::Detail::JSONNode; +using RooFit::HS3::additivePolynomialLinear; +using RooFit::HS3::Interpolation; +using RooFit::HS3::InterpolationClass; +using RooFit::HS3::multiplicativePiecewiseExponential; +using RooFit::HS3::multiplicativePolynomialExponential; + +namespace HS3 = RooFit::HS3; using namespace RooStats::HistFactory; using namespace RooStats::HistFactory::Detail; @@ -70,37 +82,12 @@ namespace Literals { constexpr auto staterror = "staterror"; } -struct Interpolation { - std::string type; - std::string in; - std::optional out; - - bool operator==(const Interpolation &other) const - { - return std::tie(type, in, out) == std::tie(other.type, other.in, other.out); - } - - bool operator!=(const Interpolation &other) const { return !(*this == other); } - - bool operator<(const Interpolation &other) const - { - return std::tie(type, in, out) < std::tie(other.type, other.in, other.out); - } -}; - -const Interpolation additivePiecewiseLinear{"add", "poly1", std::nullopt}; -const Interpolation multiplicativePiecewiseExponential{"mult", "exp", std::nullopt}; -const Interpolation additiveQuadraticLinear{"add", "poly2", "poly1"}; -const Interpolation additivePolynomialLinear{"add", "poly6", "poly1"}; -const Interpolation multiplicativePolynomialExponential{"mult", "poly6", "exp"}; -const Interpolation multiplicativePolynomialLinear{"mult", "poly6", "poly1"}; - std::string interpolationString(const Interpolation &interpolation) { std::stringstream ss; - ss << R"({"type":")" << interpolation.type << R"(","in":")" << interpolation.in << R"(","out":)"; + ss << R"({"type":")" << toString(interpolation.type) << R"(","in":")" << toString(interpolation.in) << R"(","out":)"; if (interpolation.out) { - ss << '"' << *interpolation.out << '"'; + ss << '"' << toString(*interpolation.out) << '"'; } else { ss << "null"; } @@ -108,57 +95,23 @@ std::string interpolationString(const Interpolation &interpolation) return ss.str(); } -bool isInterpolationFunction(std::string_view function) -{ - return function == "poly1" || function == "poly2" || function == "poly6" || function == "exp"; -} - Interpolation readInterpolation(const JSONNode &node, const std::string &context) { - if (!node.is_map()) { - RooJSONFactoryWSTool::error(context + " must be a struct with components 'type', 'in', and 'out'"); + HS3::Diagnostics diagnostics; + const Interpolation interpolation = HS3::parseInterpolation(node, diagnostics); + if (!diagnostics.empty()) { + RooJSONFactoryWSTool::error(context + " " + diagnostics.front().message); } - for (const char *component : {"type", "in", "out"}) { - if (!node.has_child(component)) { - RooJSONFactoryWSTool::error(context + " does not define the required '" + component + "' component"); - } - } - - const auto &typeNode = node["type"]; - const auto &inNode = node["in"]; - const auto &outNode = node["out"]; - if (typeNode.is_container() || typeNode.is_null() || inNode.is_container() || inNode.is_null()) { - RooJSONFactoryWSTool::error(context + " components 'type' and 'in' must be strings"); - } - - Interpolation interpolation{typeNode.val(), inNode.val(), std::nullopt}; - if (interpolation.type != "add" && interpolation.type != "mult") { - RooJSONFactoryWSTool::error(context + " has unknown interpolation type '" + interpolation.type + "'"); - } - if (!isInterpolationFunction(interpolation.in)) { - RooJSONFactoryWSTool::error(context + " has unknown interpolation function '" + interpolation.in + "'"); - } - - if (!outNode.is_null()) { - if (outNode.is_container()) { - RooJSONFactoryWSTool::error(context + " component 'out' must be a string or null"); - } - interpolation.out = outNode.val(); - if (!isInterpolationFunction(*interpolation.out)) { - RooJSONFactoryWSTool::error(context + " has unknown extrapolation function '" + *interpolation.out + "'"); - } - } - return interpolation; } void writeInterpolation(JSONNode &node, const Interpolation &interpolation) { node.set_map(); - node["type"] << interpolation.type; - node["in"] << interpolation.in; + node["type"] << toString(interpolation.type); + node["in"] << toString(interpolation.in); if (interpolation.out) { - node["out"] << *interpolation.out; + node["out"] << toString(*interpolation.out); } else { node["out"].set_null(); } @@ -179,40 +132,6 @@ int readLegacyInterpolationCode(const JSONNode &node, const std::string &context return code; } -enum class InterpolationClass { - Piecewise, - Flexible -}; - -// Single source of truth for the mapping between the structured HS3 interpolation -// descriptors and the RooFit integer codes of PiecewiseInterpolation and -// FlexibleInterpVar. A code of `kUnrepresentable` means the descriptor cannot be -// expressed by that class: FlexibleInterpVar internally remaps code 4 to code 5, -// so it has no additive-linear or multiplicative-linear poly6 variant. -struct InterpolationCodes { - const Interpolation &descriptor; - int piecewise; - int flexible; -}; - -constexpr int kUnrepresentable = -1; - -// clang-format off -const std::vector interpolationTable{ - {additivePiecewiseLinear, 0, 0}, - {multiplicativePiecewiseExponential, 1, 1}, - {additiveQuadraticLinear, 2, 2}, - {additivePolynomialLinear, 4, kUnrepresentable}, - {multiplicativePolynomialExponential, 5, 4}, - {multiplicativePolynomialLinear, 6, kUnrepresentable}, -}; -// clang-format on - -int codeForClass(const InterpolationCodes &row, InterpolationClass interpolationClass) -{ - return interpolationClass == InterpolationClass::Piecewise ? row.piecewise : row.flexible; -} - const char *interpolationClassName(InterpolationClass interpolationClass) { return interpolationClass == InterpolationClass::Piecewise ? "PiecewiseInterpolation" : "FlexibleInterpVar"; @@ -220,19 +139,8 @@ const char *interpolationClassName(InterpolationClass interpolationClass) Interpolation interpolationFromCode(int code, InterpolationClass interpolationClass, const std::string &context) { - // Code 3 was historically an unimplemented alias of code 2 for both classes, - // and FlexibleInterpVar treats code 5 identically to its canonical code 4. - if (code == 3) { - code = 2; - } - if (interpolationClass == InterpolationClass::Flexible && code == 5) { - code = 4; - } - for (const auto &row : interpolationTable) { - const int rowCode = codeForClass(row, interpolationClass); - if (rowCode != kUnrepresentable && rowCode == code) { - return row.descriptor; - } + if (auto interpolation = HS3::interpolationFromCode(code, interpolationClass)) { + return *interpolation; } RooJSONFactoryWSTool::error(context + " has unsupported " + interpolationClassName(interpolationClass) + " code " + std::to_string(code)); @@ -241,11 +149,8 @@ Interpolation interpolationFromCode(int code, InterpolationClass interpolationCl int codeFromInterpolation(const Interpolation &interpolation, InterpolationClass interpolationClass, const std::string &context) { - for (const auto &row : interpolationTable) { - const int rowCode = codeForClass(row, interpolationClass); - if (rowCode != kUnrepresentable && row.descriptor == interpolation) { - return rowCode; - } + if (auto code = HS3::codeFromInterpolation(interpolation, interpolationClass)) { + return *code; } RooJSONFactoryWSTool::error(context + " " + interpolationString(interpolation) + " cannot be represented by " + interpolationClassName(interpolationClass)); @@ -338,48 +243,18 @@ std::vector readInterpolations(const JSONNode &object, std::size_t nParamet return codes; } -// The fallback for modifiers without interpolation in documents that predate HS3 v0.3, see interpolationCode(). Before -// that, RooFit did not write the interpolation of modifiers that used its implicit default. -struct LegacyInterpolationDefault { - bool allowed = false; - bool warned = false; -}; - -int interpolationCode(const JSONNode &modifier, const std::optional &defaultInterpolation, - InterpolationClass interpolationClass, const std::string &context, - LegacyInterpolationDefault &legacyDefault) +int interpolationCode(std::optional const &interpolation, + std::optional const &defaultInterpolation, InterpolationClass interpolationClass, + std::string const &context) { - const auto toCode = [&](const Interpolation &interpolation) { - return codeFromInterpolation(interpolation, interpolationClass, context); - }; - - if (const auto *interpolationNode = modifier.find("interpolation")) { - if (interpolationNode->is_map()) { - return toCode(readInterpolation(*interpolationNode, context)); - } - const int legacyCode = readLegacyInterpolationCode(*interpolationNode, context); - const Interpolation interpolation = interpolationFromCode(legacyCode, interpolationClass, context); - return toCode(interpolation); - } - if (defaultInterpolation) { - return toCode(*defaultInterpolation); + if (interpolation) { + return codeFromInterpolation(*interpolation, interpolationClass, context); } - // HS3 forbids implementation-specific default interpolations. - if (!legacyDefault.allowed) { + if (!defaultInterpolation) { RooJSONFactoryWSTool::error(context + " is not given, and the channel has no \"default_interpolation\""); } - - // Before structured interpolation was introduced, both modifier classes - // used the integer code 4 as their implicit default. The meaning of code 4 - // is class-dependent. Files written by these ROOT versions are still read. - if (!legacyDefault.warned) { - RooJSONFactoryWSTool::warning("RooFitHS3: " + context + - " is not given, and the channel has no \"default_interpolation\". Using the " - "implicit default of older RooFit versions (code 4), which is deprecated."); - legacyDefault.warned = true; - } - return 4; + return codeFromInterpolation(*defaultInterpolation, interpolationClass, context); } void erasePrefix(std::string &str, std::string_view prefix) @@ -477,12 +352,6 @@ std::string constraintName(std::string const ¶mName) return paramName + "Constraint"; } -bool isLegacyConstraintType(std::string const &value) -{ - // HS3 spells "LogNormal", older RooFit versions used "Lognormal". - return value == "Gauss" || value == "Poisson" || value == "Const" || value == "LogNormal" || value == "Lognormal"; -} - RooAbsPdf *findNamedConstraint(RooJSONFactoryWSTool &tool, std::string const &constraintName, std::string const &sample) { if (auto *constraint = tool.workspace()->pdf(constraintName)) { @@ -500,210 +369,157 @@ RooAbsPdf *findNamedConstraint(RooJSONFactoryWSTool &tool, std::string const &co return nullptr; } -RooAbsPdf &createLegacyConstraint(RooJSONFactoryWSTool &tool, const JSONNode &mod, RooRealVar ¶m, - std::string const &constraintType) -{ - if (constraintType == "Gauss") { - param.setError(1.0); - RooRealVar &nominal = createNominal(*tool.workspace(), param.GetName(), 0.0, -10, 10); - return getOrCreate(*tool.workspace(), constraintName(param.GetName()), param, nominal, 1.); - } - - RooJSONFactoryWSTool::error("legacy constraint value '" + constraintType + "' for modifier '" + - RooJSONFactoryWSTool::name(mod) + - "' is a known constraint type, but it cannot be resolved in this context"); -} - -ParamHistFunc &createPHF(const std::string &phfname, std::string const &sysname, - const std::vector &parnames, const std::vector &vals, - RooJSONFactoryWSTool &tool, RooAbsCollection &constraints, const RooArgSet &observables, - const std::string &constraintType, double gammaMin, double gammaMax, double minSigma, - bool createConstraints = true) +// The parameter of a modifier, which is created if the document doesn't define it. It's a variable, or a constant if +// the document doesn't give it a domain. +RooAbsReal &getOrCreateParameter(RooJSONFactoryWSTool &tool, std::string const &name, double val, double min, + double max, std::string const &modifier) { RooWorkspace &ws = *tool.workspace(); - - size_t n = std::max(vals.size(), parnames.size()); - RooArgList gammas; - for (std::size_t i = 0; i < n; ++i) { - const std::string name = parnames.empty() ? defaultGammaName(sysname, i) : parnames[i]; - auto *e = dynamic_cast(ws.obj(name.c_str())); - if (e) - gammas.add(*e); - else - gammas.add(getOrCreate(ws, name, 1., gammaMin, gammaMax)); + TObject *existing = ws.obj(name); + if (!existing) { + return getOrCreate(ws, name, val, min, max); } - - auto &phf = tool.wsEmplace(phfname, observables, gammas); - - if (vals.size() > 0) { - if (!createConstraints) { - configureConstrainedGammas(gammas, vals, minSigma); - } else if (constraintType != "Const") { - if (constraintType == "LogNormal" || constraintType == "Lognormal") { - RooJSONFactoryWSTool::error("unsupported constraint '" + constraintType + - "' for the gamma parameters of '" + sysname + - "', supported are \"Poisson\" and \"Gauss\""); - } - auto constraintsInfo = createGammaConstraints( - gammas, vals, minSigma, constraintType == "Poisson" ? Constraint::Poisson : Constraint::Gaussian); - for (auto const &term : constraintsInfo.constraints) { - ws.import(*term, RooFit::RecycleConflictNodes(), RooFit::Silence()); - constraints.add(*ws.pdf(term->GetName())); - } - } else { - for (auto *gamma : static_range_cast(gammas)) { - gamma->setConstant(true); - } - } + if (dynamic_cast(existing) || dynamic_cast(existing)) { + return *static_cast(existing); } - - return phf; + RooJSONFactoryWSTool::error("the parameter '" + name + "' of the modifier '" + modifier + + "' is neither a variable nor a constant"); } -/// Find the staterror modifier of a sample, or return nullptr if there is none. -const JSONNode *findStaterror(const JSONNode &comp) +void setErrorIfVariable(RooAbsReal ¶m, double error) { - if (comp.has_child("modifiers")) { - for (const auto &mod : comp["modifiers"].children()) { - if (mod["type"].val() == ::Literals::staterror) - return &mod; - } + if (auto *var = dynamic_cast(¶m)) { + var->setError(error); } - return nullptr; } -// The entries of the HS3 "constraints" of a modifier: the names of the constraint distributions, or std::nullopt for -// unconstrained parameters (null entries). -std::vector> constraintNames(const JSONNode &mod) +// All constraints of parameters that control a whole sample are created here. Following the HS3 standard, the nominal +// value is defined by the type of the modifier, and the standard deviation is one. +void addConstraint(RooJSONFactoryWSTool &tool, HS3::Constraint const &constraint, RooAbsReal ¶m, double nominal, + std::string const &modifier, std::string const &sample, RooArgSet &constraints) { - std::vector> out; - auto *constraints = mod.find("constraints"); - if (!constraints) { - return out; - } - if (!constraints->is_seq()) { - RooJSONFactoryWSTool::error("\"constraints\" of modifier '" + RooJSONFactoryWSTool::name(mod) + - "' must be a list"); - } - for (const auto &entry : constraints->children()) { - out.push_back(entry.is_null() ? std::nullopt : std::make_optional(entry.val())); - } - return out; + RooWorkspace &ws = *tool.workspace(); + std::visit(HS3::overloaded{ + [](HS3::Unconstrained) {}, + [&](HS3::ConstraintType type) { + if (type != HS3::ConstraintType::Gauss) { + RooJSONFactoryWSTool::error(std::string("legacy constraint value '") + toString(type) + + "' for modifier '" + modifier + + "' is a known constraint type, but it cannot be resolved in this " + "context"); + } + RooRealVar &nom = createNominal(ws, param.GetName(), nominal, -10, 10); + setErrorIfVariable(param, 1.0); + const std::string name = constraintName(param.GetName()); + RooAbsPdf *pdf = ws.pdf(name); + constraints.add(pdf ? *pdf : tool.wsEmplace(name, param, nom, 1.)); + }, + [&](HS3::ConstraintPdf const &constraintPdf) { + RooAbsPdf *pdf = findNamedConstraint(tool, constraintPdf.pdf.name, sample); + if (!pdf) { + RooJSONFactoryWSTool::error("unable to find definition of of constraint '" + + constraintPdf.pdf.name + "' for modifier '" + modifier + "'"); + } + if (auto gauss = dynamic_cast(pdf)) { + setErrorIfVariable(param, gauss->getSigma().getVal()); + } + constraints.add(*pdf); + }, + }, + constraint); } -// The parameter of a modifier with a single parameter, given in the HS3 "parameters" or in the legacy "parameter". -std::optional modifierParameter(const JSONNode &mod) +// Create the ParamHistFunc of the gamma parameters, together with their constraints. +ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, std::string const &sysname, + std::vector const &parnames, std::vector const &vals, + HS3::GammaConstraint const &constraint, RooArgSet &constraints, RooArgSet const &observables, + double gammaMax, double minSigma, std::size_t nBins) { - if (auto *parameters = mod.find("parameters")) { - if (!parameters->is_seq() || parameters->num_children() != 1) { - RooJSONFactoryWSTool::error("\"parameters\" of the " + mod["type"].val() + - " modifier must be a list with one entry"); - } - return parameters->child(0).val(); - } - if (auto *parameter = mod.find("parameter")) { - return parameter->val(); - } - return std::nullopt; -} + RooWorkspace &ws = *tool.workspace(); -// The constraint distribution named in the HS3 "constraints" of a modifier with a single parameter. Returns an empty -// string if the parameter is explicitly unconstrained, and std::nullopt if the modifier has no "constraints". -std::optional singleConstraintName(const JSONNode &mod) -{ - if (!mod.has_child("constraints")) { - return std::nullopt; + std::size_t n = std::max(vals.size(), parnames.size()); + if (n == 0) { + n = nBins; } - const auto names = constraintNames(mod); - if (names.size() != 1) { - RooJSONFactoryWSTool::error("\"constraints\" of the " + mod["type"].val() + - " modifier must have one entry per parameter"); + if (!parnames.empty() && !vals.empty() && parnames.size() != vals.size()) { + RooJSONFactoryWSTool::error("the modifier '" + sysname + "' has " + std::to_string(vals.size()) + + " values, but " + std::to_string(parnames.size()) + " parameters"); } - return names.front().value_or(""); -} - -RooAbsPdf & -getOrCreateConstraint(RooJSONFactoryWSTool &tool, const JSONNode &mod, RooRealVar ¶m, const std::string &sample) -{ - // HS3 references the constraint distributions by name. - if (auto name = singleConstraintName(mod); name && !name->empty()) { - auto *constraint = findNamedConstraint(tool, *name, sample); - if (!constraint) { - RooJSONFactoryWSTool::error("unable to find the constraint distribution '" + *name + "' of the modifier of '" + - param.GetName() + "'"); - } - if (auto gauss = dynamic_cast(constraint)) { - param.setError(gauss->getSigma().getVal()); + RooArgList gammas; + for (std::size_t i = 0; i < n; ++i) { + const std::string name = parnames.empty() ? defaultGammaName(sysname, i) : parnames[i].name; + if (auto *existing = dynamic_cast(ws.obj(name))) { + gammas.add(*existing); + } else { + gammas.add(getOrCreate(ws, name, 1., defaultGammaMin, gammaMax)); } - return *constraint; } - JSONNode const *constrName = mod.find("constraint_name"); - if (constrName) { - auto constraint_name = constrName->val(); - auto constraint = findNamedConstraint(tool, constraint_name, sample); - if (!constraint) { - RooJSONFactoryWSTool::error("unable to find definition of of constraint '" + constraint_name + - "' for modifier '" + RooJSONFactoryWSTool::name(mod) + "'"); + // Shape modifiers of the same name in a channel share the function, so they need the same parameters. + if (RooAbsArg *existing = ws.arg(phfname.c_str())) { + auto *existingPHF = dynamic_cast(existing); + bool same = existingPHF && existingPHF->paramList().size() == gammas.size(); + for (std::size_t i = 0; same && i < gammas.size(); ++i) { + same = &existingPHF->paramList()[i] == &gammas[i]; } - if (auto gauss = dynamic_cast(constraint)) { - param.setError(gauss->getSigma().getVal()); + if (!same) { + RooJSONFactoryWSTool::error("the modifier '" + sysname + "' has other parameters than another modifier " + + "of this name in the channel"); } - return *constraint; } + auto &phf = tool.wsEmplace(phfname, observables, gammas); - if (auto constr = mod.find("constraint")) { - std::string constraintValue = constr->val(); - if (auto *constraint = findNamedConstraint(tool, constraintValue, sample)) { - if (auto gauss = dynamic_cast(constraint)) { - param.setError(gauss->getSigma().getVal()); - } - return *constraint; + auto addGammaConstraints = [&](Constraint::Type type) { + for (auto const &term : createGammaConstraints(gammas, vals, minSigma, type).constraints) { + ws.import(*term, RooFit::RecycleConflictNodes(), RooFit::Silence()); + constraints.add(*ws.pdf(term->GetName())); } + }; - if (isLegacyConstraintType(constraintValue)) { - return createLegacyConstraint(tool, mod, param, constraintValue); + if (auto const *pdfs = std::get_if(&constraint)) { + if (pdfs->pdfs.size() != n) { + RooJSONFactoryWSTool::error("modifier '" + sysname + "' has " + std::to_string(pdfs->pdfs.size()) + + " constraints, but " + std::to_string(n) + " parameters"); } + for (auto const &pdfName : pdfs->pdfs) { + if (!pdfName) { + continue; + } + auto *pdf = findNamedConstraint(tool, pdfName->name, sysname); + if (!pdf) { + RooJSONFactoryWSTool::error("unable to find definition of constraint '" + pdfName->name + + "' for modifier '" + sysname + "'"); + } + constraints.add(*pdf); + } + } + + if (vals.empty()) { + return phf; + } + + std::visit(HS3::overloaded{ + [&](HS3::Unconstrained) { configureConstrainedGammas(gammas, vals, minSigma); }, + [&](HS3::GammaConstraintPdfs const &) { configureConstrainedGammas(gammas, vals, minSigma); }, + [&](HS3::ConstantGammas) { + for (auto *gamma : static_range_cast(gammas)) { + gamma->setConstant(true); + } + }, + [&](HS3::ConstraintType type) { + switch (type) { + case HS3::ConstraintType::Gauss: addGammaConstraints(Constraint::Gaussian); break; + case HS3::ConstraintType::Poisson: addGammaConstraints(Constraint::Poisson); break; + case HS3::ConstraintType::LogNormal: + RooJSONFactoryWSTool::error(std::string("unsupported constraint '") + toString(type) + + "' for the gamma parameters of '" + sysname + + "', supported are \"Poisson\" and \"Gauss\""); + } + }, + }, + constraint); - RooJSONFactoryWSTool::error("unable to resolve constraint value '" + constraintValue + "' for modifier '" + - RooJSONFactoryWSTool::name(mod) + - "': this looks like a legacy workspace where the 'constraint' field is neither a " - "constraint pdf name nor a supported legacy constraint type"); - } - - std::string constraint_type = "Gauss"; - if (auto constrType = mod.find("constraint_type")) { - constraint_type = constrType->val(); - } - if (isLegacyConstraintType(constraint_type)) { - return createLegacyConstraint(tool, mod, param, constraint_type); - } - RooJSONFactoryWSTool::error("unknown or invalid constraint for modifier '" + RooJSONFactoryWSTool::name(mod) + "'"); -} -bool hasConstraint(const JSONNode &mod) -{ - if (auto name = singleConstraintName(mod)) { - return !name->empty(); - } - return mod.has_child("constraint") || mod.has_child("constraint_name") || mod.has_child("constraint_type"); -} - -// Create the constraint of a normsys or histosys modifier. Without a constraint, the parameter is unconstrained. Only -// documents that predate HS3 v0.2 imply a unit Gaussian, like the ones written by the former HistFactory::JSONTool. -void addSysConstraint(RooJSONFactoryWSTool &tool, const JSONNode &mod, RooRealVar ¶m, std::string const &sysname, - std::string const &sample, bool &warnedUnconstrained, RooArgSet &constraints) -{ - if (!hasConstraint(mod) && !isHS3VersionBefore(tool.hs3VersionInput(), 0, 2)) { - if (!warnedUnconstrained) { - RooJSONFactoryWSTool::warning("RooFitHS3: modifier '" + sysname + "' of sample '" + sample + - "' has no constraint, so its parameter '" + param.GetName() + - "' is unconstrained. Older RooFit versions implied a unit Gaussian. Other " - "modifiers of this channel are not reported."); - warnedUnconstrained = true; - } - return; - } - constraints.add(getOrCreateConstraint(tool, mod, param, sample)); + return phf; } double poissonTau(RooPoisson const &constraint, RooAbsArg const &gamma) @@ -751,401 +567,233 @@ double constraintRelError(RooAbsPdf const &constraint, RooAbsArg const &gamma) return std::numeric_limits::quiet_NaN(); } -bool importHistSample(RooJSONFactoryWSTool &tool, RooDataHist &dh, RooArgSet const &varlist, - RooAbsArg const *mcStatObject, const std::string &fprefix, const JSONNode &p, - const std::optional &defaultInterpolation, - LegacyInterpolationDefault &legacyInterpolationDefault, bool &warnedUnconstrained, - bool &warnedShapeSys, RooArgSet &constraints) +struct ChannelContext { + RooJSONFactoryWSTool &tool; + HS3::HistFactoryDist const &dist; + RooArgSet const &observables; + std::string fprefix; + std::string channelName; + std::size_t nBins; + RooAbsArg const *mcStatObject; + RooArgSet &constraints; +}; + +// Collects what the modifiers of a sample contribute to the RooFit objects. +struct SampleElements { + RooArgList shapeElems; + RooArgList normElems; + + RooArgList overallNps; + std::vector overallLow; + std::vector overallHigh; + std::vector overallInterp; + + RooArgList histNps; + RooArgList histoLo; + RooArgList histoHi; + std::vector histoInterp; +}; + +void buildSample(ChannelContext const &ctx, HS3::Sample const &sample, RooDataHist &dh) { + RooJSONFactoryWSTool &tool = ctx.tool; RooWorkspace &ws = *tool.workspace(); - std::string sampleName = RooJSONFactoryWSTool::name(p); - std::string prefixedName = fprefix + "_" + sampleName; + std::string const &sampleName = sample.name; + std::string prefixedName = ctx.fprefix + "_" + sampleName; - std::string channelName = fprefix; - erasePrefix(channelName, "model_"); + // The objects of a sample are not shared, so an existing object of the same name is a name clash. + auto newName = [&](std::string name) { + if (ws.arg(name.c_str())) { + RooJSONFactoryWSTool::error("the object '" + name + "' for sample '" + sampleName + "' of channel '" + + ctx.channelName + "' has the name of an existing object"); + } + return name; + }; - if (!p.has_child("data")) { - RooJSONFactoryWSTool::error("sample '" + sampleName + "' does not define a 'data' key"); - } + auto &hf = tool.wsEmplace(newName("hist_" + prefixedName), ctx.observables, dh); + hf.SetTitle(sampleName.c_str()); - auto &hf = tool.wsEmplace("hist_" + prefixedName, varlist, dh); - hf.SetTitle(RooJSONFactoryWSTool::name(p).c_str()); + SampleElements elems; + elems.shapeElems.add(tool.wsEmplace(newName(prefixedName + "_binWidth"), hf, true)); - RooArgList shapeElems; - RooArgList normElems; + const bool hasStatError = std::any_of(sample.modifiers.begin(), sample.modifiers.end(), + [](auto const &m) { return std::holds_alternative(m); }); + if (hasStatError) { + elems.shapeElems.add(*ctx.mcStatObject); + } - shapeElems.add(tool.wsEmplace(prefixedName + "_binWidth", hf, true)); - - if (findStaterror(p)) { - shapeElems.add(*mcStatObject); - } - - if (p.has_child("modifiers")) { - RooArgList overall_nps; - std::vector overall_low; - std::vector overall_high; - std::vector overall_interp; - - RooArgList histNps; - RooArgList histoLo; - RooArgList histoHi; - std::vector histoInterp; - - int idx = 0; - for (const auto &mod : p["modifiers"].children()) { - std::string const &modtype = mod["type"].val(); - const std::optional parameter = - modtype == "normfactor" || modtype == "normsys" || modtype == "histosys" ? modifierParameter(mod) - : std::nullopt; - std::string const &sysname = - mod.has_child("name") ? mod["name"].val() : parameter.value_or("syst_" + std::to_string(idx)); - ++idx; - if (modtype == "staterror") { - // this is dealt with at a different place, ignore it for now - } else if (modtype == "normfactor") { - // The "name" is only a label of the modifier, the parameter is given by "parameters" if present. - RooRealVar &constrParam = getOrCreate(ws, parameter.value_or(sysname), 1., -3, 5); - constrParam.setError(0.0); - normElems.add(constrParam); - if (hasConstraint(mod)) { + auto context = [&](const char *type, std::string const &name) { + return std::string("interpolation for ") + type + " modifier '" + name + "' in sample '" + sampleName + + "' of channel '" + ctx.channelName + "'"; + }; + + for (HS3::Modifier const &modifier : sample.modifiers) { + std::visit( + HS3::overloaded{ + [&](HS3::StatError const &) { + // this is dealt with at a different place, ignore it for now + }, + [&](HS3::NormFactor const &mod) { + RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 1., -3, 5, mod.name); + setErrorIfVariable(param, 0.0); + elems.normElems.add(param); // for norm factors, constraints are optional - constraints.add(getOrCreateConstraint(tool, mod, constrParam, sampleName)); - } - } else if (modtype == "normsys") { - std::string parname(parameter.value_or("alpha_" + sysname)); - auto &par = getOrCreate(ws, parname, 0., -5, 5); - overall_nps.add(par); - auto &data = mod["data"]; - const std::string context = "interpolation for normsys modifier '" + sysname + "' in sample '" + - sampleName + "' of channel '" + channelName + "'"; - const int interp = interpolationCode(mod, defaultInterpolation, InterpolationClass::Flexible, context, - legacyInterpolationDefault); - double low = data["lo"].val_double(); - double high = data["hi"].val_double(); - - // the below contains a a hack to cut off variations that go below 0 - // This is needed because FlexibleInterpVar code 4 interpolates in log-space. Hence, values <= 0 result in - // NaN, which propagates throughout the model and causes evaluations to fail. If you know a nicer way to - // solve this, please go ahead and fix the lines below. - if (interp == 4 && low <= 0) - low = std::numeric_limits::epsilon(); - if (interp == 4 && high <= 0) - high = std::numeric_limits::epsilon(); - - overall_low.push_back(low); - overall_high.push_back(high); - overall_interp.push_back(interp); - - addSysConstraint(tool, mod, par, sysname, sampleName, warnedUnconstrained, constraints); - } else if (modtype == "histosys") { - std::string parname(parameter.value_or("alpha_" + sysname)); - auto &par = getOrCreate(ws, parname, 0., -5, 5); - histNps.add(par); - auto &data = mod["data"]; - histoLo.add(tool.wsEmplace( - sysname + "Low_" + prefixedName, varlist, - RooJSONFactoryWSTool::readBinnedData(data["lo"], sysname + "Low_" + prefixedName, varlist))); - histoHi.add(tool.wsEmplace( - sysname + "High_" + prefixedName, varlist, - RooJSONFactoryWSTool::readBinnedData(data["hi"], sysname + "High_" + prefixedName, varlist))); - const std::string context = "interpolation for histosys modifier '" + sysname + "' in sample '" + - sampleName + "' of channel '" + channelName + "'"; - histoInterp.push_back(interpolationCode(mod, defaultInterpolation, InterpolationClass::Piecewise, context, - legacyInterpolationDefault)); - addSysConstraint(tool, mod, par, sysname, sampleName, warnedUnconstrained, constraints); - } else if (modtype == "shapesys" || modtype == "shapefactor") { - // HS3 has a single per-bin modifier type, "shapefactor", whose parameters are constrained by the - // distributions in "constraints", if given. "shapesys" is a deprecated alias. - if (modtype == "shapesys" && !isHS3VersionBefore(tool.hs3VersionInput(), 0, 3) && !warnedShapeSys) { - RooJSONFactoryWSTool::warning("RooFitHS3: the modifier type \"shapesys\" is deprecated, please use " - "\"shapefactor\" instead."); - warnedShapeSys = true; - } - std::vector vals; - if (mod["data"].has_child("vals")) { - for (const auto &v : mod["data"]["vals"].children()) { - vals.push_back(v.val_double()); - } - } - std::vector parnames; - for (const auto &v : mod["parameters"].children()) { - parnames.push_back(v.val()); - } - if (vals.empty() && parnames.empty()) { - RooJSONFactoryWSTool::error("unable to instantiate shapesys '" + sysname + - "' with neither values nor parameters!"); - } - std::string constraint = "unknown"; - std::vector constraintPdfs; - bool const hasConstraintList = mod.has_child("constraints"); - if (hasConstraintList) { - for (const auto &v : mod["constraints"].children()) { - if (v.is_null()) { - constraintPdfs.push_back(nullptr); - } else { - std::string constraintName = v.val(); - auto *constraintPdf = findNamedConstraint(tool, constraintName, sampleName); - if (!constraintPdf) { - RooJSONFactoryWSTool::error("unable to find definition of constraint '" + constraintName + - "' for modifier '" + RooJSONFactoryWSTool::name(mod) + "'"); - } - constraintPdfs.push_back(constraintPdf); - } - } - std::size_t const nGammas = std::max(vals.size(), parnames.size()); - if (constraintPdfs.size() != nGammas) { - std::stringstream ss; - ss << "modifier '" << RooJSONFactoryWSTool::name(mod) << "' has " << constraintPdfs.size() - << " constraints, but " << nGammas << " parameters"; - RooJSONFactoryWSTool::error(ss.str()); - } - } else if (mod.has_child("constraint_type")) { - constraint = mod["constraint_type"].val(); - } else if (mod.has_child("constraint")) { - std::string constraintValue = mod["constraint"].val(); - if (isLegacyConstraintType(constraintValue)) { - constraint = constraintValue; + addConstraint(tool, mod.constraint, param, 1.0, mod.name, sampleName, ctx.constraints); + }, + [&](HS3::NormSys const &mod) { + RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 0., -5, 5, mod.name); + elems.overallNps.add(param); + const int interp = interpolationCode(mod.interpolation, ctx.dist.defaultInterpolation, + InterpolationClass::Flexible, context("normsys", mod.name)); + double low = mod.lo; + double high = mod.hi; + + // the below contains a a hack to cut off variations that go below 0 + // This is needed because FlexibleInterpVar code 4 interpolates in log-space. Hence, values <= 0 result + // in NaN, which propagates throughout the model and causes evaluations to fail. If you know a nicer way + // to solve this, please go ahead and fix the lines below. + if (interp == 4 && low <= 0) + low = std::numeric_limits::epsilon(); + if (interp == 4 && high <= 0) + high = std::numeric_limits::epsilon(); + + elems.overallLow.push_back(low); + elems.overallHigh.push_back(high); + elems.overallInterp.push_back(interp); + + addConstraint(tool, mod.constraint, param, 0.0, mod.name, sampleName, ctx.constraints); + }, + [&](HS3::HistoSys const &mod) { + RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 0., -5, 5, mod.name); + elems.histNps.add(param); + auto addVariation = [&](RooArgList &list, HS3::HistoSysVariation const &variation, const char *suffix) { + const std::string name = newName(mod.name + suffix + prefixedName); + list.add( + tool.wsEmplace(name, ctx.observables, + HS3::createDataHist(name, ctx.observables, variation.contents, + variation.errors ? &*variation.errors : nullptr))); + }; + addVariation(elems.histoLo, mod.lo, "Low_"); + addVariation(elems.histoHi, mod.hi, "High_"); + elems.histoInterp.push_back(interpolationCode(mod.interpolation, ctx.dist.defaultInterpolation, + InterpolationClass::Piecewise, + context("histosys", mod.name))); + addConstraint(tool, mod.constraint, param, 0.0, mod.name, sampleName, ctx.constraints); + }, + [&](HS3::ShapeSys const &mod) { + const bool constrained = std::visit( + HS3::overloaded{[](HS3::Unconstrained) { return false; }, [](HS3::ConstantGammas) { return false; }, + [&](HS3::ConstraintType) { return !mod.vals.empty(); }, + [](HS3::GammaConstraintPdfs const &constraints) { + return std::any_of(constraints.pdfs.begin(), constraints.pdfs.end(), + [](auto const &pdf) { return pdf.has_value(); }); + }}, + mod.constraint); + // Follow the naming of HistFactory: "__ShapeSys" for constrained and + // "__shapeFactor" for free parameters. + const std::string funcName = + ctx.channelName + "_" + mod.name + (constrained ? "_ShapeSys" : "_shapeFactor"); + elems.shapeElems.add(createPHF(tool, funcName, mod.name, mod.parameters, mod.vals, mod.constraint, + ctx.constraints, ctx.observables, defaultShapeSysGammaMax, + minShapeUncertainty, ctx.nBins)); + }, + [&](HS3::CustomModifier const &mod) { + // The function can come later in the document. + RooAbsReal *obj = tool.request(mod.name, ctx.channelName); + if (obj->dependsOn(ctx.observables)) { + elems.shapeElems.add(*obj); } else { - RooJSONFactoryWSTool::error("unable to resolve constraint value '" + constraintValue + - "' for modifier '" + RooJSONFactoryWSTool::name(mod) + - "': this looks like a legacy workspace where the 'constraint' field is " - "not a supported legacy constraint type"); + elems.normElems.add(*obj); } - } - // Without "constraints", only the legacy constraint keys, or documents before HS3 0.3, make RooFit - // create constraints from "data.vals". Otherwise, the parameters are free. - const bool legacyConstraint = mod.has_child("constraint_type") || mod.has_child("constraint"); - const bool createConstraints = - !hasConstraintList && (legacyConstraint || isHS3VersionBefore(tool.hs3VersionInput(), 0, 3)); - if (!hasConstraintList && !createConstraints && !vals.empty()) { - oocoutW(nullptr, ObjectHandling) - << "RooFitHS3: the \"data.vals\" of modifier '" << sysname << "' of sample '" << sampleName - << "' are ignored, because it names no constraints, so its " - << "parameters are free" << std::endl; - vals.clear(); - } - const bool constrained = createConstraints ? !vals.empty() && constraint != "Const" - : std::any_of(constraintPdfs.begin(), constraintPdfs.end(), - [](RooAbsPdf *pdf) { return pdf != nullptr; }); - // Follow the naming of HistFactory: "__ShapeSys" for constrained and - // "__shapeFactor" for free parameters. - const std::string funcName = channelName + "_" + sysname + (constrained ? "_ShapeSys" : "_shapeFactor"); - shapeElems.add(createPHF(funcName, sysname, parnames, vals, tool, constraints, varlist, constraint, - defaultGammaMin, defaultShapeSysGammaMax, minShapeUncertainty, createConstraints)); - for (auto *constraintPdf : constraintPdfs) { - if (constraintPdf) { - constraints.add(*constraintPdf); - } - } - } else if (modtype == "custom") { - RooAbsReal *obj = ws.function(sysname); - if (!obj) { - RooJSONFactoryWSTool::error("unable to find custom modifier '" + sysname + "'"); - } - if (obj->dependsOn(varlist)) { - shapeElems.add(*obj); - } else { - normElems.add(*obj); - } - } else { - RooJSONFactoryWSTool::error("modifier '" + sysname + "' of unknown type '" + modtype + "'"); - } - } + }, + }, + modifier); + } - std::string interpName = sampleName + "_" + channelName + "_epsilon"; - if (!overall_nps.empty()) { - auto &v = tool.wsEmplace(interpName, overall_nps, 1., overall_low, - overall_high, overall_interp); - normElems.add(v); - } - if (!histNps.empty()) { - auto &v = tool.wsEmplace("histoSys_" + prefixedName, hf, histoLo, histoHi, histNps, - histoInterp); - v.setPositiveDefinite(); - shapeElems.add(v); - } else { - shapeElems.add(hf); - } + if (!elems.overallNps.empty()) { + std::string interpName = newName(sampleName + "_" + ctx.channelName + "_epsilon"); + elems.normElems.add(tool.wsEmplace( + interpName, elems.overallNps, 1., elems.overallLow, elems.overallHigh, elems.overallInterp)); + } + if (!elems.histNps.empty()) { + auto &v = tool.wsEmplace(newName("histoSys_" + prefixedName), hf, elems.histoLo, + elems.histoHi, elems.histNps, elems.histoInterp); + v.setPositiveDefinite(); + elems.shapeElems.add(v); + } else { + elems.shapeElems.add(hf); } - tool.wsEmplace(prefixedName + "_shapes", shapeElems); - if (!normElems.empty()) { - tool.wsEmplace(prefixedName + "_scaleFactors", normElems); + tool.wsEmplace(newName(prefixedName + "_shapes"), elems.shapeElems); + if (!elems.normElems.empty()) { + tool.wsEmplace(newName(prefixedName + "_scaleFactors"), elems.normElems); } else { - ws.factory("RooConstVar::" + prefixedName + "_scaleFactors(1.)"); + ws.factory("RooConstVar::" + newName(prefixedName + "_scaleFactors") + "(1.)"); } +} - return true; +// The constraint of the gamma parameters of the staterror modifiers. Without one, the gamma parameters are free. +HS3::GammaConstraint statErrorConstraint(HS3::HistFactoryDist const &dist) +{ + // The constraint distributions named by a staterror modifier take precedence over a constraint type, and a type + // given on a staterror modifier over the channel-level setting. + HS3::GammaConstraint const *modifierConstraint = nullptr; + for (auto const &sample : dist.samples) { + for (auto const &modifier : sample.modifiers) { + auto const *statError = std::get_if(&modifier); + if (!statError) { + continue; + } + if (std::holds_alternative(statError->constraint)) { + return statError->constraint; + } + if (!modifierConstraint && !std::holds_alternative(statError->constraint)) { + modifierConstraint = &statError->constraint; + } + } + } + return modifierConstraint ? *modifierConstraint : dist.statError.constraint; +} + +// Reject constraints of staterror modifiers that make no sense for gamma parameters. +void checkStatErrorConstraints(HS3::HistFactoryDist const &dist, std::string const &name) +{ + for (auto const &sample : dist.samples) { + for (auto const &modifier : sample.modifiers) { + auto const *statError = std::get_if(&modifier); + auto const *type = statError ? std::get_if(&statError->constraint) : nullptr; + if (type && *type == HS3::ConstraintType::LogNormal) { + RooJSONFactoryWSTool::error(std::string("unsupported constraint '") + toString(*type) + + "' of the staterror modifier in channel '" + name + + "', supported are \"Poisson\" and \"Gauss\""); + } + } + } } class HistFactoryImporter : public RooFit::JSONIO::Importer { public: + // This is only used for HistFactory distributions that don't come with a document. The upgrade rewrites the JSON, so + // it has to work on a copy. bool importArg(RooJSONFactoryWSTool *tool, const JSONNode &p) const override { std::string name = RooJSONFactoryWSTool::name(p); if (!p.has_child("samples")) { RooJSONFactoryWSTool::error("no samples in '" + name + "', skipping."); } - double statErrThresh = 0; - std::string statErrType = "Poisson"; - std::optional defaultInterpolation; - if (p.has_child("default_interpolation")) { - defaultInterpolation = - readInterpolation(p["default_interpolation"], "default_interpolation of channel '" + name + "'"); - } - if (p.has_child(::Literals::staterror)) { - auto &staterr = p[::Literals::staterror]; - if (staterr.has_child("relThreshold")) - statErrThresh = staterr["relThreshold"].val_double(); - if (staterr.has_child("constraint_type")) - statErrType = staterr["constraint_type"].val(); - } - std::vector sumW; - std::vector sumW2; - std::vector gammaParnames; - std::string modifierStatErrType; - std::optional>> statErrConstraintNames; - RooArgSet observables = RooJSONFactoryWSTool::readAxes(p); - - std::string fprefix = name; - - std::vector> data; - for (const auto &comp : p["samples"].children()) { - std::unique_ptr dh = RooJSONFactoryWSTool::readBinnedData( - comp["data"], fprefix + "_" + RooJSONFactoryWSTool::name(comp) + "_dataHist", observables); - size_t nbins = dh->numEntries(); - - if (const JSONNode *staterror = findStaterror(comp)) { - if (sumW.empty()) { - sumW.resize(nbins); - sumW2.resize(nbins); - } - // A sample without "errors" contributes no statistical uncertainty in HS3. Before HS3 0.3, RooFit - // assumed Poisson uncertainties for it. - const bool useWeightSquared = - comp["data"].has_child("errors") || isHS3VersionBefore(tool->hs3VersionInput(), 0, 3); - for (size_t i = 0; i < nbins; ++i) { - sumW[i] += dh->weight(i); - sumW2[i] += useWeightSquared ? dh->weightSquared(i) : 0.; - } - if (gammaParnames.empty()) { - if (auto staterrorParams = staterror->find("parameters")) { - for (const auto &v : staterrorParams->children()) { - gammaParnames.push_back(v.val()); - } - } - } - // HS3 names the constraint distributions of the gammas in "constraints". - if (!statErrConstraintNames && staterror->has_child("constraints")) { - statErrConstraintNames = constraintNames(*staterror); - } - // The legacy constraint type given on the staterror modifier takes precedence over the channel-level - // setting. - if (modifierStatErrType.empty()) { - // ROOT 6.38 and 6.40.00 wrote "constraint_type". - auto constraint = staterror->find("constraint"); - if (!constraint) { - constraint = staterror->find("constraint_type"); - } - if (constraint) { - modifierStatErrType = constraint->val(); - if (modifierStatErrType != "Poisson" && modifierStatErrType != "Gauss") { - RooJSONFactoryWSTool::error("unsupported constraint '" + modifierStatErrType + - "' of the staterror modifier in channel '" + name + - "', supported are \"Poisson\" and \"Gauss\""); - } - } - } - } - data.emplace_back(std::move(dh)); - } - - if (!modifierStatErrType.empty()) { - statErrType = modifierStatErrType; - } - // Without any constraint information, the gammas are unconstrained in HS3. Older documents relied on RooFit - // creating Poisson constraints. - const bool legacyStatErrConstraints = statErrConstraintNames || !modifierStatErrType.empty() || - p.has_child(::Literals::staterror) || - isHS3VersionBefore(tool->hs3VersionInput(), 0, 3); + std::stringstream ss; + p.writeJSON(ss); + std::unique_ptr tree = RooFit::Detail::JSONTree::create(ss.str()); - RooAbsArg *mcStatObject = nullptr; - RooArgSet constraints; - if (!sumW.empty()) { - std::string channelName = name; - erasePrefix(channelName, "model_"); - - std::vector errs(sumW.size()); - for (size_t i = 0; i < sumW.size(); ++i) { - if (sumW[i] == 0.) { - errs[i] = 0.; - continue; - } - errs[i] = std::sqrt(sumW2[i]) / sumW[i]; - // avoid negative sigma. This NP will be set constant anyway later - errs[i] = std::max(errs[i], 0.); - } + HS3::Version version; + HS3::parseVersion(tool->hs3VersionInput(), version); - const bool createConstraints = !statErrConstraintNames && legacyStatErrConstraints; - mcStatObject = &createPHF("mc_stat_" + channelName, "stat_" + channelName, gammaParnames, errs, *tool, - constraints, observables, statErrType, defaultGammaMin, defaultStatErrorGammaMax, - statErrThresh, createConstraints); - if (statErrConstraintNames) { - if (statErrConstraintNames->size() != errs.size()) { - RooJSONFactoryWSTool::error("the staterror modifier of channel '" + name + "' has " + - std::to_string(statErrConstraintNames->size()) + " constraints, but " + - std::to_string(errs.size()) + " parameters"); - } - for (auto const &constraintName : *statErrConstraintNames) { - if (!constraintName) { - continue; - } - auto *constraint = findNamedConstraint(*tool, *constraintName, name); - if (!constraint) { - RooJSONFactoryWSTool::error("unable to find the constraint distribution '" + *constraintName + - "' of the staterror modifier of channel '" + name + "'"); - } - constraints.add(*constraint); - } - } - } - - LegacyInterpolationDefault legacyInterpolationDefault; - legacyInterpolationDefault.allowed = isHS3VersionBefore(tool->hs3VersionInput(), 0, 3); - bool warnedUnconstrained = false; - bool warnedShapeSys = false; - - int idx = 0; - RooArgList funcs; - RooArgList coefs; - for (const auto &comp : p["samples"].children()) { - importHistSample(*tool, *data[idx], observables, mcStatObject, fprefix, comp, defaultInterpolation, - legacyInterpolationDefault, warnedUnconstrained, warnedShapeSys, constraints); - ++idx; - - std::string const &compName = RooJSONFactoryWSTool::name(comp); - funcs.add(*tool->request(fprefix + "_" + compName + "_shapes", name)); - coefs.add(*tool->request(fprefix + "_" + compName + "_scaleFactors", name)); - } - - // A histfactory_dist is a product of Poisson distributions of the bin counts. RooFit's binned likelihood - // computes exactly that (it differs from the event-wise likelihood only by a constant), and binned generation - // generates bin counts, so both are enabled, as HistFactory does by default. - auto enableBinned = [](RooRealSumPdf &sum) { - sum.setAttribute("BinnedLikelihood"); - sum.setAttribute("GenerateBinned"); - }; - if (constraints.empty()) { - enableBinned(tool->wsEmplace(name, funcs, coefs, true)); - } else { - std::string sumName = name + "_model"; - erasePrefix(sumName, "model_"); - auto &sum = tool->wsEmplace(sumName, funcs, coefs, true); - sum.SetTitle(name.c_str()); - enableBinned(sum); - tool->wsEmplace(name, constraints, RooFit::Conditional(sum, observables)); - } + HS3::Diagnostics diagnostics; + HS3::upgradeHistFactoryDist(tree->rootnode(), version, diagnostics); + HS3::HistFactoryDist dist = HS3::parseHistFactoryDist(tree->rootnode(), diagnostics); + HS3::report(diagnostics); + HS3::buildHistFactoryDist(*tool, name, dist); return true; } }; @@ -2330,3 +1978,103 @@ STATIC_EXECUTE([]() { }); } // namespace + +void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string const &name, HistFactoryDist const &dist) +{ + checkStatErrorConstraints(dist, name); + + RooArgSet observables = createObservables(dist.axes); + + std::vector> data; + data.reserve(dist.samples.size()); + std::vector sumW; + std::vector sumW2; + std::vector gammaParnames; + for (auto const &sample : dist.samples) { + auto dh = createDataHist(name + "_" + sample.name + "_dataHist", observables, sample.contents, + sample.errors ? &*sample.errors : nullptr); + const std::size_t nbins = dh->numEntries(); + + auto found = std::find_if(sample.modifiers.begin(), sample.modifiers.end(), + [](auto const &m) { return std::holds_alternative(m); }); + if (found != sample.modifiers.end()) { + if (sumW.empty()) { + sumW.resize(nbins); + sumW2.resize(nbins); + } + // A sample without "errors" contributes no statistical uncertainty, unless Poisson uncertainties are implied. + const bool hasErrors = sample.errors || dist.statError.impliedPoissonErrors; + for (std::size_t i = 0; i < nbins; ++i) { + sumW[i] += dh->weight(i); + sumW2[i] += hasErrors ? dh->weightSquared(i) : 0.; + } + if (gammaParnames.empty()) { + gammaParnames = std::get(*found).parameters; + } + } + data.emplace_back(std::move(dh)); + } + + std::string channelName = name; + erasePrefix(channelName, "model_"); + + std::size_t nBins = 1; + for (auto const &axis : dist.axes) { + nBins *= std::visit(overloaded{[](RegularBinning const &b) { return static_cast(b.nbins); }, + [](EdgesBinning const &b) { return b.edges.size() - 1; }}, + axis.binning); + } + + RooAbsArg *mcStatObject = nullptr; + RooArgSet constraints; + if (!sumW.empty()) { + std::vector errs(sumW.size()); + for (std::size_t i = 0; i < sumW.size(); ++i) { + if (sumW[i] == 0.) { + errs[i] = 0.; + continue; + } + errs[i] = std::sqrt(sumW2[i]) / sumW[i]; + // avoid negative sigma. This NP will be set constant anyway later + errs[i] = std::max(errs[i], 0.); + } + + mcStatObject = &createPHF(tool, "mc_stat_" + channelName, "stat_" + channelName, gammaParnames, errs, + statErrorConstraint(dist), constraints, observables, defaultStatErrorGammaMax, + dist.statError.relThreshold, nBins); + } + + ChannelContext ctx{tool, dist, observables, name, channelName, nBins, mcStatObject, constraints}; + + RooArgList funcs; + RooArgList coefs; + for (std::size_t i = 0; i < dist.samples.size(); ++i) { + HS3::Sample const &sample = dist.samples[i]; + buildSample(ctx, sample, *data[i]); + funcs.add(*tool.request(name + "_" + sample.name + "_shapes", name)); + coefs.add(*tool.request(name + "_" + sample.name + "_scaleFactors", name)); + } + + // A histfactory_dist is a product of Poisson distributions of the bin counts. RooFit's binned likelihood + // computes exactly that (it differs from the event-wise likelihood only by a constant), and binned generation + // generates bin counts, so both are enabled, as HistFactory does by default. + auto enableBinned = [](RooRealSumPdf &sum) { + sum.setAttribute("BinnedLikelihood"); + sum.setAttribute("GenerateBinned"); + }; + if (constraints.empty()) { + enableBinned(tool.wsEmplace(name, funcs, coefs, true)); + } else { + std::string sumName = name + "_model"; + erasePrefix(sumName, "model_"); + auto &sum = tool.wsEmplace(sumName, funcs, coefs, true); + sum.SetTitle(name.c_str()); + enableBinned(sum); + tool.wsEmplace(name, constraints, RooFit::Conditional(sum, observables)); + } +} + +bool RooFit::HS3::isBuiltinHistFactoryImporter(RooFit::JSONIO::Importer const &importer) +{ + return dynamic_cast(&importer) != nullptr; +} diff --git a/roofit/hs3/src/JSONInterface.cxx b/roofit/hs3/src/JSONInterface.cxx index f4a2dbf504eab..4e1d5842b44a6 100644 --- a/roofit/hs3/src/JSONInterface.cxx +++ b/roofit/hs3/src/JSONInterface.cxx @@ -210,6 +210,11 @@ void JSONNode::clear() json(_value).clear(); } +void JSONNode::erase(std::string const &key) +{ + json(_value).erase(key); +} + std::string JSONNode::key() const { return _key; diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index 2de5d34341ffe..1ea36f6562fce 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -30,6 +30,7 @@ #include "JSONIOUtils.h" #include "Domains.h" +#include "HS3Builder.h" #include "HS3Schema.h" #include @@ -148,14 +149,36 @@ struct RooFit::HS3::ToolImportState { namespace { -// Forward the warnings to the message service, and throw if there are errors. -void report(HS3::Diagnostics const &diagnostics) +// Keeps track of the objects that are being imported. Objects that depend on each other would otherwise recurse until +// the stack overflows. +class ImportGuard { +public: + ImportGuard(std::vector &stack, std::string const &name) : _stack{stack} + { + if (std::find(_stack.begin(), _stack.end(), name) != _stack.end()) { + std::string chain; + for (std::string const &ancestor : _stack) { + chain += ancestor + " -> "; + } + RooJSONFactoryWSTool::error("circular dependency in the document: " + chain + name); + } + _stack.push_back(name); + } + ~ImportGuard() { _stack.pop_back(); } + +private: + std::vector &_stack; +}; + +} // namespace + +void RooFit::HS3::report(Diagnostics const &diagnostics) { std::string errors; - for (HS3::Diagnostic const &diagnostic : diagnostics) { + for (Diagnostic const &diagnostic : diagnostics) { const std::string text = diagnostic.path.empty() ? diagnostic.message : diagnostic.path + ": " + diagnostic.message; - if (diagnostic.severity == HS3::Diagnostic::Severity::Error) { + if (diagnostic.severity == Diagnostic::Severity::Error) { errors += (errors.empty() ? "" : "\n") + text; } else { RooJSONFactoryWSTool::warning("RooFitHS3: " + text); @@ -166,6 +189,8 @@ void report(HS3::Diagnostics const &diagnostics) } } +namespace { + // True if the number of components in `data` matches the number of categories in `pdf`. bool matches(const RooJSONFactoryWSTool::CombinedData &data, const RooSimultaneous *pdf) { @@ -438,6 +463,21 @@ void exportAttributes(const RooAbsArg *arg, JSONNode &rootnode) } } +std::unique_ptr createObservable(HS3::BinnedAxis const &axis) +{ + auto obs = std::make_unique(axis.name.c_str(), axis.name.c_str(), axis.min, axis.max); + if (auto const *edges = std::get_if(&axis.binning)) { + RooBinning bins(obs->getMin(), obs->getMax()); + for (double edge : edges->edges) { + bins.addBoundary(edge); + } + obs->setBinning(bins); + } else { + obs->setBins(std::get(axis.binning).nbins); + } + return obs; +} + // Create the variables for the axes of binned data. RooArgSet createBinnedObservables(std::vector> const &axes) { @@ -447,46 +487,11 @@ RooArgSet createBinnedObservables(std::vector(axis->name.c_str(), axis->name.c_str(), axis->min, axis->max); - if (auto const *edges = std::get_if(&axis->binning)) { - RooBinning bins(obs->getMin(), obs->getMax()); - for (double edge : edges->edges) { - bins.addBoundary(edge); - } - obs->setBinning(bins); - } else { - obs->setBins(std::get(axis->binning).nbins); - } - vars.addOwned(std::move(obs)); + vars.addOwned(createObservable(*axis)); } return vars; } -// Create the RooDataHist with the contents and errors. Negative errors stand for no error. -std::unique_ptr createDataHist(std::string const &name, RooArgSet const &vars, - std::vector const &contents, std::vector const *errors) -{ - if (errors && errors->size() != contents.size()) { - RooJSONFactoryWSTool::error("inconsistent number of errors and contents in '" + name + "': errors=" + - std::to_string(errors->size()) + ", contents=" + std::to_string(contents.size())); - } - auto bins = generateBinIndices(vars); - if (contents.size() != bins.size()) { - RooJSONFactoryWSTool::error("inconsistent bin numbers: contents=" + std::to_string(contents.size()) + - ", bins=" + std::to_string(bins.size())); - } - auto dh = std::make_unique(name, name, vars); - // RooDataHist only starts to store squared weights when it sees a positive error, and then initializes them from - // the weights so far. Trigger that before setting any weight, so that errors of zero stay zero. - if (errors && !errors->empty()) { - dh->set(0, 0., 1.); - } - for (std::size_t ibin = 0; ibin < bins.size(); ++ibin) { - dh->set(ibin, contents[ibin], errors ? (*errors)[ibin] : -1); - } - return dh; -} - // Make the range of an observable of unbinned data consistent with the "min" and "max" of the axis. Variables that // didn't get their range from a domain take it from the axis. If `domains` is null, all ranges are considered explicit. void applyUnbinnedAxisRange(HS3::ContinuousAxis const &axis, std::string const &dataName, RooRealVar &var, @@ -541,8 +546,8 @@ loadData(HS3::Data const &input, RooWorkspace &workspace, RooFit::JSONIO::Detail RooJSONFactoryWSTool::testValidName(input.name, true); if (auto const *binned = std::get_if(&input.body)) { - return createDataHist(input.name, createBinnedObservables(binned->axes), binned->contents, - binned->uncertainty ? &binned->uncertainty->sigma : nullptr); + return HS3::createDataHist(input.name, createBinnedObservables(binned->axes), binned->contents, + binned->uncertainty ? &binned->uncertainty->sigma : nullptr); } if (auto const *unbinned = std::get_if(&input.body)) { return createUnbinnedData(input.name, *unbinned, workspace, domains); @@ -843,6 +848,40 @@ void checkUniqueNames(HS3Document const &doc, bool strict) } // namespace +RooArgSet RooFit::HS3::createObservables(std::vector const &axes) +{ + RooArgSet vars; + for (BinnedAxis const &axis : axes) { + vars.addOwned(createObservable(axis)); + } + return vars; +} + +std::unique_ptr RooFit::HS3::createDataHist(std::string const &name, RooArgSet const &vars, + std::vector const &contents, + std::vector const *errors) +{ + if (errors && errors->size() != contents.size()) { + RooJSONFactoryWSTool::error("inconsistent number of errors and contents in '" + name + "': errors=" + + std::to_string(errors->size()) + ", contents=" + std::to_string(contents.size())); + } + auto bins = generateBinIndices(vars); + if (contents.size() != bins.size()) { + RooJSONFactoryWSTool::error("inconsistent bin numbers: contents=" + std::to_string(contents.size()) + + ", bins=" + std::to_string(bins.size())); + } + auto dh = std::make_unique(name, name, vars); + // RooDataHist only starts to store squared weights when it sees a positive error, and then initializes them from + // the weights so far. Trigger that before setting any weight, so that errors of zero stay zero. + if (errors && !errors->empty()) { + dh->set(0, 0., 1.); + } + for (std::size_t ibin = 0; ibin < bins.size(); ++ibin) { + dh->set(ibin, contents[ibin], errors ? (*errors)[ibin] : -1); + } + return dh; +} + RooJSONFactoryWSTool::RooJSONFactoryWSTool(RooWorkspace &ws) : _workspace{ws} {} RooJSONFactoryWSTool::~RooJSONFactoryWSTool() = default; @@ -972,7 +1011,7 @@ RooAbsPdf *RooJSONFactoryWSTool::requestImpl(const std::string &objna if (_import) { auto it = _import->distributions.find(objname); if (it != _import->distributions.end()) { - this->importFunction(*it->second->body.node, true); + this->importDistribution(*it->second); if (RooAbsPdf *retval = _workspace.pdf(objname)) return retval; } @@ -1238,19 +1277,7 @@ void RooJSONFactoryWSTool::importFunction(const JSONNode &p, bool importAllDepen if (_workspace.arg(name)) { return; } - // Objects that depend on each other would otherwise recurse until the stack overflows. - if (std::find(_importStack.begin(), _importStack.end(), name) != _importStack.end()) { - std::string chain; - for (std::string const &ancestor : _importStack) { - chain += ancestor + " -> "; - } - RooJSONFactoryWSTool::error("circular dependency in the document: " + chain + name); - } - _importStack.push_back(name); - struct PopOnExit { - std::vector &stack; - ~PopOnExit() { stack.pop_back(); } - } popOnExit{_importStack}; + ImportGuard importGuard{_importStack, name}; // if the key we found is not a map, it's an error if (!p.is_map()) { RooJSONFactoryWSTool::error("RooJSONFactoryWSTool() function node " + name + " is not a map!"); @@ -1327,6 +1354,25 @@ void RooJSONFactoryWSTool::importFunction(const JSONNode &p, bool importAllDepen } } +// Import a distribution of the document, which is interpreted by the document or by the registered importers. +void RooJSONFactoryWSTool::importDistribution(HS3::Distribution const &distribution) +{ + if (auto const *generic = std::get_if(&distribution.body)) { + this->importFunction(*generic->node, true); + return; + } + std::string const &name = distribution.name; + if (hasAttribute(name, "roofit_skip")) { + return; + } + RooJSONFactoryWSTool::testValidName(name, true); + if (_workspace.arg(name)) { + return; + } + ImportGuard importGuard{_importStack, name}; + HS3::buildHistFactoryDist(*this, name, std::get(distribution.body)); +} + /** * @brief Import a function from a JSON string into the workspace. * @@ -1645,13 +1691,13 @@ RooJSONFactoryWSTool::readBinnedData(const JSONNode &n, const std::string &name, contentVals << contents; if (!n.has_child("errors")) { - return createDataHist(name, vars, contentVals, nullptr); + return HS3::createDataHist(name, vars, contentVals, nullptr); } if (!n["errors"].is_seq()) RooJSONFactoryWSTool::error("errors are not in list form"); std::vector errorVals; errorVals << n["errors"]; - return createDataHist(name, vars, contentVals, &errorVals); + return HS3::createDataHist(name, vars, contentVals, &errorVals); } // Import a single variable (RooRealVar or RooConstVar) from the parameter `p` into the workspace. @@ -2056,11 +2102,19 @@ void RooJSONFactoryWSTool::setStringAttribute(const std::string &obj, const std: } // Import the JSON document rooted at `n` into the workspace. -void RooJSONFactoryWSTool::importAllNodes(const JSONNode &n) +void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) { + HS3::ParseOptions options; + // A user can replace how HistFactory distributions are imported by registering their own importer. + auto const &importers = RooFit::JSONIO::importers(); + auto histFactoryImporters = importers.find("histfactory_dist"); + options.typedHistFactory = histFactoryImporters != importers.end() && !histFactoryImporters->second.empty() && + HS3::isBuiltinHistFactoryImporter(*histFactoryImporters->second.front()); + HS3::Diagnostics diagnostics; - HS3Document doc = HS3::parseDocument(n, diagnostics); - report(diagnostics); + HS3::upgradeDocument(n, diagnostics, options); + HS3Document doc = HS3::parseDocument(n, diagnostics, options); + HS3::report(diagnostics); // The state of the import refers to the document, so it must not survive it, not even if there is an exception. struct ImportScope { @@ -2091,7 +2145,7 @@ void RooJSONFactoryWSTool::importAllNodes(const JSONNode &n) this->importFunction(*function.body.node, true); } for (auto const &distribution : doc.distributions) { - this->importFunction(*distribution.body.node, true); + this->importDistribution(distribution); } // This creates the index categories, which the parameter points can refer to. @@ -2245,7 +2299,7 @@ void RooJSONFactoryWSTool::importJSONElement(const std::string &name, const std: HS3::Diagnostics diagnostics; HS3::Data data; readValue(HS3::Cursor{n, diagnostics}, data); - report(diagnostics); + HS3::report(diagnostics); _workspace.import(*loadData(data, _workspace, nullptr), RooFit::Silence()); } else { this->importFunction(n, false); @@ -2260,7 +2314,7 @@ void RooJSONFactoryWSTool::importVariableElement(const JSONNode &elementNode) HS3::ParseOptions options; options.requireMetadata = false; HS3Document doc = HS3::parseDocument(tree->rootnode(), diagnostics, options); - report(diagnostics); + HS3::report(diagnostics); _domains = std::make_unique(); _domains->read(doc.domains); diff --git a/roofit/hs3/test/testRooFitHS3.cxx b/roofit/hs3/test/testRooFitHS3.cxx index 699daff1ea97b..822d1edf44e65 100644 --- a/roofit/hs3/test/testRooFitHS3.cxx +++ b/roofit/hs3/test/testRooFitHS3.cxx @@ -4019,7 +4019,7 @@ TEST(RooFitHS3, HistFactoryLogNormalGammaConstraint) for (std::string key : {"constraint", "constraint_type"}) { expectImportThrowsWithError(makeHistFactoryJSON(R"({"name": "shape", "type": "shapesys", ")" + key + R"(": ")" + type + R"(", "data": {"vals": [0.1, 0.2]}})"), - "unsupported constraint '" + type + "'"); + "unsupported constraint 'LogNormal'"); } } From d3b5275245fe38d449a62a9227cc6fa5d5b87432 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Thu, 5 Nov 2026 19:18:12 +0000 Subject: [PATCH 06/20] [RF][HS3] Report components that are ignored on import A component that RooFit doesn't know, like a misspelled key, was silently ignored. Report it with its location, also for the objects that are imported by the registered importers: the JSON tree tells a JSONReadRecorder about each read of a map. Older versions of RooFit wrote the binning of the axes of unbinned data, which is now removed before parsing. Assisted-by: claude-code:claude-opus-5-5 --- roofit/hs3/CMakeLists.txt | 1 + roofit/hs3/inc/RooFit/Detail/JSONInterface.h | 1 + roofit/hs3/src/HS3Parser.cxx | 64 +++++++++++++++++++- roofit/hs3/src/HS3Schema.h | 12 +++- roofit/hs3/src/HS3Upgrade.cxx | 56 +++++++++++++++-- roofit/hs3/src/JSONInterface.cxx | 57 +++++++++++++++++ roofit/hs3/src/JSONReadRecorder.h | 50 +++++++++++++++ roofit/hs3/src/RooJSONFactoryWSTool.cxx | 7 +++ 8 files changed, 238 insertions(+), 10 deletions(-) create mode 100644 roofit/hs3/src/JSONReadRecorder.h diff --git a/roofit/hs3/CMakeLists.txt b/roofit/hs3/CMakeLists.txt index 314fec0a5d802..b71134c9f13a6 100644 --- a/roofit/hs3/CMakeLists.txt +++ b/roofit/hs3/CMakeLists.txt @@ -55,6 +55,7 @@ target_sources( src/HS3Document.h src/HS3Schema.h src/JSONIOUtils.h + src/JSONReadRecorder.h src/static_execute.h ) diff --git a/roofit/hs3/inc/RooFit/Detail/JSONInterface.h b/roofit/hs3/inc/RooFit/Detail/JSONInterface.h index e8fc73679160f..faa4aaf6c3bb1 100644 --- a/roofit/hs3/inc/RooFit/Detail/JSONInterface.h +++ b/roofit/hs3/inc/RooFit/Detail/JSONInterface.h @@ -190,6 +190,7 @@ class JSONNode { private: friend class JSONTree; + friend class JSONReadRecorder; // Owns the values and the nodes of a tree. It is defined in the only source file that knows the JSON library. class Storage; diff --git a/roofit/hs3/src/HS3Parser.cxx b/roofit/hs3/src/HS3Parser.cxx index a4129b3401e6c..5bc2089e9350b 100644 --- a/roofit/hs3/src/HS3Parser.cxx +++ b/roofit/hs3/src/HS3Parser.cxx @@ -126,6 +126,40 @@ void Cursor::warning(std::string const &message) const _diagnostics.push_back({Diagnostic::Severity::Warning, path(), message}); } +std::size_t Cursor::numErrors() const +{ + return std::count_if(_diagnostics.begin(), _diagnostics.end(), + [](Diagnostic const &d) { return d.severity == Diagnostic::Severity::Error; }); +} + +Reader::Reader(Cursor const &cursor) : _cursor{cursor}, _nErrorsBefore{cursor.numErrors()} {} + +bool Reader::has(const char *key) +{ + if (!_cursor.node().has_child(key)) { + return false; + } + if (std::find(_seen.begin(), _seen.end(), key) == _seen.end()) { + _seen.emplace_back(key); + } + return true; +} + +Reader::~Reader() +{ + JSONNode const &node = _cursor.node(); + // There is nothing to report if all components were read, and after an error it would only be noise. + if (!node.is_map() || node.num_children() == _seen.size() || _cursor.numErrors() != _nErrorsBefore) { + return; + } + for (JSONNode const &child : node.children()) { + const std::string key = child.key(); + if (std::find(_seen.begin(), _seen.end(), key) == _seen.end()) { + _cursor.warning("unknown component \"" + key + "\" is ignored"); + } + } +} + void readValue(Cursor const &c, std::string &out) { if (c.node().is_container() || c.node().is_null()) { @@ -695,8 +729,13 @@ void readValue(Cursor const &c, Interpolation &out) out.out = interpolationFunctionFromString(outNode.val()); if (!out.out) { c.error("has unknown extrapolation function '" + outNode.val() + "'"); + return; } } + Reader reader{c}; + for (const char *component : {"type", "in", "out"}) { + reader.has(component); + } } namespace { @@ -946,6 +985,10 @@ void readDistribution(Cursor const &c, Distribution &out, ParseOptions const &op out.body = std::move(body); } else { out.body = GenericObject{type, &c.node()}; + // The importers read the other components. + for (JSONNode const &child : c.node().children()) { + ar.has(child.key().c_str()); + } } } @@ -961,6 +1004,10 @@ void readValue(Cursor const &c, Function &out) ar("name", out.name); ar("type", out.body.type); out.body.node = &c.node(); + // The importers read the other components. + for (JSONNode const &child : c.node().children()) { + ar.has(child.key().c_str()); + } } void describe(Reader &ar, Attributes &attributes) @@ -1117,6 +1164,7 @@ void readMetadata(Cursor const &root, Metadata &out, ParseOptions const &options } Reader ar{metadataCursor}; + ar.has("hs3_version"); ar.list("packages", out.packages); ar.list("authors", out.authors); ar.list("publications", out.publications); @@ -1134,7 +1182,10 @@ HS3Document parseDocument(JSONNode const &root, Diagnostics &diagnostics, ParseO return doc; } - if (root.has_child("distributions") && !root["distributions"].is_null()) { + Reader ar{cursor}; + ar.has("metadata"); + ar.has("misc"); + if (ar.has("distributions") && !root["distributions"].is_null()) { Cursor distributions{root["distributions"], cursor, "distributions"}; if (!distributions.node().is_seq()) { distributions.error("must be an array"); @@ -1148,7 +1199,6 @@ HS3Document parseDocument(JSONNode const &root, Diagnostics &diagnostics, ParseO } } } - Reader ar{cursor}; ar.list("functions", doc.functions); ar.list("data", doc.data); ar.list("likelihoods", doc.likelihoods); @@ -1162,7 +1212,15 @@ HS3Document parseDocument(JSONNode const &root, Diagnostics &diagnostics, ParseO HistFactoryDist parseHistFactoryDist(JSONNode const &distribution, Diagnostics &diagnostics) { HistFactoryDist out; - readValue(Cursor{distribution, diagnostics}, out); + Cursor cursor{distribution, diagnostics}; + if (!distribution.is_map()) { + cursor.error("must be a struct"); + return out; + } + Reader ar{cursor}; + ar.has("name"); + ar.has("type"); + describe(ar, out); return out; } diff --git a/roofit/hs3/src/HS3Schema.h b/roofit/hs3/src/HS3Schema.h index adc815881b108..01663f64f370c 100644 --- a/roofit/hs3/src/HS3Schema.h +++ b/roofit/hs3/src/HS3Schema.h @@ -45,6 +45,7 @@ class Cursor { std::string path() const; void error(std::string const &message) const; void warning(std::string const &message) const; + std::size_t numErrors() const; private: Detail::JSONNode const &_node; @@ -58,7 +59,11 @@ class Cursor { /// single function can describe both directions. class Reader { public: - explicit Reader(Cursor const &cursor) : _cursor{cursor} {} + explicit Reader(Cursor const &cursor); + /// Reports the components that were not read, unless reading the struct gave an error. + ~Reader(); + Reader(Reader const &) = delete; + Reader &operator=(Reader const &) = delete; /// Read a required component, or an optional one if the field is a std::optional. template @@ -74,12 +79,15 @@ class Reader { template void nested(const char *key, Fn &&fn); - bool has(const char *key) const { return _cursor.node().has_child(key); } + /// Whether the struct has a component. Checking for a component counts as reading it. + bool has(const char *key); Cursor const &cursor() const { return _cursor; } private: Cursor const &_cursor; + std::vector _seen; + std::size_t _nErrorsBefore = 0; }; void readValue(Cursor const &c, std::string &out); diff --git a/roofit/hs3/src/HS3Upgrade.cxx b/roofit/hs3/src/HS3Upgrade.cxx index 512b62b8202f1..ca00b463c6b02 100644 --- a/roofit/hs3/src/HS3Upgrade.cxx +++ b/roofit/hs3/src/HS3Upgrade.cxx @@ -332,23 +332,69 @@ void upgradeHistFactory(JSONNode &dist, Version const &version, IsDistribution c } } -} // namespace - -void upgradeDocument(JSONNode &root, Diagnostics &diagnostics, ParseOptions const &options) +// Older versions of RooFit wrote the binning of the observables also for unbinned data, which the standard forbids. +// The edges were written without "min" and "max". +void upgradeUnbinnedAxes(JSONNode &data) { - if (!root.is_map() || !root.has_child("distributions") || !root["distributions"].is_seq()) { + if (!data.has_child("axes") || !data["axes"].is_seq()) { return; } - if (!options.typedHistFactory) { + for (JSONNode &axis : data["axes"].children()) { + if (!axis.is_map()) { + continue; + } + if (axis.has_child("edges") && axis["edges"].is_seq() && axis["edges"].num_children() > 0) { + JSONNode &edges = axis["edges"]; + JSONNode const &first = edges.child(0); + JSONNode const &last = edges.child(edges.num_children() - 1); + if (!axis.has_child("min") && first.is_number()) { + axis["min"] << first.val_double(); + } + if (!axis.has_child("max") && last.is_number()) { + axis["max"] << last.val_double(); + } + } + axis.erase("edges"); + axis.erase("nbins"); + } +} + +void upgradeData(JSONNode &root, Version const &version) +{ + if (!root.has_child("data") || !root["data"].is_seq() || !(version < explicitInterpolationVersion)) { return; } + for (JSONNode &data : root["data"].children()) { + if (data.is_map() && data.has_child("type") && isScalar(data["type"]) && data["type"].val() == "unbinned") { + upgradeUnbinnedAxes(data); + } + } +} +Version documentVersion(JSONNode const &root) +{ Version version; if (JSONNode const *versionNode = root.find("metadata", "hs3_version")) { if (isScalar(*versionNode)) { parseVersion(versionNode->val(), version); } } + return version; +} + +} // namespace + +void upgradeDocument(JSONNode &root, Diagnostics &diagnostics, ParseOptions const &options) +{ + if (!root.is_map()) { + return; + } + const Version version = documentVersion(root); + upgradeData(root, version); + + if (!root.has_child("distributions") || !root["distributions"].is_seq() || !options.typedHistFactory) { + return; + } std::set distributionNames; for (JSONNode const &dist : root["distributions"].children()) { diff --git a/roofit/hs3/src/JSONInterface.cxx b/roofit/hs3/src/JSONInterface.cxx index 4e1d5842b44a6..3ac9637b25e5c 100644 --- a/roofit/hs3/src/JSONInterface.cxx +++ b/roofit/hs3/src/JSONInterface.cxx @@ -12,8 +12,11 @@ #include +#include "JSONReadRecorder.h" + #include +#include #include #include #include @@ -88,10 +91,19 @@ class JSONNode::Storage { return list; } + // Tells the recorders that the component `key` of `value` is read, where nullptr means all components. + void recordRead(void const *value, std::string const *key) + { + for (JSONReadRecorder *recorder : recorders) { + recorder->record(value, key); + } + } + nlohmann::json root; JSONNode rootNode; std::vector> nodes; std::deque> childLists; + std::vector recorders; }; // JSONNode @@ -103,6 +115,7 @@ JSONNode::JSONNode(Storage &storage, void *value, std::string key) void JSONNode::writeJSON(std::ostream &os) const { + _storage->recordRead(_value, nullptr); os << json(_value); } @@ -143,6 +156,7 @@ JSONNode &JSONNode::operator[](std::string const &k) const JSONNode &JSONNode::operator[](std::string const &k) const { + _storage->recordRead(_value, &k); return _storage->makeNode(json(_value)[k], k); } @@ -275,6 +289,7 @@ bool JSONNode::has_val() const bool JSONNode::has_child(std::string const &c) const { + _storage->recordRead(_value, &c); return json(_value).find(c) != json(_value).end(); } @@ -296,6 +311,7 @@ JSONNode &JSONNode::child(size_t pos) const JSONNode &JSONNode::child(size_t pos) const { + _storage->recordRead(_value, nullptr); return _storage->makeNode(json(_value).at(pos), ""); } @@ -307,6 +323,7 @@ JSONNode::children_view JSONNode::children() JSONNode::const_children_view JSONNode::children() const { + _storage->recordRead(_value, nullptr); std::vector const &list = _storage->makeChildren(json(_value)); return {const_child_iterator{list.data()}, const_child_iterator{list.data() + list.size()}}; } @@ -344,5 +361,45 @@ std::unique_ptr JSONTree::create(std::string const &str) return JSONTree::create(ss); } +// JSONReadRecorder + +JSONReadRecorder::JSONReadRecorder(JSONNode const &node) : _node{node} +{ + _node._storage->recorders.push_back(this); +} + +JSONReadRecorder::~JSONReadRecorder() +{ + auto &recorders = _node._storage->recorders; + recorders.erase(std::find(recorders.begin(), recorders.end(), this)); +} + +void JSONReadRecorder::record(void const *value, std::string const *key) +{ + if (value != _node._value) { + return; + } + if (key) { + _read.insert(*key); + } else { + _readAll = true; + } +} + +std::vector JSONReadRecorder::unreadComponents() const +{ + std::vector out; + nlohmann::json const &value = json(_node._value); + if (!value.is_object() || _readAll) { + return out; + } + for (auto const &item : value.items()) { + if (_read.count(item.key()) == 0) { + out.push_back(item.key()); + } + } + return out; +} + } // namespace Detail } // namespace RooFit diff --git a/roofit/hs3/src/JSONReadRecorder.h b/roofit/hs3/src/JSONReadRecorder.h new file mode 100644 index 0000000000000..77ac8b45cedca --- /dev/null +++ b/roofit/hs3/src/JSONReadRecorder.h @@ -0,0 +1,50 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef RooFit_Detail_JSONReadRecorder_h +#define RooFit_Detail_JSONReadRecorder_h + +#include + +#include +#include +#include + +namespace RooFit { +namespace Detail { + +/// Records which components of a JSON map are read while it is alive, through any node of the tree. Like that, the +/// components that no importer understands can be reported without any change to the importers. Iterating over the +/// components or writing the map counts as reading all of them. +class JSONReadRecorder { +public: + explicit JSONReadRecorder(JSONNode const &node); + ~JSONReadRecorder(); + JSONReadRecorder(JSONReadRecorder const &) = delete; + JSONReadRecorder &operator=(JSONReadRecorder const &) = delete; + + /// The components of the map that were not read. + std::vector unreadComponents() const; + + /// Called by the tree for every read of the component `key` of `value`, where nullptr means all components. + void record(void const *value, std::string const *key); + +private: + JSONNode const &_node; + std::set _read; + bool _readAll = false; +}; + +} // namespace Detail +} // namespace RooFit + +#endif diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index 1ea36f6562fce..82a64be1fe189 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -32,12 +32,14 @@ #include "Domains.h" #include "HS3Builder.h" #include "HS3Schema.h" +#include "JSONReadRecorder.h" #include #include #include #include +#include #include #include @@ -1260,6 +1262,8 @@ void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func, std::set Date: Thu, 5 Nov 2026 19:18:12 +0000 Subject: [PATCH 07/20] [RF][HS3] Validate documents before building them Check what doesn't depend on how the objects are built before anything is created in the workspace: that names are unique, also with respect to the parameters, that references can be resolved, that binnings are valid, and that the sizes of arrays are consistent. Documents that fail these checks are not imported. Assisted-by: claude-code:claude-opus-5-5 --- roofit/hs3/CMakeLists.txt | 1 + roofit/hs3/src/HS3Document.h | 44 ++ roofit/hs3/src/HS3Validate.cxx | 531 +++++++++++++++++++ roofit/hs3/src/JSONFactories_HistFactory.cxx | 3 + roofit/hs3/src/RooJSONFactoryWSTool.cxx | 64 +-- 5 files changed, 591 insertions(+), 52 deletions(-) create mode 100644 roofit/hs3/src/HS3Validate.cxx diff --git a/roofit/hs3/CMakeLists.txt b/roofit/hs3/CMakeLists.txt index b71134c9f13a6..08dc37e6e7836 100644 --- a/roofit/hs3/CMakeLists.txt +++ b/roofit/hs3/CMakeLists.txt @@ -22,6 +22,7 @@ ROOT_STANDARD_LIBRARY_PACKAGE(RooFitHS3 src/Domains.cxx src/HS3Parser.cxx src/HS3Upgrade.cxx + src/HS3Validate.cxx src/JSONInterface.cxx src/JSONIO.cxx src/RooJSONFactoryWSTool.cxx diff --git a/roofit/hs3/src/HS3Document.h b/roofit/hs3/src/HS3Document.h index a0bea7a56c97a..9102caa38a7d3 100644 --- a/roofit/hs3/src/HS3Document.h +++ b/roofit/hs3/src/HS3Document.h @@ -13,6 +13,7 @@ #ifndef RooFit_HS3Document_h #define RooFit_HS3Document_h +#include #include #include #include @@ -436,6 +437,15 @@ bool hasErrors(Diagnostics const &diagnostics); } // namespace HS3 +class HS3Document; +class ValidatedHS3Document; + +namespace HS3 { +struct ValidateOptions; +std::optional +validate(HS3Document doc, Diagnostics &diagnostics, ValidateOptions const &options); +} // namespace HS3 + /// An HS3 document. It is move-only, because generic objects refer to the JSON tree that the document was read from. class HS3Document { public: @@ -484,6 +494,40 @@ HistFactoryDist parseHistFactoryDist(Detail::JSONNode const &distribution, Diagn /// Reads an interpolation that doesn't come with a document. Interpolation parseInterpolation(Detail::JSONNode const &interpolation, Diagnostics &diagnostics); +/// Options for validate(). +struct ValidateOptions { + /// Whether the document has to follow the current standard, or if older conventions of RooFit are tolerated. + bool strict = false; + /// Whether a name that is not defined in the document is known in some other way, e.g. in the workspace. + std::function isExternal; +}; + +} // namespace HS3 + +/// A document that passed validate(). Only validate() can create it, so the code that builds RooFit objects can rely on +/// the checks. +class ValidatedHS3Document { +public: + HS3Document const &doc() const { return _doc; } + +private: + explicit ValidatedHS3Document(HS3Document doc) : _doc{std::move(doc)} {} + friend std::optional + HS3::validate(HS3Document doc, HS3::Diagnostics &diagnostics, HS3::ValidateOptions const &options); + + HS3Document _doc; +}; + +namespace HS3 { + +/// Checks what can be checked without knowing how the objects are built: unique names, that the references can be +/// resolved, and that the sizes of arrays are consistent. Gives nothing if there are errors. +std::optional +validate(HS3Document doc, Diagnostics &diagnostics, ValidateOptions const &options = {}); + +/// Same as validate(), but for a single HistFactory distribution that doesn't come with a document. +void validateHistFactoryDist(HistFactoryDist const &dist, std::string const &name, Diagnostics &diagnostics); + /// Reads the leading "MAJOR.MINOR" and the optional ".PATCH" of an HS3 version string. bool parseVersion(std::string const &version, Version &out); diff --git a/roofit/hs3/src/HS3Validate.cxx b/roofit/hs3/src/HS3Validate.cxx new file mode 100644 index 0000000000000..8c3d1d6953b91 --- /dev/null +++ b/roofit/hs3/src/HS3Validate.cxx @@ -0,0 +1,531 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "HS3Document.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace RooFit { +namespace HS3 { +namespace { + +std::string at(const char *section, std::size_t index, std::string const &name) +{ + return std::string(section) + "[" + std::to_string(index) + "](" + name + ")"; +} + +class Validator { +public: + Validator(HS3Document const &doc, Diagnostics &diagnostics, ValidateOptions const &options) + : _doc{doc}, _diagnostics{diagnostics}, _options{options} + { + } + + void run() + { + collectNames(); + checkLikelihoods(); + checkAnalyses(); + checkData(); + for (std::size_t i = 0; i < _doc.distributions.size(); ++i) { + auto const &dist = _doc.distributions[i]; + if (auto const *histFactory = std::get_if(&dist.body)) { + checkHistFactory(*histFactory, dist.name, at("distributions", i, dist.name)); + } + } + for (auto const &item : _doc.rooFit.combinedDistributions) { + for (auto const &ref : item.second.distributions) { + requireDistribution(ref.name, "misc.ROOT_internal.combined_distributions." + item.first, "distribution"); + } + } + } + + void error(std::string const &path, std::string const &message) + { + _diagnostics.push_back({Diagnostic::Severity::Error, path, message}); + } + + void warning(std::string const &path, std::string const &message) + { + _diagnostics.push_back({Diagnostic::Severity::Warning, path, message}); + } + + // An error in documents that follow the current standard, which older versions of RooFit didn't care about. + void strictError(std::string const &path, std::string const &message) + { + _diagnostics.push_back( + {_options.strict ? Diagnostic::Severity::Error : Diagnostic::Severity::Warning, path, message}); + } + + void checkHistFactory(HistFactoryDist const &dist, std::string const &name, std::string const &path); + +private: + bool isExternal(std::string const &name) const { return _options.isExternal && _options.isExternal(name); } + + char const *sectionOf(std::string const &name) const + { + auto found = _sections.find(name); + return found != _sections.end() ? found->second.front() : nullptr; + } + + bool isDefinedIn(std::string const &name, char const *section) const + { + auto found = _sections.find(name); + if (found == _sections.end()) { + return false; + } + return std::any_of(found->second.begin(), found->second.end(), + [&](char const *s) { return std::string(s) == section; }); + } + + void requireDistribution(std::string const &name, std::string const &path, std::string const &what) + { + // RooFit refers to a RooSimultaneous by name, like the likelihood of a ModelConfig without data does. + if (!isDefinedIn(name, "distributions") && !_doc.rooFit.combinedDistributions.count(name) && !isExternal(name)) { + error(path, "the " + what + " '" + name + "' does not exist"); + } + } + + void addParameter(std::string const &name) { _parameters.insert(name); } + + // The number of bins of an axis, after reporting an invalid binning. + std::size_t checkAxis(BinnedAxis const &axis, std::string const &path); + std::size_t checkAxis(CategoricalAxis const &axis, std::string const &path); + template + std::size_t checkAxis(std::variant const &axis, std::string const &path) + { + return std::visit([&](auto const &a) { return checkAxis(a, path); }, axis); + } + // The number of bins of all axes, or nothing if it can't be stored. + template + std::optional numBins(Axes const &axes, std::string const &path); + + void collectNames(); + void checkLikelihoods(); + void checkAnalyses(); + void checkData(); + + HS3Document const &_doc; + Diagnostics &_diagnostics; + ValidateOptions const &_options; + std::map> _sections; ///< The sections where a name is defined. + std::set _parameters; +}; + +// HS3 requires the names of all objects to be unique within a document, including the parameters. RooFit writes such +// documents, e.g. for a range and a pdf of one name, so only ambiguous clashes in current documents are errors. +void Validator::collectNames() +{ + auto check = [&](char const *section, std::string const &name) { + auto §ions = _sections[name]; + sections.push_back(section); + if (sections.size() == 1) { + return; + } + std::string const firstSection = sections.front(); + // RooFit used to give analyses and likelihoods the name of the pdf or the analysis. Accept this for backwards + // compatibility: analyses are not referenced by name, and likelihoods only from analyses, so there is no + // ambiguity. + auto isLegacyNamed = [](std::string const &s) { return s == "likelihoods" || s == "analyses"; }; + // Within one section, and between distributions and functions (both can be referenced by name), a clash is + // ambiguous. Other clashes are produced by RooFit itself (a range, snapshot or data set named like a pdf). + auto isReferable = [](std::string const &s) { return s == "distributions" || s == "functions"; }; + const bool ambiguous = firstSection == section || (isReferable(firstSection) && isReferable(section)); + if (!_options.strict || !ambiguous || isLegacyNamed(firstSection) || isLegacyNamed(section)) { + warning("", "the name '" + name + "' is used in \"" + firstSection + "\" and in \"" + section + + "\", which is not allowed by the HS3 standard."); + return; + } + error("", "duplicate name '" + name + "': it is defined in \"" + firstSection + "\" and in \"" + section + + "\", but names must be unique within the document"); + }; + auto checkSection = [&](char const *section, auto const &objects) { + for (auto const &object : objects) { + check(section, object.name); + } + }; + checkSection("distributions", _doc.distributions); + checkSection("functions", _doc.functions); + checkSection("data", _doc.data); + checkSection("domains", _doc.domains); + checkSection("parameter_points", _doc.parameterPoints); + checkSection("likelihoods", _doc.likelihoods); + checkSection("analyses", _doc.analyses); + + // The parameters are the axes of the domains, the entries of the parameter points, and the parameters of + // HistFactory modifiers, which are created on the fly. + for (auto const &domain : _doc.domains) { + for (auto const &axis : domain.axes) { + addParameter(std::visit([](auto const &a) -> std::string const & { return a.name; }, axis)); + } + } + for (auto const &point : _doc.parameterPoints) { + for (auto const ¶meter : point.parameters) { + addParameter(parameter.name); + } + } + for (auto const &item : _doc.rooFit.combinedDistributions) { + addParameter(item.second.indexCat); + } + for (auto const &dist : _doc.distributions) { + auto const *histFactory = std::get_if(&dist.body); + if (!histFactory) { + continue; + } + for (auto const &sample : histFactory->samples) { + for (auto const &modifier : sample.modifiers) { + std::visit(overloaded{[&](NormFactor const &m) { addParameter(m.parameter.name); }, + [&](NormSys const &m) { addParameter(m.parameter.name); }, + [&](HistoSys const &m) { addParameter(m.parameter.name); }, + [&](ShapeSys const &m) { + for (auto const &p : m.parameters) { + addParameter(p.name); + } + }, + [&](StatError const &m) { + for (auto const &p : m.parameters) { + addParameter(p.name); + } + }, + [](CustomModifier const &) {}}, + modifier); + } + } + } + for (std::string const ¶meter : _parameters) { + if (char const *section = sectionOf(parameter)) { + strictError("", "the name '" + parameter + "' is used in \"" + section + + "\" and for a parameter, which is not allowed by the HS3 standard."); + } + } +} + +void Validator::checkLikelihoods() +{ + for (std::size_t i = 0; i < _doc.likelihoods.size(); ++i) { + auto const &likelihood = _doc.likelihoods[i]; + const std::string path = at("likelihoods", i, likelihood.name); + for (auto const &term : likelihood.terms) { + requireDistribution(term.distribution.name, path + ".distributions", "distribution"); + if (auto const *data = std::get_if(&term.data)) { + if (!isDefinedIn(data->name, "data")) { + error(path + ".data", "dataset '" + data->name + "' cannot be found!"); + } + } + } + for (auto const &aux : likelihood.auxDistributions) { + if (!isDefinedIn(aux.name, "distributions") && !isExternal(aux.name)) { + strictError(path + ".aux_distributions", "the distribution '" + aux.name + "' does not exist"); + } + } + } +} + +void Validator::checkAnalyses() +{ + for (std::size_t i = 0; i < _doc.analyses.size(); ++i) { + auto const &analysis = _doc.analyses[i]; + const std::string path = at("analyses", i, analysis.name); + if (!isDefinedIn(analysis.likelihood.name, "likelihoods")) { + error(path + ".likelihood", "likelihood node not found!"); + } + for (auto const &poi : analysis.parametersOfInterest) { + if (!_parameters.count(poi.name) && !sectionOf(poi.name) && !isExternal(poi.name)) { + error(path, "the object '" + poi.name + "' in \"parameters_of_interest\" of analysis '" + analysis.name + + "' does not exist"); + } + } + for (auto const &domain : analysis.domains) { + if (!isDefinedIn(domain.name, "domains")) { + strictError(path + ".domains", "the domain '" + domain.name + "' does not exist"); + } + } + if (analysis.init && !isDefinedIn(analysis.init->name, "parameter_points")) { + warning(path, "the parameter point '" + analysis.init->name + "' used as \"init\" of analysis '" + + analysis.name + "' does not exist."); + } + if (analysis.prior && !isDefinedIn(analysis.prior->name, "distributions") && !isExternal(analysis.prior->name)) { + strictError(path + ".prior", "the distribution '" + analysis.prior->name + "' does not exist"); + } + } +} + +std::size_t Validator::checkAxis(BinnedAxis const &axis, std::string const &path) +{ + if (auto const *regular = std::get_if(&axis.binning)) { + if (regular->nbins <= 0) { + error(path, "the axis '" + axis.name + "' needs a positive number of bins"); + return 0; + } + if (!(axis.min < axis.max)) { + error(path, "the axis '" + axis.name + "' needs \"min\" smaller than \"max\""); + } + return regular->nbins; + } + auto const &edges = std::get(axis.binning).edges; + return edges.empty() ? 0 : edges.size() - 1; +} + +std::size_t Validator::checkAxis(CategoricalAxis const &axis, std::string const &path) +{ + if (axis.categories.empty()) { + error(path, "the axis '" + axis.name + "' needs at least one category"); + } + return axis.categories.size(); +} + +template +std::optional Validator::numBins(Axes const &axes, std::string const &path) +{ + std::size_t nBins = 1; + bool valid = true; + for (std::size_t i = 0; i < axes.size(); ++i) { + const std::size_t n = checkAxis(axes[i], path + ".axes[" + std::to_string(i) + "]"); + if (n == 0) { + valid = false; // reported already + } else if (nBins > std::numeric_limits::max() / n) { + error(path + ".axes", "the axes have more bins than can be stored"); + return std::nullopt; + } + nBins *= n; + } + return valid ? std::optional{nBins} : std::nullopt; +} + +void Validator::checkData() +{ + for (std::size_t i = 0; i < _doc.data.size(); ++i) { + auto const &data = _doc.data[i]; + auto const *binned = std::get_if(&data.body); + if (!binned) { + continue; + } + const std::string path = at("data", i, data.name); + std::optional nBinsOrNone = numBins(binned->axes, path); + if (!nBinsOrNone) { + continue; + } + const std::size_t nBins = *nBinsOrNone; + if (binned->contents.size() != nBins) { + error(path, "inconsistent bin numbers: contents=" + std::to_string(binned->contents.size()) + + ", bins=" + std::to_string(nBins)); + } + if (binned->uncertainty) { + auto const &sigma = binned->uncertainty->sigma; + if (sigma.size() != binned->contents.size()) { + error(path, "inconsistent number of errors and contents in '" + data.name + + "': errors=" + std::to_string(sigma.size()) + + ", contents=" + std::to_string(binned->contents.size())); + } + auto const &correlation = binned->uncertainty->correlation; + if (!correlation.empty()) { + bool square = correlation.size() == binned->contents.size(); + for (auto const &row : correlation) { + square = square && row.size() == binned->contents.size(); + } + if (!square) { + error(path + ".uncertainty.correlation", "must be a square matrix of the size of \"contents\""); + } + } + } + } +} + +void Validator::checkHistFactory(HistFactoryDist const &dist, std::string const &name, std::string const &path) +{ + if (dist.axes.empty()) { + error(path + ".axes", "a HistFactory distribution needs at least one axis"); + return; + } + std::optional nBinsOrNone = numBins(dist.axes, path); + if (!nBinsOrNone) { + return; + } + const std::size_t nBins = *nBinsOrNone; + + std::string channelName = name; + if (channelName.rfind("model_", 0) == 0) { + channelName.erase(0, 6); + } + + auto checkSize = [&](std::string const &where, std::size_t size, char const *what) { + if (size != nBins) { + error(where, std::string(what) + " has " + std::to_string(size) + " entries, but the channel has " + + std::to_string(nBins) + " bins"); + } + }; + + // Two modifiers with the same parameter must have the same constraint. + std::map> constraintOfParameter; + auto checkCorrelated = [&](std::string const &where, std::string const ¶meter, Constraint const &constraint) { + auto inserted = constraintOfParameter.emplace(parameter, std::make_pair(constraint, where)); + if (inserted.second) { + return; + } + Constraint const &first = inserted.first->second.first; + bool same = first.index() == constraint.index(); + if (same && std::holds_alternative(first)) { + same = std::get(first) == std::get(constraint); + } else if (same && std::holds_alternative(first)) { + same = std::get(first).pdf.name == std::get(constraint).pdf.name; + } + if (!same) { + warning(where, "the parameter '" + parameter + "' has a different constraint than in " + + inserted.first->second.second + + ", but modifiers with the same parameter must share the constraint"); + } + }; + + auto checkConstraint = [&](std::string const &where, Constraint const &constraint) { + if (auto const *pdf = std::get_if(&constraint)) { + requireDistribution(pdf->pdf.name, where, "constraint"); + } + }; + + // The constraint distributions of the gamma parameters of shape modifiers. + auto checkGammaConstraint = [&](std::string const &where, GammaConstraint const &constraint) { + auto const *pdfs = std::get_if(&constraint); + if (!pdfs) { + return; + } + checkSize(where + ".constraints", pdfs->pdfs.size(), "\"constraints\""); + for (auto const &pdf : pdfs->pdfs) { + if (pdf) { + requireDistribution(pdf->name, where + ".constraints", "constraint"); + } + } + }; + + auto checkParameter = [&](std::string const &where, std::string const ¶meter, std::string const &modifier) { + if (char const *section = sectionOf(parameter)) { + error(where, "the parameter '" + parameter + "' of the modifier '" + modifier + "' is defined in \"" + + section + "\", but it must be a parameter"); + } + }; + + std::set sampleNames; + for (std::size_t iSample = 0; iSample < dist.samples.size(); ++iSample) { + auto const &sample = dist.samples[iSample]; + const std::string samplePath = path + ".samples[" + std::to_string(iSample) + "](" + sample.name + ")"; + if (!sampleNames.insert(sample.name).second) { + error(samplePath, "the name of the sample is not unique in the channel"); + } + checkSize(samplePath + ".data.contents", sample.contents.size(), "\"contents\""); + // These modifiers create objects that are named after them. + std::set objectNames; + auto checkUniqueName = [&](std::string const &where, char const *type, std::string const &modifierName) { + if (!objectNames.insert(std::string(type) + " " + modifierName).second) { + error(where, + std::string("there is another ") + type + " modifier named '" + modifierName + "' in the sample"); + } + }; + if (sample.errors && sample.errors->size() != sample.contents.size()) { + error(samplePath + ".data", "inconsistent number of errors and contents in '" + sample.name + + "': errors=" + std::to_string(sample.errors->size()) + + ", contents=" + std::to_string(sample.contents.size())); + } + for (std::size_t iModifier = 0; iModifier < sample.modifiers.size(); ++iModifier) { + std::string modifierPath = samplePath + ".modifiers[" + std::to_string(iModifier) + "]"; + auto checkInterpolation = [&](std::optional const &interpolation, char const *type, + std::string const &modifierName) { + if (!interpolation && !dist.defaultInterpolation) { + error(modifierPath, std::string("interpolation for ") + type + " modifier '" + modifierName + + "' in sample '" + sample.name + "' of channel '" + channelName + + "' is not given, and the channel has no \"default_interpolation\""); + } + }; + std::visit(overloaded{ + [&](NormFactor const &m) { + modifierPath += "(" + m.name + ")"; + checkParameter(modifierPath, m.parameter.name, m.name); + checkConstraint(modifierPath, m.constraint); + checkCorrelated(modifierPath, m.parameter.name, m.constraint); + }, + [&](NormSys const &m) { + modifierPath += "(" + m.name + ")"; + checkParameter(modifierPath, m.parameter.name, m.name); + checkConstraint(modifierPath, m.constraint); + checkCorrelated(modifierPath, m.parameter.name, m.constraint); + checkInterpolation(m.interpolation, "normsys", m.name); + }, + [&](HistoSys const &m) { + modifierPath += "(" + m.name + ")"; + checkUniqueName(modifierPath, "histosys", m.name); + checkParameter(modifierPath, m.parameter.name, m.name); + checkConstraint(modifierPath, m.constraint); + checkCorrelated(modifierPath, m.parameter.name, m.constraint); + checkInterpolation(m.interpolation, "histosys", m.name); + checkSize(modifierPath + ".data.hi.contents", m.hi.contents.size(), "\"contents\""); + checkSize(modifierPath + ".data.lo.contents", m.lo.contents.size(), "\"contents\""); + }, + [&](ShapeSys const &m) { + modifierPath += "(" + m.name + ")"; + checkUniqueName(modifierPath, "shapesys", m.name); + if (!m.vals.empty()) { + checkSize(modifierPath + ".data.vals", m.vals.size(), "\"vals\""); + } + if (!m.parameters.empty()) { + checkSize(modifierPath + ".parameters", m.parameters.size(), "\"parameters\""); + } + for (auto const &p : m.parameters) { + checkParameter(modifierPath, p.name, m.name); + } + checkGammaConstraint(modifierPath, m.constraint); + }, + [&](StatError const &m) { + modifierPath += "(" + m.name + ")"; + if (!m.parameters.empty()) { + checkSize(modifierPath + ".parameters", m.parameters.size(), "\"parameters\""); + } + checkGammaConstraint(modifierPath, m.constraint); + }, + [&](CustomModifier const &m) { + modifierPath += "(" + m.name + ")"; + if (!isDefinedIn(m.name, "functions") && !isDefinedIn(m.name, "distributions") && + !isExternal(m.name)) { + error(modifierPath, "unable to find custom modifier '" + m.name + "'"); + } + }, + }, + sample.modifiers[iModifier]); + } + } +} + +} // namespace + +std::optional validate(HS3Document doc, Diagnostics &diagnostics, ValidateOptions const &options) +{ + Validator{doc, diagnostics, options}.run(); + if (hasErrors(diagnostics)) { + return std::nullopt; + } + return ValidatedHS3Document{std::move(doc)}; +} + +void validateHistFactoryDist(HistFactoryDist const &dist, std::string const &name, Diagnostics &diagnostics) +{ + // Without a document, all names are resolved later in the workspace. + HS3Document empty; + ValidateOptions options; + options.isExternal = [](std::string const &) { return true; }; + Validator{empty, diagnostics, options}.checkHistFactory(dist, name, ""); +} + +} // namespace HS3 +} // namespace RooFit diff --git a/roofit/hs3/src/JSONFactories_HistFactory.cxx b/roofit/hs3/src/JSONFactories_HistFactory.cxx index d68553a4abfcd..be1be3599d2ef 100644 --- a/roofit/hs3/src/JSONFactories_HistFactory.cxx +++ b/roofit/hs3/src/JSONFactories_HistFactory.cxx @@ -792,6 +792,9 @@ class HistFactoryImporter : public RooFit::JSONIO::Importer { HS3::Diagnostics diagnostics; HS3::upgradeHistFactoryDist(tree->rootnode(), version, diagnostics); HS3::HistFactoryDist dist = HS3::parseHistFactoryDist(tree->rootnode(), diagnostics); + if (!HS3::hasErrors(diagnostics)) { + HS3::validateHistFactoryDist(dist, name, diagnostics); + } HS3::report(diagnostics); HS3::buildHistFactoryDist(*tool, name, dist); return true; diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index 82a64be1fe189..a6d4409caea20 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -692,14 +692,10 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo pdf->setStringAttribute("combined_data_name", mcInfo->combinedDataName->c_str()); } - if (analysis.init) { + // A missing parameter point was reported by HS3::validate(). + RooArgSet const *init = analysis.init ? workspace.getSnapshot(analysis.init->name.c_str()) : nullptr; + if (init) { const std::string &initName = analysis.init->name; - RooArgSet const *init = workspace.getSnapshot(initName.c_str()); - if (!init) { - RooJSONFactoryWSTool::warning("RooFitHS3: the parameter point '" + initName + - "' used as \"init\" of analysis '" + analysisName + "' does not exist."); - return; - } // ModelConfig::SetSnapshot() saves the snapshot as "__snapshot". If "init" already follows // this pattern, which is the case for files written by RooFit, pick the set name such that it is preserved. std::string setName = initName; @@ -805,49 +801,6 @@ RooAbsCategory *findCategoryObservable(RooAbsData const &data) return cat; } -/// HS3 requires the names of all top-level objects to be unique within a document. If `strict`, throw if a name is -/// used twice within one section or by a distribution and a function. Other clashes, and all clashes if not -/// `strict`, only warn: RooFit's own export still produces them, e.g. for a range and a pdf of one name, and it -/// must be able to read its own output. -void checkUniqueNames(HS3Document const &doc, bool strict) -{ - std::map sectionOfName; - auto check = [&](const char *section, std::string const &name) { - auto inserted = sectionOfName.emplace(name, section); - if (inserted.second) { - return; - } - std::string const &firstSection = inserted.first->second; - // RooFit used to give analyses and likelihoods the name of the pdf or the analysis. Accept this for backwards - // compatibility: analyses are not referenced by name, and likelihoods only from analyses, so there is no - // ambiguity. - auto isLegacyNamed = [](std::string const &s) { return s == "likelihoods" || s == "analyses"; }; - // Within one section, and between distributions and functions (both can be referenced by name), a clash is - // ambiguous. Other clashes are produced by RooFit itself (a range, snapshot or data set named like a pdf). - auto isReferable = [](std::string const &s) { return s == "distributions" || s == "functions"; }; - const bool ambiguous = firstSection == section || (isReferable(firstSection) && isReferable(section)); - if (!strict || !ambiguous || isLegacyNamed(firstSection) || isLegacyNamed(section)) { - RooJSONFactoryWSTool::warning("RooFitHS3: the name '" + name + "' is used in \"" + firstSection + - "\" and in \"" + section + "\", which is not allowed by the HS3 standard."); - return; - } - RooJSONFactoryWSTool::error("duplicate name '" + name + "': it is defined in \"" + firstSection + "\" and in \"" + - section + "\", but names must be unique within the document"); - }; - auto checkSection = [&](const char *section, auto const &objects) { - for (auto const &object : objects) { - check(section, object.name); - } - }; - checkSection("distributions", doc.distributions); - checkSection("functions", doc.functions); - checkSection("data", doc.data); - checkSection("domains", doc.domains); - checkSection("parameter_points", doc.parameterPoints); - checkSection("likelihoods", doc.likelihoods); - checkSection("analyses", doc.analyses); -} - } // namespace RooArgSet RooFit::HS3::createObservables(std::vector const &axes) @@ -2120,8 +2073,16 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) HS3::Diagnostics diagnostics; HS3::upgradeDocument(n, diagnostics, options); - HS3Document doc = HS3::parseDocument(n, diagnostics, options); + HS3Document parsed = HS3::parseDocument(n, diagnostics, options); + std::optional validated; + if (!HS3::hasErrors(diagnostics)) { + HS3::ValidateOptions validateOptions; + validateOptions.strict = !isHS3VersionBefore(parsed.metadata.hs3Version, 0, 3); + validateOptions.isExternal = [this](std::string const &name) { return _workspace.obj(name) != nullptr; }; + validated = HS3::validate(std::move(parsed), diagnostics, validateOptions); + } HS3::report(diagnostics); + HS3Document const &doc = validated->doc(); // The state of the import refers to the document, so it must not survive it, not even if there is an exception. struct ImportScope { @@ -2135,7 +2096,6 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) } importScope{*this}; _hs3VersionInput = doc.metadata.hs3Version; - checkUniqueNames(doc, !isHS3VersionBefore(_hs3VersionInput, 0, 3)); _import = std::make_unique(doc); From 030762f9c8d875c75e8656f9ddf7fb4b0b3ac1b3 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Thu, 5 Nov 2026 19:40:22 +0000 Subject: [PATCH 08/20] [RF][HS3] Export documents through their typed representation Collect everything but the generic objects, which the exporters still write directly, in an HS3Document, check it with the same validation as on import, and write it in one place. HistFactory distributions are exported as typed modifiers. The exported JSON doesn't change, except that an empty workspace now gets an empty array of default parameter values instead of null, and no "domains": null. Assisted-by: claude-code:claude-opus-5-5 --- roofit/hs3/CMakeLists.txt | 1 + .../hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h | 20 +- roofit/hs3/src/Domains.cxx | 69 +-- roofit/hs3/src/Domains.h | 6 +- roofit/hs3/src/HS3Builder.h | 7 + roofit/hs3/src/HS3Document.h | 16 + roofit/hs3/src/HS3Validate.cxx | 7 +- roofit/hs3/src/HS3Writer.cxx | 461 ++++++++++++++++++ roofit/hs3/src/JSONFactories_HistFactory.cxx | 204 ++++---- roofit/hs3/src/RooJSONFactoryWSTool.cxx | 419 +++++++++------- roofit/hs3/test/testRooFitHS3.cxx | 37 ++ 11 files changed, 896 insertions(+), 351 deletions(-) create mode 100644 roofit/hs3/src/HS3Writer.cxx diff --git a/roofit/hs3/CMakeLists.txt b/roofit/hs3/CMakeLists.txt index 08dc37e6e7836..ecb06ab969cb9 100644 --- a/roofit/hs3/CMakeLists.txt +++ b/roofit/hs3/CMakeLists.txt @@ -23,6 +23,7 @@ ROOT_STANDARD_LIBRARY_PACKAGE(RooFitHS3 src/HS3Parser.cxx src/HS3Upgrade.cxx src/HS3Validate.cxx + src/HS3Writer.cxx src/JSONInterface.cxx src/JSONIO.cxx src/RooJSONFactoryWSTool.cxx diff --git a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h index 5c310ab29c80e..3240887404a93 100644 --- a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h +++ b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h @@ -25,6 +25,8 @@ #include #include +class RooJSONFactoryWSTool; + namespace RooFit { namespace JSONIO { namespace Detail { @@ -33,8 +35,12 @@ class Domains; } // namespace JSONIO namespace HS3 { struct Distribution; +struct HistFactoryDist; +struct ParameterPoint; struct ParameterValue; +struct ToolExportState; struct ToolImportState; +void setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist dist); } // namespace HS3 } // namespace RooFit namespace RooStats { @@ -248,28 +254,28 @@ class RooJSONFactoryWSTool { void warnIfFreeWithoutDomain(RooRealVar const &var); void importDependants(const RooFit::Detail::JSONNode &n); - void exportVariable(const RooAbsArg *v, RooFit::Detail::JSONNode &n, bool storeConstant, bool storeBins); - void exportVariables(const RooArgSet &allElems, RooFit::Detail::JSONNode &n, bool storeConstant, bool storeBins); + void exportVariable(const RooAbsArg *v, RooFit::HS3::ParameterPoint &point, bool storeConstant); + void exportVariables(const RooArgSet &allElems, RooFit::HS3::ParameterPoint &point, bool storeConstant); void exportAllObjects(RooFit::Detail::JSONNode &n); - void exportModelConfig(RooFit::Detail::JSONNode &rootnode, RooStats::ModelConfig const &mc, + void exportModelConfig(RooStats::ModelConfig const &mc, const std::vector &combined, const std::vector &single); - void exportSingleModelConfig(RooFit::Detail::JSONNode &rootnode, RooStats::ModelConfig const &mc, - std::string const &analysisName, + void exportSingleModelConfig(RooStats::ModelConfig const &mc, std::string const &analysisName, std::map const *dataComponents); + friend void RooFit::HS3::setExportedObject(RooJSONFactoryWSTool &tool, RooFit::HS3::HistFactoryDist dist); + // member variables - RooFit::Detail::JSONNode *_rootnodeOutput = nullptr; - RooFit::Detail::JSONNode *_varsNode = nullptr; std::string _hs3VersionInput; RooWorkspace &_workspace; // objects to represent intermediate information std::unique_ptr _domains; std::unique_ptr _import; + std::unique_ptr _export; std::vector _serversToExport; std::vector _serversToDelete; diff --git a/roofit/hs3/src/Domains.cxx b/roofit/hs3/src/Domains.cxx index ac401887d0561..36286a07e6215 100644 --- a/roofit/hs3/src/Domains.cxx +++ b/roofit/hs3/src/Domains.cxx @@ -25,25 +25,14 @@ #include +#include + namespace RooFit { namespace JSONIO { namespace Detail { constexpr static auto defaultDomainName = "default_domain"; -namespace { - -// JSON can't represent infinity, so an infinite bound is expressed by omitting it. An absent bound is read as -// infinite by readBound() in JSONIOUtils. -void writeBound(RooFit::Detail::JSONNode &node, const char *key, double value) -{ - if (!RooNumber::isInfinite(value)) { - node[key] << value; - } -} - -} // namespace - void Domains::populate(RooWorkspace &ws) const { // Parameters with a single value are RooConstVars, whichever domain gives the value. @@ -142,12 +131,15 @@ double const *Domains::fixedValue(const char *name) const return nullptr; } -void Domains::writeJSON(RooFit::Detail::JSONNode &node) const +void Domains::write(std::vector &domains) const { for (auto const &domain : _map) { // Avoid writing a domain that was already written - if (!RooJSONFactoryWSTool::findNamedChild(node, domain.first)) { - domain.second.writeJSON(RooJSONFactoryWSTool::appendNamedChild(node, domain.first)); + auto found = std::find_if(domains.begin(), domains.end(), [&](auto const &d) { return d.name == domain.first; }); + if (found == domains.end()) { + domains.emplace_back(); + domains.back().name = domain.first; + domain.second.write(domains.back()); } } } @@ -213,17 +205,6 @@ void Domains::ProductDomain::applyBinning(RooRealVar &var, ProductDomainElement } } -void Domains::ProductDomain::writeBinning(RooFit::Detail::JSONNode &node, ProductDomainElement const &elem) -{ - if (!elem.edges.empty()) { - auto &edges = node["edges"].set_seq(); - for (double edge : elem.edges) { - edges.append_child() << edge; - } - } else if (elem.hasNBins && elem.nBins != 0) { - node["nbins"] << elem.nBins; - } -} void Domains::ProductDomain::writeVariable(RooRealVar &var) const { auto found = _map.find(var.GetName()); @@ -300,29 +281,33 @@ void Domains::ProductDomain::read(RooFit::HS3::Domain const &domain) } } } -void Domains::ProductDomain::writeJSON(RooFit::Detail::JSONNode &node) const - +void Domains::ProductDomain::write(RooFit::HS3::Domain &domain) const { - node.set_map(); - node["type"] << "product_domain"; - - auto &variablesNode = node["axes"]; - for (auto const &item : _map) { auto const &elem = item.second; - RooFit::Detail::JSONNode &varnode = RooJSONFactoryWSTool::appendNamedChild(variablesNode, item.first); + RooFit::HS3::DomainAxis axis; + axis.name = item.first; if (elem.hasFixedValue) { - varnode[fixedValueKey] << elem.fixedValue; + axis.fixedValue = elem.fixedValue; + domain.axes.emplace_back(std::move(axis)); continue; } - const double min = elem.hasMin ? elem.min : -RooNumber::infinity(); - const double max = elem.hasMax ? elem.max : RooNumber::infinity(); - writeBound(varnode, "min", min); - writeBound(varnode, "max", max); + // An infinite bound is expressed by omitting it. + if (elem.hasMin && !RooNumber::isInfinite(elem.min)) { + axis.min = elem.min; + } + if (elem.hasMax && !RooNumber::isInfinite(elem.max)) { + axis.max = elem.max; + } // A binning requires a bounded range. - if (!RooNumber::isInfinite(min) && !RooNumber::isInfinite(max)) { - writeBinning(varnode, elem); + if (axis.min && axis.max) { + if (!elem.edges.empty()) { + axis.binning = RooFit::HS3::EdgesBinning{elem.edges}; + } else if (elem.hasNBins && elem.nBins != 0) { + axis.binning = RooFit::HS3::RegularBinning{elem.nBins}; + } } + domain.axes.emplace_back(std::move(axis)); } } void Domains::ProductDomain::populate(RooWorkspace &ws) const diff --git a/roofit/hs3/src/Domains.h b/roofit/hs3/src/Domains.h index 96f5b7dee6781..cf3b88bf96661 100644 --- a/roofit/hs3/src/Domains.h +++ b/roofit/hs3/src/Domains.h @@ -45,7 +45,8 @@ class Domains { double const *fixedValue(const char *name) const; void read(std::vector const &); - void writeJSON(RooFit::Detail::JSONNode &) const; + /// Append the domains that are not in the list yet. + void write(std::vector &) const; bool hasVariable(const char *name) const; @@ -60,7 +61,7 @@ class Domains { void writeVariable(RooRealVar &) const; void read(RooFit::HS3::Domain const &); - void writeJSON(RooFit::Detail::JSONNode &) const; + void write(RooFit::HS3::Domain &) const; bool hasVariable(const char *name) const; double const *fixedValue(const char *name) const; @@ -86,7 +87,6 @@ class Domains { static void applyBinning(RooRealVar &var, ProductDomainElement const &elem, const char *name = nullptr); static void readBinning(ProductDomainElement &elem, RooAbsBinning const &binning); - static void writeBinning(RooFit::Detail::JSONNode &node, ProductDomainElement const &elem); std::map _map; }; diff --git a/roofit/hs3/src/HS3Builder.h b/roofit/hs3/src/HS3Builder.h index 355365decce63..7bc27acd526d4 100644 --- a/roofit/hs3/src/HS3Builder.h +++ b/roofit/hs3/src/HS3Builder.h @@ -23,6 +23,7 @@ class RooDataHist; class RooJSONFactoryWSTool; +class RooRealVar; namespace RooFit { namespace JSONIO { @@ -37,6 +38,9 @@ void report(Diagnostics const &diagnostics); /// Create the variables for the axes of binned data. RooArgSet createObservables(std::vector const &axes); +/// The axis of binned data for a variable. +BinnedAxis toBinnedAxis(RooRealVar const &var); + /// Create a RooDataHist with the contents and errors. If there are no errors, it doesn't store squared weights. std::unique_ptr createDataHist(std::string const &name, RooArgSet const &vars, std::vector const &contents, std::vector const *errors); @@ -44,6 +48,9 @@ std::unique_ptr createDataHist(std::string const &name, RooArgSet c /// Create the RooFit objects of a HistFactory distribution in the workspace of the tool. void buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string const &name, HistFactoryDist const &dist); +/// Called by the exporter of a HistFactory distribution, which is then written by the tool. +void setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist dist); + /// Whether this is the importer that RooFit registers for HistFactory distributions, as opposed to the one of a user. bool isBuiltinHistFactoryImporter(JSONIO::Importer const &importer); diff --git a/roofit/hs3/src/HS3Document.h b/roofit/hs3/src/HS3Document.h index 9102caa38a7d3..a628167ba2306 100644 --- a/roofit/hs3/src/HS3Document.h +++ b/roofit/hs3/src/HS3Document.h @@ -494,6 +494,19 @@ HistFactoryDist parseHistFactoryDist(Detail::JSONNode const &distribution, Diagn /// Reads an interpolation that doesn't come with a document. Interpolation parseInterpolation(Detail::JSONNode const &interpolation, Diagnostics &diagnostics); +/// Writes everything but the distributions and functions, which are written one by one while they are exported: the +/// generic ones by the exporters, and the others with functions like writeHistFactoryDist(). +void writeDocument(Detail::JSONNode &root, HS3Document const &doc); + +/// Writes a HistFactory distribution, without its name. +void writeHistFactoryDist(Detail::JSONNode &node, HistFactoryDist const &dist); + +/// Writes a number, as an integer if it has no fractional part. +void writeNumber(Detail::JSONNode &node, double value); + +/// Writes an array of numbers, each as an integer if it has no fractional part. +void writeContents(Detail::JSONNode &node, std::vector const &values); + /// Options for validate(). struct ValidateOptions { /// Whether the document has to follow the current standard, or if older conventions of RooFit are tolerated. @@ -525,6 +538,9 @@ namespace HS3 { std::optional validate(HS3Document doc, Diagnostics &diagnostics, ValidateOptions const &options = {}); +/// The checks of validate(), for a document that is still needed if it doesn't pass them. +void checkDocument(HS3Document const &doc, Diagnostics &diagnostics, ValidateOptions const &options = {}); + /// Same as validate(), but for a single HistFactory distribution that doesn't come with a document. void validateHistFactoryDist(HistFactoryDist const &dist, std::string const &name, Diagnostics &diagnostics); diff --git a/roofit/hs3/src/HS3Validate.cxx b/roofit/hs3/src/HS3Validate.cxx index 8c3d1d6953b91..6294b05a28c6d 100644 --- a/roofit/hs3/src/HS3Validate.cxx +++ b/roofit/hs3/src/HS3Validate.cxx @@ -509,9 +509,14 @@ void Validator::checkHistFactory(HistFactoryDist const &dist, std::string const } // namespace -std::optional validate(HS3Document doc, Diagnostics &diagnostics, ValidateOptions const &options) +void checkDocument(HS3Document const &doc, Diagnostics &diagnostics, ValidateOptions const &options) { Validator{doc, diagnostics, options}.run(); +} + +std::optional validate(HS3Document doc, Diagnostics &diagnostics, ValidateOptions const &options) +{ + checkDocument(doc, diagnostics, options); if (hasErrors(diagnostics)) { return std::nullopt; } diff --git a/roofit/hs3/src/HS3Writer.cxx b/roofit/hs3/src/HS3Writer.cxx new file mode 100644 index 0000000000000..dad43a6c098b6 --- /dev/null +++ b/roofit/hs3/src/HS3Writer.cxx @@ -0,0 +1,461 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +// Writes the typed representation of HS3 documents. The keys are the same as in HS3Parser.cxx, which the round-trip +// tests in testHS3Document.cxx check. + +#include "HS3Document.h" +#include "JSONIOUtils.h" + +#include + +#include +#include + +using RooFit::Detail::JSONNode; + +namespace RooFit { +namespace HS3 { +namespace { + +// Infinite bounds are written as null. +void writeBound(JSONNode &node, double value) +{ + if (std::isinf(value)) { + node.set_null(); + } else { + node << value; + } +} + +void writeStrings(JSONNode &node, std::vector const &values) +{ + node.fill_seq(values); +} + +template +void writeRefs(JSONNode &node, std::vector> const &refs) +{ + node.set_seq(); + for (auto const &ref : refs) { + node.append_child() << ref.name; + } +} + +JSONNode &appendNamed(JSONNode &array, std::string const &name) +{ + JSONNode &child = array.set_seq().append_child().set_map(); + child["name"] << name; + return child; +} + +void writeBinning(JSONNode &node, Binning const &binning) +{ + if (auto const *edges = std::get_if(&binning)) { + node["edges"].fill_seq(edges->edges); + } else { + node["nbins"] << std::get(binning).nbins; + } +} + +void writeAxis(JSONNode &node, BinnedAxis const &axis) +{ + node.set_map(); + node["name"] << axis.name; + // Binned axes always carry their range, also if it is redundant with the "edges" of an irregular binning. + node["min"] << axis.min; + node["max"] << axis.max; + writeBinning(node, axis.binning); +} + +void writeAxis(JSONNode &node, ContinuousAxis const &axis) +{ + node.set_map(); + node["name"] << axis.name; + if (axis.min) { + writeBound(node["min"], *axis.min); + } + if (axis.max) { + writeBound(node["max"], *axis.max); + } +} + +void writeAxis(JSONNode &node, CategoricalAxis const &axis) +{ + node.set_map(); + node["name"] << axis.name; + writeStrings(node["categories"], axis.categories); +} + +// An infinite or missing bound of a domain axis is omitted, which means that the range is unbounded. +void writeAxis(JSONNode &node, DomainAxis const &axis) +{ + node.set_map(); + node["name"] << axis.name; + if (axis.fixedValue) { + node[fixedValueKey] << *axis.fixedValue; + return; + } + if (axis.min && !std::isinf(*axis.min)) { + node["min"] << *axis.min; + } + if (axis.max && !std::isinf(*axis.max)) { + node["max"] << *axis.max; + } + if (axis.binning) { + writeBinning(node, *axis.binning); + } +} + +template +void writeAxes(JSONNode &node, std::vector> const &axes) +{ + node.set_seq(); + for (auto const &axis : axes) { + std::visit([&](auto const &a) { writeAxis(node.append_child(), a); }, axis); + } +} + +template +void writeAxes(JSONNode &node, std::vector const &axes) +{ + node.set_seq(); + for (auto const &axis : axes) { + writeAxis(node.append_child(), axis); + } +} + +void writeInterpolation(JSONNode &node, Interpolation const &interpolation) +{ + node.set_map(); + node["type"] << toString(interpolation.type); + node["in"] << toString(interpolation.in); + if (interpolation.out) { + node["out"] << toString(*interpolation.out); + } else { + node["out"].set_null(); + } +} + +// The parameter of a modifier with a single parameter, and the name of its constraint distribution. A constraint type +// is written in the form of older documents. +void writeParameter(JSONNode &mod, ParameterRef const ¶meter, Constraint const &constraint) +{ + mod["parameters"].set_seq().append_child() << parameter.name; + std::visit( + overloaded{[](Unconstrained) {}, [&](ConstraintType type) { mod["constraint"] << toString(type); }, + [&](ConstraintPdf const &pdf) { mod["constraints"].set_seq().append_child() << pdf.pdf.name; }}, + constraint); +} + +void writeGammaConstraint(JSONNode &node, GammaConstraint const &constraint, const char *typeKey) +{ + std::visit(overloaded{[](Unconstrained) {}, [&](ConstraintType type) { node[typeKey] << toString(type); }, + [&](ConstantGammas) { node[typeKey] << "Const"; }, + [&](GammaConstraintPdfs const &pdfs) { + JSONNode &names = node["constraints"].set_seq(); + for (auto const &pdf : pdfs.pdfs) { + if (pdf) { + names.append_child() << pdf->name; + } else { + names.append_child().set_null(); + } + } + }}, + constraint); +} + +JSONNode &appendModifier(JSONNode &modifiers, std::string const &name, const char *type) +{ + JSONNode &mod = modifiers.append_child().set_map(); + mod["name"] << name; + mod["type"] << type; + return mod; +} + +void writeModifier(JSONNode &modifiers, Modifier const &modifier) +{ + std::visit(overloaded{ + [&](NormFactor const &m) { + JSONNode &mod = appendModifier(modifiers, m.name, "normfactor"); + writeParameter(mod, m.parameter, m.constraint); + }, + [&](NormSys const &m) { + JSONNode &mod = appendModifier(modifiers, m.name, "normsys"); + writeParameter(mod, m.parameter, m.constraint); + if (m.interpolation) { + writeInterpolation(mod["interpolation"], *m.interpolation); + } + JSONNode &data = mod["data"].set_map(); + data["lo"] << m.lo; + data["hi"] << m.hi; + }, + [&](HistoSys const &m) { + JSONNode &mod = appendModifier(modifiers, m.name, "histosys"); + writeParameter(mod, m.parameter, m.constraint); + if (m.interpolation) { + writeInterpolation(mod["interpolation"], *m.interpolation); + } + JSONNode &data = mod["data"].set_map(); + writeContents(data["lo"].set_map()["contents"], m.lo.contents); + writeContents(data["hi"].set_map()["contents"], m.hi.contents); + }, + [&](ShapeSys const &m) { + JSONNode &mod = appendModifier(modifiers, m.name, "shapefactor"); + writeRefs(mod["parameters"], m.parameters); + writeGammaConstraint(mod, m.constraint, "constraint"); + // The values are only used by the constraint types of older documents. + if (!m.vals.empty()) { + mod["data"].set_map()["vals"].fill_seq(m.vals); + } + }, + [&](StatError const &m) { + JSONNode &mod = appendModifier(modifiers, m.name, "staterror"); + writeRefs(mod["parameters"], m.parameters); + writeGammaConstraint(mod, m.constraint, "constraint"); + }, + [&](CustomModifier const &m) { appendModifier(modifiers, m.name, "custom"); }, + }, + modifier); +} + +void writeParameterPoint(JSONNode &points, ParameterPoint const &point) +{ + JSONNode ¶meters = appendNamed(points, point.name)["parameters"].set_seq(); + for (auto const &p : point.parameters) { + JSONNode &node = appendNamed(parameters, p.name); + if (p.value) { + std::visit([&](auto const &value) { node["value"] << value; }, *p.value); + } + if (p.isConst) { + node["const"] << true; + } + } +} + +void writeData(JSONNode &dataArray, Data const &data) +{ + JSONNode &node = appendNamed(dataArray, data.name); + std::visit(overloaded{ + [&](BinnedData const &d) { + node["type"] << "binned"; + writeAxes(node["axes"], d.axes); + writeContents(node["contents"], d.contents); + if (d.uncertainty) { + JSONNode &uncertainty = node["uncertainty"].set_map(); + uncertainty["type"] << "gaussian_uncertainty"; + uncertainty["sigma"].fill_seq(d.uncertainty->sigma); + } + }, + [&](UnbinnedData const &d) { + node["type"] << "unbinned"; + writeAxes(node["axes"], d.axes); + JSONNode &entries = node["entries"].set_seq(); + const std::size_t nAxes = d.axes.size(); + for (std::size_t i = 0; i < d.numEntries(); ++i) { + JSONNode &row = entries.append_child().set_seq(); + for (std::size_t j = 0; j < nAxes; ++j) { + const double value = d.entries[i * nAxes + j]; + if (auto const *category = std::get_if(&d.axes[j])) { + row.append_child() << category->categories[static_cast(value)]; + } else { + row.append_child() << value; + } + } + } + if (d.weights) { + node["weights"].fill_seq(*d.weights); + } + }, + [&](PointData const &d) { + node["type"] << "point"; + node["value"] << d.value; + if (d.uncertainty) { + node["uncertainty"] << *d.uncertainty; + } + if (!d.axes.empty()) { + writeAxes(node["axes"], d.axes); + } + }, + }, + data.body); +} + +void writeLikelihood(JSONNode &likelihoods, Likelihood const &likelihood) +{ + JSONNode &node = appendNamed(likelihoods, likelihood.name); + JSONNode &distributions = node["distributions"].set_seq(); + JSONNode &data = node["data"].set_seq(); + for (auto const &term : likelihood.terms) { + distributions.append_child() << term.distribution.name; + std::visit(overloaded{[&](DataRef const &ref) { data.append_child() << ref.name; }, + [&](double value) { writeNumber(data.append_child(), value); }}, + term.data); + } + if (!likelihood.auxDistributions.empty()) { + writeRefs(node["aux_distributions"], likelihood.auxDistributions); + } +} + +void writeAnalysis(JSONNode &analyses, Analysis const &analysis) +{ + JSONNode &node = appendNamed(analyses, analysis.name); + writeRefs(node["domains"], analysis.domains); + node["likelihood"] << analysis.likelihood.name; + if (!analysis.parametersOfInterest.empty()) { + writeRefs(node["parameters_of_interest"], analysis.parametersOfInterest); + } + if (analysis.init) { + node["init"] << analysis.init->name; + } + if (analysis.prior) { + node["prior"] << analysis.prior->name; + } +} + +void writeRooFitInternal(JSONNode &root, RooFitInternal const &internal) +{ + auto rootInternal = [&]() -> JSONNode & { return root.get("misc", "ROOT_internal"); }; + if (!internal.attributes.empty()) { + JSONNode &attributes = rootInternal()["attributes"].set_map(); + for (auto const &item : internal.attributes) { + JSONNode &node = attributes[item.first].set_map(); + if (!item.second.dict.empty()) { + JSONNode &dict = node["dict"].set_map(); + for (auto const &entry : item.second.dict) { + dict[entry.first] << entry.second; + } + } + if (!item.second.tags.empty()) { + writeStrings(node["tags"], item.second.tags); + } + } + } + if (!internal.combinedDistributions.empty()) { + JSONNode &combined = rootInternal()["combined_distributions"].set_map(); + for (auto const &item : internal.combinedDistributions) { + JSONNode &node = combined[item.first].set_map(); + node["index_cat"] << item.second.indexCat; + writeStrings(node["labels"], item.second.labels); + node["indices"].fill_seq(item.second.indices); + writeRefs(node["distributions"], item.second.distributions); + } + } + if (!internal.combinedDatasets.empty()) { + JSONNode &combined = rootInternal()["combined_datasets"].set_map(); + for (auto const &item : internal.combinedDatasets) { + JSONNode &node = combined[item.first].set_map(); + node["index_cat"] << item.second.indexCat; + writeStrings(node["labels"], item.second.labels); + node["indices"].fill_seq(item.second.indices); + } + } + if (!internal.modelConfigs.empty()) { + JSONNode &modelConfigs = rootInternal()["ModelConfigs"].set_map(); + for (auto const &item : internal.modelConfigs) { + JSONNode &node = modelConfigs[item.first].set_map(); + if (item.second.mcName) { + node["mcName"] << *item.second.mcName; + } + if (item.second.pdfName) { + node["pdfName"] << *item.second.pdfName; + } + if (item.second.combinedDataName) { + node["combined_data_name"] << *item.second.combinedDataName; + } + } + } + if (!internal.stepWidths.empty()) { + JSONNode &stepWidths = root.get("misc", "minimization")["parameter_stepwidths"].set_seq(); + for (auto const &stepWidth : internal.stepWidths) { + appendNamed(stepWidths, stepWidth.name)["step_width"] << stepWidth.width; + } + } +} + +} // namespace + +void writeNumber(JSONNode &node, double value) +{ + // To make sure there are no unnecessary floating points in the JSON + if (static_cast(value) == value) { + node << static_cast(value); + } else { + node << value; + } +} + +void writeContents(JSONNode &node, std::vector const &values) +{ + node.set_seq(); + for (double value : values) { + writeNumber(node.append_child(), value); + } +} + +void writeHistFactoryDist(JSONNode &node, HistFactoryDist const &dist) +{ + node.set_map(); + node["type"] << "histfactory_dist"; + if (!std::holds_alternative(dist.statError.constraint)) { + writeGammaConstraint(node["staterror"].set_map(), dist.statError.constraint, "constraint_type"); + } + if (dist.defaultInterpolation) { + writeInterpolation(node["default_interpolation"], *dist.defaultInterpolation); + } + writeAxes(node["axes"], dist.axes); + JSONNode &samples = node["samples"].set_seq(); + for (auto const &sample : dist.samples) { + JSONNode &s = appendNamed(samples, sample.name); + JSONNode &modifiers = s["modifiers"].set_seq(); + for (auto const &modifier : sample.modifiers) { + writeModifier(modifiers, modifier); + } + JSONNode &data = s["data"].set_map(); + writeContents(data["contents"], sample.contents); + if (sample.errors) { + writeContents(data["errors"], *sample.errors); + } + } +} + +void writeDocument(JSONNode &root, HS3Document const &doc) +{ + root.set_map(); + JSONNode &metadata = root["metadata"].set_map(); + metadata["hs3_version"] << doc.metadata.hs3Version; + for (auto const &package : doc.metadata.packages) { + appendNamed(metadata["packages"], package.name)["version"] << package.version; + } + + for (auto const &point : doc.parameterPoints) { + writeParameterPoint(root["parameter_points"], point); + } + for (auto const &domain : doc.domains) { + JSONNode &node = appendNamed(root["domains"], domain.name); + node["type"] << "product_domain"; + writeAxes(node["axes"], domain.axes); + } + for (auto const &data : doc.data) { + writeData(root["data"], data); + } + for (auto const &likelihood : doc.likelihoods) { + writeLikelihood(root["likelihoods"], likelihood); + } + for (auto const &analysis : doc.analyses) { + writeAnalysis(root["analyses"], analysis); + } + writeRooFitInternal(root, doc.rooFit); +} + +} // namespace HS3 +} // namespace RooFit diff --git a/roofit/hs3/src/JSONFactories_HistFactory.cxx b/roofit/hs3/src/JSONFactories_HistFactory.cxx index be1be3599d2ef..e5e63e27d2902 100644 --- a/roofit/hs3/src/JSONFactories_HistFactory.cxx +++ b/roofit/hs3/src/JSONFactories_HistFactory.cxx @@ -311,33 +311,6 @@ inline std::string defaultGammaName(std::string const &sysname, std::size_t i) return "gamma_" + sysname + "_bin_" + std::to_string(i); } -/// Export the names of the gamma parameters to the modifier struct -void exportGammaParameters(JSONNode &mod, std::vector const ¶ms) -{ - std::vector paramNames; - for (RooAbsReal *param : params) { - paramNames.emplace_back(param->GetName()); - } - mod["parameters"].fill_seq(paramNames); -} - -/// Export the names of the constraint distributions of a modifier, with null for unconstrained parameters. Nothing is -/// written if all parameters are unconstrained. -void exportConstraintNames(JSONNode &mod, std::vector const &constraints) -{ - if (std::none_of(constraints.begin(), constraints.end(), [](auto *pdf) { return pdf != nullptr; })) { - return; - } - auto &names = mod["constraints"].set_seq(); - for (auto *constraint : constraints) { - if (constraint) { - names.append_child() << constraint->GetName(); - } else { - names.append_child().set_null(); - } - } -} - RooRealVar &createNominal(RooWorkspace &ws, std::string const &parname, double val, double min, double max) { RooRealVar &nom = getOrCreate(ws, "nom_" + parname, val, min, max); @@ -1640,116 +1613,122 @@ std::optional defaultInterpolation(const Channel &channel) return best->first; } -bool exportChannel(RooJSONFactoryWSTool *tool, const Channel &channel, JSONNode &elem) +HS3::Constraint toConstraint(RooAbsPdf const *constraint) { - // Write the parameter and the name of the constraint distribution of a modifier with a single parameter. - auto writeParameter = [](JSONNode &mod, RooAbsReal const ¶m, RooAbsPdf const *constraint) { - mod["parameters"].set_seq().append_child() << param.GetName(); - exportConstraintNames(mod, {constraint}); - }; - auto addModifier = [](JSONNode &modifiers, std::string const &name, const char *type) -> JSONNode & { - auto &mod = modifiers.append_child(); - mod.set_map(); - mod["name"] << name; - mod["type"] << type; - return mod; - }; + if (!constraint) { + return HS3::Unconstrained{}; + } + return HS3::ConstraintPdf{{constraint->GetName()}}; +} + +// The constraint distributions of gamma parameters, with null for the unconstrained ones. Nothing is written if all +// parameters are unconstrained. +HS3::GammaConstraint toGammaConstraint(std::vector const &constraints) +{ + if (std::none_of(constraints.begin(), constraints.end(), [](auto *pdf) { return pdf != nullptr; })) { + return HS3::Unconstrained{}; + } + HS3::GammaConstraintPdfs pdfs; + for (auto *constraint : constraints) { + pdfs.pdfs.push_back(constraint ? std::make_optional(HS3::DistributionRef{constraint->GetName()}) : std::nullopt); + } + return pdfs; +} - elem["type"] << "histfactory_dist"; - const auto channelDefaultInterpolation = defaultInterpolation(channel); - if (channelDefaultInterpolation) { - writeInterpolation(elem["default_interpolation"], *channelDefaultInterpolation); +void checkBinning(std::size_t nBins, std::size_t size, const char *what) +{ + if (nBins != size) { + RooJSONFactoryWSTool::error("inconsistent binning: " + std::to_string(nBins) + " bins expected, but " + + std::to_string(size) + " found in " + what + "!"); } +} - bool observablesWritten = false; - for (const auto &sample : channel.samples) { +// The channel as it is written to the document, which the tool writes to the JSON. +bool exportChannel(RooJSONFactoryWSTool *tool, const Channel &channel) +{ + HS3::HistFactoryDist dist; + dist.defaultInterpolation = defaultInterpolation(channel); + auto interpolationIfNotDefault = [&](Interpolation const &interpolation) -> std::optional { + if (dist.defaultInterpolation && interpolation == *dist.defaultInterpolation) { + return std::nullopt; + } + return interpolation; + }; - auto &s = RooJSONFactoryWSTool::appendNamedChild(elem["samples"], sample.name); + for (auto *obs : static_range_cast(*channel.varSet)) { + dist.axes.push_back(HS3::toBinnedAxis(*obs)); + } - auto &modifiers = s["modifiers"]; - modifiers.set_seq(); + for (const auto &sample : channel.samples) { + HS3::Sample &s = dist.samples.emplace_back(); + s.name = sample.name; for (const auto &nf : sample.normfactors) { - auto &mod = modifiers.append_child(); - mod.set_map(); - mod["name"] << nf.name; - writeParameter(mod, *nf.param, nf.constraint); - mod["type"] << "normfactor"; + s.modifiers.push_back(HS3::NormFactor{nf.name, {nf.param->GetName()}, toConstraint(nf.constraint)}); if (nf.constraint) { tool->queueExport(*nf.constraint); } } for (const auto &sys : sample.normsys) { - auto &mod = addModifier(modifiers, sys.name, "normsys"); - writeParameter(mod, *sys.param, sys.constraint); - if (!channelDefaultInterpolation || sys.interpolation != *channelDefaultInterpolation) { - writeInterpolation(mod["interpolation"], sys.interpolation); - } - auto &data = mod["data"].set_map(); - data["lo"] << sys.low; - data["hi"] << sys.high; + s.modifiers.push_back(HS3::NormSys{sys.name, + {sys.param->GetName()}, + sys.high, + sys.low, + toConstraint(sys.constraint), + interpolationIfNotDefault(sys.interpolation)}); } for (const auto &sys : sample.histosys) { - auto &mod = addModifier(modifiers, sys.name, "histosys"); - writeParameter(mod, *sys.param, sys.constraint); - if (!channelDefaultInterpolation || sys.interpolation != *channelDefaultInterpolation) { - writeInterpolation(mod["interpolation"], sys.interpolation); - } - auto &data = mod["data"].set_map(); - if (channel.nBins != sys.low.size() || channel.nBins != sys.high.size()) { - RooJSONFactoryWSTool::error("inconsistent binning: " + std::to_string(channel.nBins) + - " bins expected, but " + std::to_string(sys.low.size()) + "/" + - std::to_string(sys.high.size()) + " found in nominal histogram errors!"); - } - RooJSONFactoryWSTool::exportArray(channel.nBins, sys.low.data(), data["lo"].set_map()["contents"]); - RooJSONFactoryWSTool::exportArray(channel.nBins, sys.high.data(), data["hi"].set_map()["contents"]); + checkBinning(channel.nBins, sys.low.size(), "nominal histogram errors"); + checkBinning(channel.nBins, sys.high.size(), "nominal histogram errors"); + HS3::HistoSys mod; + mod.name = sys.name; + mod.parameter.name = sys.param->GetName(); + mod.hi.contents = sys.high; + mod.lo.contents = sys.low; + mod.constraint = toConstraint(sys.constraint); + mod.interpolation = interpolationIfNotDefault(sys.interpolation); + s.modifiers.push_back(std::move(mod)); } for (const auto &sys : sample.shapesys) { - auto &mod = addModifier(modifiers, sys.name, "shapefactor"); - exportGammaParameters(mod, sys.parameters); + HS3::ShapeSys mod; + mod.name = sys.name; + for (RooAbsReal *param : sys.parameters) { + mod.parameters.push_back({param->GetName()}); + } // The relative uncertainties are encoded in the constraint distributions. - exportConstraintNames(mod, sys.constraintPdfs); + mod.constraint = toGammaConstraint(sys.constraintPdfs); + s.modifiers.push_back(std::move(mod)); } for (const auto &other : sample.otherElements) { - addModifier(modifiers, other.name, "custom"); + s.modifiers.push_back(HS3::CustomModifier{other.name}); } for (const auto &other : sample.tmpElements) { - addModifier(modifiers, other.name, "custom"); + s.modifiers.push_back(HS3::CustomModifier{other.name}); } if (sample.useBarlowBeestonLight) { - auto &mod = addModifier(modifiers, ::Literals::staterror, ::Literals::staterror); - exportGammaParameters(mod, sample.staterrorParameters); - exportConstraintNames(mod, sample.staterrorConstraints); - } - - if (!observablesWritten) { - auto &output = elem["axes"].set_seq(); - for (auto *obs : static_range_cast(*channel.varSet)) { - RooJSONFactoryWSTool::exportAxis(output.append_child().set_map(), *obs); + HS3::StatError mod; + mod.name = ::Literals::staterror; + for (RooAbsReal *param : sample.staterrorParameters) { + mod.parameters.push_back({param->GetName()}); } - observablesWritten = true; - } - auto &dataNode = s["data"].set_map(); - if (channel.nBins != sample.hist.size()) { - RooJSONFactoryWSTool::error("inconsistent binning: " + std::to_string(channel.nBins) + " bins expected, but " + - std::to_string(sample.hist.size()) + " found in nominal histogram!"); + mod.constraint = toGammaConstraint(sample.staterrorConstraints); + s.modifiers.push_back(std::move(mod)); } - RooJSONFactoryWSTool::exportArray(channel.nBins, sample.hist.data(), dataNode["contents"]); + + checkBinning(channel.nBins, sample.hist.size(), "nominal histogram"); + s.contents = sample.hist; if (!sample.histError.empty()) { - if (channel.nBins != sample.histError.size()) { - RooJSONFactoryWSTool::error("inconsistent binning: " + std::to_string(channel.nBins) + - " bins expected, but " + std::to_string(sample.histError.size()) + - " found in nominal histogram errors!"); - } - RooJSONFactoryWSTool::exportArray(channel.nBins, sample.histError.data(), dataNode["errors"]); + checkBinning(channel.nBins, sample.histError.size(), "nominal histogram errors"); + s.errors = sample.histError; } } + HS3::setExportedObject(*tool, std::move(dist)); return true; } @@ -1825,7 +1804,7 @@ void checkUnconstrainedModifiers(const Channel &channel, const std::vector constraints, JSONNode &elem) + std::vector constraints) { // some preliminary checks if (!sumpdf) { @@ -1911,13 +1890,13 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname } } - return exportChannel(tool, channel, elem); + return exportChannel(tool, channel); } class HistFactoryStreamer_ProdPdf : public RooFit::JSONIO::Exporter { public: bool autoExportDependants() const override { return false; } - bool tryExport(RooJSONFactoryWSTool *tool, const RooProdPdf *prodpdf, JSONNode &elem) const + bool tryExport(RooJSONFactoryWSTool *tool, const RooProdPdf *prodpdf) const { std::vector constraints; RooRealSumPdf *sumpdf = nullptr; @@ -1935,36 +1914,35 @@ class HistFactoryStreamer_ProdPdf : public RooFit::JSONIO::Exporter { if (!sumpdf) return false; - bool ok = tryExportHistFactory(tool, prodpdf->GetName(), sumpdf, constraints, elem); - return ok; + return tryExportHistFactory(tool, prodpdf->GetName(), sumpdf, constraints); } std::string const &key() const override { static const std::string keystring = "histfactory_dist"; return keystring; } - bool exportObject(RooJSONFactoryWSTool *tool, const RooAbsArg *p, JSONNode &elem) const override + bool exportObject(RooJSONFactoryWSTool *tool, const RooAbsArg *p, JSONNode &) const override { - return tryExport(tool, static_cast(p), elem); + return tryExport(tool, static_cast(p)); } }; class HistFactoryStreamer_SumPdf : public RooFit::JSONIO::Exporter { public: bool autoExportDependants() const override { return false; } - bool tryExport(RooJSONFactoryWSTool *tool, const RooRealSumPdf *sumpdf, JSONNode &elem) const + bool tryExport(RooJSONFactoryWSTool *tool, const RooRealSumPdf *sumpdf) const { std::vector constraints; - return tryExportHistFactory(tool, sumpdf->GetName(), sumpdf, constraints, elem); + return tryExportHistFactory(tool, sumpdf->GetName(), sumpdf, constraints); } std::string const &key() const override { static const std::string keystring = "histfactory_dist"; return keystring; } - bool exportObject(RooJSONFactoryWSTool *tool, const RooAbsArg *p, JSONNode &elem) const override + bool exportObject(RooJSONFactoryWSTool *tool, const RooAbsArg *p, JSONNode &) const override { - return tryExport(tool, static_cast(p), elem); + return tryExport(tool, static_cast(p)); } }; diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index a6d4409caea20..6dfc7eb272f31 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -149,6 +149,19 @@ struct RooFit::HS3::ToolImportState { std::unordered_map defaultValuesByName; }; +/// The document that the tool is exporting. +struct RooFit::HS3::ToolExportState { + HS3Document doc; + JSONNode *root = nullptr; + /// Set by the exporter of an object that is interpreted by the document. + std::optional typedObject; +}; + +void RooFit::HS3::setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist dist) +{ + tool._export->typedObject = std::move(dist); +} + namespace { // Keeps track of the objects that are being imported. Objects that depend on each other would otherwise recurse until @@ -319,7 +332,7 @@ void collectParameterStepWidthCandidatesFromPdfs(std::vector const } void exportParameterStepWidths(RooWorkspace const &workspace, std::vector const &pdfs, - std::vector const &data, JSONNode &rootnode) + std::vector const &data, std::vector &out) { RooArgSet candidates; RooArgSet excluded; @@ -329,7 +342,6 @@ void exportParameterStepWidths(RooWorkspace const &workspace, std::vectorGetName()); - stepWidthNode["step_width"] << var->getError(); + out.push_back({var->GetName(), var->getError()}); } } @@ -423,48 +430,63 @@ bool isLiteralConstVar(RooAbsArg const &arg) return isRooConstVar && isNumber(arg.GetName()); } -// Export the string attributes and tags of `arg` into the ROOT-internal attributes node. -void exportAttributes(const RooAbsArg *arg, JSONNode &rootnode) +// Export the string attributes and tags of `arg` into the ROOT-internal attributes. +void exportAttributes(const RooAbsArg *arg, std::map &attributes) { // If this RooConst is a literal number, we don't need to export the attributes. - if (isLiteralConstVar(*arg)) { + if (isLiteralConstVar(*arg) || dynamic_cast(arg)) { return; } - JSONNode *node = nullptr; - - auto initializeNode = [&]() { - if (node) - return; - - node = &RooJSONFactoryWSTool::getRooFitInternal(rootnode, "attributes").set_map()[arg->GetName()].set_map(); + HS3::Attributes *out = nullptr; + auto initialize = [&]() { + if (!out) { + out = &attributes[arg->GetName()]; + } }; - if (dynamic_cast(arg)) { - return; - } - // export all string attributes of an object - if (!arg->stringAttributes().empty()) { - for (const auto &it : arg->stringAttributes()) { - // Skip some RooFit internals - if (it.first == "factory_tag" || it.first == "PROD_TERM_TYPE") - continue; - initializeNode(); - (*node)["dict"].set_map()[it.first] << it.second; - } + for (const auto &it : arg->stringAttributes()) { + // Skip some RooFit internals + if (it.first == "factory_tag" || it.first == "PROD_TERM_TYPE") + continue; + initialize(); + out->dict[it.first] = it.second; } - if (!arg->attributes().empty()) { - for (auto const &attr : arg->attributes()) { - // Skip some RooFit internals - if (attr == "SnapShot_ExtRefClone" || attr == "RooRealConstant_Factory_Object") - continue; - initializeNode(); - (*node)["tags"].set_seq().append_child() << attr; + for (auto const &attr : arg->attributes()) { + // Skip some RooFit internals + if (attr == "SnapShot_ExtRefClone" || attr == "RooRealConstant_Factory_Object") + continue; + initialize(); + out->tags.push_back(attr); + } +} + +} // namespace + +HS3::BinnedAxis RooFit::HS3::toBinnedAxis(RooRealVar const &var) +{ + RooJSONFactoryWSTool::testValidName(var.GetName(), false); + RooAbsBinning const &binning = var.getBinning(); + HS3::BinnedAxis axis; + axis.name = var.GetName(); + axis.min = binning.lowBound(); + axis.max = binning.highBound(); + if (binning.isUniform()) { + axis.binning = HS3::RegularBinning{binning.numBins()}; + } else { + HS3::EdgesBinning edges; + edges.edges.push_back(binning.binLow(0)); + for (int i = 0; i < binning.numBins(); ++i) { + edges.edges.push_back(binning.binHigh(i)); } + axis.binning = std::move(edges); } + return axis; } +namespace { + std::unique_ptr createObservable(HS3::BinnedAxis const &axis) { auto obs = std::make_unique(axis.name.c_str(), axis.name.c_str(), axis.min, axis.max); @@ -998,8 +1020,8 @@ RooAbsReal *RooJSONFactoryWSTool::requestImpl(const std::string &obj return nullptr; } -// Export a single variable (RooRealVar or RooConstVar) `v` as a named child of `node`. -void RooJSONFactoryWSTool::exportVariable(const RooAbsArg *v, JSONNode &node, bool storeConstant, bool storeBins) +// Export a single variable (RooRealVar or RooConstVar) `v` to the parameter point. +void RooJSONFactoryWSTool::exportVariable(const RooAbsArg *v, HS3::ParameterPoint &point, bool storeConstant) { auto *cv = dynamic_cast(v); auto *rrv = dynamic_cast(v); @@ -1018,30 +1040,19 @@ void RooJSONFactoryWSTool::exportVariable(const RooAbsArg *v, JSONNode &node, bo return; } - JSONNode &var = appendNamedChild(node, v->GetName()); - - if (rrv) { - var["value"] << rrv->getVal(); - if (storeConstant && (rrv->isConstant() || rrv->getMin() >= rrv->getMax())) { - var["const"] << true; - } else if (storeBins) { - var["min"] << rrv->getMin(); - var["max"] << rrv->getMax(); - } - if (rrv->getBins() != 0 && storeBins) { - var["nbins"] << rrv->getBins(); - } - _domains->readVariable(*rrv); - } + HS3::ParameterValue var; + var.name = v->GetName(); + var.value = rrv->getVal(); + var.isConst = storeConstant && (rrv->isConstant() || rrv->getMin() >= rrv->getMax()); + _domains->readVariable(*rrv); + point.parameters.push_back(std::move(var)); } -// Export all variables in `allElems` as a sequence under `n`. -void RooJSONFactoryWSTool::exportVariables(const RooArgSet &allElems, JSONNode &n, bool storeConstant, bool storeBins) +// Export all variables in `allElems` to the parameter point. +void RooJSONFactoryWSTool::exportVariables(const RooArgSet &allElems, HS3::ParameterPoint &point, bool storeConstant) { - // export a list of RooRealVar objects - n.set_seq(); for (RooAbsArg *arg : allElems) { - exportVariable(arg, n, storeConstant, storeBins); + exportVariable(arg, point, storeConstant); } } @@ -1050,16 +1061,52 @@ std::string RooJSONFactoryWSTool::exportTransformed(const RooAbsReal *original, { std::string newname = std::string(original->GetName()) + suffix; // Another object that shares the parameter might have exported the helper already. - if (findNamedChild((*_rootnodeOutput)["functions"], newname)) { + auto &functions = _export->doc.functions; + if (std::find_if(functions.begin(), functions.end(), [&](auto const &f) { return f.name == newname; }) != + functions.end()) { return newname; } - RooFit::Detail::JSONNode &trafo_node = appendNamedChild((*_rootnodeOutput)["functions"], newname); + RooFit::Detail::JSONNode &trafo_node = appendNamedChild((*_export->root)["functions"], newname); trafo_node["type"] << "generic"; trafo_node["expression"] << TString::Format(formula.c_str(), original->GetName()).Data(); + functions.push_back({newname, {"generic", nullptr}}); this->setAttribute(newname, "roofit_skip"); // this function should not be imported back in return newname; } +namespace { + +// Turn an arbitrary string into a valid variable name, but refuse to change the +// first character (which would silently rename the object). +std::string makeValidNameOrError(std::string const &in) +{ + if (!std::isalpha(in[0])) { + RooJSONFactoryWSTool::error("refusing to change first character of string '" + in + "' to make a valid name!"); + } + std::string out = RooFit::Detail::makeValidVarName(in); + if (out != in) { + oocoutW(nullptr, ObjectHandling) << "RooFitHS3: changed '" << in << "' to '" << out << "' to become a valid name"; + } + return out; +} + +} // namespace + +namespace { + +// Record an object that was written by an exporter in the document. +void recordGenericObject(HS3Document &doc, bool isPdf, std::string const &name, JSONNode const &elem) +{ + std::string type = elem.has_child("type") ? elem["type"].val() : ""; + if (isPdf) { + doc.distributions.push_back({name, HS3::GenericObject{std::move(type), nullptr}}); + } else { + doc.functions.push_back({name, {std::move(type), nullptr}}); + } +} + +} // namespace + // Export a single object `func` (pdf, function, variable or category) to the output JSON, recording its name in // `exportedObjectNames` to avoid exporting it twice. void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func, std::set &exportedObjectNames) @@ -1077,18 +1124,12 @@ void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func, std::set channelNames; + HS3::CombinedDistribution &info = _export->doc.rooFit.combinedDistributions[simPdf->GetName()]; + info.indexCat = simPdf->indexCat().GetName(); for (auto const &item : simPdf->indexCat()) { - channelNames.push_back(item.first); - } - - auto &infoNode = getRooFitInternal(*_rootnodeOutput, "combined_distributions").set_map(); - auto &child = infoNode[simPdf->GetName()].set_map(); - child["index_cat"] << simPdf->indexCat().GetName(); - exportCategory(simPdf->indexCat(), child); - child["distributions"].set_seq(); - for (auto const &item : simPdf->indexCat()) { - child["distributions"].append_child() << simPdf->getPdf(item.first.c_str())->GetName(); + info.labels.push_back(makeValidNameOrError(item.first)); + info.indices.push_back(item.second); + info.distributions.push_back({simPdf->getPdf(item.first.c_str())->GetName()}); } return; @@ -1096,11 +1137,12 @@ void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func, std::set(&func) || dynamic_cast(&func)) { - exportVariable(&func, *_varsNode, true, false); + exportVariable(&func, _export->doc.parameterPoints.front(), true); return; } - auto &collectionNode = (*_rootnodeOutput)[dynamic_cast(&func) ? "distributions" : "functions"]; + const bool isPdf = dynamic_cast(&func); + auto &collectionNode = (*_export->root)[isPdf ? "distributions" : "functions"]; auto const &exporters = RooFit::JSONIO::exporters(); auto const &exportKeys = RooFit::JSONIO::exportKeys(); @@ -1124,6 +1166,15 @@ void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func, std::settypedObject) { + HS3::writeHistFactoryDist(elem, *_export->typedObject); + elem["name"] << name; + _export->doc.distributions.push_back({name, std::move(*_export->typedObject)}); + _export->typedObject.reset(); + } else { + recordGenericObject(_export->doc, isPdf, name, elem); + } if (exp->autoExportDependants()) { exportObjects(func.servers(), exportedObjectNames); } else { @@ -1157,6 +1208,7 @@ void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func, std::setsecond.type; + recordGenericObject(_export->doc, isPdf, name, elem); size_t nprox = func.numProxies(); @@ -1417,24 +1469,6 @@ void RooJSONFactoryWSTool::exportArray(std::size_t n, double const *contents, JS } } -namespace { - -// Turn an arbitrary string into a valid variable name, but refuse to change the -// first character (which would silently rename the object). -std::string makeValidNameOrError(std::string const &in) -{ - if (!std::isalpha(in[0])) { - RooJSONFactoryWSTool::error("refusing to change first character of string '" + in + "' to make a valid name!"); - } - std::string out = RooFit::Detail::makeValidVarName(in); - if (out != in) { - oocoutW(nullptr, ObjectHandling) << "RooFitHS3: changed '" << in << "' to '" << out << "' to become a valid name"; - } - return out; -} - -} // namespace - /** * @brief Export a RooAbsCategory object to a JSONNode. * @@ -1472,18 +1506,18 @@ RooJSONFactoryWSTool::CombinedData RooJSONFactoryWSTool::exportCombinedData(RooA datamap.name = data.GetName(); // Write information necessary to reconstruct the combined dataset upon import - auto &child = getRooFitInternal(*_rootnodeOutput, "combined_datasets").set_map()[data.GetName()].set_map(); - child["index_cat"] << cat->GetName(); - exportCategory(*cat, child); + HS3::CombinedDataset &info = _export->doc.rooFit.combinedDatasets[data.GetName()]; + info.indexCat = cat->GetName(); + for (auto const &item : *cat) { + info.labels.push_back(makeValidNameOrError(item.first)); + info.indices.push_back(item.second); + } // Find a RooSimultaneous model that would fit to this dataset RooSimultaneous const *simPdf = nullptr; - auto *combinedPdfInfoNode = findRooFitInternal(*_rootnodeOutput, "combined_distributions"); - if (combinedPdfInfoNode) { - for (auto &info : combinedPdfInfoNode->children()) { - if (info["index_cat"].val() == cat->GetName()) { - simPdf = static_cast(_workspace.pdf(info.key())); - } + for (auto const &item : _export->doc.rooFit.combinedDistributions) { + if (item.second.indexCat == cat->GetName()) { + simPdf = static_cast(_workspace.pdf(item.first)); } } @@ -1503,7 +1537,7 @@ RooJSONFactoryWSTool::CombinedData RooJSONFactoryWSTool::exportCombinedData(RooA return datamap; } -// Export a single dataset `data` (binned or unbinned) to the output JSON. +// Export a single dataset `data` (binned or unbinned) to the output document. void RooJSONFactoryWSTool::exportData(RooAbsData const &data) { RooAbsCategory *cat = findCategoryObservable(data); @@ -1511,7 +1545,8 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) if (cat) return; - JSONNode &output = appendNamedChild((*_rootnodeOutput)["data"], data.GetName()); + HS3::Data output; + output.name = data.GetName(); // This works around a problem in RooStats/HistFactory that was only fixed // in ROOT 6.30: until then, the weight variable of the observed dataset, @@ -1524,13 +1559,20 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) variables.remove(*weightVar); } - // this is a regular binned dataset - if (auto dh = dynamic_cast(&data)) { - output["type"] << "binned"; + auto exportBinned = [&](std::size_t n, double const *contents) { + HS3::BinnedData binned; for (auto *var : static_range_cast(variables)) { _domains->readVariable(*var); + binned.axes.emplace_back(HS3::toBinnedAxis(*var)); } - return exportHisto(variables, dh->numEntries(), dh->weightArray(), output); + binned.contents.assign(contents, contents + n); + output.body = std::move(binned); + _export->doc.data.push_back(std::move(output)); + }; + + // this is a regular binned dataset + if (auto dh = dynamic_cast(&data)) { + return exportBinned(dh->numEntries(), dh->weightArray()); } // Check if this actually represents a binned dataset, and then import it @@ -1553,32 +1595,25 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) if (i == x.getBins()) isBinnedData = true; if (isBinnedData) { - output["type"] << "binned"; - for (auto *var : static_range_cast(variables)) { - _domains->readVariable(*var); - } - return exportHisto(variables, data.numEntries(), contents.data(), output); + return exportBinned(data.numEntries(), contents.data()); } } // this really is an unbinned dataset - output["type"] << "unbinned"; - auto &observablesNode = output["axes"].set_seq(); + HS3::UnbinnedData unbinned; for (auto *var : static_range_cast(variables)) { _domains->readVariable(*var); // HS3 forbids binning information on the axes of unbinned data. - auto &axis = observablesNode.append_child().set_map(); testValidName(var->GetName(), false); - axis["name"] << var->GetName(); - axis["min"] << var->getMin(); - axis["max"] << var->getMax(); + unbinned.axes.emplace_back(HS3::ContinuousAxis{var->GetName(), var->getMin(), var->getMax()}); } - auto &coords = output["entries"].set_seq(); std::vector weightVals; bool hasNonUnityWeights = false; for (int i = 0; i < data.numEntries(); ++i) { data.get(i); - coords.append_child().fill_seq(variables, [](auto x) { return static_cast(x)->getVal(); }); + for (auto *var : static_range_cast(variables)) { + unbinned.entries.push_back(var->getVal()); + } if (data.isWeighted()) { weightVals.push_back(data.weight()); if (data.weight() != 1.) @@ -1586,8 +1621,10 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) } } if (data.isWeighted() && hasNonUnityWeights) { - output["weights"].fill_seq(weightVals); + unbinned.weights = std::move(weightVals); } + output.body = std::move(unbinned); + _export->doc.data.push_back(std::move(output)); } /** @@ -1723,7 +1760,7 @@ void RooJSONFactoryWSTool::importDependants(const JSONNode &n) } } -void RooJSONFactoryWSTool::exportModelConfig(JSONNode &rootnode, RooFit::ModelConfig const &mc, +void RooJSONFactoryWSTool::exportModelConfig(RooFit::ModelConfig const &mc, const std::vector &combDataSets, const std::vector &singleDataSets) { @@ -1739,7 +1776,7 @@ void RooJSONFactoryWSTool::exportModelConfig(JSONNode &rootnode, RooFit::ModelCo if (hasdata) analysisName += "_" + combDataSets[i].name; - exportSingleModelConfig(rootnode, mc, analysisName, hasdata ? &combDataSets[i].components : nullptr); + exportSingleModelConfig(mc, analysisName, hasdata ? &combDataSets[i].components : nullptr); } } else { RooArgSet observables(*mc.GetObservables()); @@ -1748,87 +1785,76 @@ void RooJSONFactoryWSTool::exportModelConfig(JSONNode &rootnode, RooFit::ModelCo if (observables.equals(*(data->get()))) { std::map mapping; mapping[pdf->GetName()] = data->GetName(); - exportSingleModelConfig(rootnode, mc, std::string(mc.GetName()) + "_" + data->GetName(), &mapping); + exportSingleModelConfig(mc, std::string(mc.GetName()) + "_" + data->GetName(), &mapping); ++founddata; } } if (founddata == 0) { - exportSingleModelConfig(rootnode, mc, mc.GetName(), nullptr); + exportSingleModelConfig(mc, mc.GetName(), nullptr); } } } -void RooJSONFactoryWSTool::exportSingleModelConfig(JSONNode &rootnode, RooFit::ModelConfig const &mc, - std::string const &analysisName, +void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc, std::string const &analysisName, std::map const *dataComponents) { + HS3Document &doc = _export->doc; auto pdf = mc.GetPdf(); - JSONNode &analysisNode = appendNamedChild(rootnode["analyses"], analysisName); - - auto &domains = analysisNode["domains"].set_seq(); + HS3::Analysis analysis; + analysis.name = analysisName; // HS3 names must be unique across all top-level components, so the likelihood can't have the analysis name. - const std::string likelihoodName = analysisName + "_likelihood"; - analysisNode["likelihood"] << likelihoodName; - - auto &nllNode = appendNamedChild(rootnode["likelihoods"], likelihoodName); - nllNode["distributions"].set_seq(); - nllNode["data"].set_seq(); + HS3::Likelihood likelihood; + likelihood.name = analysisName + "_likelihood"; + analysis.likelihood.name = likelihood.name; if (dataComponents) { auto simPdf = dynamic_cast(pdf); if (simPdf) { for (auto const &item : simPdf->indexCat()) { const auto &dataComp = dataComponents->find(item.first); - nllNode["distributions"].append_child() << simPdf->getPdf(item.first)->GetName(); - nllNode["data"].append_child() << dataComp->second; + likelihood.terms.push_back({{simPdf->getPdf(item.first)->GetName()}, HS3::DataRef{dataComp->second}}); } } else { for (auto it : *dataComponents) { - nllNode["distributions"].append_child() << it.first; - nllNode["data"].append_child() << it.second; + likelihood.terms.push_back({{it.first}, HS3::DataRef{it.second}}); } } } else { - nllNode["distributions"].append_child() << pdf->GetName(); - nllNode["data"].append_child() << 0; + likelihood.terms.push_back({{pdf->GetName()}, 0.0}); } if (mc.GetExternalConstraints()) { - auto &extConstrNode = nllNode["aux_distributions"]; - extConstrNode.set_seq(); for (const auto &constr : *mc.GetExternalConstraints()) { - extConstrNode.append_child() << constr->GetName(); + likelihood.auxDistributions.push_back({constr->GetName()}); } } + doc.likelihoods.push_back(std::move(likelihood)); - auto writeList = [&](const char *name, RooArgSet const *args) { - if (!args || !args->size()) - return; - + if (RooArgSet const *pois = mc.GetParametersOfInterest()) { std::vector names; - names.reserve(args->size()); - for (RooAbsArg const *arg : *args) + names.reserve(pois->size()); + for (RooAbsArg const *arg : *pois) names.push_back(arg->GetName()); std::sort(names.begin(), names.end()); - analysisNode[name].fill_seq(names); - }; - - writeList("parameters_of_interest", mc.GetParametersOfInterest()); - - auto &domainsNode = rootnode["domains"]; + for (auto const &name : names) { + analysis.parametersOfInterest.push_back({name}); + } + } auto writeProductDomain = [&](const char *suffix, RooArgSet const *args) { if (!args || args->empty()) return; const std::string domainName = analysisName + suffix; - domains.append_child() << domainName; + analysis.domains.push_back({domainName}); RooFit::JSONIO::Detail::Domains::ProductDomain domain; for (auto *var : static_range_cast(*args)) { domain.readVariable(*var); } - domain.writeJSON(appendNamedChild(domainsNode, domainName)); + doc.domains.emplace_back(); + doc.domains.back().name = domainName; + domain.write(doc.domains.back()); }; writeProductDomain("_nuisance_parameters", mc.GetNuisanceParameters()); @@ -1837,21 +1863,39 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(JSONNode &rootnode, RooFit::M // The snapshots are exported as parameter points. Reference the one of the ModelConfig as the initial values. if (_workspace.getSnapshot(mc.GetSnapshotName().c_str())) { - analysisNode["init"] << mc.GetSnapshotName(); + analysis.init = HS3::ParameterPointRef{mc.GetSnapshotName()}; } + doc.analyses.push_back(std::move(analysis)); - auto &modelConfigAux = getRooFitInternal(rootnode, "ModelConfigs", analysisName); - modelConfigAux.set_map(); - modelConfigAux["pdfName"] << pdf->GetName(); - modelConfigAux["mcName"] << mc.GetName(); + HS3::ModelConfigInfo &info = doc.rooFit.modelConfigs[analysisName]; + info.pdfName = pdf->GetName(); + info.mcName = mc.GetName(); } // Export all top-level pdfs, functions, datasets and ModelConfigs of the workspace into `n`. void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) { _domains = std::make_unique(); - _varsNode = &makeVariablesNode(n); - _rootnodeOutput = &n; + _export = std::make_unique(); + _export->root = &n; + // The state of the export must not survive it, not even if there is an exception. + struct ExportScope { + RooJSONFactoryWSTool &tool; + ~ExportScope() + { + tool._export.reset(); + tool._domains.reset(); + } + } exportScope{*this}; + + HS3Document &doc = _export->doc; + doc.metadata.hs3Version = hs3VersionTag; + // Add information about the ROOT version that was used to generate this file. We want to consistently use dots + // such that the version name can be easily digested automatically. + std::string versionName = gROOT->GetVersion(); + std::replace(versionName.begin(), versionName.end(), '/', '.'); + doc.metadata.packages.push_back({"ROOT", versionName}); + doc.parameterPoints.push_back({"default_values", {}}); // export all toplevel pdfs std::vector allpdfs; @@ -1880,7 +1924,7 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) // export attributes of all objects for (RooAbsArg *arg : _workspace.components()) { - exportAttributes(arg, n); + exportAttributes(arg, doc.rooFit.attributes); } // collect all datasets @@ -1907,11 +1951,11 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) // export all ModelConfig objects and attached Pdfs for (TObject *obj : _workspace.allGenericObjects()) { if (auto mc = dynamic_cast(obj)) { - exportModelConfig(n, *mc, combData, singleData); + exportModelConfig(*mc, combData, singleData); } } - exportParameterStepWidths(_workspace, allpdfs, alldata, n); + exportParameterStepWidths(_workspace, allpdfs, alldata, doc.rooFit.stepWidths); for (auto *snsh : static_range_cast(_workspace.getSnapshots())) { RooArgSet snapshotSorted; @@ -1933,14 +1977,25 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) snapshotSorted.sort(); std::string name(snsh->GetName()); if (name != "default_values") { - this->exportVariables(snapshotSorted, appendNamedChild(n["parameter_points"], name)["parameters"], true, - false); + doc.parameterPoints.push_back({name, {}}); + this->exportVariables(snapshotSorted, doc.parameterPoints.back(), true); } } - _varsNode = nullptr; - _domains->writeJSON(n["domains"]); - _domains.reset(); - _rootnodeOutput = nullptr; + _domains->write(doc.domains); + + // The document that RooFit writes has to pass the same checks as the ones that it reads. Until RooFit writes + // documents that follow the current standard, problems are only reported. + HS3::Diagnostics diagnostics; + HS3::ValidateOptions validateOptions; + validateOptions.isExternal = [](std::string const &) { return false; }; + doc.metadata.version = HS3::Version{}; + HS3::checkDocument(doc, diagnostics, validateOptions); + for (auto &diagnostic : diagnostics) { + diagnostic.severity = HS3::Diagnostic::Severity::Warning; + diagnostic.message = "the exported document is not valid: " + diagnostic.message; + } + HS3::report(diagnostics); + HS3::writeDocument(n, doc); } /** @@ -2001,8 +2056,8 @@ std::unique_ptr RooJSONFactoryWSTool::createNewJSONTree() */ bool RooJSONFactoryWSTool::exportJSON(std::ostream &os) { - std::unique_ptr tree = createNewJSONTree(); - JSONNode &n = tree->rootnode(); + std::unique_ptr tree = JSONTree::create(); + JSONNode &n = tree->rootnode().set_map(); this->exportAllObjects(n); n.writeJSON(os); return true; @@ -2035,10 +2090,7 @@ bool RooJSONFactoryWSTool::hasAttribute(const std::string &obj, const std::strin } void RooJSONFactoryWSTool::setAttribute(const std::string &obj, const std::string &attrib) { - auto node = &RooJSONFactoryWSTool::getRooFitInternal(*_rootnodeOutput, "attributes").set_map()[obj].set_map(); - auto &tags = (*node)["tags"]; - tags.set_seq(); - tags.append_child() << attrib; + _export->doc.rooFit.attributes[obj].tags.push_back(attrib); } std::string RooJSONFactoryWSTool::getStringAttribute(const std::string &obj, const std::string &attrib) @@ -2055,10 +2107,7 @@ std::string RooJSONFactoryWSTool::getStringAttribute(const std::string &obj, con void RooJSONFactoryWSTool::setStringAttribute(const std::string &obj, const std::string &attrib, const std::string &value) { - auto node = &RooJSONFactoryWSTool::getRooFitInternal(*_rootnodeOutput, "attributes").set_map()[obj].set_map(); - auto &dict = (*node)["dict"]; - dict.set_map(); - dict[attrib] << value; + _export->doc.rooFit.attributes[obj].dict[attrib] = value; } // Import the JSON document rooted at `n` into the workspace. diff --git a/roofit/hs3/test/testRooFitHS3.cxx b/roofit/hs3/test/testRooFitHS3.cxx index 822d1edf44e65..212d3a7c597e8 100644 --- a/roofit/hs3/test/testRooFitHS3.cxx +++ b/roofit/hs3/test/testRooFitHS3.cxx @@ -4986,6 +4986,43 @@ TEST(RooFitHS3, HistFactoryLegacyBinWidthFactor) EXPECT_EQ(normfactorsOf(jsonWithFactors), renamedFactors.size()) << jsonWithFactors; } +// If the exported document is not valid, this is reported, but everything that can be exported is still written. Here +// the pdf of the ModelConfig has no exporter, so the likelihood refers to a distribution that doesn't exist. +TEST(RooFitHS3, InvalidExportIsStillWritten) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + + RooWorkspace ws{"ws"}; + ws.factory("Voigtian::voigt(x[0, -10, 10], mu[1, -5, 5], width[1, 0.1, 5], sigma[1, 0.1, 5])"); + ws.factory("Gaussian::gauss(x, mu, sigma)"); + RooDataSet data{"data", "data", *ws.var("x")}; + data.add(*ws.var("x")); + ws.import(data); + ws.saveSnapshot("snap", *ws.var("mu")); + RooFit::ModelConfig mc{"mc", &ws}; + mc.SetPdf("voigt"); + mc.SetParametersOfInterest("mu"); + mc.SetObservables("x"); + ws.import(mc); + + std::string json; + const std::string warnings = captureMessages(RooFit::WARNING, RooFit::ObjectHandling, + [&] { json = RooJSONFactoryWSTool{ws}.exportJSONtoString(); }); + EXPECT_NE(warnings.find("the exported document is not valid: the distribution 'voigt' does not exist"), + std::string::npos) + << warnings; + + auto tree = RooFit::Detail::JSONTree::create(json); + auto const &root = tree->rootnode(); + EXPECT_EQ(root["metadata"]["hs3_version"].val(), "0.3") << json; + for (const char *section : {"domains", "parameter_points", "data", "likelihoods", "analyses"}) { + ASSERT_TRUE(root.has_child(section)) << section << ": " << json; + EXPECT_GT(root[section].num_children(), 0u) << section << ": " << json; + } + EXPECT_NE(RooJSONFactoryWSTool::findNamedChild(root["parameter_points"], "snap"), nullptr) << json; + EXPECT_NE(RooJSONFactoryWSTool::findNamedChild(root["distributions"], "gauss"), nullptr) << json; +} + // A fixed domain axis (HS3 #131): the parameter only takes the given "value", and RooFit reads it as a RooConstVar. // It needs no entry in a parameter point. A parameter point must not give it another value, the axis has no range, // binning or categories, and all domains that list the parameter give it the same value. From 5749b5f7306cfa48172b335438b83ada58e79e8b Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Thu, 5 Nov 2026 19:45:15 +0000 Subject: [PATCH 09/20] [RF][HS3] Use the type keys of the standard for resolution models Export RooGaussModel as "gauss_model_function" and RooTruthModel as "truth_model_function", which the standard lists as functions, and RooAddModel as "mixture_model". The former keys are still read, with a deprecation warning. Assisted-by: claude-code:claude-opus-5-5 --- roofit/hs3/src/JSONFactories_RooFitCore.cxx | 18 ++++++++++++++---- roofit/hs3/src/RooJSONFactoryWSTool.cxx | 11 ++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/roofit/hs3/src/JSONFactories_RooFitCore.cxx b/roofit/hs3/src/JSONFactories_RooFitCore.cxx index 16ae38d8569d5..e24e6b0530dd8 100644 --- a/roofit/hs3/src/JSONFactories_RooFitCore.cxx +++ b/roofit/hs3/src/JSONFactories_RooFitCore.cxx @@ -548,7 +548,8 @@ bool importDecay(RooJSONFactoryWSTool *tool, const JSONNode &p) std::string name(RooJSONFactoryWSTool::name(p)); RooRealVar *t = tool->requestArg(p, "t"); RooAbsReal *tau = tool->requestArg(p, "tau"); - RooResolutionModel *model = dynamic_cast(tool->requestArg(p, "resolutionModel")); + // HS3 lists some resolution models as functions. + RooResolutionModel *model = dynamic_cast(tool->requestArg(p, "resolutionModel")); if (!model) { RooJSONFactoryWSTool::error("resolutionModel of '" + name + "' is not a RooResolutionModel"); } @@ -1611,10 +1612,17 @@ STATIC_EXECUTE([]() { RooFit::JSONIO::registerImporter("ARGUS_dist", std::make_unique("ARGUS_dist", "argus_dist")); RooFit::JSONIO::registerImporter("binsampling", std::make_unique("binsampling", "binsampling_dist")); + RooFit::JSONIO::registerImporter( + "mixture_resolution_model", std::make_unique("mixture_resolution_model", "mixture_model")); + RooFit::JSONIO::registerImporter("delta_resolution_model", std::make_unique( + "delta_resolution_model", "truth_model_function")); + RooFit::JSONIO::registerImporter("gauss_resolution_model", std::make_unique( + "gauss_resolution_model", "gauss_model_function")); registerImporter("product", false); registerImporter("product_dist", false); registerImporter("sum", false); registerImporter("mixture_dist", false); + registerImporter("mixture_model", false); registerImporter("mixture_resolution_model", false); registerImporter("binsampling_dist", false); registerImporter("binsampling", false); @@ -1632,7 +1640,9 @@ STATIC_EXECUTE([]() { registerImporter("multivariate_normal_dist", false); registerImporter("poisson_dist", false); registerImporter("decay_dist", false); + registerImporter("truth_model_function", false); registerImporter("delta_resolution_model", false); + registerImporter("gauss_model_function", false); registerImporter("gauss_resolution_model", false); registerImporter>("polynomial_dist", false); registerImporter>("polynomial", false); @@ -1647,7 +1657,7 @@ STATIC_EXECUTE([]() { registerExporter(RooWrapperPdf::Class(), "density_function_dist"); registerExporter>(RooAddPdf::Class(), "mixture_dist", false); - registerExporter>(RooAddModel::Class(), "mixture_resolution_model", false); + registerExporter>(RooAddModel::Class(), "mixture_model", false); registerExporter(RooBinSamplingPdf::Class(), "binsampling_dist", false); registerExporter(RooCrystalBall::Class(), "crystalball_doublesided_dist", false); registerExporter(RooBinWidthFunction::Class(), "binvolume", false); @@ -1661,8 +1671,8 @@ STATIC_EXECUTE([]() { registerExporter(RooMultiVarGaussian::Class(), "multivariate_normal_dist", false); registerExporter(RooPoisson::Class(), "poisson_dist", false); registerExporter(RooDecay::Class(), "decay_dist", false, /*autoExportDependants=*/false); - registerExporter(RooTruthModel::Class(), "delta_resolution_model", false); - registerExporter(RooGaussModel::Class(), "gauss_resolution_model", false); + registerExporter(RooTruthModel::Class(), "truth_model_function", false); + registerExporter(RooGaussModel::Class(), "gauss_model_function", false); registerExporter>(RooPolynomial::Class(), "polynomial_dist", false); registerExporter>(RooPolyVar::Class(), "polynomial", false); registerExporter>(RooRealSumFunc::Class(), "weighted_sum", false); diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index 6dfc7eb272f31..9487d28593b17 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -992,6 +992,13 @@ RooAbsPdf *RooJSONFactoryWSTool::requestImpl(const std::string &objna if (RooAbsPdf *retval = _workspace.pdf(objname)) return retval; } + // HS3 lists some resolution models, which are pdfs in RooFit, as functions. + auto function = _import->functions.find(objname); + if (function != _import->functions.end()) { + this->importFunction(*function->second->body.node, true); + if (RooAbsPdf *retval = _workspace.pdf(objname)) + return retval; + } } return nullptr; } @@ -1141,7 +1148,9 @@ void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func, std::set(&func); + // HS3 lists some resolution models as functions. + const bool isPdf = dynamic_cast(&func) && !func.InheritsFrom("RooGaussModel") && + !func.InheritsFrom("RooTruthModel"); auto &collectionNode = (*_export->root)[isPdf ? "distributions" : "functions"]; auto const &exporters = RooFit::JSONIO::exporters(); From 1952c39de55d9c87edcab9ab6a30807bfa94b762 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Tue, 10 Nov 2026 15:12:38 +0000 Subject: [PATCH 10/20] [RF][HS3] Check "min" and "max" of irregular binnings The standard requires them for all continuous axes of binned data, also if they have "edges". Bounds that don't agree with the edges are now rejected, like axes with both "edges" and "nbins". Older documents without them are still read. Generic formulas accept "min" and "max" next to "edges", and write them like the other binned axes. Assisted-by: claude-code:claude-opus-5-5 --- roofit/hs3/src/HS3Parser.cxx | 11 +++++++++++ roofit/hs3/src/JSONFactories_RooFitCore.cxx | 10 ++++++++-- roofit/hs3/src/JSONIOUtils.cxx | 4 ++-- roofit/hs3/src/RooJSONFactoryWSTool.cxx | 5 ----- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/roofit/hs3/src/HS3Parser.cxx b/roofit/hs3/src/HS3Parser.cxx index 5bc2089e9350b..b58c3df7ef043 100644 --- a/roofit/hs3/src/HS3Parser.cxx +++ b/roofit/hs3/src/HS3Parser.cxx @@ -339,6 +339,17 @@ void describe(Reader &ar, BinnedAxis &axis) axis.min = binning.edges.front(); axis.max = binning.edges.back(); } + // The standard requires "min" and "max" also for irregular binnings, which older versions of RooFit didn't write. + std::optional min; + std::optional max; + ar("min", min); + ar("max", max); + if ((min && *min != axis.min) || (max && *max != axis.max)) { + ar.cursor().error("\"min\" and \"max\" of axis '" + axis.name + "' must agree with the \"edges\""); + } + if (ar.has("nbins")) { + ar.cursor().error("axis '" + axis.name + "' must not have both \"edges\" and \"nbins\""); + } axis.binning = std::move(binning); return; } diff --git a/roofit/hs3/src/JSONFactories_RooFitCore.cxx b/roofit/hs3/src/JSONFactories_RooFitCore.cxx index e24e6b0530dd8..baf0ac79acc38 100644 --- a/roofit/hs3/src/JSONFactories_RooFitCore.cxx +++ b/roofit/hs3/src/JSONFactories_RooFitCore.cxx @@ -285,8 +285,8 @@ readFormulaAxisBinning(const JSONNode &axis, const std::string &axisName, const const bool hasMax = axis.has_child("max"); const bool hasNBins = axis.has_child("nbins"); - if (hasEdges && (hasMin || hasMax || hasNBins)) { - RooJSONFactoryWSTool::error(context + " must use either \"edges\" or \"min\"/\"max\"/\"nbins\""); + if (hasEdges && hasNBins) { + RooJSONFactoryWSTool::error(context + " must use either \"edges\" or \"nbins\""); } if (hasEdges) { @@ -313,6 +313,12 @@ readFormulaAxisBinning(const JSONNode &axis, const std::string &axisName, const if (edges.size() < 2) { RooJSONFactoryWSTool::error("\"edges\" in " + context + " must contain at least two values"); } + // The HS3 standard requires "min" and "max" also for irregular binnings, which older versions of RooFit didn't + // write. + if ((hasMin && (!axis["min"].is_number() || axis["min"].val_double() != edges.front())) || + (hasMax && (!axis["max"].is_number() || axis["max"].val_double() != edges.back()))) { + RooJSONFactoryWSTool::error("\"min\" and \"max\" in " + context + " must agree with the \"edges\""); + } return std::make_unique(static_cast(edges.size() - 1), edges.data()); } diff --git a/roofit/hs3/src/JSONIOUtils.cxx b/roofit/hs3/src/JSONIOUtils.cxx index 757c9a9ba02fc..bcd5e311ee694 100644 --- a/roofit/hs3/src/JSONIOUtils.cxx +++ b/roofit/hs3/src/JSONIOUtils.cxx @@ -86,9 +86,9 @@ std::unique_ptr varJSONString(const JSONNode &treeRoot void writeAxisBinning(JSONNode &node, const RooAbsBinning &binning) { + node["min"] << binning.lowBound(); + node["max"] << binning.highBound(); if (binning.isUniform()) { - node["min"] << binning.lowBound(); - node["max"] << binning.highBound(); node["nbins"] << binning.numBins(); return; } diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index 9487d28593b17..4849b7a433ed0 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -1423,11 +1423,6 @@ void RooJSONFactoryWSTool::exportAxis(JSONNode &obsNode, RooRealVar const &var) obsNode["name"] << name; writeAxisBinning(obsNode, var.getBinning()); - // Binned axes always carry their range, also if it is redundant with the "edges" of an irregular binning. - if (!obsNode.has_child("min")) { - obsNode["min"] << var.getBinning().lowBound(); - obsNode["max"] << var.getBinning().highBound(); - } } /** From cf615f019cf9b7d0b06076cf8b4f44927ac3430d Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Tue, 10 Nov 2026 15:14:26 +0000 Subject: [PATCH 11/20] [RF][HS3] Read and write the prior of analyses Map the "prior" of an analysis to the prior pdf of the ModelConfig. Assisted-by: claude-code:claude-opus-5-5 --- roofit/hs3/src/RooJSONFactoryWSTool.cxx | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index 4849b7a433ed0..85aeb3b261f64 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -661,6 +661,12 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo if (!extConstraints.empty()) mc->SetExternalConstraints(extConstraints); + if (analysis.prior) { + if (RooAbsPdf *prior = workspace.pdf(analysis.prior->name)) { + mc->SetPriorPdf(*prior); + } + } + RooArgSet parametersOfInterest; for (auto const &poi : analysis.parametersOfInterest) { RooAbsArg *arg = workspace.arg(poi.name); @@ -1869,6 +1875,10 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc if (_workspace.getSnapshot(mc.GetSnapshotName().c_str())) { analysis.init = HS3::ParameterPointRef{mc.GetSnapshotName()}; } + // The prior is a top-level pdf of the workspace or a component of one, so it is exported already. + if (RooAbsPdf const *prior = mc.GetPriorPdf()) { + analysis.prior = HS3::DistributionRef{prior->GetName()}; + } doc.analyses.push_back(std::move(analysis)); HS3::ModelConfigInfo &info = doc.rooFit.modelConfigs[analysisName]; From 991789ac1434727f734ca7a67aed217d6669bd0a Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Tue, 10 Nov 2026 15:22:17 +0000 Subject: [PATCH 12/20] [RF][HS3] Keep the constraint terms of HistFactory channels Constraint terms of a channel that no modifier names in its "constraints" were lost with a warning on export, and the export failed if they depended on the parameter of a modifier. They are now exported as "aux_distributions" of the likelihoods, which the standard adds to the constraint terms of the prediction, and recorded in misc.ROOT_internal.histfactory_constraints, which puts them back into the channel on import. A Gaussian of a normfactor parameter that is not a term of the channel, like an external constraint or a prior, is no longer exported as the constraint of the modifier. Assisted-by: claude-code:claude-opus-5-5 --- .../hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h | 9 +- roofit/hs3/src/HS3Builder.h | 16 +- roofit/hs3/src/HS3Document.h | 3 + roofit/hs3/src/HS3Parser.cxx | 1 + roofit/hs3/src/HS3Validate.cxx | 6 + roofit/hs3/src/HS3Writer.cxx | 6 + roofit/hs3/src/JSONFactories_HistFactory.cxx | 213 ++++++++---------- roofit/hs3/src/RooJSONFactoryWSTool.cxx | 63 +++++- 8 files changed, 190 insertions(+), 127 deletions(-) diff --git a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h index 3240887404a93..7b311e8f93f17 100644 --- a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h +++ b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h @@ -40,7 +40,11 @@ struct ParameterPoint; struct ParameterValue; struct ToolExportState; struct ToolImportState; -void setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist dist); +template +struct Ref; +using DistributionRef = Ref; +void setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist dist, + std::vector channelConstraints); } // namespace HS3 } // namespace RooFit namespace RooStats { @@ -266,7 +270,8 @@ class RooJSONFactoryWSTool { void exportSingleModelConfig(RooStats::ModelConfig const &mc, std::string const &analysisName, std::map const *dataComponents); - friend void RooFit::HS3::setExportedObject(RooJSONFactoryWSTool &tool, RooFit::HS3::HistFactoryDist dist); + friend void RooFit::HS3::setExportedObject(RooJSONFactoryWSTool &tool, RooFit::HS3::HistFactoryDist dist, + std::vector channelConstraints); // member variables std::string _hs3VersionInput; diff --git a/roofit/hs3/src/HS3Builder.h b/roofit/hs3/src/HS3Builder.h index 7bc27acd526d4..dd856e87a033d 100644 --- a/roofit/hs3/src/HS3Builder.h +++ b/roofit/hs3/src/HS3Builder.h @@ -45,15 +45,23 @@ BinnedAxis toBinnedAxis(RooRealVar const &var); std::unique_ptr createDataHist(std::string const &name, RooArgSet const &vars, std::vector const &contents, std::vector const *errors); -/// Create the RooFit objects of a HistFactory distribution in the workspace of the tool. -void buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string const &name, HistFactoryDist const &dist); +/// Create the RooFit objects of a HistFactory distribution in the workspace of the tool. The constraint terms that no +/// modifier names are passed separately. +void buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string const &name, HistFactoryDist const &dist, + std::vector const *channelConstraints = nullptr); -/// Called by the exporter of a HistFactory distribution, which is then written by the tool. -void setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist dist); +/// Called by the exporter of a HistFactory distribution, which is then written by the tool. The constraint terms that +/// no modifier names are passed separately. +void setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist dist, + std::vector channelConstraints); /// Whether this is the importer that RooFit registers for HistFactory distributions, as opposed to the one of a user. bool isBuiltinHistFactoryImporter(JSONIO::Importer const &importer); +/// What the built-in importer does with a HistFactory distribution, plus the constraint terms of the channel. +void importHistFactoryNode(RooJSONFactoryWSTool &tool, Detail::JSONNode const &node, + std::vector const *channelConstraints); + } // namespace HS3 } // namespace RooFit diff --git a/roofit/hs3/src/HS3Document.h b/roofit/hs3/src/HS3Document.h index a628167ba2306..01fa1609d3b68 100644 --- a/roofit/hs3/src/HS3Document.h +++ b/roofit/hs3/src/HS3Document.h @@ -418,6 +418,9 @@ struct RooFitInternal { std::map combinedDatasets; std::map modelConfigs; std::vector stepWidths; + /// The constraint terms of HistFactory distributions that no modifier names, by distribution. They are + /// also listed in the "aux_distributions" of the likelihoods, which is what other implementations use. + std::map> channelConstraints; }; struct Diagnostic { diff --git a/roofit/hs3/src/HS3Parser.cxx b/roofit/hs3/src/HS3Parser.cxx index b58c3df7ef043..ae61a75a05efd 100644 --- a/roofit/hs3/src/HS3Parser.cxx +++ b/roofit/hs3/src/HS3Parser.cxx @@ -1135,6 +1135,7 @@ void readMisc(Cursor const &root, RooFitInternal &out) readMap(ar, "combined_distributions", out.combinedDistributions); readMap(ar, "combined_datasets", out.combinedDatasets); readMap(ar, "ModelConfigs", out.modelConfigs); + readMap(ar, "histfactory_constraints", out.channelConstraints); } void readMetadata(Cursor const &root, Metadata &out, ParseOptions const &options) diff --git a/roofit/hs3/src/HS3Validate.cxx b/roofit/hs3/src/HS3Validate.cxx index 6294b05a28c6d..007ace60c491e 100644 --- a/roofit/hs3/src/HS3Validate.cxx +++ b/roofit/hs3/src/HS3Validate.cxx @@ -53,6 +53,12 @@ class Validator { requireDistribution(ref.name, "misc.ROOT_internal.combined_distributions." + item.first, "distribution"); } } + for (auto const &item : _doc.rooFit.channelConstraints) { + requireDistribution(item.first, "misc.ROOT_internal.histfactory_constraints", "distribution"); + for (auto const &ref : item.second) { + requireDistribution(ref.name, "misc.ROOT_internal.histfactory_constraints." + item.first, "constraint"); + } + } } void error(std::string const &path, std::string const &message) diff --git a/roofit/hs3/src/HS3Writer.cxx b/roofit/hs3/src/HS3Writer.cxx index dad43a6c098b6..ec8d25b027ebe 100644 --- a/roofit/hs3/src/HS3Writer.cxx +++ b/roofit/hs3/src/HS3Writer.cxx @@ -374,6 +374,12 @@ void writeRooFitInternal(JSONNode &root, RooFitInternal const &internal) } } } + if (!internal.channelConstraints.empty()) { + JSONNode &constraints = rootInternal()["histfactory_constraints"].set_map(); + for (auto const &item : internal.channelConstraints) { + writeRefs(constraints[item.first], item.second); + } + } if (!internal.stepWidths.empty()) { JSONNode &stepWidths = root.get("misc", "minimization")["parameter_stepwidths"].set_seq(); for (auto const &stepWidth : internal.stepWidths) { diff --git a/roofit/hs3/src/JSONFactories_HistFactory.cxx b/roofit/hs3/src/JSONFactories_HistFactory.cxx index e5e63e27d2902..470a75a464d69 100644 --- a/roofit/hs3/src/JSONFactories_HistFactory.cxx +++ b/roofit/hs3/src/JSONFactories_HistFactory.cxx @@ -747,29 +747,10 @@ void checkStatErrorConstraints(HS3::HistFactoryDist const &dist, std::string con class HistFactoryImporter : public RooFit::JSONIO::Importer { public: - // This is only used for HistFactory distributions that don't come with a document. The upgrade rewrites the JSON, so - // it has to work on a copy. + // This is only used for HistFactory distributions that don't come with a document. bool importArg(RooJSONFactoryWSTool *tool, const JSONNode &p) const override { - std::string name = RooJSONFactoryWSTool::name(p); - if (!p.has_child("samples")) { - RooJSONFactoryWSTool::error("no samples in '" + name + "', skipping."); - } - std::stringstream ss; - p.writeJSON(ss); - std::unique_ptr tree = RooFit::Detail::JSONTree::create(ss.str()); - - HS3::Version version; - HS3::parseVersion(tool->hs3VersionInput(), version); - - HS3::Diagnostics diagnostics; - HS3::upgradeHistFactoryDist(tree->rootnode(), version, diagnostics); - HS3::HistFactoryDist dist = HS3::parseHistFactoryDist(tree->rootnode(), diagnostics); - if (!HS3::hasErrors(diagnostics)) { - HS3::validateHistFactoryDist(dist, name, diagnostics); - } - HS3::report(diagnostics); - HS3::buildHistFactoryDist(*tool, name, dist); + RooFit::HS3::importHistFactoryNode(*tool, p, nullptr); return true; } }; @@ -1643,9 +1624,32 @@ void checkBinning(std::size_t nBins, std::size_t size, const char *what) } } -// The channel as it is written to the document, which the tool writes to the JSON. -bool exportChannel(RooJSONFactoryWSTool *tool, const Channel &channel) +// Whether a pdf is a constraint term of the channel. A pdf of a parameter that is not, like an external constraint or a +// prior, stays where it is. +bool isChannelConstraint(RooAbsPdf const *pdf, std::vector const &constraints) { + return pdf && std::find(constraints.begin(), constraints.end(), pdf) != constraints.end(); +} + +// The channel as it is written to the document, which the tool writes to the JSON. The modifiers name the constraint +// terms of their parameters. The other constraint terms of the channel are written separately. +bool exportChannel(RooJSONFactoryWSTool *tool, const Channel &channel, std::vector const &constraints) +{ + std::set namedConstraints; + auto name = [&](RooAbsPdf const *pdf) -> RooAbsPdf const * { + if (!isChannelConstraint(pdf, constraints)) { + return nullptr; + } + namedConstraints.insert(pdf); + return pdf; + }; + auto nameAll = [&](std::vector pdfs) { + for (auto &pdf : pdfs) { + pdf = name(pdf); + } + return toGammaConstraint(pdfs); + }; + HS3::HistFactoryDist dist; dist.defaultInterpolation = defaultInterpolation(channel); auto interpolationIfNotDefault = [&](Interpolation const &interpolation) -> std::optional { @@ -1664,10 +1668,8 @@ bool exportChannel(RooJSONFactoryWSTool *tool, const Channel &channel) s.name = sample.name; for (const auto &nf : sample.normfactors) { - s.modifiers.push_back(HS3::NormFactor{nf.name, {nf.param->GetName()}, toConstraint(nf.constraint)}); - if (nf.constraint) { - tool->queueExport(*nf.constraint); - } + s.modifiers.push_back( + HS3::NormFactor{nf.name, {nf.param->GetName()}, toConstraint(name(nf.constraint))}); } for (const auto &sys : sample.normsys) { @@ -1675,7 +1677,7 @@ bool exportChannel(RooJSONFactoryWSTool *tool, const Channel &channel) {sys.param->GetName()}, sys.high, sys.low, - toConstraint(sys.constraint), + toConstraint(name(sys.constraint)), interpolationIfNotDefault(sys.interpolation)}); } @@ -1687,7 +1689,7 @@ bool exportChannel(RooJSONFactoryWSTool *tool, const Channel &channel) mod.parameter.name = sys.param->GetName(); mod.hi.contents = sys.high; mod.lo.contents = sys.low; - mod.constraint = toConstraint(sys.constraint); + mod.constraint = toConstraint(name(sys.constraint)); mod.interpolation = interpolationIfNotDefault(sys.interpolation); s.modifiers.push_back(std::move(mod)); } @@ -1699,7 +1701,7 @@ bool exportChannel(RooJSONFactoryWSTool *tool, const Channel &channel) mod.parameters.push_back({param->GetName()}); } // The relative uncertainties are encoded in the constraint distributions. - mod.constraint = toGammaConstraint(sys.constraintPdfs); + mod.constraint = nameAll(sys.constraintPdfs); s.modifiers.push_back(std::move(mod)); } @@ -1716,7 +1718,7 @@ bool exportChannel(RooJSONFactoryWSTool *tool, const Channel &channel) for (RooAbsReal *param : sample.staterrorParameters) { mod.parameters.push_back({param->GetName()}); } - mod.constraint = toGammaConstraint(sample.staterrorConstraints); + mod.constraint = nameAll(sample.staterrorConstraints); s.modifiers.push_back(std::move(mod)); } @@ -1728,73 +1730,28 @@ bool exportChannel(RooJSONFactoryWSTool *tool, const Channel &channel) } } - HS3::setExportedObject(*tool, std::move(dist)); - return true; -} - -std::vector findLostConstraints(const Channel &channel, const std::vector &constraints) -{ - // collect all the vars that are used by the model - std::set vars; - for (const auto &sample : channel.samples) { - for (const auto &nf : sample.normfactors) { - vars.insert(nf.param); - } - for (const auto &sys : sample.normsys) { - vars.insert(sys.param); - } - - for (const auto &sys : sample.histosys) { - vars.insert(sys.param); - } - for (const auto &sys : sample.shapesys) { - for (const auto &par : sys.parameters) { - vars.insert(par); - } - } - if (sample.useBarlowBeestonLight) { - for (const auto &par : sample.staterrorParameters) { - vars.insert(par); - } + // All other constraint terms of the channel. + std::vector channelConstraintNames; + for (RooAbsPdf const *pdf : constraints) { + if (!namedConstraints.count(pdf)) { + tool->queueExport(*pdf); + channelConstraintNames.push_back({pdf->GetName()}); } } - // check if there is any constraint present that is unrelated to these vars - std::vector lostConstraints; - for (auto *pdf : constraints) { - bool related = false; - for (const auto *var : vars) { - if (pdf->dependsOn(*var)) { - related = true; - } - } - if (!related) { - lostConstraints.push_back(pdf); - } - } - // return the constraints that would be "lost" when exporting the model - return lostConstraints; + HS3::setExportedObject(*tool, std::move(dist), std::move(channelConstraintNames)); + return true; } -// A normsys or histosys modifier without a recognized constraint is exported as unconstrained. That is only correct if -// its parameter is a variable, and none of the other constraint terms of the channel depends on it. -void checkUnconstrainedModifiers(const Channel &channel, const std::vector &constraints) +// The parameters of normsys and histosys modifiers must be variables. +void checkModifierParameters(const Channel &channel) { auto check = [&](auto const &modifiers) { for (auto const &sys : modifiers) { - if (sys.constraint || sys.param->isConstant()) { - continue; - } - if (!dynamic_cast(sys.param)) { + if (!sys.constraint && !sys.param->isConstant() && !dynamic_cast(sys.param)) { RooJSONFactoryWSTool::error("cannot find constraint for " + std::string(sys.param->GetName()) + ": functions are not supported as parameters of this modifier"); } - for (RooAbsPdf const *pdf : constraints) { - if (pdf->dependsOn(*sys.param)) { - RooJSONFactoryWSTool::error("cannot find constraint for " + std::string(sys.param->GetName()) + - ": the constraint term '" + pdf->GetName() + "' is not supported"); - } - } } }; for (const auto &sample : channel.samples) { @@ -1829,44 +1786,34 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname } canonicalizeModifiers(channel); - checkUnconstrainedModifiers(channel, constraints); + checkModifierParameters(channel); // stat error handling configureStatError(channel); - auto lostConstraints = findLostConstraints(channel, constraints); - // Export all the lost constraints - for (const auto *constraint : lostConstraints) { - RooJSONFactoryWSTool::warning( - "losing constraint term '" + std::string(constraint->GetName()) + - "', implicit constraints are not supported by HS3 yet! The term will appear in the HS3 file, but will not be " - "picked up when creating a likelihood from it! You will have to add it manually as an external constraint."); - tool->queueExport(*constraint); - } - - // Export all the regular modifiers - auto queueConstraints = [&](auto const &modifiers) { - for (auto &modifier : modifiers) { - if (modifier.constraint) { - tool->queueExport(*modifier.constraint); - } + // Export the constraint terms that the modifiers name. + auto queueConstraint = [&](RooAbsPdf const *constraint) { + if (isChannelConstraint(constraint, constraints)) { + tool->queueExport(*constraint); } }; for (const auto &sample : channel.samples) { - queueConstraints(sample.normfactors); - queueConstraints(sample.normsys); - queueConstraints(sample.histosys); + for (auto &modifier : sample.normfactors) { + queueConstraint(modifier.constraint); + } + for (auto &modifier : sample.normsys) { + queueConstraint(modifier.constraint); + } + for (auto &modifier : sample.histosys) { + queueConstraint(modifier.constraint); + } for (auto &modifier : sample.shapesys) { for (auto *constraint : modifier.constraintPdfs) { - if (constraint) { - tool->queueExport(*constraint); - } + queueConstraint(constraint); } } for (auto *constraint : sample.staterrorConstraints) { - if (constraint) { - tool->queueExport(*constraint); - } + queueConstraint(constraint); } } @@ -1890,7 +1837,7 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname } } - return exportChannel(tool, channel); + return exportChannel(tool, channel, constraints); } class HistFactoryStreamer_ProdPdf : public RooFit::JSONIO::Exporter { @@ -1960,7 +1907,8 @@ STATIC_EXECUTE([]() { } // namespace -void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string const &name, HistFactoryDist const &dist) +void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string const &name, HistFactoryDist const &dist, + std::vector const *channelConstraints) { checkStatErrorConstraints(dist, name); @@ -2036,6 +1984,16 @@ void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string c coefs.add(*tool.request(name + "_" + sample.name + "_scaleFactors", name)); } + if (channelConstraints) { + for (auto const &ref : *channelConstraints) { + RooAbsPdf *pdf = findNamedConstraint(tool, ref.name, name); + if (!pdf) { + RooJSONFactoryWSTool::error("unable to find the constraint '" + ref.name + "' of '" + name + "'"); + } + constraints.add(*pdf); + } + } + // A histfactory_dist is a product of Poisson distributions of the bin counts. RooFit's binned likelihood // computes exactly that (it differs from the event-wise likelihood only by a constant), and binned generation // generates bin counts, so both are enabled, as HistFactory does by default. @@ -2059,3 +2017,28 @@ bool RooFit::HS3::isBuiltinHistFactoryImporter(RooFit::JSONIO::Importer const &i { return dynamic_cast(&importer) != nullptr; } + +// The upgrade rewrites the JSON, so it has to work on a copy. +void RooFit::HS3::importHistFactoryNode(RooJSONFactoryWSTool &tool, JSONNode const &p, + std::vector const *channelConstraints) +{ + std::string name = RooJSONFactoryWSTool::name(p); + if (!p.has_child("samples")) { + RooJSONFactoryWSTool::error("no samples in '" + name + "', skipping."); + } + std::stringstream ss; + p.writeJSON(ss); + std::unique_ptr tree = RooFit::Detail::JSONTree::create(ss.str()); + + HS3::Version version; + HS3::parseVersion(tool.hs3VersionInput(), version); + + HS3::Diagnostics diagnostics; + HS3::upgradeHistFactoryDist(tree->rootnode(), version, diagnostics); + HS3::HistFactoryDist dist = HS3::parseHistFactoryDist(tree->rootnode(), diagnostics); + if (!HS3::hasErrors(diagnostics)) { + HS3::validateHistFactoryDist(dist, name, diagnostics); + } + HS3::report(diagnostics); + HS3::buildHistFactoryDist(tool, name, dist, channelConstraints); +} diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index 85aeb3b261f64..d4d9fc11955e0 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -155,11 +155,14 @@ struct RooFit::HS3::ToolExportState { JSONNode *root = nullptr; /// Set by the exporter of an object that is interpreted by the document. std::optional typedObject; + std::vector channelConstraints; }; -void RooFit::HS3::setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist dist) +void RooFit::HS3::setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist dist, + std::vector channelConstraints) { tool._export->typedObject = std::move(dist); + tool._export->channelConstraints = std::move(channelConstraints); } namespace { @@ -608,11 +611,29 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo for (auto const &term : likelihood.terms) { nllDistNames.push_back(term.distribution.name); } + // The constraint terms of HistFactory distributions are listed as auxiliary distributions, but they are part of the + // distributions in RooFit. + std::set channelConstraints; + for (auto const &item : doc.rooFit.channelConstraints) { + for (auto const &ref : item.second) { + channelConstraints.insert(ref.name); + } + } RooArgSet extConstraints; for (auto const &aux : likelihood.auxDistributions) { - if (RooAbsArg *extConstraint = workspace.arg(aux.name)) { - extConstraints.add(*extConstraint); + RooAbsArg *extConstraint = workspace.arg(aux.name); + if (!extConstraint) { + continue; } + // Only if the constraint really ended up in a distribution of the likelihood. + if (channelConstraints.count(aux.name) && + std::any_of(nllDistNames.begin(), nllDistNames.end(), [&](std::string const &distName) { + RooAbsArg const *dist = workspace.arg(distName); + return dist && dist->dependsOn(*extConstraint); + })) { + continue; + } + extConstraints.add(*extConstraint); } RooArgSet observables; for (auto const &term : likelihood.terms) { @@ -1187,6 +1208,10 @@ void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func, std::setdoc.distributions.push_back({name, std::move(*_export->typedObject)}); _export->typedObject.reset(); + if (!_export->channelConstraints.empty()) { + _export->doc.rooFit.channelConstraints[name] = std::move(_export->channelConstraints); + } + _export->channelConstraints.clear(); } else { recordGenericObject(_export->doc, isPdf, name, elem); } @@ -1326,7 +1351,15 @@ void RooJSONFactoryWSTool::importFunction(const JSONNode &p, bool importAllDepen if (it != importers.end()) { for (auto &imp : it->second) { try { - ok = imp->importArg(this, p); + // The built-in importer doesn't see the constraint terms that the document adds to the channel. + if (_import && HS3::isBuiltinHistFactoryImporter(*imp)) { + auto const &channelConstraints = _import->document.rooFit.channelConstraints; + auto found = channelConstraints.find(name); + HS3::importHistFactoryNode(*this, p, found != channelConstraints.end() ? &found->second : nullptr); + ok = true; + } else { + ok = imp->importArg(this, p); + } } catch (const std::exception &e) { std::stringstream ss; const auto *ptr = imp.get(); @@ -1397,7 +1430,10 @@ void RooJSONFactoryWSTool::importDistribution(HS3::Distribution const &distribut return; } ImportGuard importGuard{_importStack, name}; - HS3::buildHistFactoryDist(*this, name, std::get(distribution.body)); + auto const &channelConstraints = _import->document.rooFit.channelConstraints; + auto found = channelConstraints.find(name); + HS3::buildHistFactoryDist(*this, name, std::get(distribution.body), + found != channelConstraints.end() ? &found->second : nullptr); } /** @@ -1835,9 +1871,24 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc likelihood.terms.push_back({{pdf->GetName()}, 0.0}); } + auto addAux = [&](std::string const &name) { + auto &aux = likelihood.auxDistributions; + if (std::find_if(aux.begin(), aux.end(), [&](auto const &ref) { return ref.name == name; }) == aux.end()) { + aux.push_back({name}); + } + }; + // The constraint terms of HistFactory channels that no modifier names are part of the likelihood. + for (auto const &term : likelihood.terms) { + auto found = doc.rooFit.channelConstraints.find(term.distribution.name); + if (found != doc.rooFit.channelConstraints.end()) { + for (auto const &ref : found->second) { + addAux(ref.name); + } + } + } if (mc.GetExternalConstraints()) { for (const auto &constr : *mc.GetExternalConstraints()) { - likelihood.auxDistributions.push_back({constr->GetName()}); + addAux(constr->GetName()); } } doc.likelihoods.push_back(std::move(likelihood)); From a1cb2adfe28cf5b1000179fb6a9ed495d67084f6 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Thu, 5 Nov 2026 21:27:43 +0000 Subject: [PATCH 13/20] [RF][HS3] Warn about uncertainties of data that RooFit ignores The correlations of the uncertainties of binned data and the uncertainties of the entries of unbinned data were dropped silently, because RooDataHist and RooDataSet can't store them. Datasets of type "point" get a clear error. Assisted-by: claude-code:claude-opus-5-5 --- roofit/hs3/src/RooJSONFactoryWSTool.cxx | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index d4d9fc11955e0..fc0a8f1eadab6 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -572,14 +572,23 @@ loadData(HS3::Data const &input, RooWorkspace &workspace, RooFit::JSONIO::Detail { RooJSONFactoryWSTool::testValidName(input.name, true); + // What RooFit can't store is reported, instead of being dropped silently. if (auto const *binned = std::get_if(&input.body)) { + if (binned->uncertainty && !binned->uncertainty->correlation.empty()) { + RooJSONFactoryWSTool::warning("RooFitHS3: the correlations of the uncertainties of the dataset '" + + input.name + "' are ignored, because RooDataHist can't store them."); + } return HS3::createDataHist(input.name, createBinnedObservables(binned->axes), binned->contents, binned->uncertainty ? &binned->uncertainty->sigma : nullptr); } if (auto const *unbinned = std::get_if(&input.body)) { + if (unbinned->hasEntriesUncertainties) { + RooJSONFactoryWSTool::warning("RooFitHS3: the \"entries_uncertainties\" of the dataset '" + input.name + + "' are ignored, because RooDataSet can't store them."); + } return createUnbinnedData(input.name, *unbinned, workspace, domains); } - RooJSONFactoryWSTool::error("RooJSONFactoryWSTool() failed to create dataset " + input.name); + RooJSONFactoryWSTool::error("datasets of type \"point\" like '" + input.name + "' are not supported yet"); } // Import an analysis (likelihood + domains) as one or more ModelConfig objects into the workspace. From e6344532d4040ef2af23c9b10db7879d1e8f6592 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Mon, 9 Nov 2026 15:55:01 +0000 Subject: [PATCH 14/20] [RF][HS3] Reject analyses with objects of the wrong kind The document can't tell what RooFit makes of its objects. Parameters of interest that name another object, or that are constants because no domain lists them, and likelihood terms, priors, auxiliary distributions or components of combined distributions that are functions in RooFit or skipped made the import crash or were dropped silently. They are reported now. Assisted-by: claude-code:claude-opus-5-5 --- roofit/hs3/src/HS3Validate.cxx | 5 ++++- roofit/hs3/src/RooJSONFactoryWSTool.cxx | 30 ++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/roofit/hs3/src/HS3Validate.cxx b/roofit/hs3/src/HS3Validate.cxx index 007ace60c491e..4f32caf0bc685 100644 --- a/roofit/hs3/src/HS3Validate.cxx +++ b/roofit/hs3/src/HS3Validate.cxx @@ -251,7 +251,10 @@ void Validator::checkAnalyses() error(path + ".likelihood", "likelihood node not found!"); } for (auto const &poi : analysis.parametersOfInterest) { - if (!_parameters.count(poi.name) && !sectionOf(poi.name) && !isExternal(poi.name)) { + if (char const *section = sectionOf(poi.name)) { + error(path, "the parameter of interest '" + poi.name + "' of analysis '" + analysis.name + + "' is defined in \"" + section + "\", but it must be a parameter"); + } else if (!_parameters.count(poi.name) && !isExternal(poi.name)) { error(path, "the object '" + poi.name + "' in \"parameters_of_interest\" of analysis '" + analysis.name + "' does not exist"); } diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index fc0a8f1eadab6..d38dad95d00b3 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -591,6 +591,15 @@ loadData(HS3::Data const &input, RooWorkspace &workspace, RooFit::JSONIO::Detail RooJSONFactoryWSTool::error("datasets of type \"point\" like '" + input.name + "' are not supported yet"); } +// A distribution of the document can be a function in RooFit, or it can be skipped. +void requirePdf(RooAbsArg const *arg, std::string const &what) +{ + if (!dynamic_cast(arg)) { + RooJSONFactoryWSTool::error(what + " is " + + (arg ? std::string("a ") + arg->ClassName() + ", not a pdf" : "not imported")); + } +} + // Import an analysis (likelihood + domains) as one or more ModelConfig objects into the workspace. void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWorkspace &workspace, const std::vector> &datasets) @@ -615,9 +624,14 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo } HS3::Likelihood const &likelihood = *foundLikelihood; + auto checkPdf = [&](RooAbsArg const *arg, std::string const &name, std::string const &what) { + requirePdf(arg, "the " + what + " '" + name + "' of analysis '" + analysisName + "'"); + }; + std::vector nllDistNames; nllDistNames.reserve(likelihood.terms.size()); for (auto const &term : likelihood.terms) { + checkPdf(workspace.arg(term.distribution.name), term.distribution.name, "distribution"); nllDistNames.push_back(term.distribution.name); } // The constraint terms of HistFactory distributions are listed as auxiliary distributions, but they are part of the @@ -634,6 +648,7 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo if (!extConstraint) { continue; } + checkPdf(extConstraint, aux.name, "auxiliary distribution"); // Only if the constraint really ended up in a distribution of the likelihood. if (channelConstraints.count(aux.name) && std::any_of(nllDistNames.begin(), nllDistNames.end(), [&](std::string const &distName) { @@ -692,8 +707,9 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo mc->SetExternalConstraints(extConstraints); if (analysis.prior) { - if (RooAbsPdf *prior = workspace.pdf(analysis.prior->name)) { - mc->SetPriorPdf(*prior); + if (RooAbsArg *prior = workspace.arg(analysis.prior->name)) { + checkPdf(prior, analysis.prior->name, "prior"); + mc->SetPriorPdf(static_cast(*prior)); } } @@ -704,6 +720,11 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo RooJSONFactoryWSTool::error("the object '" + poi.name + "' in \"parameters_of_interest\" of analysis '" + analysisName + "' does not exist"); } + if (!dynamic_cast(arg)) { + RooJSONFactoryWSTool::error("the parameter of interest '" + poi.name + "' of analysis '" + analysisName + + "' is a " + arg->ClassName() + ", not a variable" + + (dynamic_cast(arg) ? ", because no domain lists it" : "")); + } parametersOfInterest.add(*arg); } @@ -782,7 +803,10 @@ void combinePdfs(HS3Document const &doc, RooWorkspace &ws) for (std::size_t iChannel = 0; iChannel < info.labels.size(); ++iChannel) { indexCat.defineType(info.labels[iChannel], info.indices[iChannel]); - pdfMap[info.labels[iChannel]] = ws.pdf(info.distributions[iChannel].name); + std::string const &name = info.distributions[iChannel].name; + requirePdf(ws.arg(name), + "the distribution '" + name + "' of the combined distribution '" + combinedName + "'"); + pdfMap[info.labels[iChannel]] = ws.pdf(name); } RooSimultaneous simPdf{combinedName.c_str(), combinedName.c_str(), pdfMap, indexCat}; From 835f4b53b7345625db3dfd8002ed1856bd02da40 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Tue, 10 Nov 2026 16:16:57 +0000 Subject: [PATCH 15/20] [RF][HS3] Move the Domains helper into RooJSONFactoryWSTool.cxx Only the tool uses it, and its header forward-declares it for the _domains member already. Assisted-by: claude-code:claude-opus-5-5 --- roofit/hs3/CMakeLists.txt | 2 - roofit/hs3/src/Domains.cxx | 354 ---------------------- roofit/hs3/src/Domains.h | 101 ------- roofit/hs3/src/RooJSONFactoryWSTool.cxx | 386 +++++++++++++++++++++++- 4 files changed, 385 insertions(+), 458 deletions(-) delete mode 100644 roofit/hs3/src/Domains.cxx delete mode 100644 roofit/hs3/src/Domains.h diff --git a/roofit/hs3/CMakeLists.txt b/roofit/hs3/CMakeLists.txt index ecb06ab969cb9..c8af1a0850c77 100644 --- a/roofit/hs3/CMakeLists.txt +++ b/roofit/hs3/CMakeLists.txt @@ -19,7 +19,6 @@ ROOT_STANDARD_LIBRARY_PACKAGE(RooFitHS3 HEADERS ${ROOFITHS3_HEADERS} SOURCES - src/Domains.cxx src/HS3Parser.cxx src/HS3Upgrade.cxx src/HS3Validate.cxx @@ -52,7 +51,6 @@ target_sources( FILES ${RELATIVE_INC_HEADERS} # LinkDef.h # seemingly not being used by ROOT_STANDARD_LIBRARY_PACKAGE call - src/Domains.h src/HS3Builder.h src/HS3Document.h src/HS3Schema.h diff --git a/roofit/hs3/src/Domains.cxx b/roofit/hs3/src/Domains.cxx deleted file mode 100644 index 36286a07e6215..0000000000000 --- a/roofit/hs3/src/Domains.cxx +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Project: RooFit - * Authors: - * Jonas Rembser, CERN, Jan 2023 - * - * Copyright (c) 2023, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#include "Domains.h" -#include "HS3Document.h" -#include "JSONIOUtils.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace RooFit { -namespace JSONIO { -namespace Detail { - -constexpr static auto defaultDomainName = "default_domain"; - -void Domains::populate(RooWorkspace &ws) const -{ - // Parameters with a single value are RooConstVars, whichever domain gives the value. - for (const auto &domain : _map) { - domain.second.populateFixedValues(ws); - } - auto default_domain = _map.find(defaultDomainName); - if (default_domain != _map.end()) { - default_domain->second.populate(ws); - } else { - // Without the RooFit-specific "default_domain", the variables are created with the range of the domain that - // mentions them. - for (const auto &domain : _map) { - domain.second.populate(ws); - } - } - for (const auto &domain : _map) { - if (domain.first == defaultDomainName) - continue; - domain.second.registerBinnings(domain.first.c_str(), ws); - } -} - -void Domains::readVariable(RooRealVar const &var) -{ - _map[defaultDomainName].readVariable(var.GetName(), var.getBinning()); - for (const auto &bname : var.getBinningNames()) { - if (bname.empty()) - continue; - auto &binning = var.getBinning(bname.c_str()); - _map[bname].readVariable(var.GetName(), binning); - } -} - -void Domains::writeVariable(RooRealVar &var) const -{ - auto default_domain = _map.find(defaultDomainName); - if (default_domain != _map.end()) { - default_domain->second.writeVariable(var); - return; - } - // Without the RooFit-specific "default_domain", use the first domain that mentions the variable. - for (const auto &domain : _map) { - if (domain.second.hasVariable(var.GetName())) { - domain.second.writeVariable(var); - return; - } - } -} - -void Domains::read(std::vector const &domains) -{ - // The "default_domain" is written by RooFit, but it is not required by the HS3 standard. - for (auto const &domain : domains) { - _map[domain.name].read(domain); - } - // A parameter with a single value has no other domain: every domain that lists it must give it the same value. - for (auto const &[domainName, domain] : _map) { - for (auto const &[varName, elem] : domain._map) { - if (!elem.hasFixedValue) { - continue; - } - for (auto const &[otherName, other] : _map) { - auto found = other._map.find(varName); - if (found == other._map.end()) { - continue; - } - if (!found->second.hasFixedValue) { - RooJSONFactoryWSTool::error("the parameter '" + varName + "' has the single value " + - std::to_string(elem.fixedValue) + " in the domain '" + domainName + - "', but a range in the domain '" + otherName + "'"); - } - if (found->second.fixedValue != elem.fixedValue) { - RooJSONFactoryWSTool::error("the parameter '" + varName + - "' has different single values in the " - "domains '" + - domainName + "' and '" + otherName + "'"); - } - } - } - } -} - -void Domains::readFixedValue(const char *name, double value) -{ - _map[defaultDomainName].readFixedValue(name, value); -} - -double const *Domains::fixedValue(const char *name) const -{ - for (auto const &domain : _map) { - if (double const *value = domain.second.fixedValue(name)) { - return value; - } - } - return nullptr; -} - -void Domains::write(std::vector &domains) const -{ - for (auto const &domain : _map) { - // Avoid writing a domain that was already written - auto found = std::find_if(domains.begin(), domains.end(), [&](auto const &d) { return d.name == domain.first; }); - if (found == domains.end()) { - domains.emplace_back(); - domains.back().name = domain.first; - domain.second.write(domains.back()); - } - } -} - -bool Domains::hasVariable(const char *name) const -{ - for (auto const &domain : _map) { - if (domain.second.hasVariable(name)) { - return true; - } - } - return false; -} - -void Domains::ProductDomain::readVariable(RooRealVar const &var) -{ - readVariable(var.GetName(), var.getBinning()); -} - -void Domains::ProductDomain::readBinning(ProductDomainElement &elem, RooAbsBinning const &binning) -{ - elem.hasNBins = false; - elem.nBins = 0; - elem.edges.clear(); - - const int nBins = binning.numBins(); - if (nBins <= 0) { - return; - } - - if (binning.isUniform()) { - elem.hasNBins = true; - elem.nBins = nBins; - } else { - elem.edges.push_back(binning.binLow(0)); - for (int i = 0; i < nBins; ++i) { - elem.edges.push_back(binning.binHigh(i)); - } - } -} - -void Domains::ProductDomain::readVariable(const char *name, RooAbsBinning const &binning) -{ - auto &elem = _map[name]; - - elem.hasMin = true; - elem.min = binning.lowBound(); - elem.hasMax = true; - elem.max = binning.highBound(); - readBinning(elem, binning); -} - -void Domains::ProductDomain::applyBinning(RooRealVar &var, ProductDomainElement const &elem, const char *name) -{ - if (!elem.edges.empty()) { - RooBinning binning(elem.edges.front(), elem.edges.back()); - for (double edge : elem.edges) { - binning.addBoundary(edge); - } - var.setBinning(binning, name); - } else if (elem.hasNBins && elem.nBins != 0) { - var.setBins(elem.nBins, name); - } -} - -void Domains::ProductDomain::writeVariable(RooRealVar &var) const -{ - auto found = _map.find(var.GetName()); - if (found != _map.end() && !found->second.hasFixedValue) { - auto const &elem = found->second; - if (elem.hasMin) { - if (RooNumber::isInfinite(elem.min)) { - var.removeMin(); - } else { - var.setMin(elem.min); - } - } - if (elem.hasMax) { - if (RooNumber::isInfinite(elem.max)) { - var.removeMax(); - } else { - var.setMax(elem.max); - } - } - applyBinning(var, elem); - } -} - -bool Domains::ProductDomain::hasVariable(const char *name) const -{ - return _map.find(name) != _map.end(); -} - -double const *Domains::ProductDomain::fixedValue(const char *name) const -{ - auto found = _map.find(name); - return found != _map.end() && found->second.hasFixedValue ? &found->second.fixedValue : nullptr; -} - -void Domains::ProductDomain::readFixedValue(const char *name, double value) -{ - auto &elem = _map[name]; - elem = ProductDomainElement{}; - elem.hasFixedValue = true; - elem.fixedValue = value; -} - -void Domains::ProductDomain::read(RooFit::HS3::Domain const &domain) -{ - for (auto const &axisVariant : domain.axes) { - auto const *axis = std::get_if(&axisVariant); - if (!axis) { - continue; - } - if (axis->fixedValue) { - readFixedValue(axis->name.c_str(), *axis->fixedValue); - continue; - } - auto &elem = _map[axis->name]; - - if (axis->min) { - elem.min = *axis->min; - elem.hasMin = true; - } - if (axis->max) { - elem.max = *axis->max; - elem.hasMax = true; - } - if (!axis->binning) { - continue; - } - if (auto const *edges = std::get_if(&*axis->binning)) { - elem.hasNBins = false; - elem.edges = edges->edges; - } else { - elem.nBins = std::get(*axis->binning).nbins; - elem.hasNBins = elem.nBins != 0; - elem.edges.clear(); - } - } -} -void Domains::ProductDomain::write(RooFit::HS3::Domain &domain) const -{ - for (auto const &item : _map) { - auto const &elem = item.second; - RooFit::HS3::DomainAxis axis; - axis.name = item.first; - if (elem.hasFixedValue) { - axis.fixedValue = elem.fixedValue; - domain.axes.emplace_back(std::move(axis)); - continue; - } - // An infinite bound is expressed by omitting it. - if (elem.hasMin && !RooNumber::isInfinite(elem.min)) { - axis.min = elem.min; - } - if (elem.hasMax && !RooNumber::isInfinite(elem.max)) { - axis.max = elem.max; - } - // A binning requires a bounded range. - if (axis.min && axis.max) { - if (!elem.edges.empty()) { - axis.binning = RooFit::HS3::EdgesBinning{elem.edges}; - } else if (elem.hasNBins && elem.nBins != 0) { - axis.binning = RooFit::HS3::RegularBinning{elem.nBins}; - } - } - domain.axes.emplace_back(std::move(axis)); - } -} -void Domains::ProductDomain::populate(RooWorkspace &ws) const -{ - for (auto const &item : _map) { - const auto &name = item.first; - if (item.second.hasFixedValue) { - continue; // see populateFixedValues() - } - if (!ws.arg(name)) { - const auto &elem = item.second; - const double vMin = elem.hasMin ? elem.min : -RooNumber::infinity(); - const double vMax = elem.hasMax ? elem.max : RooNumber::infinity(); - RooRealVar var{name.c_str(), name.c_str(), vMin, vMax}; - applyBinning(var, elem); - ws.import(var, RooFit::Silence()); - } - } -} -void Domains::ProductDomain::populateFixedValues(RooWorkspace &ws) const -{ - for (auto const &[name, elem] : _map) { - if (elem.hasFixedValue && !ws.arg(name)) { - ws.import(RooConstVar{name.c_str(), name.c_str(), elem.fixedValue}, RooFit::Silence()); - } - } -} - -void Domains::ProductDomain::registerBinnings(const char *name, RooWorkspace &ws) const -{ - for (auto const &item : _map) { - auto *var = ws.var(item.first); - if (!var) - continue; - const double vMin = item.second.hasMin ? item.second.min : -RooNumber::infinity(); - const double vMax = item.second.hasMax ? item.second.max : RooNumber::infinity(); - var->setRange(name, vMin, vMax); - applyBinning(*var, item.second, name); - } -} - -} // namespace Detail -} // namespace JSONIO -} // namespace RooFit diff --git a/roofit/hs3/src/Domains.h b/roofit/hs3/src/Domains.h deleted file mode 100644 index cf3b88bf96661..0000000000000 --- a/roofit/hs3/src/Domains.h +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Project: RooFit - * Authors: - * Jonas Rembser, CERN, Jan 2023 - * - * Copyright (c) 2023, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#ifndef RooFit_JSONIO_Detail_Domains_h -#define RooFit_JSONIO_Detail_Domains_h - -#include -#include -#include - -class RooAbsBinning; -class RooRealVar; -class RooWorkspace; - -namespace RooFit { -namespace Detail { -class JSONNode; -class JSONTree; -} // namespace Detail -namespace HS3 { -struct Domain; -} -} // namespace RooFit - -namespace RooFit { -namespace JSONIO { -namespace Detail { - -class Domains { -public: - void readVariable(RooRealVar const &); - void writeVariable(RooRealVar &) const; - // A parameter that only takes the value `value` (a RooConstVar), written as an axis with a single value. - void readFixedValue(const char *name, double value); - // The single value of the parameter `name` if a domain has one for it, otherwise nullptr. - double const *fixedValue(const char *name) const; - - void read(std::vector const &); - /// Append the domains that are not in the list yet. - void write(std::vector &) const; - - bool hasVariable(const char *name) const; - - void populate(RooWorkspace &ws) const; - - class ProductDomain { - friend class Domains; - - public: - void readVariable(const RooRealVar &); - void readVariable(const char *name, RooAbsBinning const &binning); - void writeVariable(RooRealVar &) const; - - void read(RooFit::HS3::Domain const &); - void write(RooFit::HS3::Domain &) const; - - bool hasVariable(const char *name) const; - double const *fixedValue(const char *name) const; - void readFixedValue(const char *name, double value); - - void populate(RooWorkspace &ws) const; - void populateFixedValues(RooWorkspace &ws) const; - void registerBinnings(const char *name, RooWorkspace &ws) const; - - private: - struct ProductDomainElement { - // An axis with a single value: the parameter only takes this value (HS3 #131). - bool hasFixedValue = false; - double fixedValue = 0.0; - bool hasMin = false; - bool hasMax = false; - double min = 0.0; - double max = 0.0; - bool hasNBins = false; - int nBins = 0; - std::vector edges; - }; - - static void applyBinning(RooRealVar &var, ProductDomainElement const &elem, const char *name = nullptr); - static void readBinning(ProductDomainElement &elem, RooAbsBinning const &binning); - - std::map _map; - }; - - std::map _map; -}; - -} // namespace Detail -} // namespace JSONIO -} // namespace RooFit - -#endif diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index d38dad95d00b3..a16ad20440f56 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -29,7 +29,6 @@ #include #include "JSONIOUtils.h" -#include "Domains.h" #include "HS3Builder.h" #include "HS3Schema.h" #include "JSONReadRecorder.h" @@ -165,6 +164,391 @@ void RooFit::HS3::setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist tool._export->channelConstraints = std::move(channelConstraints); } +namespace RooFit { +namespace JSONIO { +namespace Detail { + +class Domains { +public: + void readVariable(RooRealVar const &); + void writeVariable(RooRealVar &) const; + // A parameter that only takes the value `value` (a RooConstVar), written as an axis with a single value. + void readFixedValue(const char *name, double value); + // The single value of the parameter `name` if a domain has one for it, otherwise nullptr. + double const *fixedValue(const char *name) const; + + void read(std::vector const &); + /// Append the domains that are not in the list yet. + void write(std::vector &) const; + + bool hasVariable(const char *name) const; + + void populate(RooWorkspace &ws) const; + + class ProductDomain { + friend class Domains; + + public: + void readVariable(const RooRealVar &); + void readVariable(const char *name, RooAbsBinning const &binning); + void writeVariable(RooRealVar &) const; + + void read(RooFit::HS3::Domain const &); + void write(RooFit::HS3::Domain &) const; + + bool hasVariable(const char *name) const; + double const *fixedValue(const char *name) const; + void readFixedValue(const char *name, double value); + + void populate(RooWorkspace &ws) const; + void populateFixedValues(RooWorkspace &ws) const; + void registerBinnings(const char *name, RooWorkspace &ws) const; + + private: + struct ProductDomainElement { + // An axis with a single value: the parameter only takes this value (HS3 #131). + bool hasFixedValue = false; + double fixedValue = 0.0; + bool hasMin = false; + bool hasMax = false; + double min = 0.0; + double max = 0.0; + bool hasNBins = false; + int nBins = 0; + std::vector edges; + }; + + static void applyBinning(RooRealVar &var, ProductDomainElement const &elem, const char *name = nullptr); + static void readBinning(ProductDomainElement &elem, RooAbsBinning const &binning); + + std::map _map; + }; + + std::map _map; +}; + +constexpr static auto defaultDomainName = "default_domain"; + +void Domains::populate(RooWorkspace &ws) const +{ + // Parameters with a single value are RooConstVars, whichever domain gives the value. + for (const auto &domain : _map) { + domain.second.populateFixedValues(ws); + } + auto default_domain = _map.find(defaultDomainName); + if (default_domain != _map.end()) { + default_domain->second.populate(ws); + } else { + // Without the RooFit-specific "default_domain", the variables are created with the range of the domain that + // mentions them. + for (const auto &domain : _map) { + domain.second.populate(ws); + } + } + for (const auto &domain : _map) { + if (domain.first == defaultDomainName) + continue; + domain.second.registerBinnings(domain.first.c_str(), ws); + } +} + +void Domains::readVariable(RooRealVar const &var) +{ + _map[defaultDomainName].readVariable(var.GetName(), var.getBinning()); + for (const auto &bname : var.getBinningNames()) { + if (bname.empty()) + continue; + auto &binning = var.getBinning(bname.c_str()); + _map[bname].readVariable(var.GetName(), binning); + } +} + +void Domains::writeVariable(RooRealVar &var) const +{ + auto default_domain = _map.find(defaultDomainName); + if (default_domain != _map.end()) { + default_domain->second.writeVariable(var); + return; + } + // Without the RooFit-specific "default_domain", use the first domain that mentions the variable. + for (const auto &domain : _map) { + if (domain.second.hasVariable(var.GetName())) { + domain.second.writeVariable(var); + return; + } + } +} + +void Domains::read(std::vector const &domains) +{ + // The "default_domain" is written by RooFit, but it is not required by the HS3 standard. + for (auto const &domain : domains) { + _map[domain.name].read(domain); + } + // A parameter with a single value has no other domain: every domain that lists it must give it the same value. + for (auto const &[domainName, domain] : _map) { + for (auto const &[varName, elem] : domain._map) { + if (!elem.hasFixedValue) { + continue; + } + for (auto const &[otherName, other] : _map) { + auto found = other._map.find(varName); + if (found == other._map.end()) { + continue; + } + if (!found->second.hasFixedValue) { + RooJSONFactoryWSTool::error("the parameter '" + varName + "' has the single value " + + std::to_string(elem.fixedValue) + " in the domain '" + domainName + + "', but a range in the domain '" + otherName + "'"); + } + if (found->second.fixedValue != elem.fixedValue) { + RooJSONFactoryWSTool::error("the parameter '" + varName + + "' has different single values in the " + "domains '" + + domainName + "' and '" + otherName + "'"); + } + } + } + } +} + +void Domains::readFixedValue(const char *name, double value) +{ + _map[defaultDomainName].readFixedValue(name, value); +} + +double const *Domains::fixedValue(const char *name) const +{ + for (auto const &domain : _map) { + if (double const *value = domain.second.fixedValue(name)) { + return value; + } + } + return nullptr; +} + +void Domains::write(std::vector &domains) const +{ + for (auto const &domain : _map) { + // Avoid writing a domain that was already written + auto found = std::find_if(domains.begin(), domains.end(), [&](auto const &d) { return d.name == domain.first; }); + if (found == domains.end()) { + domains.emplace_back(); + domains.back().name = domain.first; + domain.second.write(domains.back()); + } + } +} + +bool Domains::hasVariable(const char *name) const +{ + for (auto const &domain : _map) { + if (domain.second.hasVariable(name)) { + return true; + } + } + return false; +} + +void Domains::ProductDomain::readVariable(RooRealVar const &var) +{ + readVariable(var.GetName(), var.getBinning()); +} + +void Domains::ProductDomain::readBinning(ProductDomainElement &elem, RooAbsBinning const &binning) +{ + elem.hasNBins = false; + elem.nBins = 0; + elem.edges.clear(); + + const int nBins = binning.numBins(); + if (nBins <= 0) { + return; + } + + if (binning.isUniform()) { + elem.hasNBins = true; + elem.nBins = nBins; + } else { + elem.edges.push_back(binning.binLow(0)); + for (int i = 0; i < nBins; ++i) { + elem.edges.push_back(binning.binHigh(i)); + } + } +} + +void Domains::ProductDomain::readVariable(const char *name, RooAbsBinning const &binning) +{ + auto &elem = _map[name]; + + elem.hasMin = true; + elem.min = binning.lowBound(); + elem.hasMax = true; + elem.max = binning.highBound(); + readBinning(elem, binning); +} + +void Domains::ProductDomain::applyBinning(RooRealVar &var, ProductDomainElement const &elem, const char *name) +{ + if (!elem.edges.empty()) { + RooBinning binning(elem.edges.front(), elem.edges.back()); + for (double edge : elem.edges) { + binning.addBoundary(edge); + } + var.setBinning(binning, name); + } else if (elem.hasNBins && elem.nBins != 0) { + var.setBins(elem.nBins, name); + } +} + +void Domains::ProductDomain::writeVariable(RooRealVar &var) const +{ + auto found = _map.find(var.GetName()); + if (found != _map.end() && !found->second.hasFixedValue) { + auto const &elem = found->second; + if (elem.hasMin) { + if (RooNumber::isInfinite(elem.min)) { + var.removeMin(); + } else { + var.setMin(elem.min); + } + } + if (elem.hasMax) { + if (RooNumber::isInfinite(elem.max)) { + var.removeMax(); + } else { + var.setMax(elem.max); + } + } + applyBinning(var, elem); + } +} + +bool Domains::ProductDomain::hasVariable(const char *name) const +{ + return _map.find(name) != _map.end(); +} + +double const *Domains::ProductDomain::fixedValue(const char *name) const +{ + auto found = _map.find(name); + return found != _map.end() && found->second.hasFixedValue ? &found->second.fixedValue : nullptr; +} + +void Domains::ProductDomain::readFixedValue(const char *name, double value) +{ + auto &elem = _map[name]; + elem = ProductDomainElement{}; + elem.hasFixedValue = true; + elem.fixedValue = value; +} + +void Domains::ProductDomain::read(RooFit::HS3::Domain const &domain) +{ + for (auto const &axisVariant : domain.axes) { + auto const *axis = std::get_if(&axisVariant); + if (!axis) { + continue; + } + if (axis->fixedValue) { + readFixedValue(axis->name.c_str(), *axis->fixedValue); + continue; + } + auto &elem = _map[axis->name]; + + if (axis->min) { + elem.min = *axis->min; + elem.hasMin = true; + } + if (axis->max) { + elem.max = *axis->max; + elem.hasMax = true; + } + if (!axis->binning) { + continue; + } + if (auto const *edges = std::get_if(&*axis->binning)) { + elem.hasNBins = false; + elem.edges = edges->edges; + } else { + elem.nBins = std::get(*axis->binning).nbins; + elem.hasNBins = elem.nBins != 0; + elem.edges.clear(); + } + } +} +void Domains::ProductDomain::write(RooFit::HS3::Domain &domain) const +{ + for (auto const &item : _map) { + auto const &elem = item.second; + RooFit::HS3::DomainAxis axis; + axis.name = item.first; + if (elem.hasFixedValue) { + axis.fixedValue = elem.fixedValue; + domain.axes.emplace_back(std::move(axis)); + continue; + } + // An infinite bound is expressed by omitting it. + if (elem.hasMin && !RooNumber::isInfinite(elem.min)) { + axis.min = elem.min; + } + if (elem.hasMax && !RooNumber::isInfinite(elem.max)) { + axis.max = elem.max; + } + // A binning requires a bounded range. + if (axis.min && axis.max) { + if (!elem.edges.empty()) { + axis.binning = RooFit::HS3::EdgesBinning{elem.edges}; + } else if (elem.hasNBins && elem.nBins != 0) { + axis.binning = RooFit::HS3::RegularBinning{elem.nBins}; + } + } + domain.axes.emplace_back(std::move(axis)); + } +} +void Domains::ProductDomain::populate(RooWorkspace &ws) const +{ + for (auto const &item : _map) { + const auto &name = item.first; + if (item.second.hasFixedValue) { + continue; // see populateFixedValues() + } + if (!ws.arg(name)) { + const auto &elem = item.second; + const double vMin = elem.hasMin ? elem.min : -RooNumber::infinity(); + const double vMax = elem.hasMax ? elem.max : RooNumber::infinity(); + RooRealVar var{name.c_str(), name.c_str(), vMin, vMax}; + applyBinning(var, elem); + ws.import(var, RooFit::Silence()); + } + } +} +void Domains::ProductDomain::populateFixedValues(RooWorkspace &ws) const +{ + for (auto const &[name, elem] : _map) { + if (elem.hasFixedValue && !ws.arg(name)) { + ws.import(RooConstVar{name.c_str(), name.c_str(), elem.fixedValue}, RooFit::Silence()); + } + } +} + +void Domains::ProductDomain::registerBinnings(const char *name, RooWorkspace &ws) const +{ + for (auto const &item : _map) { + auto *var = ws.var(item.first); + if (!var) + continue; + const double vMin = item.second.hasMin ? item.second.min : -RooNumber::infinity(); + const double vMax = item.second.hasMax ? item.second.max : RooNumber::infinity(); + var->setRange(name, vMin, vMax); + applyBinning(*var, item.second, name); + } +} + +} // namespace Detail +} // namespace JSONIO +} // namespace RooFit + namespace { // Keeps track of the objects that are being imported. Objects that depend on each other would otherwise recurse until From d154847c8f3c94fc0d32e2ee651ffdc98c77c4c1 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Tue, 10 Nov 2026 16:49:53 +0000 Subject: [PATCH 16/20] [RF][HS3] Replace the Domains helper with functions on the document The ranges of the variables are now looked up with HS3::DomainRanges, which doesn't depend on RooFit, and the domains are written directly as HS3::Domain. Variables are created in the order of the document. Assisted-by: claude-code:claude-opus-5-5 --- roofit/hs3/CMakeLists.txt | 1 + .../hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h | 7 - roofit/hs3/src/HS3Document.cxx | 54 ++ roofit/hs3/src/HS3Document.h | 19 + roofit/hs3/src/HS3Validate.cxx | 66 +++ roofit/hs3/src/RooJSONFactoryWSTool.cxx | 551 +++++------------- roofit/hs3/test/testRooFitHS3.cxx | 25 + 7 files changed, 302 insertions(+), 421 deletions(-) create mode 100644 roofit/hs3/src/HS3Document.cxx diff --git a/roofit/hs3/CMakeLists.txt b/roofit/hs3/CMakeLists.txt index c8af1a0850c77..e0c92c1e927d6 100644 --- a/roofit/hs3/CMakeLists.txt +++ b/roofit/hs3/CMakeLists.txt @@ -19,6 +19,7 @@ ROOT_STANDARD_LIBRARY_PACKAGE(RooFitHS3 HEADERS ${ROOFITHS3_HEADERS} SOURCES + src/HS3Document.cxx src/HS3Parser.cxx src/HS3Upgrade.cxx src/HS3Validate.cxx diff --git a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h index 7b311e8f93f17..63306209bd401 100644 --- a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h +++ b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h @@ -28,11 +28,6 @@ class RooJSONFactoryWSTool; namespace RooFit { -namespace JSONIO { -namespace Detail { -class Domains; -} -} // namespace JSONIO namespace HS3 { struct Distribution; struct HistFactoryDist; @@ -254,7 +249,6 @@ class RooJSONFactoryWSTool { void importDistribution(RooFit::HS3::Distribution const &distribution); void importVariable(RooFit::HS3::ParameterValue const &p); - void checkFixedValue(RooFit::HS3::ParameterValue const &p, std::string const &pointName); void warnIfFreeWithoutDomain(RooRealVar const &var); void importDependants(const RooFit::Detail::JSONNode &n); @@ -278,7 +272,6 @@ class RooJSONFactoryWSTool { RooWorkspace &_workspace; // objects to represent intermediate information - std::unique_ptr _domains; std::unique_ptr _import; std::unique_ptr _export; std::vector _serversToExport; diff --git a/roofit/hs3/src/HS3Document.cxx b/roofit/hs3/src/HS3Document.cxx new file mode 100644 index 0000000000000..7cb26bc402b6e --- /dev/null +++ b/roofit/hs3/src/HS3Document.cxx @@ -0,0 +1,54 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +// Operations on documents that RooFit needs besides reading, upgrading, validating and writing them. + +#include "HS3Document.h" + +#include + +namespace RooFit { +namespace HS3 { + +DomainRanges::DomainRanges(std::vector const &domains) +{ + std::vector sorted; + for (Domain const &domain : domains) { + sorted.push_back(&domain); + } + std::stable_sort(sorted.begin(), sorted.end(), [](Domain const *l, Domain const *r) { return l->name < r->name; }); + const bool hasDefault = + std::any_of(domains.begin(), domains.end(), [](Domain const &d) { return d.name == defaultDomainName; }); + + for (Domain const *domain : sorted) { + // Of several axes of a variable in one domain, the last one counts. + std::map axes; + for (auto const &axis : domain->axes) { + if (auto const *continuous = std::get_if(&axis)) { + axes[continuous->name] = continuous; + _names.insert(continuous->name); + } + } + if (!hasDefault || domain->name == defaultDomainName) { + _ranges.insert(axes.begin(), axes.end()); + } + } +} + +DomainAxis const *DomainRanges::find(std::string const &name) const +{ + auto found = _ranges.find(name); + return found != _ranges.end() ? found->second : nullptr; +} + +} // namespace HS3 +} // namespace RooFit diff --git a/roofit/hs3/src/HS3Document.h b/roofit/hs3/src/HS3Document.h index 01fa1609d3b68..3783d36d5d1cb 100644 --- a/roofit/hs3/src/HS3Document.h +++ b/roofit/hs3/src/HS3Document.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -127,6 +128,24 @@ struct Domain { std::vector> axes; }; +/// The domain in which RooFit writes the ranges of all variables. It is a convention of RooFit, not of the standard. +inline constexpr char const *defaultDomainName = "default_domain"; + +/// Where RooFit takes the ranges of variables from: the "default_domain" if there is one, otherwise the first domain by +/// name that has the variable. The domains must outlive this object. +class DomainRanges { +public: + explicit DomainRanges(std::vector const &domains); + /// The axis that defines the range of a variable, or null. + DomainAxis const *find(std::string const &name) const; + /// Whether any domain has the variable. + bool contains(std::string const &name) const { return _names.count(name); } + +private: + std::map _ranges; + std::set _names; +}; + struct ParameterValue { std::string name; std::optional> value; ///< A string selects the state of a category. diff --git a/roofit/hs3/src/HS3Validate.cxx b/roofit/hs3/src/HS3Validate.cxx index 4f32caf0bc685..6683608c635e6 100644 --- a/roofit/hs3/src/HS3Validate.cxx +++ b/roofit/hs3/src/HS3Validate.cxx @@ -13,6 +13,7 @@ #include "HS3Document.h" #include +#include #include #include #include @@ -39,6 +40,7 @@ class Validator { void run() { collectNames(); + checkFixedAxes(); checkLikelihoods(); checkAnalyses(); checkData(); @@ -122,6 +124,7 @@ class Validator { std::optional numBins(Axes const &axes, std::string const &path); void collectNames(); + void checkFixedAxes(); void checkLikelihoods(); void checkAnalyses(); void checkData(); @@ -221,6 +224,69 @@ void Validator::collectNames() } } +// A fixed axis says that the parameter only takes a single value. No other domain can give it a range or another +// value, and no parameter point another value. +void Validator::checkFixedAxes() +{ + struct FixedAxis { + double value = 0.0; + std::string const *domain = nullptr; + }; + std::map fixedAxes; + auto forEachAxis = [&](auto &&fn) { + for (std::size_t i = 0; i < _doc.domains.size(); ++i) { + for (auto const &axis : _doc.domains[i].axes) { + // A categorical axis is passed as null. + fn(i, std::visit([](auto const &a) -> std::string const & { return a.name; }, axis), + std::get_if(&axis)); + } + } + }; + forEachAxis([&](std::size_t i, std::string const &, DomainAxis const *continuous) { + if (!continuous || !continuous->fixedValue) { + return; + } + DomainAxis const &axis = *continuous; + Domain const &domain = _doc.domains[i]; + if (!std::isfinite(*axis.fixedValue)) { + error(at("domains", i, domain.name), + "the single value of the parameter '" + axis.name + "' is not finite, which JSON can't represent"); + } + auto inserted = fixedAxes.insert({axis.name, FixedAxis{*axis.fixedValue, &domain.name}}); + if (!inserted.second && inserted.first->second.value != *axis.fixedValue) { + error(at("domains", i, domain.name), + "the parameter '" + axis.name + "' has different single values in the domains '" + + *inserted.first->second.domain + "' and '" + domain.name + "'"); + } + }); + if (fixedAxes.empty()) { + return; + } + forEachAxis([&](std::size_t i, std::string const &name, DomainAxis const *continuous) { + auto found = fixedAxes.find(name); + if (found != fixedAxes.end() && !(continuous && continuous->fixedValue)) { + Domain const &domain = _doc.domains[i]; + error(at("domains", i, domain.name), + "the parameter '" + name + "' has the single value " + std::to_string(found->second.value) + + " in the domain '" + *found->second.domain + "', but " + (continuous ? "a range" : "categories") + + " in the domain '" + domain.name + "'"); + } + }); + for (std::size_t i = 0; i < _doc.parameterPoints.size(); ++i) { + ParameterPoint const &point = _doc.parameterPoints[i]; + for (ParameterValue const ¶meter : point.parameters) { + auto found = fixedAxes.find(parameter.name); + double const *value = parameter.value ? std::get_if(&*parameter.value) : nullptr; + if (value && found != fixedAxes.end() && *value != found->second.value) { + error(at("parameter_points", i, point.name), + "the parameter point '" + point.name + "' gives the parameter '" + parameter.name + "' the value " + + std::to_string(*value) + ", but its domain only has the value " + + std::to_string(found->second.value)); + } + } + } +} + void Validator::checkLikelihoods() { for (std::size_t i = 0; i < _doc.likelihoods.size(); ++i) { diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index a16ad20440f56..06d2b44f84210 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -146,12 +146,15 @@ struct RooFit::HS3::ToolImportState { std::unordered_map distributions; HS3::ParameterPoint const *defaultValues = nullptr; std::unordered_map defaultValuesByName; + HS3::DomainRanges ranges{document.domains}; }; /// The document that the tool is exporting. struct RooFit::HS3::ToolExportState { HS3Document doc; JSONNode *root = nullptr; + /// The ranges and binnings of the exported variables, by domain and variable name. + std::map> domains; /// Set by the exporter of an object that is interpreted by the document. std::optional typedObject; std::vector channelConstraints; @@ -164,390 +167,139 @@ void RooFit::HS3::setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist tool._export->channelConstraints = std::move(channelConstraints); } -namespace RooFit { -namespace JSONIO { -namespace Detail { - -class Domains { -public: - void readVariable(RooRealVar const &); - void writeVariable(RooRealVar &) const; - // A parameter that only takes the value `value` (a RooConstVar), written as an axis with a single value. - void readFixedValue(const char *name, double value); - // The single value of the parameter `name` if a domain has one for it, otherwise nullptr. - double const *fixedValue(const char *name) const; - - void read(std::vector const &); - /// Append the domains that are not in the list yet. - void write(std::vector &) const; - - bool hasVariable(const char *name) const; - - void populate(RooWorkspace &ws) const; - - class ProductDomain { - friend class Domains; - - public: - void readVariable(const RooRealVar &); - void readVariable(const char *name, RooAbsBinning const &binning); - void writeVariable(RooRealVar &) const; - - void read(RooFit::HS3::Domain const &); - void write(RooFit::HS3::Domain &) const; - - bool hasVariable(const char *name) const; - double const *fixedValue(const char *name) const; - void readFixedValue(const char *name, double value); - - void populate(RooWorkspace &ws) const; - void populateFixedValues(RooWorkspace &ws) const; - void registerBinnings(const char *name, RooWorkspace &ws) const; - - private: - struct ProductDomainElement { - // An axis with a single value: the parameter only takes this value (HS3 #131). - bool hasFixedValue = false; - double fixedValue = 0.0; - bool hasMin = false; - bool hasMax = false; - double min = 0.0; - double max = 0.0; - bool hasNBins = false; - int nBins = 0; - std::vector edges; - }; - - static void applyBinning(RooRealVar &var, ProductDomainElement const &elem, const char *name = nullptr); - static void readBinning(ProductDomainElement &elem, RooAbsBinning const &binning); - - std::map _map; - }; - - std::map _map; -}; - -constexpr static auto defaultDomainName = "default_domain"; +namespace { -void Domains::populate(RooWorkspace &ws) const +HS3::Binning toBinning(RooAbsBinning const &binning) { - // Parameters with a single value are RooConstVars, whichever domain gives the value. - for (const auto &domain : _map) { - domain.second.populateFixedValues(ws); - } - auto default_domain = _map.find(defaultDomainName); - if (default_domain != _map.end()) { - default_domain->second.populate(ws); - } else { - // Without the RooFit-specific "default_domain", the variables are created with the range of the domain that - // mentions them. - for (const auto &domain : _map) { - domain.second.populate(ws); - } + if (binning.isUniform()) { + return HS3::RegularBinning{binning.numBins()}; } - for (const auto &domain : _map) { - if (domain.first == defaultDomainName) - continue; - domain.second.registerBinnings(domain.first.c_str(), ws); + HS3::EdgesBinning edges; + edges.edges.push_back(binning.binLow(0)); + for (int i = 0; i < binning.numBins(); ++i) { + edges.edges.push_back(binning.binHigh(i)); } + return edges; } -void Domains::readVariable(RooRealVar const &var) +// The axis of a domain for a range and binning. Infinite bounds are omitted, and only a bounded range has a binning. +HS3::DomainAxis toDomainAxis(std::string const &name, RooAbsBinning const &binning) { - _map[defaultDomainName].readVariable(var.GetName(), var.getBinning()); - for (const auto &bname : var.getBinningNames()) { - if (bname.empty()) - continue; - auto &binning = var.getBinning(bname.c_str()); - _map[bname].readVariable(var.GetName(), binning); + HS3::DomainAxis axis; + axis.name = name; + if (!RooNumber::isInfinite(binning.lowBound())) { + axis.min = binning.lowBound(); } -} - -void Domains::writeVariable(RooRealVar &var) const -{ - auto default_domain = _map.find(defaultDomainName); - if (default_domain != _map.end()) { - default_domain->second.writeVariable(var); - return; + if (!RooNumber::isInfinite(binning.highBound())) { + axis.max = binning.highBound(); } - // Without the RooFit-specific "default_domain", use the first domain that mentions the variable. - for (const auto &domain : _map) { - if (domain.second.hasVariable(var.GetName())) { - domain.second.writeVariable(var); - return; - } + if (axis.min && axis.max && binning.numBins() > 0) { + axis.binning = toBinning(binning); } + return axis; } -void Domains::read(std::vector const &domains) +// Record the range and binning of a variable in the "default_domain", and its named ranges in the domains of their +// names. +void recordDomains(std::map> &domains, RooRealVar const &var) { - // The "default_domain" is written by RooFit, but it is not required by the HS3 standard. - for (auto const &domain : domains) { - _map[domain.name].read(domain); - } - // A parameter with a single value has no other domain: every domain that lists it must give it the same value. - for (auto const &[domainName, domain] : _map) { - for (auto const &[varName, elem] : domain._map) { - if (!elem.hasFixedValue) { - continue; - } - for (auto const &[otherName, other] : _map) { - auto found = other._map.find(varName); - if (found == other._map.end()) { - continue; - } - if (!found->second.hasFixedValue) { - RooJSONFactoryWSTool::error("the parameter '" + varName + "' has the single value " + - std::to_string(elem.fixedValue) + " in the domain '" + domainName + - "', but a range in the domain '" + otherName + "'"); - } - if (found->second.fixedValue != elem.fixedValue) { - RooJSONFactoryWSTool::error("the parameter '" + varName + - "' has different single values in the " - "domains '" + - domainName + "' and '" + otherName + "'"); - } - } + domains[HS3::defaultDomainName][var.GetName()] = toDomainAxis(var.GetName(), var.getBinning()); + for (auto const &name : var.getBinningNames()) { + if (!name.empty()) { + domains[name][var.GetName()] = toDomainAxis(var.GetName(), var.getBinning(name.c_str())); } } } -void Domains::readFixedValue(const char *name, double value) -{ - _map[defaultDomainName].readFixedValue(name, value); -} - -double const *Domains::fixedValue(const char *name) const +void appendDomain(std::vector &domains, std::string const &name, + std::map axes) { - for (auto const &domain : _map) { - if (double const *value = domain.second.fixedValue(name)) { - return value; - } + HS3::Domain &domain = domains.emplace_back(); + domain.name = name; + for (auto &item : axes) { + domain.axes.emplace_back(std::move(item.second)); } - return nullptr; } -void Domains::write(std::vector &domains) const +// Set the binning of a variable, or the binning of one of its named ranges. +void applyBinning(RooRealVar &var, HS3::Binning const &binning, const char *name = nullptr) { - for (auto const &domain : _map) { - // Avoid writing a domain that was already written - auto found = std::find_if(domains.begin(), domains.end(), [&](auto const &d) { return d.name == domain.first; }); - if (found == domains.end()) { - domains.emplace_back(); - domains.back().name = domain.first; - domain.second.write(domains.back()); + if (auto const *edges = std::get_if(&binning)) { + if (edges->edges.empty()) { + return; } - } -} - -bool Domains::hasVariable(const char *name) const -{ - for (auto const &domain : _map) { - if (domain.second.hasVariable(name)) { - return true; + RooBinning bins(edges->edges.front(), edges->edges.back()); + for (double edge : edges->edges) { + bins.addBoundary(edge); } + var.setBinning(bins, name); + } else if (int nbins = std::get(binning).nbins) { + var.setBins(nbins, name); } - return false; } -void Domains::ProductDomain::readVariable(RooRealVar const &var) +void applyRange(RooRealVar &var, HS3::DomainAxis const &axis) { - readVariable(var.GetName(), var.getBinning()); -} - -void Domains::ProductDomain::readBinning(ProductDomainElement &elem, RooAbsBinning const &binning) -{ - elem.hasNBins = false; - elem.nBins = 0; - elem.edges.clear(); - - const int nBins = binning.numBins(); - if (nBins <= 0) { - return; - } - - if (binning.isUniform()) { - elem.hasNBins = true; - elem.nBins = nBins; - } else { - elem.edges.push_back(binning.binLow(0)); - for (int i = 0; i < nBins; ++i) { - elem.edges.push_back(binning.binHigh(i)); + if (axis.min) { + if (RooNumber::isInfinite(*axis.min)) { + var.removeMin(); + } else { + var.setMin(*axis.min); } } -} - -void Domains::ProductDomain::readVariable(const char *name, RooAbsBinning const &binning) -{ - auto &elem = _map[name]; - - elem.hasMin = true; - elem.min = binning.lowBound(); - elem.hasMax = true; - elem.max = binning.highBound(); - readBinning(elem, binning); -} - -void Domains::ProductDomain::applyBinning(RooRealVar &var, ProductDomainElement const &elem, const char *name) -{ - if (!elem.edges.empty()) { - RooBinning binning(elem.edges.front(), elem.edges.back()); - for (double edge : elem.edges) { - binning.addBoundary(edge); + if (axis.max) { + if (RooNumber::isInfinite(*axis.max)) { + var.removeMax(); + } else { + var.setMax(*axis.max); } - var.setBinning(binning, name); - } else if (elem.hasNBins && elem.nBins != 0) { - var.setBins(elem.nBins, name); } -} - -void Domains::ProductDomain::writeVariable(RooRealVar &var) const -{ - auto found = _map.find(var.GetName()); - if (found != _map.end() && !found->second.hasFixedValue) { - auto const &elem = found->second; - if (elem.hasMin) { - if (RooNumber::isInfinite(elem.min)) { - var.removeMin(); - } else { - var.setMin(elem.min); - } - } - if (elem.hasMax) { - if (RooNumber::isInfinite(elem.max)) { - var.removeMax(); - } else { - var.setMax(elem.max); - } - } - applyBinning(var, elem); + if (axis.binning) { + applyBinning(var, *axis.binning); } } -bool Domains::ProductDomain::hasVariable(const char *name) const +// Create the variables of the domains that don't exist yet, and the domains other than the "default_domain" as named +// ranges. +void createVariables(RooWorkspace &ws, std::vector const &domains, HS3::DomainRanges const &ranges) { - return _map.find(name) != _map.end(); -} - -double const *Domains::ProductDomain::fixedValue(const char *name) const -{ - auto found = _map.find(name); - return found != _map.end() && found->second.hasFixedValue ? &found->second.fixedValue : nullptr; -} - -void Domains::ProductDomain::readFixedValue(const char *name, double value) -{ - auto &elem = _map[name]; - elem = ProductDomainElement{}; - elem.hasFixedValue = true; - elem.fixedValue = value; -} - -void Domains::ProductDomain::read(RooFit::HS3::Domain const &domain) -{ - for (auto const &axisVariant : domain.axes) { - auto const *axis = std::get_if(&axisVariant); - if (!axis) { - continue; - } - if (axis->fixedValue) { - readFixedValue(axis->name.c_str(), *axis->fixedValue); - continue; - } - auto &elem = _map[axis->name]; - - if (axis->min) { - elem.min = *axis->min; - elem.hasMin = true; - } - if (axis->max) { - elem.max = *axis->max; - elem.hasMax = true; - } - if (!axis->binning) { - continue; - } - if (auto const *edges = std::get_if(&*axis->binning)) { - elem.hasNBins = false; - elem.edges = edges->edges; - } else { - elem.nBins = std::get(*axis->binning).nbins; - elem.hasNBins = elem.nBins != 0; - elem.edges.clear(); - } - } -} -void Domains::ProductDomain::write(RooFit::HS3::Domain &domain) const -{ - for (auto const &item : _map) { - auto const &elem = item.second; - RooFit::HS3::DomainAxis axis; - axis.name = item.first; - if (elem.hasFixedValue) { - axis.fixedValue = elem.fixedValue; - domain.axes.emplace_back(std::move(axis)); - continue; - } - // An infinite bound is expressed by omitting it. - if (elem.hasMin && !RooNumber::isInfinite(elem.min)) { - axis.min = elem.min; - } - if (elem.hasMax && !RooNumber::isInfinite(elem.max)) { - axis.max = elem.max; - } - // A binning requires a bounded range. - if (axis.min && axis.max) { - if (!elem.edges.empty()) { - axis.binning = RooFit::HS3::EdgesBinning{elem.edges}; - } else if (elem.hasNBins && elem.nBins != 0) { - axis.binning = RooFit::HS3::RegularBinning{elem.nBins}; + auto forEachAxis = [&](auto &&fn) { + for (HS3::Domain const &domain : domains) { + for (auto const &axis : domain.axes) { + if (auto const *continuous = std::get_if(&axis)) { + fn(domain.name, *continuous); + } } } - domain.axes.emplace_back(std::move(axis)); - } -} -void Domains::ProductDomain::populate(RooWorkspace &ws) const -{ - for (auto const &item : _map) { - const auto &name = item.first; - if (item.second.hasFixedValue) { - continue; // see populateFixedValues() - } - if (!ws.arg(name)) { - const auto &elem = item.second; - const double vMin = elem.hasMin ? elem.min : -RooNumber::infinity(); - const double vMax = elem.hasMax ? elem.max : RooNumber::infinity(); - RooRealVar var{name.c_str(), name.c_str(), vMin, vMax}; - applyBinning(var, elem); + }; + const double inf = RooNumber::infinity(); + // Parameters with a single value are RooConstVars, whichever domain gives the value. + forEachAxis([&](std::string const &, HS3::DomainAxis const &axis) { + if (axis.fixedValue && !ws.arg(axis.name)) { + ws.import(RooConstVar{axis.name.c_str(), axis.name.c_str(), *axis.fixedValue}, RooFit::Silence()); + } + }); + forEachAxis([&](std::string const &, HS3::DomainAxis const &axis) { + if (!axis.fixedValue && ranges.find(axis.name) == &axis && !ws.arg(axis.name)) { + RooRealVar var{axis.name.c_str(), axis.name.c_str(), axis.min.value_or(-inf), axis.max.value_or(inf)}; + if (axis.binning) { + applyBinning(var, *axis.binning); + } ws.import(var, RooFit::Silence()); } - } -} -void Domains::ProductDomain::populateFixedValues(RooWorkspace &ws) const -{ - for (auto const &[name, elem] : _map) { - if (elem.hasFixedValue && !ws.arg(name)) { - ws.import(RooConstVar{name.c_str(), name.c_str(), elem.fixedValue}, RooFit::Silence()); + }); + forEachAxis([&](std::string const &domain, HS3::DomainAxis const &axis) { + RooRealVar *var = ws.var(axis.name); + if (var && domain != HS3::defaultDomainName) { + var->setRange(domain.c_str(), axis.min.value_or(-inf), axis.max.value_or(inf)); + if (axis.binning) { + applyBinning(*var, *axis.binning, domain.c_str()); + } } - } + }); } -void Domains::ProductDomain::registerBinnings(const char *name, RooWorkspace &ws) const -{ - for (auto const &item : _map) { - auto *var = ws.var(item.first); - if (!var) - continue; - const double vMin = item.second.hasMin ? item.second.min : -RooNumber::infinity(); - const double vMax = item.second.hasMax ? item.second.max : RooNumber::infinity(); - var->setRange(name, vMin, vMax); - applyBinning(*var, item.second, name); - } -} - -} // namespace Detail -} // namespace JSONIO -} // namespace RooFit +} // namespace namespace { @@ -611,14 +363,16 @@ bool isNumber(const std::string &str) } // Configure `v` (value, error, binning, constness) from the parameter `p`. -void configureVariable(RooFit::JSONIO::Detail::Domains &domains, HS3::ParameterValue const &p, RooRealVar &v) +void configureVariable(HS3::DomainRanges const &ranges, HS3::ParameterValue const &p, RooRealVar &v) { if (p.value) { if (auto const *value = std::get_if(&*p.value)) { v.setVal(*value); } } - domains.writeVariable(v); + if (HS3::DomainAxis const *range = ranges.find(v.GetName())) { + applyRange(v, *range); + } if (p.nbins) v.setBins(*p.nbins); if (p.relError) @@ -859,16 +613,7 @@ HS3::BinnedAxis RooFit::HS3::toBinnedAxis(RooRealVar const &var) axis.name = var.GetName(); axis.min = binning.lowBound(); axis.max = binning.highBound(); - if (binning.isUniform()) { - axis.binning = HS3::RegularBinning{binning.numBins()}; - } else { - HS3::EdgesBinning edges; - edges.edges.push_back(binning.binLow(0)); - for (int i = 0; i < binning.numBins(); ++i) { - edges.edges.push_back(binning.binHigh(i)); - } - axis.binning = std::move(edges); - } + axis.binning = toBinning(binning); return axis; } @@ -877,15 +622,7 @@ namespace { std::unique_ptr createObservable(HS3::BinnedAxis const &axis) { auto obs = std::make_unique(axis.name.c_str(), axis.name.c_str(), axis.min, axis.max); - if (auto const *edges = std::get_if(&axis.binning)) { - RooBinning bins(obs->getMin(), obs->getMax()); - for (double edge : edges->edges) { - bins.addBoundary(edge); - } - obs->setBinning(bins); - } else { - obs->setBins(std::get(axis.binning).nbins); - } + applyBinning(*obs, axis.binning); return obs; } @@ -904,14 +641,14 @@ RooArgSet createBinnedObservables(std::vectorhasVariable(axis.name.c_str())) { + if (ranges && !ranges->contains(axis.name)) { var.setRange(*axis.min, *axis.max); } else if (var.getMin() != *axis.min || var.getMax() != *axis.max) { std::stringstream ss; @@ -923,7 +660,7 @@ void applyUnbinnedAxisRange(HS3::ContinuousAxis const &axis, std::string const & } std::unique_ptr createUnbinnedData(std::string const &name, HS3::UnbinnedData const &input, - RooWorkspace &workspace, RooFit::JSONIO::Detail::Domains const *domains) + RooWorkspace &workspace, HS3::DomainRanges const *ranges) { RooArgList varlist; for (auto const &axisVariant : input.axes) { @@ -936,7 +673,7 @@ std::unique_ptr createUnbinnedData(std::string const &name, HS3::Unb RooJSONFactoryWSTool::error("The observable \"" + axis->name + "\" could not be found in the workspace!"); } varlist.add(*var); - applyUnbinnedAxisRange(*axis, name, *var, domains); + applyUnbinnedAxisRange(*axis, name, *var, ranges); } RooArgSet vars(varlist); auto data = std::make_unique(name, name, vars, RooFit::WeightVar()); @@ -952,7 +689,7 @@ std::unique_ptr createUnbinnedData(std::string const &name, HS3::Unb // Create a RooAbsData (binned or unbinned) from its description. std::unique_ptr -loadData(HS3::Data const &input, RooWorkspace &workspace, RooFit::JSONIO::Detail::Domains const *domains) +loadData(HS3::Data const &input, RooWorkspace &workspace, HS3::DomainRanges const *ranges) { RooJSONFactoryWSTool::testValidName(input.name, true); @@ -970,7 +707,7 @@ loadData(HS3::Data const &input, RooWorkspace &workspace, RooFit::JSONIO::Detail RooJSONFactoryWSTool::warning("RooFitHS3: the \"entries_uncertainties\" of the dataset '" + input.name + "' are ignored, because RooDataSet can't store them."); } - return createUnbinnedData(input.name, *unbinned, workspace, domains); + return createUnbinnedData(input.name, *unbinned, workspace, ranges); } RooJSONFactoryWSTool::error("datasets of type \"point\" like '" + input.name + "' are not supported yet"); } @@ -1487,7 +1224,10 @@ void RooJSONFactoryWSTool::exportVariable(const RooAbsArg *v, HS3::ParameterPoin // A named RooConstVar only takes its value: it is written as a domain axis with that single value, not in the // parameter points. if (cv) { - _domains->readFixedValue(cv->GetName(), cv->getVal()); + HS3::DomainAxis axis; + axis.name = cv->GetName(); + axis.fixedValue = cv->getVal(); + _export->domains[HS3::defaultDomainName][axis.name] = std::move(axis); return; } @@ -1495,7 +1235,7 @@ void RooJSONFactoryWSTool::exportVariable(const RooAbsArg *v, HS3::ParameterPoin var.name = v->GetName(); var.value = rrv->getVal(); var.isConst = storeConstant && (rrv->isConstant() || rrv->getMin() >= rrv->getMax()); - _domains->readVariable(*rrv); + recordDomains(_export->domains, *rrv); point.parameters.push_back(std::move(var)); } @@ -2025,7 +1765,7 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) auto exportBinned = [&](std::size_t n, double const *contents) { HS3::BinnedData binned; for (auto *var : static_range_cast(variables)) { - _domains->readVariable(*var); + recordDomains(_export->domains, *var); binned.axes.emplace_back(HS3::toBinnedAxis(*var)); } binned.contents.assign(contents, contents + n); @@ -2065,7 +1805,7 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) // this really is an unbinned dataset HS3::UnbinnedData unbinned; for (auto *var : static_range_cast(variables)) { - _domains->readVariable(*var); + recordDomains(_export->domains, *var); // HS3 forbids binning information on the axes of unbinned data. testValidName(var->GetName(), false); unbinned.axes.emplace_back(HS3::ContinuousAxis{var->GetName(), var->getMin(), var->getMax()}); @@ -2166,13 +1906,12 @@ void RooJSONFactoryWSTool::importVariable(HS3::ParameterValue const &p) // import a RooRealVar object RooJSONFactoryWSTool::testValidName(p.name, true); - checkFixedValue(p, "default_values"); if (_workspace.arg(p.name)) return; // A string selects the state of a category, which is done together with the other parameter points. if (p.value && std::holds_alternative(*p.value)) return; - if (!_domains->hasVariable(p.name.c_str())) { + if (!_import->ranges.contains(p.name)) { // ROOT versions before HS3 0.3 marked named RooConstVars with an attribute. auto const &attributes = _import->document.rooFit.attributes; auto found = attributes.find(p.name); @@ -2185,26 +1924,14 @@ void RooJSONFactoryWSTool::importVariable(HS3::ParameterValue const &p) } } RooRealVar &var = wsEmplace(p.name, 1.); - configureVariable(*_domains, p, var); + configureVariable(_import->ranges, p, var); warnIfFreeWithoutDomain(var); } -// A parameter with a single value in its domain only takes that value: a parameter point must not give it another. -void RooJSONFactoryWSTool::checkFixedValue(HS3::ParameterValue const &p, std::string const &pointName) -{ - double const *fixed = _domains ? _domains->fixedValue(p.name.c_str()) : nullptr; - double const *value = p.value ? std::get_if(&*p.value) : nullptr; - if (fixed && value && *value != *fixed) { - RooJSONFactoryWSTool::error("the parameter point '" + pointName + "' gives the parameter '" + p.name + - "' the value " + std::to_string(*value) + ", but its domain only has the value " + - std::to_string(*fixed)); - } -} - // HS3 requires every parameter to be listed in a domain or constant ("const": true) in the parameter point. void RooJSONFactoryWSTool::warnIfFreeWithoutDomain(RooRealVar const &var) { - if (!var.isConstant() && _domains && !_domains->hasVariable(var.GetName())) { + if (!var.isConstant() && !_import->ranges.contains(var.GetName())) { warning(std::string("RooFitHS3: the parameter '") + var.GetName() + "' is not listed in any domain and is not constant: HS3 requires a domain, or \"const\": true. It " "is read as a free parameter without range."); @@ -2326,13 +2053,11 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc return; const std::string domainName = analysisName + suffix; analysis.domains.push_back({domainName}); - RooFit::JSONIO::Detail::Domains::ProductDomain domain; + std::map axes; for (auto *var : static_range_cast(*args)) { - domain.readVariable(*var); + axes[var->GetName()] = toDomainAxis(var->GetName(), var->getBinning()); } - doc.domains.emplace_back(); - doc.domains.back().name = domainName; - domain.write(doc.domains.back()); + appendDomain(doc.domains, domainName, std::move(axes)); }; writeProductDomain("_nuisance_parameters", mc.GetNuisanceParameters()); @@ -2357,17 +2082,12 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc // Export all top-level pdfs, functions, datasets and ModelConfigs of the workspace into `n`. void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) { - _domains = std::make_unique(); _export = std::make_unique(); _export->root = &n; // The state of the export must not survive it, not even if there is an exception. struct ExportScope { RooJSONFactoryWSTool &tool; - ~ExportScope() - { - tool._export.reset(); - tool._domains.reset(); - } + ~ExportScope() { tool._export.reset(); } } exportScope{*this}; HS3Document &doc = _export->doc; @@ -2463,7 +2183,13 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) this->exportVariables(snapshotSorted, doc.parameterPoints.back(), true); } } - _domains->write(doc.domains); + // The domains of the analyses take precedence. + for (auto &item : _export->domains) { + if (std::none_of(doc.domains.begin(), doc.domains.end(), + [&](HS3::Domain const &domain) { return domain.name == item.first; })) { + appendDomain(doc.domains, item.first, std::move(item.second)); + } + } // The document that RooFit writes has to pass the same checks as the ones that it reads. Until RooFit writes // documents that follow the current standard, problems are only reported. @@ -2621,7 +2347,6 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) ~ImportScope() { tool._import.reset(); - tool._domains.reset(); tool._hs3VersionInput.clear(); } } importScope{*this}; @@ -2630,9 +2355,7 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) _import = std::make_unique(doc); - _domains = std::make_unique(); - _domains->read(doc.domains); - _domains->populate(_workspace); + createVariables(_workspace, _import->document.domains, _import->ranges); if (_import->defaultValues) { for (auto const &p : _import->defaultValues->parameters) { @@ -2669,17 +2392,16 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) point.name + "', because there is no category with that name."); continue; } - checkFixedValue(var, point.name); RooRealVar *rrv = _workspace.var(var.name); if (!rrv && !_workspace.arg(var.name)) { // The parameter is not used by any other component: create it. "const": false is the same as no // "const" at all. testValidName(var.name, true); rrv = &wsEmplace(var.name, 1.); - configureVariable(*_domains, var, *rrv); + configureVariable(_import->ranges, var, *rrv); warnIfFreeWithoutDomain(*rrv); } else if (rrv) { - configureVariable(*_domains, var, *rrv); + configureVariable(_import->ranges, var, *rrv); } if (rrv) { vars.add(*rrv); @@ -2700,7 +2422,7 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) // don't get imported. std::vector> datasets; for (auto const &data : doc.data) { - datasets.push_back(loadData(data, _workspace, _domains.get())); + datasets.push_back(loadData(data, _workspace, &_import->ranges)); } // Now, read in analyses and likelihoods if there are any @@ -2814,16 +2536,17 @@ void RooJSONFactoryWSTool::importVariableElement(const JSONNode &elementNode) HS3Document doc = HS3::parseDocument(tree->rootnode(), diagnostics, options); HS3::report(diagnostics); - _domains = std::make_unique(); - _domains->read(doc.domains); + _import = std::make_unique(doc); + struct ImportScope { + RooJSONFactoryWSTool &tool; + ~ImportScope() { tool._import.reset(); } + } importScope{*this}; HS3::ParameterValue const ¶meter = doc.parameterPoints.front().parameters.front(); importVariable(parameter); if (RooRealVar *rrv = _workspace.var(parameter.name)) { - configureVariable(*_domains, parameter, *rrv); + configureVariable(_import->ranges, parameter, *rrv); } - - _domains.reset(); } /** diff --git a/roofit/hs3/test/testRooFitHS3.cxx b/roofit/hs3/test/testRooFitHS3.cxx index 212d3a7c597e8..15648932e00ee 100644 --- a/roofit/hs3/test/testRooFitHS3.cxx +++ b/roofit/hs3/test/testRooFitHS3.cxx @@ -49,6 +49,7 @@ #include #include +#include #include #include #include @@ -5023,6 +5024,26 @@ TEST(RooFitHS3, InvalidExportIsStillWritten) EXPECT_NE(RooJSONFactoryWSTool::findNamedChild(root["distributions"], "gauss"), nullptr) << json; } +// A named RooConstVar is exported as a fixed domain axis. JSON can't represent a value that is not finite, which is +// reported on export, and such an axis is not read back as a parameter without range. +TEST(RooFitHS3, NonFiniteRooConstVar) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + + RooWorkspace ws{"ws"}; + RooRealVar x{"x", "x", 0.0, -10.0, 10.0}; + RooConstVar cinf{"cinf", "cinf", std::numeric_limits::infinity()}; + RooFormulaVar formula{"formula", "x[0] + x[1]", {x, cinf}}; + ws.import(formula, RooFit::Silence()); + + std::string json; + const std::string warnings = captureMessages(RooFit::WARNING, RooFit::ObjectHandling, + [&] { json = RooJSONFactoryWSTool{ws}.exportJSONtoString(); }); + EXPECT_NE(warnings.find("the single value of the parameter 'cinf' is not finite"), std::string::npos) << warnings; + + expectImportThrowsWithError(json, "must be a number"); +} + // A fixed domain axis (HS3 #131): the parameter only takes the given "value", and RooFit reads it as a RooConstVar. // It needs no entry in a parameter point. A parameter point must not give it another value, the axis has no range, // binning or categories, and all domains that list the parameter give it the same value. @@ -5096,6 +5117,10 @@ TEST(RooFitHS3, FixedDomainAxis) document(R"({"name": "mZ", "value": 91.188})", R"(, {"name": "other", "type": "product_domain", "axes": [{"name": "mZ", "min": 80.0, "max": 100.0}]})"), "but a range in the domain 'other'"); + expectImportThrowsWithError( + document(R"({"name": "mZ", "value": 91.188})", + R"(, {"name": "other", "type": "product_domain", "axes": [{"name": "mZ", "categories": ["a", "b"]}]})"), + "but categories in the domain 'other'"); // A parameter without domain is a RooRealVar: constant with "const": true, otherwise free, with a warning. { From f441c5435b0f6300d2aa05d762e7566934f36a39 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Tue, 10 Nov 2026 16:59:26 +0000 Subject: [PATCH 17/20] [RF][HS3] Export HistFactory channels directly as typed distributions The exporter fills an HS3::HistFactoryDist instead of its own channel model. Merging duplicate modifiers and choosing the default interpolation now work on the document, so they don't depend on RooFit. Inconsistent binnings are reported by the validation of the exported document. Assisted-by: claude-code:claude-opus-5-5 --- roofit/hs3/src/HS3Document.cxx | 171 ++++ roofit/hs3/src/HS3Document.h | 17 + roofit/hs3/src/HS3Validate.cxx | 9 +- roofit/hs3/src/JSONFactories_HistFactory.cxx | 793 ++++++------------- 4 files changed, 430 insertions(+), 560 deletions(-) diff --git a/roofit/hs3/src/HS3Document.cxx b/roofit/hs3/src/HS3Document.cxx index 7cb26bc402b6e..1f1a1f8ab966e 100644 --- a/roofit/hs3/src/HS3Document.cxx +++ b/roofit/hs3/src/HS3Document.cxx @@ -50,5 +50,176 @@ DomainAxis const *DomainRanges::find(std::string const &name) const return found != _ranges.end() ? found->second : nullptr; } +namespace { + +std::string const &nameOf(Modifier const &modifier) +{ + return std::visit([](auto const &m) -> std::string const & { return m.name; }, modifier); +} + +char const *typeOf(Modifier const &modifier) +{ + return std::visit( + overloaded{[](NormFactor const &) { return "normfactor"; }, [](NormSys const &) { return "normsys"; }, + [](HistoSys const &) { return "histosys"; }, [](ShapeSys const &) { return "shapefactor"; }, + [](StatError const &) { return "staterror"; }, [](CustomModifier const &) { return "custom"; }}, + modifier); +} + +// What duplicates must have in common to be combined. +template +char const *differences(Modifier_t const &l, Modifier_t const &r) +{ + if (!(l.constraint == r.constraint)) { + return "constraint metadata differs"; + } + if (l.parameter.name != r.parameter.name) { + return "parameter metadata differs"; + } + if (l.interpolation != r.interpolation) { + return "interpolation behaviours differ"; + } + return nullptr; +} + +// Combines `modifier` into `merged`, or returns why that's not possible. +char const *combine(NormSys &merged, NormSys const &modifier, std::vector const &) +{ + if (char const *reason = differences(merged, modifier)) { + return reason; + } + // Multiplicatively combining two normsys is only faithful when the interpolation is done in log-space, so that + // f1(alpha) * f2(alpha) is again representable by a single normsys with the multiplied lo/hi factors. This holds + // for the piecewise exponential (exact everywhere) and for the polynomial exponential (exact at the +-1 sigma + // anchors and in the exponential extrapolation region). The linear-space interpolations would turn the product into + // a shape that cannot be represented by a single normsys. + if (merged.interpolation != multiplicativePiecewiseExponential && + merged.interpolation != multiplicativePolynomialExponential) { + return "multiplicative combination is only valid for log-space interpolation"; + } + merged.lo *= modifier.lo; + merged.hi *= modifier.hi; + return nullptr; +} + +char const *combine(HistoSys &merged, HistoSys const &modifier, std::vector const &nominal) +{ + if (char const *reason = differences(merged, modifier)) { + return reason; + } + if (merged.interpolation != additivePolynomialLinear) { + return "this interpolation cannot currently be combined for duplicate histosys modifiers"; + } + const std::size_t nBins = nominal.size(); + auto fits = [&](HistoSysVariation const &variation) { return variation.contents.size() == nBins; }; + if (!fits(merged.lo) || !fits(merged.hi) || !fits(modifier.lo) || !fits(modifier.hi)) { + return "histogram binning differs"; + } + for (std::size_t bin = 0; bin < nBins; ++bin) { + merged.lo.contents[bin] += modifier.lo.contents[bin] - nominal[bin]; + merged.hi.contents[bin] += modifier.hi.contents[bin] - nominal[bin]; + } + return nullptr; +} + +} // namespace + +void mergeDuplicateModifiers(HistFactoryDist &dist, std::string const &channel, Diagnostics &diagnostics) +{ + for (Sample &sample : dist.samples) { + auto error = [&](Modifier const &modifier, std::string const &reason) { + diagnostics.push_back({Diagnostic::Severity::Error, "", + "cannot combine duplicate modifier '" + nameOf(modifier) + "' of type '" + + typeOf(modifier) + "' in sample '" + sample.name + "' of channel '" + channel + + "': " + reason}); + }; + + // The duplicates of a type are next to each other, because the modifiers of a type are sorted by name. + std::vector merged; + std::size_t count = 0; // of the modifiers that the last merged one combines + bool failed = false; + auto warnIfCombined = [&]() { + if (count > 1 && !failed) { + diagnostics.push_back({Diagnostic::Severity::Warning, "", + "combined " + std::to_string(count) + " duplicate modifiers named '" + + nameOf(merged.back()) + "' of type '" + typeOf(merged.back()) + "' in sample '" + + sample.name + "' of channel '" + channel + "'"}); + } + }; + for (Modifier &modifier : sample.modifiers) { + Modifier *last = merged.empty() ? nullptr : &merged.back(); + const bool combinable = + std::holds_alternative(modifier) || std::holds_alternative(modifier); + if (combinable && last && last->index() == modifier.index() && nameOf(*last) == nameOf(modifier)) { + char const *reason = nullptr; + if (auto *normSys = std::get_if(last)) { + reason = combine(*normSys, std::get(modifier), sample.contents); + } else { + reason = combine(std::get(*last), std::get(modifier), sample.contents); + } + if (reason) { + error(modifier, reason); + failed = true; + } + ++count; + continue; + } + warnIfCombined(); + merged.push_back(std::move(modifier)); + count = 1; + failed = false; + } + warnIfCombined(); + sample.modifiers = std::move(merged); + + std::set> seen; + for (Modifier const &modifier : sample.modifiers) { + if (!seen.emplace(typeOf(modifier), nameOf(modifier)).second) { + error(modifier, "this modifier type cannot be combined without changing its meaning"); + } + } + } +} + +void factorOutDefaultInterpolation(HistFactoryDist &dist) +{ + auto forEachInterpolation = [&](auto &&fn) { + for (Sample &sample : dist.samples) { + for (Modifier &modifier : sample.modifiers) { + if (auto *normSys = std::get_if(&modifier)) { + fn(normSys->interpolation); + } else if (auto *histoSys = std::get_if(&modifier)) { + fn(histoSys->interpolation); + } + } + } + }; + + std::map counts; + forEachInterpolation([&](std::optional &interpolation) { + if (interpolation) { + ++counts[*interpolation]; + } + }); + if (counts.empty()) { + return; + } + // A tie is decided in favor of the default of HistFactory. + auto best = counts.begin(); + for (auto current = std::next(counts.begin()); current != counts.end(); ++current) { + if (current->second > best->second || + (current->second == best->second && current->first == multiplicativePolynomialExponential && + best->first != multiplicativePolynomialExponential)) { + best = current; + } + } + dist.defaultInterpolation = best->first; + forEachInterpolation([&](std::optional &interpolation) { + if (interpolation == dist.defaultInterpolation) { + interpolation.reset(); + } + }); +} + } // namespace HS3 } // namespace RooFit diff --git a/roofit/hs3/src/HS3Document.h b/roofit/hs3/src/HS3Document.h index 3783d36d5d1cb..b847a1f4b1c26 100644 --- a/roofit/hs3/src/HS3Document.h +++ b/roofit/hs3/src/HS3Document.h @@ -304,6 +304,15 @@ struct ConstraintPdf { using Constraint = std::variant; +inline bool operator==(Unconstrained, Unconstrained) +{ + return true; +} +inline bool operator==(ConstraintPdf const &l, ConstraintPdf const &r) +{ + return l.pdf.name == r.pdf.name; +} + struct NormFactor { std::string name; ParameterRef parameter; @@ -569,6 +578,14 @@ void validateHistFactoryDist(HistFactoryDist const &dist, std::string const &nam /// Reads the leading "MAJOR.MINOR" and the optional ".PATCH" of an HS3 version string. bool parseVersion(std::string const &version, Version &out); +/// Combines the modifiers of a sample that have the same type and name, which HistFactory models can have but +/// documents can't: normsys by multiplying their variations, and histosys by adding up their differences to the +/// nominal contents. The modifiers of a type must be sorted by name. Other duplicates are errors. +void mergeDuplicateModifiers(HistFactoryDist &dist, std::string const &channel, Diagnostics &diagnostics); + +/// Makes the most common interpolation of the modifiers the "default_interpolation", which they then don't repeat. +void factorOutDefaultInterpolation(HistFactoryDist &dist); + } // namespace HS3 } // namespace RooFit diff --git a/roofit/hs3/src/HS3Validate.cxx b/roofit/hs3/src/HS3Validate.cxx index 6683608c635e6..1028458f7f163 100644 --- a/roofit/hs3/src/HS3Validate.cxx +++ b/roofit/hs3/src/HS3Validate.cxx @@ -452,14 +452,7 @@ void Validator::checkHistFactory(HistFactoryDist const &dist, std::string const if (inserted.second) { return; } - Constraint const &first = inserted.first->second.first; - bool same = first.index() == constraint.index(); - if (same && std::holds_alternative(first)) { - same = std::get(first) == std::get(constraint); - } else if (same && std::holds_alternative(first)) { - same = std::get(first).pdf.name == std::get(constraint).pdf.name; - } - if (!same) { + if (!(inserted.first->second.first == constraint)) { warning(where, "the parameter '" + parameter + "' has a different constraint than in " + inserted.first->second.second + ", but modifiers with the same parameter must share the constraint"); diff --git a/roofit/hs3/src/JSONFactories_HistFactory.cxx b/roofit/hs3/src/JSONFactories_HistFactory.cxx index 470a75a464d69..f274f2f623849 100644 --- a/roofit/hs3/src/JSONFactories_HistFactory.cxx +++ b/roofit/hs3/src/JSONFactories_HistFactory.cxx @@ -76,12 +76,6 @@ double round_prec(double d, int nSig) return sf * std::round(d / sf); } -// To avoid repeating the same string literals that can potentially get out of -// sync. -namespace Literals { -constexpr auto staterror = "staterror"; -} - std::string interpolationString(const Interpolation &interpolation) { std::stringstream ss; @@ -873,59 +867,6 @@ class FlexibleInterpVarFactory : public RooFit::JSONIO::Importer { } }; -struct NormFactor { - std::string name; - RooAbsReal const *param = nullptr; - RooAbsPdf const *constraint = nullptr; - NormFactor(RooAbsReal const &par, const RooAbsPdf *constr = nullptr) - : name{par.GetName()}, param{&par}, constraint{constr} - { - } -}; - -struct NormSys { - std::string name = ""; - RooAbsReal const *param = nullptr; - double low = 1.; - double high = 1.; - Interpolation interpolation = multiplicativePolynomialExponential; - RooAbsPdf const *constraint = nullptr; - NormSys() {}; - NormSys(const std::string &n, RooAbsReal *const p, double h, double l, Interpolation i, const RooAbsPdf *c) - : name(n), param(p), low(l), high(h), interpolation(std::move(i)), constraint(c) - { - } -}; - -struct HistoSys { - std::string name; - RooAbsReal const *param = nullptr; - std::vector low; - std::vector high; - Interpolation interpolation = additivePolynomialLinear; - RooAbsPdf const *constraint = nullptr; - HistoSys(const std::string &n, RooAbsReal *const p, RooHistFunc *l, RooHistFunc *h, Interpolation i, - const RooAbsPdf *c) - : name(n), param(p), interpolation(std::move(i)), constraint(c) - { - low.assign(l->dataHist().weightArray(), l->dataHist().weightArray() + l->dataHist().numEntries()); - high.assign(h->dataHist().weightArray(), h->dataHist().weightArray() + h->dataHist().numEntries()); - } -}; -// A per-bin modifier ("shapefactor"): HistFactory's ShapeSys (constrained) or ShapeFactor (free). -struct ShapeSys { - std::string name; - std::vector constraintPdfs; - std::vector parameters; - ShapeSys(const std::string &n) : name{n} {} -}; - -struct GenericElement { - std::string name; - RooAbsReal *function = nullptr; - GenericElement(RooAbsReal *e) : name(e->GetName()), function(e) {}; -}; - std::string stripOuterParens(const std::string &s) { size_t start = 0; @@ -1012,19 +953,22 @@ std::vector splitTopLevelProduct(const std::string &expr) return parts; } -NormSys parseOverallModifierFormula(const std::string &s, RooFormulaVar *formula) +// A normsys that is written as the formula "1 +- a * b", where one of a and b is a constrained parameter. +struct FormulaNormSys { + RooAbsReal *param = nullptr; + RooAbsPdf *constraint = nullptr; + double variation = 0.0; +}; + +FormulaNormSys parseOverallModifierFormula(const std::string &s, RooFormulaVar *formula) { static const std::regex pattern( R"(^\s*1(?:\.0)?\s*([\+\-])\s*([a-zA-Z_][a-zA-Z0-9_]*|[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)\s*\*\s*([a-zA-Z_][a-zA-Z0-9_]*|[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)\s*$)"); - NormSys sys; - double sign = 1.0; - + FormulaNormSys sys; std::smatch match; if (std::regex_match(s, match, pattern)) { - if (match[1].str() == "-") { - sign = -1.0; - } + const double sign = match[1].str() == "-" ? -1.0 : 1.0; std::string token2 = match[2].str(); std::string token3 = match[4].str(); @@ -1038,34 +982,21 @@ NormSys parseOverallModifierFormula(const std::string &s, RooFormulaVar *formula auto *constr3 = findConstraint(v3); if (constr2 && !p3) { - sys.name = p2->GetName(); sys.param = p2; - sys.high = 1.0 + sign * toDouble(token3); - sys.low = 1.0 - sign * toDouble(token3); + sys.variation = sign * toDouble(token3); } else if (!p2 && constr3) { - sys.name = p3->GetName(); sys.param = p3; - sys.high = 1.0 + sign * toDouble(token2); - sys.low = 1.0 - sign * toDouble(token2); + sys.variation = sign * toDouble(token2); } else if (constr2 && p3 && !constr3) { - sys.name = v2->GetName(); sys.param = v2; - sys.high = 1.0 + sign * p3->getVal(); - sys.low = 1.0 - sign * p3->getVal(); + sys.variation = sign * p3->getVal(); } else if (p2 && !constr2 && constr3) { - sys.name = v3->GetName(); sys.param = v3; - sys.high = 1.0 + sign * p2->getVal(); - sys.low = 1.0 - sign * p2->getVal(); + sys.variation = sign * p2->getVal(); } // A recognized formula has exactly one constrained parameter. sys.constraint = constr2 ? constr2 : constr3; - - // Preserve the legacy export behaviour for recognized explicit formulae. - sys.interpolation = multiplicativePiecewiseExponential; - - erasePrefix(sys.name, "alpha_"); } return sys; } @@ -1091,46 +1022,34 @@ bool allRooRealVar(const RooAbsCollection &list) return true; } -struct Sample { - std::string name; - std::vector hist; - std::vector histError; - std::vector normfactors; - std::vector normsys; - std::vector histosys; - std::vector shapesys; - std::vector tmpElements; - std::vector otherElements; - bool useBarlowBeestonLight = false; - std::vector staterrorParameters; - std::vector staterrorConstraints; - Sample(const std::string &n) : name{n} {} -}; - -void addNormFactor(RooRealVar const *par, Sample &sample, RooWorkspace *ws) +// The parameters of normsys and histosys modifiers must be variables. +void checkModifierParameter(RooAbsReal const ¶m, RooAbsPdf const *constraint) { - std::string parname = par->GetName(); - bool isConstrained = false; - for (RooAbsArg const *pdf : ws->allPdfs()) { - if (auto gauss = dynamic_cast(pdf)) { - if (parname == gauss->getX().GetName()) { - sample.normfactors.emplace_back(*par, gauss); - isConstrained = true; - } - } + if (!constraint && !param.isConstant() && !dynamic_cast(¶m)) { + RooJSONFactoryWSTool::error("cannot find constraint for " + std::string(param.GetName()) + + ": functions are not supported as parameters of this modifier"); } - if (!isConstrained) - sample.normfactors.emplace_back(*par); } -struct Channel { +std::vector contents(RooDataHist const &dataHist) +{ + return {dataHist.weightArray(), dataHist.weightArray() + dataHist.numEntries()}; +} + +// A channel as it is exported, and what the exporter needs to know about its RooFit objects. +struct ExportedChannel { std::string name; - std::vector samples; - std::map tot_yield; - std::map tot_yield2; - std::map rel_errors; - RooArgSet const *varSet = nullptr; - long unsigned int nBins = 0; + HS3::HistFactoryDist dist; + RooArgSet const *observables = nullptr; + /// The pdfs that the modifiers name as constraints, by name. + std::map constraints; + std::vector customModifiers; + /// Parts of formulas that are exported as custom modifiers. They are created for the export. + std::vector> temporaries; + // To reconstruct the statistical uncertainties, by bin starting at one. + std::map totYield; + std::map totYield2; + std::map relErrors; // Constant inverse bin widths of HistFactory before ROOT 6.26, which histfactory_dist implies. std::set legacyBinWidthFactors; }; @@ -1195,9 +1114,9 @@ bool isLegacyBinWidthFactor(RooAbsArg const &factor, RooRealSumPdf const &sumpdf [&](double volume) { return volume > 0. && std::abs(value * volume - 1.) < 1e-6; }); } -Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, const RooRealSumPdf *sumpdf) +ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, const RooRealSumPdf *sumpdf) { - Channel channel; + ExportedChannel channel; RooWorkspace *ws = tool->workspace(); @@ -1209,25 +1128,56 @@ Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, cons sumpdf->treeNodeServerList(&channelNodes); const std::vector volumes = binVolumes(*sumpdf); + auto constraint = [&](RooAbsPdf const *pdf) -> HS3::Constraint { + if (!pdf) { + return HS3::Unconstrained{}; + } + channel.constraints.emplace(pdf->GetName(), pdf); + return HS3::ConstraintPdf{{pdf->GetName()}}; + }; + auto gammaConstraint = [&](RooAbsPdf const *pdf) -> std::optional { + if (!pdf) { + return std::nullopt; + } + channel.constraints.emplace(pdf->GetName(), pdf); + return HS3::DistributionRef{pdf->GetName()}; + }; + for (size_t sampleidx = 0; sampleidx < sumpdf->funcList().size(); ++sampleidx) { PiecewiseInterpolation *pip = nullptr; std::vector phfs; const auto func = sumpdf->funcList().at(sampleidx); - Sample sample(func->GetName()); + HS3::Sample sample; + sample.name = func->GetName(); erasePrefix(sample.name, "L_x_"); eraseSuffix(sample.name, "_shapes"); eraseSuffix(sample.name, "_" + channel.name); erasePrefix(sample.name, pdfname + "_"); + // The modifiers by type, which are written in this order. + std::vector normFactors; + std::vector normSys; + std::vector histoSys; + std::vector shapeSys; + std::vector custom; + std::vector temporaries; + std::optional statError; + + auto addNormSys = [&](RooAbsReal ¶m, double hi, double lo, HS3::Interpolation interpolation, + RooAbsPdf const *pdf) { + checkModifierParameter(param, pdf); + std::string sysname(param.GetName()); + erasePrefix(sysname, "alpha_"); + normSys.push_back({sysname, {param.GetName()}, hi, lo, constraint(pdf), interpolation}); + }; + auto updateObservables = [&](RooDataHist const &dataHist) { - if (channel.varSet == nullptr) { - channel.varSet = dataHist.get(); - channel.nBins = dataHist.numEntries(); + if (channel.observables == nullptr) { + channel.observables = dataHist.get(); } - if (sample.hist.empty()) { - auto *w = dataHist.weightArray(); - sample.hist.assign(w, w + dataHist.numEntries()); + if (sample.contents.empty()) { + sample.contents = contents(dataHist); } }; auto processElements = [&](const auto &elements, auto &&self) -> void { @@ -1249,10 +1199,22 @@ Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, cons } } else if (auto constVar = dynamic_cast(e)) { if (constVar->getVal() != 1.) { - sample.normfactors.emplace_back(*constVar); + normFactors.push_back({constVar->GetName(), {constVar->GetName()}, HS3::Unconstrained{}}); } } else if (auto par = dynamic_cast(e)) { - addNormFactor(par, sample, ws); + // Every Gaussian of the parameter is a candidate for its constraint. + bool isConstrained = false; + for (RooAbsArg const *pdf : ws->allPdfs()) { + if (auto gauss = dynamic_cast(pdf)) { + if (std::string(par->GetName()) == gauss->getX().GetName()) { + normFactors.push_back({par->GetName(), {par->GetName()}, constraint(gauss)}); + isConstrained = true; + } + } + } + if (!isConstrained) { + normFactors.push_back({par->GetName(), {par->GetName()}, HS3::Unconstrained{}}); + } } else if (auto hf = dynamic_cast(e)) { updateObservables(hf->dataHist()); } else if (ParamHistFunc *phf = dynamic_cast(e); phf && allRooRealVar(phf->paramList())) { @@ -1264,14 +1226,12 @@ Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, cons RooAbsReal *var = static_cast(fip->variables().at(i)); std::string sysname(var->GetName()); erasePrefix(sysname, "alpha_"); - // A parameter without constraint is unconstrained, see checkUnconstrainedModifiers(). - const auto *constraint = findConstraint(var); const std::string context = "normsys modifier '" + sysname + "' in sample '" + sample.name + "' of channel '" + channel.name + "'"; - sample.normsys.emplace_back( - sysname, var, fip->high()[i], fip->low()[i], - interpolationFromCode(fip->interpolationCodes()[i], InterpolationClass::Flexible, context), - constraint); + // A parameter without constraint is unconstrained, see checkUnconstrainedModifiers(). + addNormSys(*var, fip->high()[i], fip->low()[i], + interpolationFromCode(fip->interpolationCodes()[i], InterpolationClass::Flexible, context), + findConstraint(var)); } } else if (!pip && (pip = dynamic_cast(e))) { // nothing to do here, already assigned @@ -1287,7 +1247,8 @@ Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, cons auto components = splitTopLevelProduct(expression.Data()); if (components.size() == 0) { // it's not a product, let's just treat it as an unknown element - sample.otherElements.push_back(formula); + custom.push_back({formula->GetName()}); + channel.customModifiers.push_back(formula); } else { // it is a prododuct, we can try to handle the elements separately std::vector realComponents; @@ -1300,22 +1261,26 @@ Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, cons continue; } // check if this is an attempt at explicitly encoding an overallSys - auto normsys = parseOverallModifierFormula(comp, formula); - if (normsys.param) { - sample.normsys.emplace_back(std::move(normsys)); + FormulaNormSys sys = parseOverallModifierFormula(comp, formula); + if (sys.param) { + // Preserve the legacy export behaviour for recognized explicit formulae. + addNormSys(*sys.param, 1.0 + sys.variation, 1.0 - sys.variation, + multiplicativePiecewiseExponential, sys.constraint); continue; } // this is something non-trivial, let's deal with it separately std::string name = std::string(formula->GetName()) + "_part" + std::to_string(idx); ++idx; - auto *var = new RooFormulaVar(name.c_str(), name.c_str(), comp.c_str(), formula->dependents()); - sample.tmpElements.push_back({var}); + temporaries.push_back({name}); + channel.temporaries.push_back(std::make_unique( + name.c_str(), name.c_str(), comp.c_str(), formula->dependents())); } self(realComponents, self); } } else if (auto real = dynamic_cast(e)) { - sample.otherElements.push_back(real); + custom.push_back({real->GetName()}); + channel.customModifiers.push_back(real); } } }; @@ -1332,11 +1297,6 @@ Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, cons } } - // sort and configure norms - sortByName(sample.normfactors); - sortByName(sample.normsys); - - // sort and configure the histosys if (pip) { for (size_t i = 0; i < pip->paramList().size(); ++i) { RooAbsReal *var = static_cast(pip->paramList().at(i)); @@ -1345,36 +1305,43 @@ Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, cons if (auto lo = dynamic_cast(pip->lowList().at(i))) { if (auto hi = dynamic_cast(pip->highList().at(i))) { // A parameter without constraint is unconstrained, see checkUnconstrainedModifiers(). - const auto *constraint = findConstraint(var); + const auto *pdf = findConstraint(var); + checkModifierParameter(*var, pdf); const std::string context = "histosys modifier '" + sysname + "' in sample '" + sample.name + "' of channel '" + channel.name + "'"; - sample.histosys.emplace_back( - sysname, var, lo, hi, - interpolationFromCode(pip->interpolationCodes()[i], InterpolationClass::Piecewise, context), - constraint); + histoSys.push_back( + {sysname, + {var->GetName()}, + {contents(hi->dataHist()), std::nullopt}, + {contents(lo->dataHist()), std::nullopt}, + constraint(pdf), + interpolationFromCode(pip->interpolationCodes()[i], InterpolationClass::Piecewise, context)}); } } } - sortByName(sample.histosys); } for (ParamHistFunc *phf : phfs) { if (startsWith(std::string(phf->GetName()), "mc_stat_")) { // MC stat uncertainty + statError.emplace(); + statError->name = "staterror"; + HS3::GammaConstraintPdfs pdfs; int idx = 0; for (const auto &g : phf->paramList()) { - sample.staterrorParameters.push_back(static_cast(g)); + statError->parameters.push_back({g->GetName()}); ++idx; - RooAbsPdf *constraint = findConstraint(g); - sample.staterrorConstraints.push_back(constraint); - channel.tot_yield[idx] += sample.hist[idx - 1]; - channel.tot_yield2[idx] += (sample.hist[idx - 1] * sample.hist[idx - 1]); - if (constraint) { - channel.rel_errors[idx] = constraintRelError(*constraint, *g); + RooAbsPdf *pdf = findConstraint(g); + pdfs.pdfs.push_back(gammaConstraint(pdf)); + channel.totYield[idx] += sample.contents[idx - 1]; + channel.totYield2[idx] += (sample.contents[idx - 1] * sample.contents[idx - 1]); + if (pdf) { + channel.relErrors[idx] = constraintRelError(*pdf, *g); } } - sample.useBarlowBeestonLight = true; + statError->constraint = std::move(pdfs); } else { // ShapeSys or ShapeFactor - ShapeSys sys(phf->GetName()); + HS3::ShapeSys &sys = shapeSys.emplace_back(); + sys.name = phf->GetName(); erasePrefix(sys.name, channel.name + "_"); for (std::string_view suffix : {"_ShapeSys", "_shapeSys", "_ShapeFactor", "_shapeFactor"}) { if (eraseSuffix(sys.name, suffix)) @@ -1383,381 +1350,112 @@ Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, cons // Both are written as "shapefactor": the constrained parameters name their constraint distribution in // "constraints", the others are free. + HS3::GammaConstraintPdfs pdfs; for (const auto &g : phf->paramList()) { - sys.parameters.push_back(static_cast(g)); - RooAbsPdf *constraint = findConstraint(g); - if (!constraint) - constraint = ws->pdf(constraintName(g->GetName())); - sys.constraintPdfs.push_back(constraint); + sys.parameters.push_back({g->GetName()}); + RooAbsPdf *pdf = findConstraint(g); + if (!pdf) + pdf = ws->pdf(constraintName(g->GetName())); + pdfs.pdfs.push_back(gammaConstraint(pdf)); } - sample.shapesys.emplace_back(std::move(sys)); - } - } - sortByName(sample.shapesys); - - // add the sample - channel.samples.emplace_back(std::move(sample)); - } - - sortByName(channel.samples); - return channel; -} - -bool hasSameMetadata(const RooAbsArg *lhs, const RooAbsArg *rhs) -{ - if (!lhs || !rhs) { - return lhs == rhs; - } - return std::string{lhs->GetName()} == rhs->GetName() && lhs->IsA() == rhs->IsA(); -} - -[[noreturn]] void duplicateModifierError(const Channel &channel, const Sample &sample, std::string_view type, - std::string_view name, std::string_view reason) -{ - std::stringstream ss; - ss << "cannot combine duplicate modifier '" << name << "' of type '" << type << "' in sample '" << sample.name - << "' of channel '" << channel.name << "': " << reason; - RooJSONFactoryWSTool::error(ss.str().c_str()); -} - -void warnDuplicateModifiersCombined(const Channel &channel, const Sample &sample, std::string_view type, - std::string_view name, std::size_t count) -{ - std::stringstream ss; - ss << "combined " << count << " duplicate modifiers named '" << name << "' of type '" << type << "' in sample '" - << sample.name << "' of channel '" << channel.name << "'"; - RooJSONFactoryWSTool::warning(ss.str()); -} - -// Multiplicatively combining two normsys is only faithful when the interpolation is done in log-space, so that -// f1(alpha) * f2(alpha) is again representable by a single normsys with the multiplied lo/hi factors. This holds for -// the piecewise-exponential code 1 (exact everywhere) and for the default code 4 (exact at the +-1 sigma anchors and in -// the exponential extrapolation region). The linear-space codes (e.g. 0 and 2) would turn the product into a shape that -// cannot be represented by a single normsys, so those must not be merged. -bool normSysSupportsMultiplicativeMerge(const Interpolation &interpolation) -{ - return interpolation == multiplicativePiecewiseExponential || interpolation == multiplicativePolynomialExponential; -} - -// Combines runs of adjacent modifiers that share the same name (the container is sorted by name beforehand) into a -// single modifier. The shared metadata (constraint, parameter and interpolation behaviour) must be identical across the -// duplicates; the type-specific `combine` callable performs the actual merge and any additional validation. -template -void mergeDuplicateModifiers(const Channel &channel, const Sample &sample, Modifiers &modifiers, std::string_view type, - CombineFn combine) -{ - Modifiers mergedModifiers; - mergedModifiers.reserve(modifiers.size()); - - for (std::size_t begin = 0; begin < modifiers.size();) { - std::size_t end = begin + 1; - while (end < modifiers.size() && modifiers[end].name == modifiers[begin].name) { - ++end; - } - - auto merged = modifiers[begin]; - for (std::size_t i = begin + 1; i < end; ++i) { - const auto &modifier = modifiers[i]; - if (!hasSameMetadata(merged.constraint, modifier.constraint)) { - duplicateModifierError(channel, sample, type, merged.name, "constraint metadata differs"); + sys.constraint = std::move(pdfs); } - if (!hasSameMetadata(merged.param, modifier.param)) { - duplicateModifierError(channel, sample, type, merged.name, "parameter metadata differs"); - } - if (merged.interpolation != modifier.interpolation) { - duplicateModifierError(channel, sample, type, merged.name, "interpolation behaviours differ"); - } - combine(merged, modifier); } - if (end - begin > 1) { - warnDuplicateModifiersCombined(channel, sample, type, merged.name, end - begin); - } - mergedModifiers.emplace_back(std::move(merged)); - begin = end; - } - - modifiers = std::move(mergedModifiers); -} - -void mergeDuplicateNormSys(const Channel &channel, Sample &sample) -{ - mergeDuplicateModifiers(channel, sample, sample.normsys, "normsys", [&](NormSys &merged, const NormSys &modifier) { - if (!normSysSupportsMultiplicativeMerge(merged.interpolation)) { - duplicateModifierError(channel, sample, "normsys", merged.name, - "multiplicative combination is only valid for log-space interpolation"); - } - merged.low *= modifier.low; - merged.high *= modifier.high; - }); -} - -void mergeDuplicateHistoSys(const Channel &channel, Sample &sample) -{ - const std::size_t nBins = sample.hist.size(); - mergeDuplicateModifiers( - channel, sample, sample.histosys, "histosys", [&](HistoSys &merged, const HistoSys &modifier) { - if (merged.interpolation != additivePolynomialLinear) { - duplicateModifierError(channel, sample, "histosys", merged.name, - "this interpolation cannot currently be combined for duplicate histosys " - "modifiers"); - } - if (merged.low.size() != nBins || merged.high.size() != nBins || modifier.low.size() != nBins || - modifier.high.size() != nBins) { - duplicateModifierError(channel, sample, "histosys", merged.name, "histogram binning differs"); - } - for (std::size_t bin = 0; bin < nBins; ++bin) { - merged.low[bin] += modifier.low[bin] - sample.hist[bin]; - merged.high[bin] += modifier.high[bin] - sample.hist[bin]; - } - }); -} - -void ensureUniqueModifiers(const Channel &channel, const Sample &sample) -{ - std::set> seen; - auto add = [&](std::string type, const std::string &name) { - if (!seen.emplace(type, name).second) { - duplicateModifierError(channel, sample, type, name, - "this modifier type cannot be combined without changing its meaning"); + sortByName(normFactors); + sortByName(normSys); + sortByName(histoSys); + sortByName(shapeSys); + auto append = [&](auto &modifiers) { + sample.modifiers.insert(sample.modifiers.end(), std::make_move_iterator(modifiers.begin()), + std::make_move_iterator(modifiers.end())); + }; + append(normFactors); + append(normSys); + append(histoSys); + append(shapeSys); + append(custom); + append(temporaries); + if (statError) { + sample.modifiers.push_back(std::move(*statError)); } - }; - - for (const auto &modifier : sample.normfactors) - add("normfactor", modifier.name); - for (const auto &modifier : sample.normsys) - add("normsys", modifier.name); - for (const auto &modifier : sample.histosys) - add("histosys", modifier.name); - for (const auto &modifier : sample.shapesys) - add("shapefactor", modifier.name); - for (const auto &modifier : sample.otherElements) - add("custom", modifier.name); - for (const auto &modifier : sample.tmpElements) - add("custom", modifier.name); - if (sample.useBarlowBeestonLight) - add(::Literals::staterror, ::Literals::staterror); -} -void canonicalizeModifiers(Channel &channel) -{ - for (auto &sample : channel.samples) { - mergeDuplicateNormSys(channel, sample); - mergeDuplicateHistoSys(channel, sample); - ensureUniqueModifiers(channel, sample); + channel.dist.samples.push_back(std::move(sample)); } -} -void configureStatError(Channel &channel) -{ - for (auto &sample : channel.samples) { - if (sample.useBarlowBeestonLight) { - sample.histError.resize(sample.hist.size()); - for (auto bin : channel.rel_errors) { - // reverse engineering the correct partial error - // the (arbitrary) convention used here is that all samples should have the same relative error - const int i = bin.first; - const double relerr_tot = bin.second; - const double count = sample.hist[i - 1]; - // this reconstruction is inherently imprecise, so we truncate it at some decimal places to make sure that - // we don't carry around too many useless digits - sample.histError[i - 1] = - round_prec(relerr_tot * channel.tot_yield[i] / std::sqrt(channel.tot_yield2[i]) * count, 7); - } - } - } + sortByName(channel.dist.samples); + return channel; } -std::optional defaultInterpolation(const Channel &channel) +// The statistical uncertainties of the samples are reconstructed from the constraints of the gamma parameters, with +// the (arbitrary) convention that all samples have the same relative uncertainty. +void configureStatError(ExportedChannel &channel) { - std::map counts; - for (const auto &sample : channel.samples) { - for (const auto &modifier : sample.normsys) { - ++counts[modifier.interpolation]; - } - for (const auto &modifier : sample.histosys) { - ++counts[modifier.interpolation]; + for (HS3::Sample &sample : channel.dist.samples) { + if (std::none_of(sample.modifiers.begin(), sample.modifiers.end(), + [](auto const &m) { return std::holds_alternative(m); })) { + continue; } - } - if (counts.empty()) { - return std::nullopt; - } - - auto best = counts.begin(); - for (auto current = std::next(counts.begin()); current != counts.end(); ++current) { - if (current->second > best->second || - (current->second == best->second && current->first == multiplicativePolynomialExponential && - best->first != multiplicativePolynomialExponential)) { - best = current; + std::vector errors(sample.contents.size()); + for (auto const &bin : channel.relErrors) { + const int i = bin.first; + // this reconstruction is inherently imprecise, so we truncate it at some decimal places to make sure that + // we don't carry around too many useless digits + errors[i - 1] = + round_prec(bin.second * channel.totYield[i] / std::sqrt(channel.totYield2[i]) * sample.contents[i - 1], 7); } + sample.errors = std::move(errors); } - return best->first; -} - -HS3::Constraint toConstraint(RooAbsPdf const *constraint) -{ - if (!constraint) { - return HS3::Unconstrained{}; - } - return HS3::ConstraintPdf{{constraint->GetName()}}; -} - -// The constraint distributions of gamma parameters, with null for the unconstrained ones. Nothing is written if all -// parameters are unconstrained. -HS3::GammaConstraint toGammaConstraint(std::vector const &constraints) -{ - if (std::none_of(constraints.begin(), constraints.end(), [](auto *pdf) { return pdf != nullptr; })) { - return HS3::Unconstrained{}; - } - HS3::GammaConstraintPdfs pdfs; - for (auto *constraint : constraints) { - pdfs.pdfs.push_back(constraint ? std::make_optional(HS3::DistributionRef{constraint->GetName()}) : std::nullopt); - } - return pdfs; -} - -void checkBinning(std::size_t nBins, std::size_t size, const char *what) -{ - if (nBins != size) { - RooJSONFactoryWSTool::error("inconsistent binning: " + std::to_string(nBins) + " bins expected, but " + - std::to_string(size) + " found in " + what + "!"); - } -} - -// Whether a pdf is a constraint term of the channel. A pdf of a parameter that is not, like an external constraint or a -// prior, stays where it is. -bool isChannelConstraint(RooAbsPdf const *pdf, std::vector const &constraints) -{ - return pdf && std::find(constraints.begin(), constraints.end(), pdf) != constraints.end(); } -// The channel as it is written to the document, which the tool writes to the JSON. The modifiers name the constraint -// terms of their parameters. The other constraint terms of the channel are written separately. -bool exportChannel(RooJSONFactoryWSTool *tool, const Channel &channel, std::vector const &constraints) +// The modifiers name only the constraint terms of the channel. A pdf of a parameter that is not, like an external +// constraint or a prior, stays where it is. Returns the named constraint terms. +std::set +nameChannelConstraints(ExportedChannel &channel, std::vector const &constraints) { - std::set namedConstraints; - auto name = [&](RooAbsPdf const *pdf) -> RooAbsPdf const * { - if (!isChannelConstraint(pdf, constraints)) { - return nullptr; - } - namedConstraints.insert(pdf); - return pdf; - }; - auto nameAll = [&](std::vector pdfs) { - for (auto &pdf : pdfs) { - pdf = name(pdf); + std::set named; + auto isNamed = [&](std::string const &name) { + RooAbsPdf const *pdf = channel.constraints.at(name); + if (std::find(constraints.begin(), constraints.end(), pdf) == constraints.end()) { + return false; } - return toGammaConstraint(pdfs); + named.insert(pdf); + return true; }; - - HS3::HistFactoryDist dist; - dist.defaultInterpolation = defaultInterpolation(channel); - auto interpolationIfNotDefault = [&](Interpolation const &interpolation) -> std::optional { - if (dist.defaultInterpolation && interpolation == *dist.defaultInterpolation) { - return std::nullopt; + auto filter = [&](HS3::Constraint &constraint) { + auto const *pdf = std::get_if(&constraint); + if (pdf && !isNamed(pdf->pdf.name)) { + constraint = HS3::Unconstrained{}; } - return interpolation; }; - - for (auto *obs : static_range_cast(*channel.varSet)) { - dist.axes.push_back(HS3::toBinnedAxis(*obs)); - } - - for (const auto &sample : channel.samples) { - HS3::Sample &s = dist.samples.emplace_back(); - s.name = sample.name; - - for (const auto &nf : sample.normfactors) { - s.modifiers.push_back( - HS3::NormFactor{nf.name, {nf.param->GetName()}, toConstraint(name(nf.constraint))}); - } - - for (const auto &sys : sample.normsys) { - s.modifiers.push_back(HS3::NormSys{sys.name, - {sys.param->GetName()}, - sys.high, - sys.low, - toConstraint(name(sys.constraint)), - interpolationIfNotDefault(sys.interpolation)}); - } - - for (const auto &sys : sample.histosys) { - checkBinning(channel.nBins, sys.low.size(), "nominal histogram errors"); - checkBinning(channel.nBins, sys.high.size(), "nominal histogram errors"); - HS3::HistoSys mod; - mod.name = sys.name; - mod.parameter.name = sys.param->GetName(); - mod.hi.contents = sys.high; - mod.lo.contents = sys.low; - mod.constraint = toConstraint(name(sys.constraint)); - mod.interpolation = interpolationIfNotDefault(sys.interpolation); - s.modifiers.push_back(std::move(mod)); - } - - for (const auto &sys : sample.shapesys) { - HS3::ShapeSys mod; - mod.name = sys.name; - for (RooAbsReal *param : sys.parameters) { - mod.parameters.push_back({param->GetName()}); - } - // The relative uncertainties are encoded in the constraint distributions. - mod.constraint = nameAll(sys.constraintPdfs); - s.modifiers.push_back(std::move(mod)); - } - - for (const auto &other : sample.otherElements) { - s.modifiers.push_back(HS3::CustomModifier{other.name}); - } - for (const auto &other : sample.tmpElements) { - s.modifiers.push_back(HS3::CustomModifier{other.name}); - } - - if (sample.useBarlowBeestonLight) { - HS3::StatError mod; - mod.name = ::Literals::staterror; - for (RooAbsReal *param : sample.staterrorParameters) { - mod.parameters.push_back({param->GetName()}); + auto filterGammas = [&](HS3::GammaConstraint &constraint) { + auto *pdfs = std::get_if(&constraint); + if (!pdfs) { + return; + } + bool any = false; + for (auto &pdf : pdfs->pdfs) { + if (pdf && !isNamed(pdf->name)) { + pdf.reset(); } - mod.constraint = nameAll(sample.staterrorConstraints); - s.modifiers.push_back(std::move(mod)); + any = any || pdf; } - - checkBinning(channel.nBins, sample.hist.size(), "nominal histogram"); - s.contents = sample.hist; - if (!sample.histError.empty()) { - checkBinning(channel.nBins, sample.histError.size(), "nominal histogram errors"); - s.errors = sample.histError; - } - } - - // All other constraint terms of the channel. - std::vector channelConstraintNames; - for (RooAbsPdf const *pdf : constraints) { - if (!namedConstraints.count(pdf)) { - tool->queueExport(*pdf); - channelConstraintNames.push_back({pdf->GetName()}); - } - } - - HS3::setExportedObject(*tool, std::move(dist), std::move(channelConstraintNames)); - return true; -} - -// The parameters of normsys and histosys modifiers must be variables. -void checkModifierParameters(const Channel &channel) -{ - auto check = [&](auto const &modifiers) { - for (auto const &sys : modifiers) { - if (!sys.constraint && !sys.param->isConstant() && !dynamic_cast(sys.param)) { - RooJSONFactoryWSTool::error("cannot find constraint for " + std::string(sys.param->GetName()) + - ": functions are not supported as parameters of this modifier"); - } + // Nothing is written if all parameters are unconstrained. + if (!any) { + constraint = HS3::Unconstrained{}; } }; - for (const auto &sample : channel.samples) { - check(sample.normsys); - check(sample.histosys); + for (HS3::Sample &sample : channel.dist.samples) { + for (HS3::Modifier &modifier : sample.modifiers) { + std::visit(HS3::overloaded{ + [&](HS3::NormFactor &m) { filter(m.constraint); }, + [&](HS3::NormSys &m) { filter(m.constraint); }, [&](HS3::HistoSys &m) { filter(m.constraint); }, + [&](HS3::ShapeSys &m) { filterGammas(m.constraint); }, + [&](HS3::StatError &m) { filterGammas(m.constraint); }, [](HS3::CustomModifier &) {}}, + modifier); + } } + return named; } bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname, const RooRealSumPdf *sumpdf, @@ -1774,62 +1472,43 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname } } - auto channel = readChannel(tool, pdfname, sumpdf); + ExportedChannel channel = readChannel(tool, pdfname, sumpdf); + HS3::HistFactoryDist &dist = channel.dist; // sanity checks - if (channel.samples.size() == 0) + if (dist.samples.empty()) return false; - for (auto &sample : channel.samples) { - if (sample.hist.empty()) { + for (auto &sample : dist.samples) { + if (sample.contents.empty()) { return false; } } - canonicalizeModifiers(channel); - checkModifierParameters(channel); - - // stat error handling + HS3::Diagnostics diagnostics; + HS3::mergeDuplicateModifiers(dist, channel.name, diagnostics); + HS3::report(diagnostics); configureStatError(channel); + HS3::factorOutDefaultInterpolation(dist); - // Export the constraint terms that the modifiers name. - auto queueConstraint = [&](RooAbsPdf const *constraint) { - if (isChannelConstraint(constraint, constraints)) { - tool->queueExport(*constraint); - } - }; - for (const auto &sample : channel.samples) { - for (auto &modifier : sample.normfactors) { - queueConstraint(modifier.constraint); - } - for (auto &modifier : sample.normsys) { - queueConstraint(modifier.constraint); - } - for (auto &modifier : sample.histosys) { - queueConstraint(modifier.constraint); - } - for (auto &modifier : sample.shapesys) { - for (auto *constraint : modifier.constraintPdfs) { - queueConstraint(constraint); - } - } - for (auto *constraint : sample.staterrorConstraints) { - queueConstraint(constraint); - } + for (auto *obs : static_range_cast(*channel.observables)) { + dist.axes.push_back(HS3::toBinnedAxis(*obs)); } - // Export all the custom modifiers - for (const auto &sample : channel.samples) { - for (auto &modifier : sample.otherElements) { - tool->queueExport(*modifier.function); - } - for (auto &modifier : sample.tmpElements) { - tool->queueExportTemporary(modifier.function); - } + std::set namedConstraints = nameChannelConstraints(channel, constraints); + for (RooAbsPdf const *pdf : namedConstraints) { + tool->queueExport(*pdf); + } + + for (RooAbsReal *modifier : channel.customModifiers) { + tool->queueExport(*modifier); + } + for (auto &modifier : channel.temporaries) { + tool->queueExportTemporary(modifier.release()); } // Export all model parameters RooArgSet parameters; - sumpdf->getParameters(channel.varSet, parameters); + sumpdf->getParameters(channel.observables, parameters); for (RooAbsArg *param : parameters) { // This should exclude the global observables. The legacy inverse bin widths are not written at all. if (!startsWith(std::string{param->GetName()}, "nom_") && !channel.legacyBinWidthFactors.count(param)) { @@ -1837,7 +1516,17 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname } } - return exportChannel(tool, channel, constraints); + // All other constraint terms of the channel are written separately. + std::vector channelConstraintNames; + for (RooAbsPdf const *pdf : constraints) { + if (!namedConstraints.count(pdf)) { + tool->queueExport(*pdf); + channelConstraintNames.push_back({pdf->GetName()}); + } + } + + HS3::setExportedObject(*tool, std::move(dist), std::move(channelConstraintNames)); + return true; } class HistFactoryStreamer_ProdPdf : public RooFit::JSONIO::Exporter { From 11e97a2b1b79a18dbe96307c925601e6e1c04b11 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Tue, 10 Nov 2026 17:09:45 +0000 Subject: [PATCH 18/20] [RF][HS3] Read and write the interpolation functions as typed structs The functions "interpolation" and "interpolation0d" are read into HS3::InterpolatedFunction and HS3::InterpolatedNumber, which also handle the legacy "interpolationCodes", and written from them. The upgrade of legacy interpolation codes of modifiers shares the code. Assisted-by: claude-code:claude-opus-5-5 --- roofit/hs3/src/HS3Builder.h | 4 + roofit/hs3/src/HS3Document.h | 36 ++- roofit/hs3/src/HS3Parser.cxx | 121 ++++++++- roofit/hs3/src/HS3Schema.h | 4 + roofit/hs3/src/HS3Upgrade.cxx | 32 +-- roofit/hs3/src/HS3Writer.cxx | 78 +++++- roofit/hs3/src/JSONFactories_HistFactory.cxx | 245 ++++++------------- roofit/hs3/src/RooJSONFactoryWSTool.cxx | 8 + 8 files changed, 308 insertions(+), 220 deletions(-) diff --git a/roofit/hs3/src/HS3Builder.h b/roofit/hs3/src/HS3Builder.h index dd856e87a033d..dda8410ddc2a9 100644 --- a/roofit/hs3/src/HS3Builder.h +++ b/roofit/hs3/src/HS3Builder.h @@ -21,6 +21,7 @@ #include #include +class RooAbsArg; class RooDataHist; class RooJSONFactoryWSTool; class RooRealVar; @@ -41,6 +42,9 @@ RooArgSet createObservables(std::vector const &axes); /// The axis of binned data for a variable. BinnedAxis toBinnedAxis(RooRealVar const &var); +/// The name of an object, or the value of a literal constant. +NameOrNumber toNameOrNumber(RooAbsArg const &arg); + /// Create a RooDataHist with the contents and errors. If there are no errors, it doesn't store squared weights. std::unique_ptr createDataHist(std::string const &name, RooArgSet const &vars, std::vector const &contents, std::vector const *errors); diff --git a/roofit/hs3/src/HS3Document.h b/roofit/hs3/src/HS3Document.h index b847a1f4b1c26..52ca3394a6ac3 100644 --- a/roofit/hs3/src/HS3Document.h +++ b/roofit/hs3/src/HS3Document.h @@ -407,6 +407,30 @@ struct Function { GenericObject body; }; +/// A name, or a number for a constant. +using NameOrNumber = std::variant; + +/// The function "interpolation" of RooFit, implemented by PiecewiseInterpolation: a nominal function plus the +/// interpolated differences to the variations "high" and "low" for each parameter. +struct InterpolatedFunction { + std::string nominal; + std::vector parameters; + std::vector high; + std::vector low; + std::vector interpolations; ///< One per parameter. + bool positiveDefinite = false; +}; + +/// The function "interpolation0d" of RooFit, implemented by FlexibleInterpVar: a nominal value plus the interpolated +/// differences to the variations "high" and "low" for each parameter. +struct InterpolatedNumber { + double nominal = 0.0; + std::vector parameters; + std::vector high; + std::vector low; + std::vector interpolations; ///< One per parameter. +}; + // The following types are stored in the "misc" section, for which the HS3 standard doesn't prescribe a format. struct Attributes { @@ -522,8 +546,10 @@ HS3Document parseDocument(Detail::JSONNode const &root, Diagnostics &diagnostics /// Reads a HistFactory distribution that doesn't come with a document. HistFactoryDist parseHistFactoryDist(Detail::JSONNode const &distribution, Diagnostics &diagnostics); -/// Reads an interpolation that doesn't come with a document. -Interpolation parseInterpolation(Detail::JSONNode const &interpolation, Diagnostics &diagnostics); +/// Reads the functions "interpolation" and "interpolation0d", which don't come with a document. Older versions of +/// RooFit wrote integer "interpolationCodes" instead of "interpolations". +InterpolatedFunction parseInterpolatedFunction(Detail::JSONNode const &node, Diagnostics &diagnostics); +InterpolatedNumber parseInterpolatedNumber(Detail::JSONNode const &node, Diagnostics &diagnostics); /// Writes everything but the distributions and functions, which are written one by one while they are exported: the /// generic ones by the exporters, and the others with functions like writeHistFactoryDist(). @@ -532,6 +558,12 @@ void writeDocument(Detail::JSONNode &root, HS3Document const &doc); /// Writes a HistFactory distribution, without its name. void writeHistFactoryDist(Detail::JSONNode &node, HistFactoryDist const &dist); +/// Writes the functions "interpolation" and "interpolation0d", with a single interpolation if they are all the same. +void writeInterpolatedFunction(Detail::JSONNode &node, InterpolatedFunction const &function); +void writeInterpolatedNumber(Detail::JSONNode &node, InterpolatedNumber const &function); + +void writeInterpolation(Detail::JSONNode &node, Interpolation const &interpolation); + /// Writes a number, as an integer if it has no fractional part. void writeNumber(Detail::JSONNode &node, double value); diff --git a/roofit/hs3/src/HS3Parser.cxx b/roofit/hs3/src/HS3Parser.cxx index ae61a75a05efd..60457d8c294ad 100644 --- a/roofit/hs3/src/HS3Parser.cxx +++ b/roofit/hs3/src/HS3Parser.cxx @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -1236,12 +1237,126 @@ HistFactoryDist parseHistFactoryDist(JSONNode const &distribution, Diagnostics & return out; } -Interpolation parseInterpolation(JSONNode const &interpolation, Diagnostics &diagnostics) +void readValue(Cursor const &c, NameOrNumber &out) { - Interpolation out; - readValue(Cursor{interpolation, diagnostics}, out); + if (c.node().is_number()) { + out = c.node().val_double(); + } else { + readValue(c, out.emplace()); + } +} + +std::optional readInterpolationCode(Cursor const &c, InterpolationClass interpolationClass) +{ + JSONNode const &node = c.node(); + if (node.is_container() || node.is_null() || !node.has_val()) { + c.error("must be a structured interpolation or a legacy integer code"); + return std::nullopt; + } + const std::string value = node.val(); + int code = 0; + const auto result = std::from_chars(value.data(), value.data() + value.size(), code); + if (result.ec != std::errc{} || result.ptr != value.data() + value.size()) { + c.error("has invalid legacy interpolation code '" + value + "'"); + return std::nullopt; + } + auto interpolation = interpolationFromCode(code, interpolationClass); + if (!interpolation) { + c.error(std::string("has unsupported ") + + (interpolationClass == InterpolationClass::Piecewise ? "PiecewiseInterpolation" : "FlexibleInterpVar") + + " code " + value); + } + return interpolation; +} + +namespace { + +// The "interpolations" of the functions "interpolation" and "interpolation0d": one for all parameters, or one per +// parameter. Older versions of RooFit wrote integer "interpolationCodes". Without both, the interpolation is linear. +void readInterpolations(Reader &ar, std::size_t nParameters, InterpolationClass interpolationClass, + std::vector &out) +{ + out.assign(nParameters, additivePiecewiseLinear); + if (ar.has("interpolations")) { + Cursor c{ar.cursor().node()["interpolations"], ar.cursor(), "interpolations"}; + const std::size_t errorsBefore = c.numErrors(); + std::vector interpolations; + readValue(c, interpolations); + const std::size_t size = interpolations.size(); + if (c.numErrors() != errorsBefore) { + return; + } + if (nParameters == 0 ? size != 0 : size != 1 && size != nParameters) { + c.error("component 'interpolations' must contain either one descriptor or one descriptor per parameter (got " + + std::to_string(size) + " for " + std::to_string(nParameters) + " parameters)"); + } else if (size == 1) { + out.assign(nParameters, interpolations.front()); + } else { + out = std::move(interpolations); + } + return; + } + if (!ar.has("interpolationCodes")) { + return; + } + Cursor c{ar.cursor().node()["interpolationCodes"], ar.cursor(), "interpolationCodes"}; + if (!c.node().is_seq() || c.node().num_children() != nParameters) { + c.error("legacy component 'interpolationCodes' must contain one code per parameter (got " + + std::to_string(c.node().is_seq() ? c.node().num_children() : 0) + " for " + std::to_string(nParameters) + + " parameters)"); + return; + } + std::size_t i = 0; + for (JSONNode const &child : c.node().children()) { + if (auto interpolation = readInterpolationCode(Cursor{child, c, i}, interpolationClass)) { + out[i] = *interpolation; + } + ++i; + } +} + +// The functions "interpolation" and "interpolation0d" have the variations "high" and "low" for each of the "vars". +template +Function_t parseInterpolated(JSONNode const &node, Diagnostics &diagnostics, InterpolationClass interpolationClass, + Fn &&readOthers) +{ + Function_t out; + Cursor cursor{node, diagnostics}; + if (!node.is_map()) { + cursor.error("must be a struct"); + return out; + } + const std::size_t errorsBefore = cursor.numErrors(); + Reader ar{cursor}; + ar.has("name"); + ar.has("type"); + ar("nom", out.nominal); + ar("vars", out.parameters); + ar("high", out.high); + ar("low", out.low); + readOthers(ar, out); + if (cursor.numErrors() == errorsBefore && + (out.high.size() != out.parameters.size() || out.low.size() != out.parameters.size())) { + cursor.error("has non-matching lengths of 'vars', 'high' and 'low'"); + } + readInterpolations(ar, out.parameters.size(), interpolationClass, out.interpolations); return out; } +} // namespace + +InterpolatedFunction parseInterpolatedFunction(JSONNode const &node, Diagnostics &diagnostics) +{ + return parseInterpolated( + node, diagnostics, InterpolationClass::Piecewise, + [](Reader &ar, InterpolatedFunction &out) { ar("positiveDefinite", out.positiveDefinite); }); +} + +InterpolatedNumber parseInterpolatedNumber(JSONNode const &node, Diagnostics &diagnostics) +{ + return parseInterpolated(node, diagnostics, InterpolationClass::Flexible, + [](Reader &, InterpolatedNumber &) {}); +} + } // namespace HS3 } // namespace RooFit diff --git a/roofit/hs3/src/HS3Schema.h b/roofit/hs3/src/HS3Schema.h index 01663f64f370c..c807f41535504 100644 --- a/roofit/hs3/src/HS3Schema.h +++ b/roofit/hs3/src/HS3Schema.h @@ -97,6 +97,10 @@ void readValue(Cursor const &c, bool &out); void readValue(Cursor const &c, ParameterValue &out); void readValue(Cursor const &c, Data &out); void readValue(Cursor const &c, Interpolation &out); +void readValue(Cursor const &c, NameOrNumber &out); + +/// Reads an integer interpolation code, as older versions of RooFit wrote them. +std::optional readInterpolationCode(Cursor const &c, InterpolationClass interpolationClass); template void readValue(Cursor const &c, Ref &out) diff --git a/roofit/hs3/src/HS3Upgrade.cxx b/roofit/hs3/src/HS3Upgrade.cxx index ca00b463c6b02..d5be0bcf733f9 100644 --- a/roofit/hs3/src/HS3Upgrade.cxx +++ b/roofit/hs3/src/HS3Upgrade.cxx @@ -16,7 +16,6 @@ #include "HS3Schema.h" -#include #include #include #include @@ -127,34 +126,9 @@ void upgradeInterpolationCode(JSONNode &mod, InterpolationClass interpolationCla return; } JSONNode &node = mod["interpolation"]; - Cursor nodeCursor{node, c, "interpolation"}; - if (!isScalar(node)) { - nodeCursor.error("must be a structured interpolation or a legacy integer code"); - return; - } - const std::string value = node.val(); - int code = 0; - const auto result = std::from_chars(value.data(), value.data() + value.size(), code); - if (result.ec != std::errc{} || result.ptr != value.data() + value.size()) { - nodeCursor.error("has invalid legacy interpolation code '" + value + "'"); - return; - } - auto interpolation = interpolationFromCode(code, interpolationClass); - if (!interpolation) { - nodeCursor.error( - std::string("has unsupported ") + - (interpolationClass == InterpolationClass::Piecewise ? "PiecewiseInterpolation" : "FlexibleInterpVar") + - " code " + value); - return; - } - node.set_null(); - node.set_map(); - node["type"] << toString(interpolation->type); - node["in"] << toString(interpolation->in); - if (interpolation->out) { - node["out"] << toString(*interpolation->out); - } else { - node["out"].set_null(); + if (auto interpolation = readInterpolationCode(Cursor{node, c, "interpolation"}, interpolationClass)) { + node.set_null(); + writeInterpolation(node, *interpolation); } } diff --git a/roofit/hs3/src/HS3Writer.cxx b/roofit/hs3/src/HS3Writer.cxx index ec8d25b027ebe..7fb2c1f021a09 100644 --- a/roofit/hs3/src/HS3Writer.cxx +++ b/roofit/hs3/src/HS3Writer.cxx @@ -18,6 +18,7 @@ #include +#include #include #include @@ -134,18 +135,6 @@ void writeAxes(JSONNode &node, std::vector const &axes) } } -void writeInterpolation(JSONNode &node, Interpolation const &interpolation) -{ - node.set_map(); - node["type"] << toString(interpolation.type); - node["in"] << toString(interpolation.in); - if (interpolation.out) { - node["out"] << toString(*interpolation.out); - } else { - node["out"].set_null(); - } -} - // The parameter of a modifier with a single parameter, and the name of its constraint distribution. A constraint type // is written in the form of older documents. void writeParameter(JSONNode &mod, ParameterRef const ¶meter, Constraint const &constraint) @@ -408,6 +397,71 @@ void writeContents(JSONNode &node, std::vector const &values) } } +void writeInterpolation(JSONNode &node, Interpolation const &interpolation) +{ + node.set_map(); + node["type"] << toString(interpolation.type); + node["in"] << toString(interpolation.in); + if (interpolation.out) { + node["out"] << toString(*interpolation.out); + } else { + node["out"].set_null(); + } +} + +namespace { + +void writeValue(JSONNode &node, NameOrNumber const &value) +{ + std::visit([&](auto const &v) { node << v; }, value); +} + +template +void writeValue(JSONNode &node, T const &value) +{ + node << value; +} + +template +void writeList(JSONNode &node, std::vector const &values) +{ + node.set_seq(); + for (auto const &value : values) { + writeValue(node.append_child(), value); + } +} + +template +void writeInterpolated(JSONNode &node, Function_t const &function, char const *type) +{ + node["type"] << type; + auto const &interpolations = function.interpolations; + const bool allEqual = std::all_of(interpolations.begin(), interpolations.end(), + [&](Interpolation const &i) { return i == interpolations.front(); }); + JSONNode &interpolationsNode = node["interpolations"].set_seq(); + for (std::size_t i = 0; i < (allEqual ? std::min(interpolations.size(), 1) : interpolations.size()); + ++i) { + writeInterpolation(interpolationsNode.append_child(), interpolations[i]); + } + writeList(node["vars"], function.parameters); + node["nom"] << function.nominal; + writeList(node["high"], function.high); + writeList(node["low"], function.low); +} + +} // namespace + +void writeInterpolatedFunction(JSONNode &node, InterpolatedFunction const &function) +{ + writeInterpolated(node, function, "interpolation"); + node["positiveDefinite"] << function.positiveDefinite; +} + +void writeInterpolatedNumber(JSONNode &node, InterpolatedNumber const &function) +{ + writeInterpolated(node, function, "interpolation0d"); +} + void writeHistFactoryDist(JSONNode &node, HistFactoryDist const &dist) { node.set_map(); diff --git a/roofit/hs3/src/JSONFactories_HistFactory.cxx b/roofit/hs3/src/JSONFactories_HistFactory.cxx index f274f2f623849..ce759f2b3f74f 100644 --- a/roofit/hs3/src/JSONFactories_HistFactory.cxx +++ b/roofit/hs3/src/JSONFactories_HistFactory.cxx @@ -89,43 +89,6 @@ std::string interpolationString(const Interpolation &interpolation) return ss.str(); } -Interpolation readInterpolation(const JSONNode &node, const std::string &context) -{ - HS3::Diagnostics diagnostics; - const Interpolation interpolation = HS3::parseInterpolation(node, diagnostics); - if (!diagnostics.empty()) { - RooJSONFactoryWSTool::error(context + " " + diagnostics.front().message); - } - return interpolation; -} - -void writeInterpolation(JSONNode &node, const Interpolation &interpolation) -{ - node.set_map(); - node["type"] << toString(interpolation.type); - node["in"] << toString(interpolation.in); - if (interpolation.out) { - node["out"] << toString(*interpolation.out); - } else { - node["out"].set_null(); - } -} - -int readLegacyInterpolationCode(const JSONNode &node, const std::string &context) -{ - if (node.is_container() || node.is_null() || !node.has_val()) { - RooJSONFactoryWSTool::error(context + " must be a structured interpolation or a legacy integer code"); - } - - const std::string value = node.val(); - int code = 0; - const auto result = std::from_chars(value.data(), value.data() + value.size(), code); - if (result.ec != std::errc{} || result.ptr != value.data() + value.size()) { - RooJSONFactoryWSTool::error(context + " has invalid legacy interpolation code '" + value + "'"); - } - return code; -} - const char *interpolationClassName(InterpolationClass interpolationClass) { return interpolationClass == InterpolationClass::Piecewise ? "PiecewiseInterpolation" : "FlexibleInterpVar"; @@ -150,91 +113,36 @@ int codeFromInterpolation(const Interpolation &interpolation, InterpolationClass interpolationClassName(interpolationClass)); } -void writeInterpolations(JSONNode &node, const std::vector &codes, InterpolationClass interpolationClass, - const std::string &context) +// The interpolations of the parameters of a PiecewiseInterpolation or FlexibleInterpVar. +std::vector +toInterpolations(std::vector const &codes, InterpolationClass interpolationClass, std::string const &context) { - auto &interpolations = node.set_seq(); - if (codes.empty()) { - return; - } - - std::vector descriptors; - descriptors.reserve(codes.size()); + std::vector out; for (std::size_t i = 0; i < codes.size(); ++i) { - descriptors.push_back( + out.push_back( interpolationFromCode(codes[i], interpolationClass, context + " at parameter index " + std::to_string(i))); } - - bool allEqual = true; - for (std::size_t i = 1; i < descriptors.size(); ++i) { - if (descriptors[i] != descriptors.front()) { - allEqual = false; - break; - } - } - - const std::size_t outputSize = allEqual ? 1 : descriptors.size(); - for (std::size_t i = 0; i < outputSize; ++i) { - writeInterpolation(interpolations.append_child(), descriptors[i]); - } + return out; } -std::vector readInterpolations(const JSONNode &object, std::size_t nParameters, - InterpolationClass interpolationClass, const std::string &context) +std::vector toCodes(std::vector const &interpolations, InterpolationClass interpolationClass, + std::string const &context) { - if (const auto *interpolations = object.find("interpolations")) { - if (!interpolations->is_seq()) { - RooJSONFactoryWSTool::error(context + " component 'interpolations' must be an array"); - } - - const std::size_t size = interpolations->num_children(); - const bool validSize = nParameters == 0 ? size == 0 : size == 1 || size == nParameters; - if (!validSize) { - RooJSONFactoryWSTool::error(context + - " component 'interpolations' must contain either one descriptor or one " - "descriptor per parameter (got " + - std::to_string(size) + " for " + std::to_string(nParameters) + " parameters)"); - } - - std::vector codes; - codes.reserve(size); - std::size_t i = 0; - for (const auto &node : interpolations->children()) { - const std::string entryContext = context + " component 'interpolations' at index " + std::to_string(i); - codes.push_back( - codeFromInterpolation(readInterpolation(node, entryContext), interpolationClass, entryContext)); - ++i; - } - if (size == 1) { - codes.resize(nParameters, codes.front()); - } - return codes; + std::vector out; + for (std::size_t i = 0; i < interpolations.size(); ++i) { + out.push_back(codeFromInterpolation(interpolations[i], interpolationClass, + context + " component 'interpolations' at index " + std::to_string(i))); } + return out; +} - std::vector codes(nParameters, 0); - if (const auto *legacyCodes = object.find("interpolationCodes")) { - if (!legacyCodes->is_seq()) { - RooJSONFactoryWSTool::error(context + " legacy component 'interpolationCodes' must be an array"); - } - if (legacyCodes->num_children() != nParameters) { - RooJSONFactoryWSTool::error(context + - " legacy component 'interpolationCodes' must contain one code per " - "parameter (got " + - std::to_string(legacyCodes->num_children()) + " for " + - std::to_string(nParameters) + " parameters)"); - } - - std::size_t i = 0; - for (const auto &node : legacyCodes->children()) { - const std::string entryContext = - context + " legacy component 'interpolationCodes' at index " + std::to_string(i); - const Interpolation interpolation = - interpolationFromCode(readLegacyInterpolationCode(node, entryContext), interpolationClass, entryContext); - codes[i] = codeFromInterpolation(interpolation, interpolationClass, entryContext); - ++i; - } +// Reports the problems with an object that doesn't come with a document. +void report(std::string const &name, HS3::Diagnostics &diagnostics) +{ + for (auto &diagnostic : diagnostics) { + diagnostic.path = name + (diagnostic.path.empty() ? "" : "." + diagnostic.path); } - return codes; + HS3::report(diagnostics); } int interpolationCode(std::optional const &interpolation, @@ -759,19 +667,21 @@ class FlexibleInterpVarStreamer : public RooFit::JSONIO::Exporter { bool exportObject(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem) const override { auto fip = static_cast(func); + const std::string context = "FlexibleInterpVar '" + std::string{fip->GetName()} + "'"; const std::size_t nParameters = fip->variables().size(); if (fip->low().size() != nParameters || fip->high().size() != nParameters || fip->interpolationCodes().size() != nParameters) { - RooJSONFactoryWSTool::error("FlexibleInterpVar '" + std::string{fip->GetName()} + - "' has non-matching parameter, variation, and interpolation lengths"); + RooJSONFactoryWSTool::error(context + " has non-matching parameter, variation, and interpolation lengths"); + } + HS3::InterpolatedNumber out; + for (RooAbsArg const *var : fip->variables()) { + out.parameters.push_back(var->GetName()); } - elem["type"] << key(); - writeInterpolations(elem["interpolations"], fip->interpolationCodes(), InterpolationClass::Flexible, - "FlexibleInterpVar '" + std::string{fip->GetName()} + "'"); - RooJSONFactoryWSTool::fillSeq(elem["vars"], fip->variables()); - elem["nom"] << fip->nominal(); - elem["high"].fill_seq(fip->high(), fip->variables().size()); - elem["low"].fill_seq(fip->low(), fip->variables().size()); + out.nominal = fip->nominal(); + out.high = fip->high(); + out.low = fip->low(); + out.interpolations = toInterpolations(fip->interpolationCodes(), InterpolationClass::Flexible, context); + HS3::writeInterpolatedNumber(elem, out); return true; } }; @@ -786,20 +696,22 @@ class PiecewiseInterpolationStreamer : public RooFit::JSONIO::Exporter { bool exportObject(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem) const override { const PiecewiseInterpolation *pip = static_cast(func); + const std::string context = "PiecewiseInterpolation '" + std::string{pip->GetName()} + "'"; const std::size_t nParameters = pip->paramList().size(); if (pip->lowList().size() != nParameters || pip->highList().size() != nParameters || pip->interpolationCodes().size() != nParameters) { - RooJSONFactoryWSTool::error("PiecewiseInterpolation '" + std::string{pip->GetName()} + - "' has non-matching parameter, variation, and interpolation lengths"); + RooJSONFactoryWSTool::error(context + " has non-matching parameter, variation, and interpolation lengths"); } - elem["type"] << key(); - writeInterpolations(elem["interpolations"], pip->interpolationCodes(), InterpolationClass::Piecewise, - "PiecewiseInterpolation '" + std::string{pip->GetName()} + "'"); - elem["positiveDefinite"] << pip->positiveDefinite(); - RooJSONFactoryWSTool::fillSeq(elem["vars"], pip->paramList()); - elem["nom"] << pip->nominalHist()->GetName(); - RooJSONFactoryWSTool::fillSeq(elem["high"], pip->highList(), pip->paramList().size()); - RooJSONFactoryWSTool::fillSeq(elem["low"], pip->lowList(), pip->paramList().size()); + HS3::InterpolatedFunction out; + out.nominal = pip->nominalHist()->GetName(); + for (std::size_t i = 0; i < nParameters; ++i) { + out.parameters.push_back(HS3::toNameOrNumber(pip->paramList()[i])); + out.high.push_back(HS3::toNameOrNumber(pip->highList()[i])); + out.low.push_back(HS3::toNameOrNumber(pip->lowList()[i])); + } + out.interpolations = toInterpolations(pip->interpolationCodes(), InterpolationClass::Piecewise, context); + out.positiveDefinite = pip->positiveDefinite(); + HS3::writeInterpolatedFunction(elem, out); return true; } }; @@ -809,22 +721,25 @@ class PiecewiseInterpolationFactory : public RooFit::JSONIO::Importer { bool importArg(RooJSONFactoryWSTool *tool, const JSONNode &p) const override { std::string name(RooJSONFactoryWSTool::name(p)); - - RooArgList vars{tool->requestArgList(p, "vars")}; - RooArgList low{tool->requestArgList(p, "low")}; - RooArgList high{tool->requestArgList(p, "high")}; - if (vars.size() != low.size() || vars.size() != high.size()) { - RooJSONFactoryWSTool::error("PiecewiseInterpolation '" + name + - "' has non-matching lengths of 'vars', 'high' and 'low'"); - } - const std::vector codes = - readInterpolations(p, vars.size(), InterpolationClass::Piecewise, "PiecewiseInterpolation '" + name + "'"); - - auto &pip = - tool->wsEmplace(name, *tool->requestArg(p, "nom"), low, high, vars, codes); - - pip.setPositiveDefinite(p["positiveDefinite"].val_bool()); - + HS3::Diagnostics diagnostics; + const HS3::InterpolatedFunction in = HS3::parseInterpolatedFunction(p, diagnostics); + report(name, diagnostics); + + auto request = [&](std::vector const &refs) { + RooArgList out; + for (auto const &ref : refs) { + auto const *value = std::get_if(&ref); + out.add(value ? RooFit::RooConst(*value) : *tool->request(std::get(ref), name)); + } + return out; + }; + RooArgList vars{request(in.parameters)}; + RooArgList low{request(in.low)}; + RooArgList high{request(in.high)}; + auto &pip = tool->wsEmplace( + name, *tool->request(in.nominal, name), low, high, vars, + toCodes(in.interpolations, InterpolationClass::Piecewise, "PiecewiseInterpolation '" + name + "'")); + pip.setPositiveDefinite(in.positiveDefinite); return true; } }; @@ -834,35 +749,17 @@ class FlexibleInterpVarFactory : public RooFit::JSONIO::Importer { bool importArg(RooJSONFactoryWSTool *tool, const JSONNode &p) const override { std::string name(RooJSONFactoryWSTool::name(p)); - if (!p.has_child("high")) { - RooJSONFactoryWSTool::error("no high variations of '" + name + "'"); - } - if (!p.has_child("low")) { - RooJSONFactoryWSTool::error("no low variations of '" + name + "'"); - } - if (!p.has_child("nom")) { - RooJSONFactoryWSTool::error("no nominal variation of '" + name + "'"); - } - - double nom(p["nom"].val_double()); - - RooArgList vars{tool->requestArgList(p, "vars")}; - - std::vector high; - high << p["high"]; + HS3::Diagnostics diagnostics; + const HS3::InterpolatedNumber in = HS3::parseInterpolatedNumber(p, diagnostics); + report(name, diagnostics); - std::vector low; - low << p["low"]; - - if (vars.size() != low.size() || vars.size() != high.size()) { - RooJSONFactoryWSTool::error("FlexibleInterpVar '" + name + - "' has non-matching lengths of 'vars', 'high' and 'low'!"); + RooArgList vars; + for (std::string const &var : in.parameters) { + vars.add(*tool->request(var, name)); } - const std::vector codes = - readInterpolations(p, vars.size(), InterpolationClass::Flexible, "FlexibleInterpVar '" + name + "'"); - - tool->wsEmplace(name, vars, nom, low, high, codes); - + tool->wsEmplace( + name, vars, in.nominal, in.low, in.high, + toCodes(in.interpolations, InterpolationClass::Flexible, "FlexibleInterpVar '" + name + "'")); return true; } }; diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index 06d2b44f84210..47a2ea670f1e9 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -617,6 +617,14 @@ HS3::BinnedAxis RooFit::HS3::toBinnedAxis(RooRealVar const &var) return axis; } +HS3::NameOrNumber RooFit::HS3::toNameOrNumber(RooAbsArg const &arg) +{ + if (isLiteralConstVar(arg)) { + return static_cast(arg).getVal(); + } + return std::string{arg.GetName()}; +} + namespace { std::unique_ptr createObservable(HS3::BinnedAxis const &axis) From a0c9288a1d68464afaa8f514f80b30ac11010c30 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Tue, 10 Nov 2026 17:24:29 +0000 Subject: [PATCH 19/20] [RF][HS3] Remove duplicated and dead code from the HS3 tool - Import a single variable without writing and parsing a document for it. - Read the axes of histograms with the parser instead of a second reader. - Drop the branches for useListsInsteadOfDicts, which is always true. - Check the size of channel-level staterror constraints in the validation instead of when building the model. Assisted-by: claude-code:claude-opus-5-5 --- .../hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h | 1 + roofit/hs3/src/HS3Document.h | 3 + roofit/hs3/src/HS3Parser.cxx | 7 ++ roofit/hs3/src/HS3Validate.cxx | 2 + roofit/hs3/src/JSONFactories_HistFactory.cxx | 15 +--- roofit/hs3/src/JSONIOUtils.cxx | 87 ------------------ roofit/hs3/src/JSONIOUtils.h | 3 - roofit/hs3/src/RooJSONFactoryWSTool.cxx | 90 ++++++++----------- 8 files changed, 51 insertions(+), 157 deletions(-) diff --git a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h index 63306209bd401..91e6e8389dd60 100644 --- a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h +++ b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h @@ -49,6 +49,7 @@ class RooRealVar; class RooJSONFactoryWSTool { public: + /// Named objects are always stored in lists, as the HS3 standard requires. Kept for backwards compatibility. static constexpr bool useListsInsteadOfDicts = true; struct Config { diff --git a/roofit/hs3/src/HS3Document.h b/roofit/hs3/src/HS3Document.h index 52ca3394a6ac3..317e558a0d1e3 100644 --- a/roofit/hs3/src/HS3Document.h +++ b/roofit/hs3/src/HS3Document.h @@ -546,6 +546,9 @@ HS3Document parseDocument(Detail::JSONNode const &root, Diagnostics &diagnostics /// Reads a HistFactory distribution that doesn't come with a document. HistFactoryDist parseHistFactoryDist(Detail::JSONNode const &distribution, Diagnostics &diagnostics); +/// Reads the axes of binned data that doesn't come with a document. +std::vector parseBinnedAxes(Detail::JSONNode const &axes, Diagnostics &diagnostics); + /// Reads the functions "interpolation" and "interpolation0d", which don't come with a document. Older versions of /// RooFit wrote integer "interpolationCodes" instead of "interpolations". InterpolatedFunction parseInterpolatedFunction(Detail::JSONNode const &node, Diagnostics &diagnostics); diff --git a/roofit/hs3/src/HS3Parser.cxx b/roofit/hs3/src/HS3Parser.cxx index 60457d8c294ad..527c701a63415 100644 --- a/roofit/hs3/src/HS3Parser.cxx +++ b/roofit/hs3/src/HS3Parser.cxx @@ -1345,6 +1345,13 @@ Function_t parseInterpolated(JSONNode const &node, Diagnostics &diagnostics, Int } // namespace +std::vector parseBinnedAxes(JSONNode const &axes, Diagnostics &diagnostics) +{ + std::vector out; + readValue(Cursor{axes, diagnostics}, out); + return out; +} + InterpolatedFunction parseInterpolatedFunction(JSONNode const &node, Diagnostics &diagnostics) { return parseInterpolated( diff --git a/roofit/hs3/src/HS3Validate.cxx b/roofit/hs3/src/HS3Validate.cxx index 1028458f7f163..eaebe10785ca0 100644 --- a/roofit/hs3/src/HS3Validate.cxx +++ b/roofit/hs3/src/HS3Validate.cxx @@ -486,6 +486,8 @@ void Validator::checkHistFactory(HistFactoryDist const &dist, std::string const } }; + checkGammaConstraint(path + ".staterror", dist.statError.constraint); + std::set sampleNames; for (std::size_t iSample = 0; iSample < dist.samples.size(); ++iSample) { auto const &sample = dist.samples[iSample]; diff --git a/roofit/hs3/src/JSONFactories_HistFactory.cxx b/roofit/hs3/src/JSONFactories_HistFactory.cxx index ce759f2b3f74f..d5414b2edbd03 100644 --- a/roofit/hs3/src/JSONFactories_HistFactory.cxx +++ b/roofit/hs3/src/JSONFactories_HistFactory.cxx @@ -312,16 +312,9 @@ ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, { RooWorkspace &ws = *tool.workspace(); - std::size_t n = std::max(vals.size(), parnames.size()); - if (n == 0) { - n = nBins; - } - if (!parnames.empty() && !vals.empty() && parnames.size() != vals.size()) { - RooJSONFactoryWSTool::error("the modifier '" + sysname + "' has " + std::to_string(vals.size()) + - " values, but " + std::to_string(parnames.size()) + " parameters"); - } + // The validation made sure that there is a value, parameter and constraint for each bin, if they are given. RooArgList gammas; - for (std::size_t i = 0; i < n; ++i) { + for (std::size_t i = 0; i < nBins; ++i) { const std::string name = parnames.empty() ? defaultGammaName(sysname, i) : parnames[i].name; if (auto *existing = dynamic_cast(ws.obj(name))) { gammas.add(*existing); @@ -352,10 +345,6 @@ ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, }; if (auto const *pdfs = std::get_if(&constraint)) { - if (pdfs->pdfs.size() != n) { - RooJSONFactoryWSTool::error("modifier '" + sysname + "' has " + std::to_string(pdfs->pdfs.size()) + - " constraints, but " + std::to_string(n) + " parameters"); - } for (auto const &pdfName : pdfs->pdfs) { if (!pdfName) { continue; diff --git a/roofit/hs3/src/JSONIOUtils.cxx b/roofit/hs3/src/JSONIOUtils.cxx index bcd5e311ee694..51dc6e597c3fb 100644 --- a/roofit/hs3/src/JSONIOUtils.cxx +++ b/roofit/hs3/src/JSONIOUtils.cxx @@ -1,14 +1,12 @@ #include "JSONIOUtils.h" #include -#include #include #include #include using RooFit::Detail::JSONNode; -using RooFit::Detail::JSONTree; bool startsWith(std::string_view str, std::string_view prefix) { @@ -28,62 +26,6 @@ std::string removeSuffix(std::string_view str, std::string_view suffix) return out; } -std::unique_ptr varJSONString(const JSONNode &treeRoot) -{ - std::string varName = treeRoot.find("name")->val(); - double val = 0; - double maxVal = 0; - double minVal = 0; - bool isConstant = false; - bool isRange = false; - - if (auto n = treeRoot.find("value")) { - val = n->val_double(); - isConstant = true; - } - - auto maxNode = treeRoot.find("max"); - auto minNode = treeRoot.find("min"); - if (maxNode && minNode) { - maxVal = maxNode->val_double(); - minVal = minNode->val_double(); - isRange = true; - } - if (!isConstant) { - val = (maxVal + minVal) / 2; - } - // Check if variable is at least a range or constant else throw error - if (!isConstant && !isRange) { - throw std::invalid_argument("Invalid Syntax: Please provide either 'value' or 'min' and 'max' or both"); - } - - std::unique_ptr jsonDict = RooFit::Detail::JSONTree::create(); - JSONNode &n = jsonDict->rootnode().set_map(); - JSONNode &_domains = n["domains"].set_seq().append_child().set_map(); - JSONNode &_parameterPoints = n["parameter_points"].set_seq().append_child().set_map(); - - _domains["name"] << "default_domain"; - _domains["type"] << "product_domain"; - JSONNode &_axes = _domains["axes"].set_seq().append_child().set_map(); - _axes["name"] << varName; - - _parameterPoints["name"] << "default_values"; - JSONNode &_parameters = _parameterPoints["parameters"].set_seq().append_child().set_map(); - _parameters["name"] << varName; - _parameters["value"] << val; - - if (isRange) { - _axes["max"] << maxVal; - _axes["min"] << minVal; - } - - if (isConstant && !isRange) { - _parameters["const"] << true; - } - - return jsonDict; -} - void writeAxisBinning(JSONNode &node, const RooAbsBinning &binning) { node["min"] << binning.lowBound(); @@ -100,16 +42,6 @@ void writeAxisBinning(JSONNode &node, const RooAbsBinning &binning) } } -// Read a bound of an axis. Infinite bounds are written as null. -double readBound(JSONNode const &node, const char *key, double defaultValue) -{ - if (!node.has_child(key)) { - return defaultValue; - } - auto const &bound = node[key]; - return bound.is_null() ? defaultValue : bound.val_double(); -} - // Parse the leading "MAJOR.MINOR" of an HS3 version string like "0.2", "0.2.9" or "0.3". bool parseHS3Version(std::string const &version, int &major, int &minor) { @@ -144,22 +76,3 @@ bool isHS3VersionBefore(std::string const &version, int major, int minor) } return std::make_pair(versionMajor, versionMinor) < std::make_pair(major, minor); } - -// Read the bin edges of an axis, which must be at least two strictly ascending numbers. -std::vector readEdges(const JSONNode &node, std::string const &context) -{ - if (!node.is_seq()) { - RooJSONFactoryWSTool::error("\"edges\" of " + context + " must be a list"); - } - std::vector edges; - for (auto const &edge : node.children()) { - edges.push_back(edge.val_double()); - if (edges.size() > 1 && !(edges.back() > edges[edges.size() - 2])) { - RooJSONFactoryWSTool::error("\"edges\" of " + context + " must be strictly ascending"); - } - } - if (edges.size() < 2) { - RooJSONFactoryWSTool::error("\"edges\" of " + context + " must contain at least two values"); - } - return edges; -} diff --git a/roofit/hs3/src/JSONIOUtils.h b/roofit/hs3/src/JSONIOUtils.h index 2e8dd007c30ae..ac2f807345680 100644 --- a/roofit/hs3/src/JSONIOUtils.h +++ b/roofit/hs3/src/JSONIOUtils.h @@ -10,15 +10,12 @@ class RooAbsBinning; bool startsWith(std::string_view str, std::string_view prefix); bool endsWith(std::string_view str, std::string_view suffix); std::string removeSuffix(std::string_view str, std::string_view suffix); -std::unique_ptr varJSONString(const RooFit::Detail::JSONNode &treeRoot); void writeAxisBinning(RooFit::Detail::JSONNode &node, const RooAbsBinning &binning); -double readBound(RooFit::Detail::JSONNode const &node, const char *key, double defaultValue); // The key of a fixed domain axis (HS3 #131): the parameter only takes this value, and RooFit reads it as a // RooConstVar. inline constexpr const char *fixedValueKey = "value"; bool parseHS3Version(std::string const &version, int &major, int &minor); bool isHS3VersionBefore(std::string const &version, int major, int minor); -std::vector readEdges(const RooFit::Detail::JSONNode &node, std::string const &context); #endif diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index 47a2ea670f1e9..1a3bcd8514465 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -720,6 +720,14 @@ loadData(HS3::Data const &input, RooWorkspace &workspace, HS3::DomainRanges cons RooJSONFactoryWSTool::error("datasets of type \"point\" like '" + input.name + "' are not supported yet"); } +// The version of ROOT that writes the document, with dots only such that it can be read automatically. +std::string rootVersion() +{ + std::string versionName = gROOT->GetVersion(); + std::replace(versionName.begin(), versionName.end(), '/', '.'); + return versionName; +} + // A distribution of the document can be a function in RooFit, or it can be skipped. void requirePdf(RooAbsArg const *arg, std::string const &what) { @@ -1074,9 +1082,6 @@ void RooJSONFactoryWSTool::fillSeq(JSONNode &node, RooAbsCollection const &coll, JSONNode &RooJSONFactoryWSTool::appendNamedChild(JSONNode &node, std::string const &name) { - if (!useListsInsteadOfDicts) { - return node.set_map()[name].set_map(); - } JSONNode &child = node.set_seq().append_child().set_map(); child["name"] << name; return child; @@ -1084,11 +1089,6 @@ JSONNode &RooJSONFactoryWSTool::appendNamedChild(JSONNode &node, std::string con JSONNode const *RooJSONFactoryWSTool::findNamedChild(JSONNode const &node, std::string const &name) { - if (!useListsInsteadOfDicts) { - if (!node.is_map()) - return nullptr; - return node.find(name); - } if (!node.is_seq()) return nullptr; for (JSONNode const &child : node.children()) { @@ -1145,7 +1145,7 @@ bool RooJSONFactoryWSTool::testValidName(const std::string &name, bool forceErro std::string RooJSONFactoryWSTool::name(const JSONNode &n) { - return useListsInsteadOfDicts ? n["name"].val() : n.key(); + return n["name"].val(); } JSONNode &RooJSONFactoryWSTool::makeVariablesNode(JSONNode &rootNode) @@ -1362,9 +1362,7 @@ void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func, std::set edges = readEdges(node["edges"], "axis '" + node["name"].val() + "'"); - auto obs = std::make_unique(node["name"].val().c_str(), node["name"].val().c_str(), edges[0], - edges[edges.size() - 1]); - RooBinning bins(obs->getMin(), obs->getMax()); - for (auto b : edges) { - bins.addBoundary(b); - } - obs->setBinning(bins); - vars.addOwned(std::move(obs)); - } else { - auto obs = std::make_unique(node["name"].val().c_str(), node["name"].val().c_str(), - node["min"].val_double(), node["max"].val_double()); - obs->setBins(node["nbins"].val_int()); - vars.addOwned(std::move(obs)); - } - } - - return vars; + HS3::Diagnostics diagnostics; + std::vector axes = HS3::parseBinnedAxes(topNode["axes"], diagnostics); + HS3::report(diagnostics); + return HS3::createObservables(axes); } /** @@ -2100,11 +2080,7 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) HS3Document &doc = _export->doc; doc.metadata.hs3Version = hs3VersionTag; - // Add information about the ROOT version that was used to generate this file. We want to consistently use dots - // such that the version name can be easily digested automatically. - std::string versionName = gROOT->GetVersion(); - std::replace(versionName.begin(), versionName.end(), '/', '.'); - doc.metadata.packages.push_back({"ROOT", versionName}); + doc.metadata.packages.push_back({"ROOT", rootVersion()}); doc.parameterPoints.push_back({"default_values", {}}); // export all toplevel pdfs @@ -2253,13 +2229,7 @@ std::unique_ptr RooJSONFactoryWSTool::createNewJSONTree() // add the mandatory hs3 version number metadata["hs3_version"] << hs3VersionTag; - // Add information about the ROOT version that was used to generate this file - auto &rootInfo = appendNamedChild(metadata["packages"], "ROOT"); - std::string versionName = gROOT->GetVersion(); - // We want to consistently use dots such that the version name can be easily - // digested automatically. - std::replace(versionName.begin(), versionName.end(), '/', '.'); - rootInfo["version"] << versionName; + appendNamedChild(metadata["packages"], "ROOT")["version"] << rootVersion(); return tree; } @@ -2536,13 +2506,26 @@ void RooJSONFactoryWSTool::importJSONElement(const std::string &name, const std: void RooJSONFactoryWSTool::importVariableElement(const JSONNode &elementNode) { - std::unique_ptr tree = varJSONString(elementNode); - - HS3::Diagnostics diagnostics; - HS3::ParseOptions options; - options.requireMetadata = false; - HS3Document doc = HS3::parseDocument(tree->rootnode(), diagnostics, options); - HS3::report(diagnostics); + // A variable is given by its "value", by its range "min" and "max", or by both. With only a value, it is constant. + JSONNode const *value = elementNode.find("value"); + JSONNode const *min = elementNode.find("min"); + JSONNode const *max = elementNode.find("max"); + if (!value && !(min && max)) { + throw std::invalid_argument("Invalid Syntax: Please provide either 'value' or 'min' and 'max' or both"); + } + HS3::DomainAxis axis; + axis.name = elementNode["name"].val(); + HS3::ParameterValue parameter; + parameter.name = axis.name; + if (min && max) { + axis.min = min->val_double(); + axis.max = max->val_double(); + } + parameter.value = value ? value->val_double() : (*axis.max + *axis.min) / 2; + parameter.isConst = value && !(min && max); + HS3Document doc; + doc.domains.push_back({HS3::defaultDomainName, {std::move(axis)}}); + doc.parameterPoints.push_back({"default_values", {parameter}}); _import = std::make_unique(doc); struct ImportScope { @@ -2550,7 +2533,6 @@ void RooJSONFactoryWSTool::importVariableElement(const JSONNode &elementNode) ~ImportScope() { tool._import.reset(); } } importScope{*this}; - HS3::ParameterValue const ¶meter = doc.parameterPoints.front().parameters.front(); importVariable(parameter); if (RooRealVar *rrv = _workspace.var(parameter.name)) { configureVariable(_import->ranges, parameter, *rrv); From ce3c9569dab6350c3d933b801ba47abd361d731b Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Tue, 10 Nov 2026 17:24:38 +0000 Subject: [PATCH 20/20] [RF][HS3] Don't promise a Writer in the documentation of HS3::Reader There is no Writer that shares the describe() functions. Most structs are read with legacy keys and defaults that the writer doesn't mirror, so sharing them would not save code. Assisted-by: claude-code:claude-opus-5-5 --- roofit/hs3/src/HS3Schema.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roofit/hs3/src/HS3Schema.h b/roofit/hs3/src/HS3Schema.h index c807f41535504..6dfcebed75f60 100644 --- a/roofit/hs3/src/HS3Schema.h +++ b/roofit/hs3/src/HS3Schema.h @@ -55,8 +55,7 @@ class Cursor { std::size_t _index = 0; }; -/// Reads the components of a struct. The functions take the same arguments as the ones of the Writer, such that a -/// single function can describe both directions. +/// Reads the components of a struct. class Reader { public: explicit Reader(Cursor const &cursor);