diff --git a/roofit/hs3/CMakeLists.txt b/roofit/hs3/CMakeLists.txt index ffeea5e753c31..e0c92c1e927d6 100644 --- a/roofit/hs3/CMakeLists.txt +++ b/roofit/hs3/CMakeLists.txt @@ -19,7 +19,11 @@ ROOT_STANDARD_LIBRARY_PACKAGE(RooFitHS3 HEADERS ${ROOFITHS3_HEADERS} SOURCES - src/Domains.cxx + src/HS3Document.cxx + src/HS3Parser.cxx + src/HS3Upgrade.cxx + src/HS3Validate.cxx + src/HS3Writer.cxx src/JSONInterface.cxx src/JSONIO.cxx src/RooJSONFactoryWSTool.cxx @@ -48,8 +52,11 @@ target_sources( FILES ${RELATIVE_INC_HEADERS} # LinkDef.h # seemingly not being used by ROOT_STANDARD_LIBRARY_PACKAGE call - src/Domains.h + src/HS3Builder.h + src/HS3Document.h + src/HS3Schema.h src/JSONIOUtils.h + src/JSONReadRecorder.h src/static_execute.h ) diff --git a/roofit/hs3/inc/RooFit/Detail/JSONInterface.h b/roofit/hs3/inc/RooFit/Detail/JSONInterface.h index 10abfbd941ec3..faa4aaf6c3bb1 100644 --- a/roofit/hs3/inc/RooFit/Detail/JSONInterface.h +++ b/roofit/hs3/inc/RooFit/Detail/JSONInterface.h @@ -92,6 +92,8 @@ class JSONNode { JSONNode &set_seq(); JSONNode &set_null(); void clear(); + /// Remove a child of a map, if it exists. + void erase(std::string const &key); std::string key() const; std::string val() const; @@ -188,6 +190,7 @@ class JSONNode { private: friend class JSONTree; + friend class JSONReadRecorder; // Owns the values and the nodes of a tree. It is defined in the only source file that knows the JSON library. class Storage; diff --git a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h index b0bd15cf2208f..91e6e8389dd60 100644 --- a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h +++ b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h @@ -23,14 +23,24 @@ #include #include #include -#include +#include + +class RooJSONFactoryWSTool; namespace RooFit { -namespace JSONIO { -namespace Detail { -class Domains; -} -} // namespace JSONIO +namespace HS3 { +struct Distribution; +struct HistFactoryDist; +struct ParameterPoint; +struct ParameterValue; +struct ToolExportState; +struct ToolImportState; +template +struct Ref; +using DistributionRef = Ref; +void setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist dist, + std::vector channelConstraints); +} // namespace HS3 } // namespace RooFit namespace RooStats { class ModelConfig; @@ -39,6 +49,7 @@ class RooRealVar; class RooJSONFactoryWSTool { public: + /// Named objects are always stored in lists, as the HS3 standard requires. Kept for backwards compatibility. static constexpr bool useListsInsteadOfDicts = true; struct Config { @@ -235,44 +246,39 @@ class RooJSONFactoryWSTool { void exportData(RooAbsData const &data); RooJSONFactoryWSTool::CombinedData exportCombinedData(RooAbsData const &data); - void importAllNodes(const RooFit::Detail::JSONNode &n); + void importAllNodes(RooFit::Detail::JSONNode &n); + void importDistribution(RooFit::HS3::Distribution const &distribution); - void importVariable(const RooFit::Detail::JSONNode &p); - void checkFixedValue(std::string const &name, const RooFit::Detail::JSONNode &p, std::string const &pointName); + void importVariable(RooFit::HS3::ParameterValue const &p); void warnIfFreeWithoutDomain(RooRealVar const &var); void importDependants(const RooFit::Detail::JSONNode &n); - void exportVariable(const RooAbsArg *v, RooFit::Detail::JSONNode &n, bool storeConstant, bool storeBins); - void exportVariables(const RooArgSet &allElems, RooFit::Detail::JSONNode &n, bool storeConstant, bool storeBins); + void exportVariable(const RooAbsArg *v, RooFit::HS3::ParameterPoint &point, bool storeConstant); + void exportVariables(const RooArgSet &allElems, RooFit::HS3::ParameterPoint &point, bool storeConstant); void exportAllObjects(RooFit::Detail::JSONNode &n); - void exportModelConfig(RooFit::Detail::JSONNode &rootnode, RooStats::ModelConfig const &mc, + void exportModelConfig(RooStats::ModelConfig const &mc, const std::vector &combined, const std::vector &single); - void exportSingleModelConfig(RooFit::Detail::JSONNode &rootnode, RooStats::ModelConfig const &mc, - std::string const &analysisName, + void exportSingleModelConfig(RooStats::ModelConfig const &mc, std::string const &analysisName, std::map const *dataComponents); + friend void RooFit::HS3::setExportedObject(RooJSONFactoryWSTool &tool, RooFit::HS3::HistFactoryDist dist, + std::vector channelConstraints); + // member variables - const RooFit::Detail::JSONNode *_rootnodeInput = nullptr; - const RooFit::Detail::JSONNode *_attributesNode = nullptr; - RooFit::Detail::JSONNode *_rootnodeOutput = nullptr; - RooFit::Detail::JSONNode *_varsNode = nullptr; std::string _hs3VersionInput; RooWorkspace &_workspace; // objects to represent intermediate information - std::unique_ptr _domains; + std::unique_ptr _import; + std::unique_ptr _export; std::vector _serversToExport; std::vector _serversToDelete; - // Name-keyed indices over the top-level "functions" and "distributions" - // sequences of the input JSON. Built once at the start of importAllNodes() - // so that requestImpl() lookups become O(1) instead of an O(N) scan over - // every sibling node. - std::unordered_map _functionsByName; - std::unordered_map _distributionsByName; + // The objects that are currently being imported, innermost last, to detect circular dependencies. + std::vector _importStack; }; #endif diff --git a/roofit/hs3/src/Domains.cxx b/roofit/hs3/src/Domains.cxx deleted file mode 100644 index b02197eb38471..0000000000000 --- a/roofit/hs3/src/Domains.cxx +++ /dev/null @@ -1,398 +0,0 @@ -/* - * Project: RooFit - * Authors: - * Jonas Rembser, CERN, Jan 2023 - * - * Copyright (c) 2023, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#include "Domains.h" -#include "JSONIOUtils.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace RooFit { -namespace JSONIO { -namespace Detail { - -constexpr static auto defaultDomainName = "default_domain"; - -namespace { - -// JSON can't represent infinity, so an infinite bound is expressed by omitting it. An absent bound is read as -// infinite by readBound() in JSONIOUtils. -void writeBound(RooFit::Detail::JSONNode &node, const char *key, double value) -{ - if (!RooNumber::isInfinite(value)) { - node[key] << value; - } -} - -} // namespace - -void Domains::populate(RooWorkspace &ws) const -{ - // Parameters with a single value are RooConstVars, whichever domain gives the value. - for (const auto &domain : _map) { - domain.second.populateFixedValues(ws); - } - auto default_domain = _map.find(defaultDomainName); - if (default_domain != _map.end()) { - default_domain->second.populate(ws); - } else { - // Without the RooFit-specific "default_domain", the variables are created with the range of the domain that - // mentions them. - for (const auto &domain : _map) { - domain.second.populate(ws); - } - } - for (const auto &domain : _map) { - if (domain.first == defaultDomainName) - continue; - domain.second.registerBinnings(domain.first.c_str(), ws); - } -} - -void Domains::readVariable(RooRealVar const &var) -{ - _map[defaultDomainName].readVariable(var.GetName(), var.getBinning()); - for (const auto &bname : var.getBinningNames()) { - if (bname.empty()) - continue; - auto &binning = var.getBinning(bname.c_str()); - _map[bname].readVariable(var.GetName(), binning); - } -} - -void Domains::writeVariable(RooRealVar &var) const -{ - auto default_domain = _map.find(defaultDomainName); - if (default_domain != _map.end()) { - default_domain->second.writeVariable(var); - return; - } - // Without the RooFit-specific "default_domain", use the first domain that mentions the variable. - for (const auto &domain : _map) { - if (domain.second.hasVariable(var.GetName())) { - domain.second.writeVariable(var); - return; - } - } -} - -void Domains::readJSON(RooFit::Detail::JSONNode const &node) -{ - // The "default_domain" is written by RooFit, but it is not required by the HS3 standard. - for (auto &domain : node.children()) { - if (!domain.has_child("name")) { - RooJSONFactoryWSTool::error("encountered domain without \"name\""); - } - auto &name = domain["name"]; - _map[name.val()].readJSON(domain); - } - // A parameter with a single value has no other domain: every domain that lists it must give it the same value. - for (auto const &[domainName, domain] : _map) { - for (auto const &[varName, elem] : domain._map) { - if (!elem.hasFixedValue) { - continue; - } - for (auto const &[otherName, other] : _map) { - auto found = other._map.find(varName); - if (found == other._map.end()) { - continue; - } - if (!found->second.hasFixedValue) { - RooJSONFactoryWSTool::error("the parameter '" + varName + "' has the single value " + - std::to_string(elem.fixedValue) + " in the domain '" + domainName + - "', but a range in the domain '" + otherName + "'"); - } - if (found->second.fixedValue != elem.fixedValue) { - RooJSONFactoryWSTool::error("the parameter '" + varName + - "' has different single values in the " - "domains '" + - domainName + "' and '" + otherName + "'"); - } - } - } - } -} - -void Domains::readFixedValue(const char *name, double value) -{ - _map[defaultDomainName].readFixedValue(name, value); -} - -double const *Domains::fixedValue(const char *name) const -{ - for (auto const &domain : _map) { - if (double const *value = domain.second.fixedValue(name)) { - return value; - } - } - return nullptr; -} - -void Domains::writeJSON(RooFit::Detail::JSONNode &node) const -{ - for (auto const &domain : _map) { - // Avoid writing a domain that was already written - if (!RooJSONFactoryWSTool::findNamedChild(node, domain.first)) { - domain.second.writeJSON(RooJSONFactoryWSTool::appendNamedChild(node, domain.first)); - } - } -} - -bool Domains::hasVariable(const char *name) const -{ - for (auto const &domain : _map) { - if (domain.second.hasVariable(name)) { - return true; - } - } - return false; -} - -void Domains::ProductDomain::readVariable(RooRealVar const &var) -{ - readVariable(var.GetName(), var.getBinning()); -} - -void Domains::ProductDomain::readBinning(ProductDomainElement &elem, RooAbsBinning const &binning) -{ - elem.hasNBins = false; - elem.nBins = 0; - elem.edges.clear(); - - const int nBins = binning.numBins(); - if (nBins <= 0) { - return; - } - - if (binning.isUniform()) { - elem.hasNBins = true; - elem.nBins = nBins; - } else { - elem.edges.push_back(binning.binLow(0)); - for (int i = 0; i < nBins; ++i) { - elem.edges.push_back(binning.binHigh(i)); - } - } -} - -void Domains::ProductDomain::readVariable(const char *name, RooAbsBinning const &binning) -{ - auto &elem = _map[name]; - - elem.hasMin = true; - elem.min = binning.lowBound(); - elem.hasMax = true; - elem.max = binning.highBound(); - readBinning(elem, binning); -} - -void Domains::ProductDomain::applyBinning(RooRealVar &var, ProductDomainElement const &elem, const char *name) -{ - if (!elem.edges.empty()) { - RooBinning binning(elem.edges.front(), elem.edges.back()); - for (double edge : elem.edges) { - binning.addBoundary(edge); - } - var.setBinning(binning, name); - } else if (elem.hasNBins && elem.nBins != 0) { - var.setBins(elem.nBins, name); - } -} - -void Domains::ProductDomain::writeBinning(RooFit::Detail::JSONNode &node, ProductDomainElement const &elem) -{ - if (!elem.edges.empty()) { - auto &edges = node["edges"].set_seq(); - for (double edge : elem.edges) { - edges.append_child() << edge; - } - } else if (elem.hasNBins && elem.nBins != 0) { - node["nbins"] << elem.nBins; - } -} -void Domains::ProductDomain::writeVariable(RooRealVar &var) const -{ - auto found = _map.find(var.GetName()); - if (found != _map.end() && !found->second.hasFixedValue) { - auto const &elem = found->second; - if (elem.hasMin) { - if (RooNumber::isInfinite(elem.min)) { - var.removeMin(); - } else { - var.setMin(elem.min); - } - } - if (elem.hasMax) { - if (RooNumber::isInfinite(elem.max)) { - var.removeMax(); - } else { - var.setMax(elem.max); - } - } - applyBinning(var, elem); - } -} - -bool Domains::ProductDomain::hasVariable(const char *name) const -{ - return _map.find(name) != _map.end(); -} - -double const *Domains::ProductDomain::fixedValue(const char *name) const -{ - auto found = _map.find(name); - return found != _map.end() && found->second.hasFixedValue ? &found->second.fixedValue : nullptr; -} - -void Domains::ProductDomain::readFixedValue(const char *name, double value) -{ - auto &elem = _map[name]; - elem = ProductDomainElement{}; - elem.hasFixedValue = true; - elem.fixedValue = value; -} - -void Domains::ProductDomain::readJSON(RooFit::Detail::JSONNode const &node) -{ - if (!node.has_child("type") || node["type"].val() != "product_domain") { - RooJSONFactoryWSTool::error("only domains of type \"product_domain\" are currently supported!"); - } - for (auto const &varNode : node["axes"].children()) { - const std::string varName = RooJSONFactoryWSTool::name(varNode); - auto &elem = _map[varName]; - - if (varNode.has_child(fixedValueKey)) { - // The domain of the parameter is the set with this single value: no range, binning or categories. - for (const char *key : {"min", "max", "nbins", "edges", "categories"}) { - if (varNode.has_child(key)) { - RooJSONFactoryWSTool::error("the axis '" + varName + "' of the domain '" + - RooJSONFactoryWSTool::name(node) + "' has \"" + fixedValueKey + "\" and \"" + - key + "\", which exclude each other"); - } - } - auto const &valueNode = varNode[fixedValueKey]; - if (!valueNode.is_number()) { - RooJSONFactoryWSTool::error("the \"" + std::string(fixedValueKey) + "\" of the axis '" + varName + - "' must be a number"); - } - elem = ProductDomainElement{}; - elem.hasFixedValue = true; - elem.fixedValue = valueNode.val_double(); - continue; - } - - if (varNode.has_child("min")) { - elem.min = readBound(varNode, "min", -RooNumber::infinity()); - elem.hasMin = true; - } - if (varNode.has_child("max")) { - elem.max = readBound(varNode, "max", RooNumber::infinity()); - elem.hasMax = true; - } - if (varNode.has_child("edges")) { - elem.hasNBins = false; - elem.edges.clear(); - for (auto const &edge : varNode["edges"].children()) { - elem.edges.push_back(edge.val_double()); - } - if (!elem.edges.empty()) { - if (!elem.hasMin) { - elem.min = elem.edges.front(); - elem.hasMin = true; - } - if (!elem.hasMax) { - elem.max = elem.edges.back(); - elem.hasMax = true; - } - } - } else if (varNode.has_child("nbins")) { - elem.nBins = varNode["nbins"].val_int(); - elem.hasNBins = elem.nBins != 0; - elem.edges.clear(); - } - } -} -void Domains::ProductDomain::writeJSON(RooFit::Detail::JSONNode &node) const - -{ - node.set_map(); - node["type"] << "product_domain"; - - auto &variablesNode = node["axes"]; - - for (auto const &item : _map) { - auto const &elem = item.second; - RooFit::Detail::JSONNode &varnode = RooJSONFactoryWSTool::appendNamedChild(variablesNode, item.first); - if (elem.hasFixedValue) { - varnode[fixedValueKey] << elem.fixedValue; - continue; - } - const double min = elem.hasMin ? elem.min : -RooNumber::infinity(); - const double max = elem.hasMax ? elem.max : RooNumber::infinity(); - writeBound(varnode, "min", min); - writeBound(varnode, "max", max); - // A binning requires a bounded range. - if (!RooNumber::isInfinite(min) && !RooNumber::isInfinite(max)) { - writeBinning(varnode, elem); - } - } -} -void Domains::ProductDomain::populate(RooWorkspace &ws) const -{ - for (auto const &item : _map) { - const auto &name = item.first; - if (item.second.hasFixedValue) { - continue; // see populateFixedValues() - } - if (!ws.arg(name)) { - const auto &elem = item.second; - const double vMin = elem.hasMin ? elem.min : -RooNumber::infinity(); - const double vMax = elem.hasMax ? elem.max : RooNumber::infinity(); - RooRealVar var{name.c_str(), name.c_str(), vMin, vMax}; - applyBinning(var, elem); - ws.import(var, RooFit::Silence()); - } - } -} -void Domains::ProductDomain::populateFixedValues(RooWorkspace &ws) const -{ - for (auto const &[name, elem] : _map) { - if (elem.hasFixedValue && !ws.arg(name)) { - ws.import(RooConstVar{name.c_str(), name.c_str(), elem.fixedValue}, RooFit::Silence()); - } - } -} - -void Domains::ProductDomain::registerBinnings(const char *name, RooWorkspace &ws) const -{ - for (auto const &item : _map) { - auto *var = ws.var(item.first); - if (!var) - continue; - const double vMin = item.second.hasMin ? item.second.min : -RooNumber::infinity(); - const double vMax = item.second.hasMax ? item.second.max : RooNumber::infinity(); - var->setRange(name, vMin, vMax); - applyBinning(*var, item.second, name); - } -} - -} // namespace Detail -} // namespace JSONIO -} // namespace RooFit diff --git a/roofit/hs3/src/Domains.h b/roofit/hs3/src/Domains.h deleted file mode 100644 index b0825c0d8198c..0000000000000 --- a/roofit/hs3/src/Domains.h +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Project: RooFit - * Authors: - * Jonas Rembser, CERN, Jan 2023 - * - * Copyright (c) 2023, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#ifndef RooFit_JSONIO_Detail_Domains_h -#define RooFit_JSONIO_Detail_Domains_h - -#include -#include -#include - -class RooAbsBinning; -class RooRealVar; -class RooWorkspace; - -namespace RooFit { -namespace Detail { -class JSONNode; -class JSONTree; -} // namespace Detail -} // namespace RooFit - -namespace RooFit { -namespace JSONIO { -namespace Detail { - -class Domains { -public: - void readVariable(RooRealVar const &); - void writeVariable(RooRealVar &) const; - // A parameter that only takes the value `value` (a RooConstVar), written as an axis with a single value. - void readFixedValue(const char *name, double value); - // The single value of the parameter `name` if a domain has one for it, otherwise nullptr. - double const *fixedValue(const char *name) const; - - void readJSON(RooFit::Detail::JSONNode const &); - void writeJSON(RooFit::Detail::JSONNode &) const; - - bool hasVariable(const char *name) const; - - void populate(RooWorkspace &ws) const; - - class ProductDomain { - friend class Domains; - - public: - void readVariable(const RooRealVar &); - void readVariable(const char *name, RooAbsBinning const &binning); - void writeVariable(RooRealVar &) const; - - void readJSON(RooFit::Detail::JSONNode const &); - void writeJSON(RooFit::Detail::JSONNode &) const; - - bool hasVariable(const char *name) const; - double const *fixedValue(const char *name) const; - void readFixedValue(const char *name, double value); - - void populate(RooWorkspace &ws) const; - void populateFixedValues(RooWorkspace &ws) const; - void registerBinnings(const char *name, RooWorkspace &ws) const; - - private: - struct ProductDomainElement { - // An axis with a single value: the parameter only takes this value (HS3 #131). - bool hasFixedValue = false; - double fixedValue = 0.0; - bool hasMin = false; - bool hasMax = false; - double min = 0.0; - double max = 0.0; - bool hasNBins = false; - int nBins = 0; - std::vector edges; - }; - - static void applyBinning(RooRealVar &var, ProductDomainElement const &elem, const char *name = nullptr); - static void readBinning(ProductDomainElement &elem, RooAbsBinning const &binning); - static void writeBinning(RooFit::Detail::JSONNode &node, ProductDomainElement const &elem); - - std::map _map; - }; - - std::map _map; -}; - -} // namespace Detail -} // namespace JSONIO -} // namespace RooFit - -#endif diff --git a/roofit/hs3/src/HS3Builder.h b/roofit/hs3/src/HS3Builder.h new file mode 100644 index 0000000000000..dda8410ddc2a9 --- /dev/null +++ b/roofit/hs3/src/HS3Builder.h @@ -0,0 +1,72 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef RooFit_HS3Builder_h +#define RooFit_HS3Builder_h + +#include "HS3Document.h" + +#include + +#include +#include +#include + +class RooAbsArg; +class RooDataHist; +class RooJSONFactoryWSTool; +class RooRealVar; + +namespace RooFit { +namespace JSONIO { +class Importer; +} + +namespace HS3 { + +/// Forward the warnings to the message service, and throw if there are errors. +void report(Diagnostics const &diagnostics); + +/// Create the variables for the axes of binned data. +RooArgSet createObservables(std::vector const &axes); + +/// The axis of binned data for a variable. +BinnedAxis toBinnedAxis(RooRealVar const &var); + +/// The name of an object, or the value of a literal constant. +NameOrNumber toNameOrNumber(RooAbsArg const &arg); + +/// Create a RooDataHist with the contents and errors. If there are no errors, it doesn't store squared weights. +std::unique_ptr createDataHist(std::string const &name, RooArgSet const &vars, + std::vector const &contents, std::vector const *errors); + +/// Create the RooFit objects of a HistFactory distribution in the workspace of the tool. The constraint terms that no +/// modifier names are passed separately. +void buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string const &name, HistFactoryDist const &dist, + std::vector const *channelConstraints = nullptr); + +/// Called by the exporter of a HistFactory distribution, which is then written by the tool. The constraint terms that +/// no modifier names are passed separately. +void setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist dist, + std::vector channelConstraints); + +/// Whether this is the importer that RooFit registers for HistFactory distributions, as opposed to the one of a user. +bool isBuiltinHistFactoryImporter(JSONIO::Importer const &importer); + +/// What the built-in importer does with a HistFactory distribution, plus the constraint terms of the channel. +void importHistFactoryNode(RooJSONFactoryWSTool &tool, Detail::JSONNode const &node, + std::vector const *channelConstraints); + +} // namespace HS3 +} // namespace RooFit + +#endif diff --git a/roofit/hs3/src/HS3Document.cxx b/roofit/hs3/src/HS3Document.cxx new file mode 100644 index 0000000000000..1f1a1f8ab966e --- /dev/null +++ b/roofit/hs3/src/HS3Document.cxx @@ -0,0 +1,225 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +// Operations on documents that RooFit needs besides reading, upgrading, validating and writing them. + +#include "HS3Document.h" + +#include + +namespace RooFit { +namespace HS3 { + +DomainRanges::DomainRanges(std::vector const &domains) +{ + std::vector sorted; + for (Domain const &domain : domains) { + sorted.push_back(&domain); + } + std::stable_sort(sorted.begin(), sorted.end(), [](Domain const *l, Domain const *r) { return l->name < r->name; }); + const bool hasDefault = + std::any_of(domains.begin(), domains.end(), [](Domain const &d) { return d.name == defaultDomainName; }); + + for (Domain const *domain : sorted) { + // Of several axes of a variable in one domain, the last one counts. + std::map axes; + for (auto const &axis : domain->axes) { + if (auto const *continuous = std::get_if(&axis)) { + axes[continuous->name] = continuous; + _names.insert(continuous->name); + } + } + if (!hasDefault || domain->name == defaultDomainName) { + _ranges.insert(axes.begin(), axes.end()); + } + } +} + +DomainAxis const *DomainRanges::find(std::string const &name) const +{ + auto found = _ranges.find(name); + return found != _ranges.end() ? found->second : nullptr; +} + +namespace { + +std::string const &nameOf(Modifier const &modifier) +{ + return std::visit([](auto const &m) -> std::string const & { return m.name; }, modifier); +} + +char const *typeOf(Modifier const &modifier) +{ + return std::visit( + overloaded{[](NormFactor const &) { return "normfactor"; }, [](NormSys const &) { return "normsys"; }, + [](HistoSys const &) { return "histosys"; }, [](ShapeSys const &) { return "shapefactor"; }, + [](StatError const &) { return "staterror"; }, [](CustomModifier const &) { return "custom"; }}, + modifier); +} + +// What duplicates must have in common to be combined. +template +char const *differences(Modifier_t const &l, Modifier_t const &r) +{ + if (!(l.constraint == r.constraint)) { + return "constraint metadata differs"; + } + if (l.parameter.name != r.parameter.name) { + return "parameter metadata differs"; + } + if (l.interpolation != r.interpolation) { + return "interpolation behaviours differ"; + } + return nullptr; +} + +// Combines `modifier` into `merged`, or returns why that's not possible. +char const *combine(NormSys &merged, NormSys const &modifier, std::vector const &) +{ + if (char const *reason = differences(merged, modifier)) { + return reason; + } + // Multiplicatively combining two normsys is only faithful when the interpolation is done in log-space, so that + // f1(alpha) * f2(alpha) is again representable by a single normsys with the multiplied lo/hi factors. This holds + // for the piecewise exponential (exact everywhere) and for the polynomial exponential (exact at the +-1 sigma + // anchors and in the exponential extrapolation region). The linear-space interpolations would turn the product into + // a shape that cannot be represented by a single normsys. + if (merged.interpolation != multiplicativePiecewiseExponential && + merged.interpolation != multiplicativePolynomialExponential) { + return "multiplicative combination is only valid for log-space interpolation"; + } + merged.lo *= modifier.lo; + merged.hi *= modifier.hi; + return nullptr; +} + +char const *combine(HistoSys &merged, HistoSys const &modifier, std::vector const &nominal) +{ + if (char const *reason = differences(merged, modifier)) { + return reason; + } + if (merged.interpolation != additivePolynomialLinear) { + return "this interpolation cannot currently be combined for duplicate histosys modifiers"; + } + const std::size_t nBins = nominal.size(); + auto fits = [&](HistoSysVariation const &variation) { return variation.contents.size() == nBins; }; + if (!fits(merged.lo) || !fits(merged.hi) || !fits(modifier.lo) || !fits(modifier.hi)) { + return "histogram binning differs"; + } + for (std::size_t bin = 0; bin < nBins; ++bin) { + merged.lo.contents[bin] += modifier.lo.contents[bin] - nominal[bin]; + merged.hi.contents[bin] += modifier.hi.contents[bin] - nominal[bin]; + } + return nullptr; +} + +} // namespace + +void mergeDuplicateModifiers(HistFactoryDist &dist, std::string const &channel, Diagnostics &diagnostics) +{ + for (Sample &sample : dist.samples) { + auto error = [&](Modifier const &modifier, std::string const &reason) { + diagnostics.push_back({Diagnostic::Severity::Error, "", + "cannot combine duplicate modifier '" + nameOf(modifier) + "' of type '" + + typeOf(modifier) + "' in sample '" + sample.name + "' of channel '" + channel + + "': " + reason}); + }; + + // The duplicates of a type are next to each other, because the modifiers of a type are sorted by name. + std::vector merged; + std::size_t count = 0; // of the modifiers that the last merged one combines + bool failed = false; + auto warnIfCombined = [&]() { + if (count > 1 && !failed) { + diagnostics.push_back({Diagnostic::Severity::Warning, "", + "combined " + std::to_string(count) + " duplicate modifiers named '" + + nameOf(merged.back()) + "' of type '" + typeOf(merged.back()) + "' in sample '" + + sample.name + "' of channel '" + channel + "'"}); + } + }; + for (Modifier &modifier : sample.modifiers) { + Modifier *last = merged.empty() ? nullptr : &merged.back(); + const bool combinable = + std::holds_alternative(modifier) || std::holds_alternative(modifier); + if (combinable && last && last->index() == modifier.index() && nameOf(*last) == nameOf(modifier)) { + char const *reason = nullptr; + if (auto *normSys = std::get_if(last)) { + reason = combine(*normSys, std::get(modifier), sample.contents); + } else { + reason = combine(std::get(*last), std::get(modifier), sample.contents); + } + if (reason) { + error(modifier, reason); + failed = true; + } + ++count; + continue; + } + warnIfCombined(); + merged.push_back(std::move(modifier)); + count = 1; + failed = false; + } + warnIfCombined(); + sample.modifiers = std::move(merged); + + std::set> seen; + for (Modifier const &modifier : sample.modifiers) { + if (!seen.emplace(typeOf(modifier), nameOf(modifier)).second) { + error(modifier, "this modifier type cannot be combined without changing its meaning"); + } + } + } +} + +void factorOutDefaultInterpolation(HistFactoryDist &dist) +{ + auto forEachInterpolation = [&](auto &&fn) { + for (Sample &sample : dist.samples) { + for (Modifier &modifier : sample.modifiers) { + if (auto *normSys = std::get_if(&modifier)) { + fn(normSys->interpolation); + } else if (auto *histoSys = std::get_if(&modifier)) { + fn(histoSys->interpolation); + } + } + } + }; + + std::map counts; + forEachInterpolation([&](std::optional &interpolation) { + if (interpolation) { + ++counts[*interpolation]; + } + }); + if (counts.empty()) { + return; + } + // A tie is decided in favor of the default of HistFactory. + auto best = counts.begin(); + for (auto current = std::next(counts.begin()); current != counts.end(); ++current) { + if (current->second > best->second || + (current->second == best->second && current->first == multiplicativePolynomialExponential && + best->first != multiplicativePolynomialExponential)) { + best = current; + } + } + dist.defaultInterpolation = best->first; + forEachInterpolation([&](std::optional &interpolation) { + if (interpolation == dist.defaultInterpolation) { + interpolation.reset(); + } + }); +} + +} // namespace HS3 +} // namespace RooFit diff --git a/roofit/hs3/src/HS3Document.h b/roofit/hs3/src/HS3Document.h new file mode 100644 index 0000000000000..317e558a0d1e3 --- /dev/null +++ b/roofit/hs3/src/HS3Document.h @@ -0,0 +1,627 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef RooFit_HS3Document_h +#define RooFit_HS3Document_h + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace RooFit { + +namespace Detail { +class JSONNode; +class JSONTree; +} // namespace Detail + +/// Types that represent an HS3 document. They follow the structure of the HS3 standard, and this header must not depend +/// on RooFit: creating RooFit objects from a document is a separate step. +namespace HS3 { + +struct Version { + int major = 0; + int minor = 0; + int patch = 0; +}; + +inline bool operator<(Version const &l, Version const &r) +{ + if (l.major != r.major) { + return l.major < r.major; + } + if (l.minor != r.minor) { + return l.minor < r.minor; + } + return l.patch < r.patch; +} + +/// Combines lambdas into one visitor for std::visit(). The compiler complains about the alternatives without lambda. +template +struct overloaded : Ts... { + using Ts::operator()...; +}; +template +overloaded(Ts...) -> overloaded; + +/// A name that says what it refers to, so that for example a DataRef can't be used as a DomainRef. +template +struct Ref { + std::string name; +}; + +using DistributionRef = Ref; +using DataRef = Ref; +using DomainRef = Ref; +using LikelihoodRef = Ref; +using ParameterPointRef = Ref; +using ParameterRef = Ref; + +struct Package { + std::string name; + std::string version; +}; + +struct Metadata { + std::string hs3Version; ///< As it was written in the document, empty for fragments of a document. + Version version; ///< The parsed version. Versions that can't be parsed count as the oldest one. + std::vector packages; + std::vector authors; + std::vector publications; + std::optional description; +}; + +struct RegularBinning { + int nbins = 0; +}; +struct EdgesBinning { + std::vector edges; +}; +using Binning = std::variant; + +/// The axis of a domain. Infinite bounds are written as null, and bounds can be missing in older documents. The binning +/// is a RooFit extension. A fixed axis has a single value instead, which excludes bounds and a binning. +struct DomainAxis { + std::string name; + std::optional fixedValue; + std::optional min; + std::optional max; + std::optional binning; +}; + +/// The axis of point data or unbinned data, which can't have a binning. +struct ContinuousAxis { + std::string name; + std::optional min; + std::optional max; +}; + +/// The axis of binned data. +struct BinnedAxis { + std::string name; + double min = 0.0; + double max = 0.0; + Binning binning; +}; + +struct CategoricalAxis { + std::string name; + std::vector categories; +}; + +struct Domain { + std::string name; + std::vector> axes; +}; + +/// The domain in which RooFit writes the ranges of all variables. It is a convention of RooFit, not of the standard. +inline constexpr char const *defaultDomainName = "default_domain"; + +/// Where RooFit takes the ranges of variables from: the "default_domain" if there is one, otherwise the first domain by +/// name that has the variable. The domains must outlive this object. +class DomainRanges { +public: + explicit DomainRanges(std::vector const &domains); + /// The axis that defines the range of a variable, or null. + DomainAxis const *find(std::string const &name) const; + /// Whether any domain has the variable. + bool contains(std::string const &name) const { return _names.count(name); } + +private: + std::map _ranges; + std::set _names; +}; + +struct ParameterValue { + std::string name; + std::optional> value; ///< A string selects the state of a category. + bool isConst = false; + // RooFit extensions that were written by older versions: + std::optional error; + std::optional relError; + std::optional nbins; +}; + +struct ParameterPoint { + std::string name; + std::vector parameters; +}; + +struct GaussianUncertainty { + std::vector sigma; + std::vector> correlation; ///< Empty if there is no correlation. +}; + +struct PointData { + double value = 0.0; + std::optional uncertainty; + std::vector axes; +}; + +struct UnbinnedData { + std::vector> axes; + /// The coordinates, row by row. A value on a categorical axis is the index in the list of categories of that axis. + std::vector entries; + std::optional> weights; + bool hasEntriesUncertainties = false; + + std::size_t numEntries() const { return axes.empty() ? 0 : entries.size() / axes.size(); } +}; + +struct BinnedData { + std::vector> axes; + std::vector contents; + std::optional uncertainty; +}; + +struct Data { + std::string name; + std::variant body; +}; + +struct LikelihoodTerm { + DistributionRef distribution; + std::variant data; +}; + +struct Likelihood { + std::string name; + std::vector terms; + std::vector auxDistributions; +}; + +struct Analysis { + std::string name; + LikelihoodRef likelihood; + std::vector parametersOfInterest; + std::vector domains; + std::optional init; + std::optional prior; +}; + +/// A distribution or function that is not interpreted by the document, but by the importers that are registered for +/// its type in RooFit::JSONIO::importers(), or by the factory expressions. +struct GenericObject { + std::string type; + Detail::JSONNode const *node = nullptr; +}; + +enum class InterpolationType { + Add, + Mult +}; + +enum class InterpolationFunction { + Poly1, + Poly2, + Poly6, + Exp +}; + +/// How a modifier of a HistFactory distribution interpolates and extrapolates its effect. +struct Interpolation { + InterpolationType type = InterpolationType::Add; + InterpolationFunction in = InterpolationFunction::Poly1; + std::optional out; ///< If not set, the function inside the bounds is used outside as well. +}; + +inline bool operator==(Interpolation const &l, Interpolation const &r) +{ + return l.type == r.type && l.in == r.in && l.out == r.out; +} +inline bool operator!=(Interpolation const &l, Interpolation const &r) +{ + return !(l == r); +} +inline bool operator<(Interpolation const &l, Interpolation const &r) +{ + if (l.type != r.type) { + return l.type < r.type; + } + if (l.in != r.in) { + return l.in < r.in; + } + return l.out < r.out; +} + +inline constexpr Interpolation additivePiecewiseLinear{InterpolationType::Add, InterpolationFunction::Poly1, + std::nullopt}; +inline constexpr Interpolation multiplicativePiecewiseExponential{InterpolationType::Mult, InterpolationFunction::Exp, + std::nullopt}; +inline constexpr Interpolation additiveQuadraticLinear{InterpolationType::Add, InterpolationFunction::Poly2, + InterpolationFunction::Poly1}; +inline constexpr Interpolation additivePolynomialLinear{InterpolationType::Add, InterpolationFunction::Poly6, + InterpolationFunction::Poly1}; +inline constexpr Interpolation multiplicativePolynomialExponential{ + InterpolationType::Mult, InterpolationFunction::Poly6, InterpolationFunction::Exp}; +inline constexpr Interpolation multiplicativePolynomialLinear{InterpolationType::Mult, InterpolationFunction::Poly6, + InterpolationFunction::Poly1}; + +char const *toString(InterpolationType type); +char const *toString(InterpolationFunction function); + +/// The RooFit classes that implement the interpolations, which encode them with integer codes. +enum class InterpolationClass { + Piecewise, + Flexible +}; + +/// The interpolation for an integer code, or nothing if the class doesn't support the code. +std::optional interpolationFromCode(int code, InterpolationClass interpolationClass); +/// The integer code of an interpolation, or nothing if the class can't represent the interpolation. +std::optional codeFromInterpolation(Interpolation const &interpolation, InterpolationClass interpolationClass); + +/// The constraint types that older documents named, for which RooFit creates the constraint terms. +enum class ConstraintType { + Gauss, + Poisson, + LogNormal +}; + +char const *toString(ConstraintType type); + +/// The parameter of the modifier is not constrained. +struct Unconstrained {}; + +/// The constraint term is a distribution of the document, named in "constraints". +struct ConstraintPdf { + DistributionRef pdf; +}; + +using Constraint = std::variant; + +inline bool operator==(Unconstrained, Unconstrained) +{ + return true; +} +inline bool operator==(ConstraintPdf const &l, ConstraintPdf const &r) +{ + return l.pdf.name == r.pdf.name; +} + +struct NormFactor { + std::string name; + ParameterRef parameter; + Constraint constraint; +}; + +struct NormSys { + std::string name; + ParameterRef parameter; + double hi = 1.0; + double lo = 1.0; + Constraint constraint; + std::optional interpolation; +}; + +struct HistoSysVariation { + std::vector contents; + std::optional> errors; ///< Not in the standard, but older versions of RooFit read them. +}; + +struct HistoSys { + std::string name; + ParameterRef parameter; + HistoSysVariation hi; + HistoSysVariation lo; + Constraint constraint; + std::optional interpolation; +}; + +/// The gamma parameters are constant, which is a RooFit extension. +struct ConstantGammas {}; + +/// Every gamma parameter has its own constraint term, or none, as named in "constraints". +struct GammaConstraintPdfs { + std::vector> pdfs; +}; + +using GammaConstraint = std::variant; + +/// Scales each bin by its own parameter. The modifier type "shapesys" is a deprecated alias of "shapefactor". +struct ShapeSys { + std::string name; + std::vector parameters; ///< If empty, the parameters are named after the modifier. + std::vector vals; + GammaConstraint constraint; +}; + +/// Scales each bin by a parameter, that is constrained by the statistical uncertainty of all samples that have this +/// modifier. +struct StatError { + std::string name; + std::vector parameters; ///< If empty, the parameters are named after the channel. + GammaConstraint constraint; +}; + +/// Multiplies the sample with a function of the document. +struct CustomModifier { + std::string name; +}; + +using Modifier = std::variant; + +struct Sample { + std::string name; + std::vector contents; + std::optional> errors; + std::vector modifiers; +}; + +/// Settings of the staterror modifiers that apply to the whole channel. This is a RooFit extension. +struct StatErrorConfig { + double relThreshold = 0.0; + GammaConstraint constraint; + /// Samples without "errors" have Poisson uncertainties, as in documents before HS3 0.3. + bool impliedPoissonErrors = false; +}; + +struct HistFactoryDist { + std::vector axes; + std::optional defaultInterpolation; + std::vector samples; + StatErrorConfig statError; +}; + +struct Distribution { + std::string name; + std::variant body; +}; + +struct Function { + std::string name; + GenericObject body; +}; + +/// A name, or a number for a constant. +using NameOrNumber = std::variant; + +/// The function "interpolation" of RooFit, implemented by PiecewiseInterpolation: a nominal function plus the +/// interpolated differences to the variations "high" and "low" for each parameter. +struct InterpolatedFunction { + std::string nominal; + std::vector parameters; + std::vector high; + std::vector low; + std::vector interpolations; ///< One per parameter. + bool positiveDefinite = false; +}; + +/// The function "interpolation0d" of RooFit, implemented by FlexibleInterpVar: a nominal value plus the interpolated +/// differences to the variations "high" and "low" for each parameter. +struct InterpolatedNumber { + double nominal = 0.0; + std::vector parameters; + std::vector high; + std::vector low; + std::vector interpolations; ///< One per parameter. +}; + +// The following types are stored in the "misc" section, for which the HS3 standard doesn't prescribe a format. + +struct Attributes { + std::map dict; + std::vector tags; + bool isConstVar = false; ///< How ROOT versions before HS3 0.3 marked named RooConstVars. +}; + +struct CombinedDistribution { + std::string indexCat; + std::vector labels; + std::vector indices; + std::vector distributions; +}; + +struct CombinedDataset { + std::string indexCat; + std::vector labels; + std::vector indices; +}; + +struct ModelConfigInfo { + std::optional mcName; + std::optional pdfName; + std::optional combinedDataName; +}; + +struct StepWidth { + std::string name; + double width = 0.0; +}; + +/// The content of misc.ROOT_internal and misc.minimization. +struct RooFitInternal { + std::map attributes; + std::map combinedDistributions; + std::map combinedDatasets; + std::map modelConfigs; + std::vector stepWidths; + /// The constraint terms of HistFactory distributions that no modifier names, by distribution. They are + /// also listed in the "aux_distributions" of the likelihoods, which is what other implementations use. + std::map> channelConstraints; +}; + +struct Diagnostic { + enum class Severity { + Warning, + Error + }; + Severity severity = Severity::Warning; + std::string path; ///< Where in the document, for example "distributions[3](pdf).x". + std::string message; +}; + +using Diagnostics = std::vector; + +/// Whether there is a diagnostic with error severity. +bool hasErrors(Diagnostics const &diagnostics); + +} // namespace HS3 + +class HS3Document; +class ValidatedHS3Document; + +namespace HS3 { +struct ValidateOptions; +std::optional +validate(HS3Document doc, Diagnostics &diagnostics, ValidateOptions const &options); +} // namespace HS3 + +/// An HS3 document. It is move-only, because generic objects refer to the JSON tree that the document was read from. +class HS3Document { +public: + HS3::Metadata metadata; + std::vector distributions; + std::vector functions; + std::vector data; + std::vector likelihoods; + std::vector domains; + std::vector parameterPoints; + std::vector analyses; + HS3::RooFitInternal rooFit; + + HS3Document() = default; + HS3Document(HS3Document &&) = default; + HS3Document &operator=(HS3Document &&) = default; + HS3Document(HS3Document const &) = delete; + HS3Document &operator=(HS3Document const &) = delete; +}; + +namespace HS3 { + +/// Options for upgradeDocument() and parseDocument(). +struct ParseOptions { + /// Fragments of a document can be imported without metadata. + bool requireMetadata = true; + /// If false, HistFactory distributions stay generic objects, because a user registered their own importer. + bool typedHistFactory = true; +}; + +/// Rewrites everything that older versions of RooFit wrote in a way that is not part of the standard into the form that +/// parseDocument() understands. +void upgradeDocument(Detail::JSONNode &root, Diagnostics &diagnostics, ParseOptions const &options = {}); + +/// Same as upgradeDocument(), but for a single HistFactory distribution that doesn't come with a document. Names that +/// can't be resolved in the document are assumed to refer to distributions. +void upgradeHistFactoryDist(Detail::JSONNode &distribution, Version const &version, Diagnostics &diagnostics); + +/// Reads a document. The generic objects in the document refer to nodes of the JSON tree, so the tree must outlive the +/// document. Problems are reported in `diagnostics`, and if there are errors the document must not be used. +HS3Document parseDocument(Detail::JSONNode const &root, Diagnostics &diagnostics, ParseOptions const &options = {}); + +/// Reads a HistFactory distribution that doesn't come with a document. +HistFactoryDist parseHistFactoryDist(Detail::JSONNode const &distribution, Diagnostics &diagnostics); + +/// Reads the axes of binned data that doesn't come with a document. +std::vector parseBinnedAxes(Detail::JSONNode const &axes, Diagnostics &diagnostics); + +/// Reads the functions "interpolation" and "interpolation0d", which don't come with a document. Older versions of +/// RooFit wrote integer "interpolationCodes" instead of "interpolations". +InterpolatedFunction parseInterpolatedFunction(Detail::JSONNode const &node, Diagnostics &diagnostics); +InterpolatedNumber parseInterpolatedNumber(Detail::JSONNode const &node, Diagnostics &diagnostics); + +/// Writes everything but the distributions and functions, which are written one by one while they are exported: the +/// generic ones by the exporters, and the others with functions like writeHistFactoryDist(). +void writeDocument(Detail::JSONNode &root, HS3Document const &doc); + +/// Writes a HistFactory distribution, without its name. +void writeHistFactoryDist(Detail::JSONNode &node, HistFactoryDist const &dist); + +/// Writes the functions "interpolation" and "interpolation0d", with a single interpolation if they are all the same. +void writeInterpolatedFunction(Detail::JSONNode &node, InterpolatedFunction const &function); +void writeInterpolatedNumber(Detail::JSONNode &node, InterpolatedNumber const &function); + +void writeInterpolation(Detail::JSONNode &node, Interpolation const &interpolation); + +/// Writes a number, as an integer if it has no fractional part. +void writeNumber(Detail::JSONNode &node, double value); + +/// Writes an array of numbers, each as an integer if it has no fractional part. +void writeContents(Detail::JSONNode &node, std::vector const &values); + +/// Options for validate(). +struct ValidateOptions { + /// Whether the document has to follow the current standard, or if older conventions of RooFit are tolerated. + bool strict = false; + /// Whether a name that is not defined in the document is known in some other way, e.g. in the workspace. + std::function isExternal; +}; + +} // namespace HS3 + +/// A document that passed validate(). Only validate() can create it, so the code that builds RooFit objects can rely on +/// the checks. +class ValidatedHS3Document { +public: + HS3Document const &doc() const { return _doc; } + +private: + explicit ValidatedHS3Document(HS3Document doc) : _doc{std::move(doc)} {} + friend std::optional + HS3::validate(HS3Document doc, HS3::Diagnostics &diagnostics, HS3::ValidateOptions const &options); + + HS3Document _doc; +}; + +namespace HS3 { + +/// Checks what can be checked without knowing how the objects are built: unique names, that the references can be +/// resolved, and that the sizes of arrays are consistent. Gives nothing if there are errors. +std::optional +validate(HS3Document doc, Diagnostics &diagnostics, ValidateOptions const &options = {}); + +/// The checks of validate(), for a document that is still needed if it doesn't pass them. +void checkDocument(HS3Document const &doc, Diagnostics &diagnostics, ValidateOptions const &options = {}); + +/// Same as validate(), but for a single HistFactory distribution that doesn't come with a document. +void validateHistFactoryDist(HistFactoryDist const &dist, std::string const &name, Diagnostics &diagnostics); + +/// Reads the leading "MAJOR.MINOR" and the optional ".PATCH" of an HS3 version string. +bool parseVersion(std::string const &version, Version &out); + +/// Combines the modifiers of a sample that have the same type and name, which HistFactory models can have but +/// documents can't: normsys by multiplying their variations, and histosys by adding up their differences to the +/// nominal contents. The modifiers of a type must be sorted by name. Other duplicates are errors. +void mergeDuplicateModifiers(HistFactoryDist &dist, std::string const &channel, Diagnostics &diagnostics); + +/// Makes the most common interpolation of the modifiers the "default_interpolation", which they then don't repeat. +void factorOutDefaultInterpolation(HistFactoryDist &dist); + +} // namespace HS3 +} // namespace RooFit + +#endif diff --git a/roofit/hs3/src/HS3Parser.cxx b/roofit/hs3/src/HS3Parser.cxx new file mode 100644 index 0000000000000..527c701a63415 --- /dev/null +++ b/roofit/hs3/src/HS3Parser.cxx @@ -0,0 +1,1369 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "HS3Schema.h" +#include "JSONIOUtils.h" + +#include +#include +#include +#include +#include +#include +#include + +using RooFit::Detail::JSONNode; + +namespace RooFit { +namespace HS3 { + +constexpr auto supportedVersion = "0.3"; + +namespace { + +// Single source of truth for the mapping between the structured HS3 interpolation +// descriptors and the RooFit integer codes of PiecewiseInterpolation and +// FlexibleInterpVar. A code of `kUnrepresentable` means the descriptor cannot be +// expressed by that class: FlexibleInterpVar internally remaps code 4 to code 5, +// so it has no additive-linear or multiplicative-linear poly6 variant. +struct InterpolationCodes { + Interpolation descriptor; + int piecewise; + int flexible; +}; + +constexpr int kUnrepresentable = -1; + +// clang-format off +constexpr InterpolationCodes interpolationTable[]{ + {additivePiecewiseLinear, 0, 0}, + {multiplicativePiecewiseExponential, 1, 1}, + {additiveQuadraticLinear, 2, 2}, + {additivePolynomialLinear, 4, kUnrepresentable}, + {multiplicativePolynomialExponential, 5, 4}, + {multiplicativePolynomialLinear, 6, kUnrepresentable}, +}; +// clang-format on + +int codeForClass(InterpolationCodes const &row, InterpolationClass interpolationClass) +{ + return interpolationClass == InterpolationClass::Piecewise ? row.piecewise : row.flexible; +} + +} // namespace + +std::optional interpolationFromCode(int code, InterpolationClass interpolationClass) +{ + // Code 3 was historically an unimplemented alias of code 2 for both classes, + // and FlexibleInterpVar treats code 5 identically to its canonical code 4. + if (code == 3) { + code = 2; + } + if (interpolationClass == InterpolationClass::Flexible && code == 5) { + code = 4; + } + for (auto const &row : interpolationTable) { + const int rowCode = codeForClass(row, interpolationClass); + if (rowCode != kUnrepresentable && rowCode == code) { + return row.descriptor; + } + } + return std::nullopt; +} + +std::optional codeFromInterpolation(Interpolation const &interpolation, InterpolationClass interpolationClass) +{ + for (auto const &row : interpolationTable) { + const int rowCode = codeForClass(row, interpolationClass); + if (rowCode != kUnrepresentable && row.descriptor == interpolation) { + return rowCode; + } + } + return std::nullopt; +} + +bool hasErrors(Diagnostics const &diagnostics) +{ + return std::any_of(diagnostics.begin(), diagnostics.end(), + [](Diagnostic const &d) { return d.severity == Diagnostic::Severity::Error; }); +} + +std::string Cursor::path() const +{ + if (!_parent) { + return ""; + } + std::string out = _parent->path(); + if (_key) { + if (!out.empty()) { + out += "."; + } + out += _key; + return out; + } + out += "[" + std::to_string(_index) + "]"; + if (_node.is_map() && _node.has_child("name") && !_node["name"].is_container() && !_node["name"].is_null()) { + out += "(" + _node["name"].val() + ")"; + } + return out; +} + +void Cursor::error(std::string const &message) const +{ + _diagnostics.push_back({Diagnostic::Severity::Error, path(), message}); +} + +void Cursor::warning(std::string const &message) const +{ + _diagnostics.push_back({Diagnostic::Severity::Warning, path(), message}); +} + +std::size_t Cursor::numErrors() const +{ + return std::count_if(_diagnostics.begin(), _diagnostics.end(), + [](Diagnostic const &d) { return d.severity == Diagnostic::Severity::Error; }); +} + +Reader::Reader(Cursor const &cursor) : _cursor{cursor}, _nErrorsBefore{cursor.numErrors()} {} + +bool Reader::has(const char *key) +{ + if (!_cursor.node().has_child(key)) { + return false; + } + if (std::find(_seen.begin(), _seen.end(), key) == _seen.end()) { + _seen.emplace_back(key); + } + return true; +} + +Reader::~Reader() +{ + JSONNode const &node = _cursor.node(); + // There is nothing to report if all components were read, and after an error it would only be noise. + if (!node.is_map() || node.num_children() == _seen.size() || _cursor.numErrors() != _nErrorsBefore) { + return; + } + for (JSONNode const &child : node.children()) { + const std::string key = child.key(); + if (std::find(_seen.begin(), _seen.end(), key) == _seen.end()) { + _cursor.warning("unknown component \"" + key + "\" is ignored"); + } + } +} + +void readValue(Cursor const &c, std::string &out) +{ + if (c.node().is_container() || c.node().is_null()) { + c.error("must be a string"); + return; + } + out = c.node().val(); +} + +void readValue(Cursor const &c, double &out) +{ + if (!c.node().is_number()) { + c.error("must be a number"); + return; + } + out = c.node().val_double(); +} + +void readValue(Cursor const &c, int &out) +{ + // Read through double, such that an integer that is encoded as a JSON float like 1e6 is accepted. + const double value = c.node().is_number() ? c.node().val_double() : 0.5; + if (value != std::floor(value) || std::abs(value) > static_cast(std::numeric_limits::max())) { + c.error("must be an integer"); + return; + } + out = static_cast(value); +} + +void readValue(Cursor const &c, bool &out) +{ + if (c.node().is_container() || c.node().is_null()) { + c.error("must be a boolean"); + return; + } + try { + out = c.node().val_bool(); + } catch (std::exception const &) { + c.error("must be a boolean"); + } +} + +bool parseVersion(std::string const &version, Version &out) +{ + std::size_t pos = 0; + auto readNumber = [&](int &number) { + const std::size_t begin = pos; + while (pos < version.size() && std::isdigit(static_cast(version[pos]))) { + ++pos; + } + // Reject empty and absurdly long numbers, which std::stoi could not represent. + if (pos == begin || pos - begin > 9) { + return false; + } + number = std::stoi(version.substr(begin, pos - begin)); + return true; + }; + Version parsed; + if (!readNumber(parsed.major) || pos >= version.size() || version[pos] != '.') { + return false; + } + ++pos; + if (!readNumber(parsed.minor)) { + return false; + } + if (pos < version.size() && version[pos] == '.') { + ++pos; + if (!readNumber(parsed.patch)) { + parsed.patch = 0; + } + } + out = parsed; + return true; +} + +namespace { + +// Read a bound of an axis. Infinite bounds are written as null. +void readBound(Reader &ar, const char *key, std::optional &out, double valueIfNull) +{ + if (!ar.has(key)) { + return; + } + Cursor c{ar.cursor().node()[key], ar.cursor(), key}; + if (c.node().is_null()) { + out = valueIfNull; + return; + } + out.emplace(); + readValue(c, *out); +} + +constexpr double infinity = std::numeric_limits::infinity(); + +// Read bin edges, which must be at least two strictly ascending numbers. +void readEdges(Reader &ar, std::string const &axisName, std::vector &edges) +{ + Cursor c{ar.cursor().node()["edges"], ar.cursor(), "edges"}; + const std::string context = "\"edges\" of axis '" + axisName + "' must "; + if (!c.node().is_seq()) { + c.error(context + "be a list"); + return; + } + readValue(c, edges); + for (std::size_t i = 1; i < edges.size(); ++i) { + if (!(edges[i] > edges[i - 1])) { + c.error(context + "be strictly ascending"); + return; + } + } + if (edges.size() < 2) { + c.error(context + "contain at least two values"); + } +} + +} // namespace + +void describe(Reader &ar, Package &p) +{ + ar("name", p.name); + ar("version", p.version); +} + +void describe(Reader &ar, DomainAxis &axis) +{ + ar("name", axis.name); + if (ar.has(fixedValueKey)) { + // The domain of the parameter is the set with this single value: no range, binning or categories. + for (const char *key : {"min", "max", "nbins", "edges", "categories"}) { + if (ar.has(key)) { + ar.cursor().error("the axis '" + axis.name + "' has \"" + fixedValueKey + "\" and \"" + key + + "\", which exclude each other"); + } + } + // Not read as an optional component, for which null would be the same as leaving it out. + ar(fixedValueKey, axis.fixedValue.emplace()); + return; + } + readBound(ar, "min", axis.min, -infinity); + readBound(ar, "max", axis.max, infinity); + if (ar.has("edges")) { + // Not necessarily ascending, because RooFit also writes parameterized ranges as edges. + EdgesBinning binning; + ar("edges", binning.edges); + if (!binning.edges.empty()) { + if (!axis.min) { + axis.min = binning.edges.front(); + } + if (!axis.max) { + axis.max = binning.edges.back(); + } + } + axis.binning = std::move(binning); + } else if (ar.has("nbins")) { + RegularBinning binning; + ar("nbins", binning.nbins); + if (binning.nbins != 0) { + axis.binning = binning; + } + } +} + +void describe(Reader &ar, ContinuousAxis &axis) +{ + ar("name", axis.name); + readBound(ar, "min", axis.min, -infinity); + readBound(ar, "max", axis.max, infinity); +} + +void describe(Reader &ar, BinnedAxis &axis) +{ + ar("name", axis.name); + if (ar.has("edges")) { + EdgesBinning binning; + readEdges(ar, axis.name, binning.edges); + if (binning.edges.size() >= 2) { + axis.min = binning.edges.front(); + axis.max = binning.edges.back(); + } + // The standard requires "min" and "max" also for irregular binnings, which older versions of RooFit didn't write. + std::optional min; + std::optional max; + ar("min", min); + ar("max", max); + if ((min && *min != axis.min) || (max && *max != axis.max)) { + ar.cursor().error("\"min\" and \"max\" of axis '" + axis.name + "' must agree with the \"edges\""); + } + if (ar.has("nbins")) { + ar.cursor().error("axis '" + axis.name + "' must not have both \"edges\" and \"nbins\""); + } + axis.binning = std::move(binning); + return; + } + ar("min", axis.min); + ar("max", axis.max); + RegularBinning binning; + ar("nbins", binning.nbins); + axis.binning = binning; +} + +void describe(Reader &ar, CategoricalAxis &axis) +{ + ar("name", axis.name); + ar("categories", axis.categories); +} + +namespace { + +// The kind of an axis is determined by its components. +template +void readAxis(Cursor const &c, std::variant &out) +{ + if (!c.node().is_map()) { + c.error("must be a struct"); + return; + } + Reader reader{c}; + // A fixed axis is not categorical. That it has no categories is checked when it is read. + if (reader.has("categories") && !(std::is_same_v && reader.has(fixedValueKey))) { + CategoricalAxis axis; + describe(reader, axis); + out = std::move(axis); + } else { + ContinuousAxis_t axis; + describe(reader, axis); + out = std::move(axis); + } +} + +} // namespace + +void readValue(Cursor const &c, std::variant &out) +{ + readAxis(c, out); +} + +void readValue(Cursor const &c, std::variant &out) +{ + readAxis(c, out); +} + +void readValue(Cursor const &c, std::variant &out) +{ + readAxis(c, out); +} + +void describe(Reader &ar, Domain &domain) +{ + ar("name", domain.name); + std::string type; + if (ar.has("type")) { + ar("type", type); + } + if (type != "product_domain") { + ar.cursor().error("only domains of type \"product_domain\" are currently supported!"); + return; + } + ar.list("axes", domain.axes); +} + +void readValue(Cursor const &c, ParameterValue &p) +{ + if (!c.node().is_map()) { + c.error("must be a struct"); + return; + } + Reader ar{c}; + ar("name", p.name); + if (ar.has("value")) { + Cursor crsr{ar.cursor().node()["value"], ar.cursor(), "value"}; + if (crsr.node().is_number()) { + p.value = crsr.node().val_double(); + } else { + std::string label; + readValue(crsr, label); + p.value = std::move(label); + } + } + std::optional isConst; + ar("const", isConst); + p.isConst = isConst.value_or(false); + ar("err", p.error); + ar("relErr", p.relError); + ar("nbins", p.nbins); +} + +void describe(Reader &ar, ParameterPoint &point) +{ + ar("name", point.name); + ar.list("parameters", point.parameters); +} + +void describe(Reader &ar, PointData &data) +{ + ar("value", data.value); + ar("uncertainty", data.uncertainty); + ar.list("axes", data.axes); +} + +void describe(Reader &ar, UnbinnedData &data) +{ + ar.list("axes", data.axes); + ar("weights", data.weights); + data.hasEntriesUncertainties = ar.has("entries_uncertainties"); + + if (!ar.has("entries")) { + ar.cursor().error("does not define the required component \"entries\""); + return; + } + Cursor entries{ar.cursor().node()["entries"], ar.cursor(), "entries"}; + if (!entries.node().is_seq()) { + entries.error("key 'entries' is not a list!"); + return; + } + if (data.weights && data.weights->size() != entries.node().num_children()) { + ar.cursor().error("inconsistent number of entries and weights!"); + return; + } + data.entries.reserve(entries.node().num_children() * data.axes.size()); + std::size_t iRow = 0; + for (JSONNode const &row : entries.node().children()) { + Cursor rowCursor{row, entries, iRow}; + if (!row.is_seq()) { + rowCursor.error("coordinate point '" + std::to_string(iRow) + "' is not a list!"); + return; + } + if (row.num_children() != data.axes.size()) { + rowCursor.error("inconsistent number of entries and observables!"); + return; + } + std::size_t iAxis = 0; + for (JSONNode const &value : row.children()) { + Cursor valueCursor{value, rowCursor, iAxis}; + if (auto const *category = std::get_if(&data.axes[iAxis])) { + std::string label; + readValue(valueCursor, label); + auto found = std::find(category->categories.begin(), category->categories.end(), label); + if (found == category->categories.end()) { + valueCursor.error("'" + label + "' is not a category of the axis '" + category->name + "'"); + return; + } + data.entries.push_back(static_cast(std::distance(category->categories.begin(), found))); + } else { + double coordinate = 0.0; + readValue(valueCursor, coordinate); + data.entries.push_back(coordinate); + } + ++iAxis; + } + ++iRow; + } +} + +void describe(Reader &ar, GaussianUncertainty &uncertainty) +{ + if (ar.has("type")) { + std::string type; + ar("type", type); + if (type != "gaussian_uncertainty") { + ar.cursor().error("unsupported type of uncertainty '" + type + "'"); + } + } + ar("sigma", uncertainty.sigma); + if (ar.has("correlation")) { + Cursor c{ar.cursor().node()["correlation"], ar.cursor(), "correlation"}; + // Besides a matrix, it can be the number 0 to indicate that there is no correlation. + if (c.node().is_seq()) { + readValue(c, uncertainty.correlation); + } else if (!c.node().is_number() || c.node().val_double() != 0.0) { + c.error("must be a matrix or 0"); + } + } +} + +void describe(Reader &ar, BinnedData &data) +{ + ar.list("axes", data.axes); + ar("contents", data.contents); + ar("uncertainty", data.uncertainty); + // RooFit extension: older versions read the errors like in the samples of a HistFactory distribution. + if (ar.has("errors")) { + Cursor errors{ar.cursor().node()["errors"], ar.cursor(), "errors"}; + if (!errors.node().is_seq()) { + errors.error("errors are not in list form"); + } else if (!data.uncertainty) { + data.uncertainty.emplace(); + readValue(errors, data.uncertainty->sigma); + } + } +} + +void readValue(Cursor const &c, Data &out) +{ + if (!c.node().is_map()) { + c.error("must be a struct"); + return; + } + Reader ar{c}; + ar("name", out.name); + std::string type; + ar("type", type); + if (type == "binned") { + BinnedData body; + describe(ar, body); + out.body = std::move(body); + } else if (type == "unbinned") { + UnbinnedData body; + describe(ar, body); + out.body = std::move(body); + } else if (type == "point") { + PointData body; + describe(ar, body); + out.body = std::move(body); + } else if (ar.has("type")) { + c.error("unknown type of dataset '" + type + "'"); + } +} + +void describe(Reader &ar, Likelihood &likelihood) +{ + ar("name", likelihood.name); + ar.list("aux_distributions", likelihood.auxDistributions); + + std::vector distributions; + ar("distributions", distributions); + if (!ar.has("data")) { + ar.cursor().error("does not define the required component \"data\""); + return; + } + Cursor data{ar.cursor().node()["data"], ar.cursor(), "data"}; + if (!data.node().is_seq()) { + data.error("must be an array"); + return; + } + if (data.node().num_children() != distributions.size()) { + ar.cursor().error("\"distributions\" and \"data\" must have the same length"); + return; + } + likelihood.terms.reserve(distributions.size()); + std::size_t i = 0; + for (JSONNode const &node : data.node().children()) { + LikelihoodTerm term; + term.distribution = std::move(distributions[i]); + // Older versions of RooFit used the string "0" to say that there is no dataset. + if (node.is_number() || (!node.is_container() && !node.is_null() && node.val() == "0")) { + term.data = node.is_number() ? node.val_double() : 0.0; + } else { + DataRef ref; + readValue(Cursor{node, data, i}, ref); + term.data = std::move(ref); + } + likelihood.terms.push_back(std::move(term)); + ++i; + } +} + +void describe(Reader &ar, Analysis &analysis) +{ + ar("name", analysis.name); + ar("likelihood", analysis.likelihood); + // HS3 v0.2.9 used the singular keys "parameter_of_interest" (a list) and "domain" (a single name). They are still + // accepted, but the plural keys take precedence. + auto isLegacy = [&](const char *key, const char *legacyKey) { + if (ar.has(key) || !ar.has(legacyKey)) { + return false; + } + ar.cursor().warning(std::string("the key \"") + legacyKey + "\" of analysis '" + analysis.name + + "' is deprecated, please use \"" + key + "\" instead."); + return true; + }; + ar.list(isLegacy("parameters_of_interest", "parameter_of_interest") ? "parameter_of_interest" + : "parameters_of_interest", + analysis.parametersOfInterest); + if (!isLegacy("domains", "domain")) { + ar.list("domains", analysis.domains); + } else if (ar.cursor().node()["domain"].is_seq()) { + ar.list("domain", analysis.domains); + } else { + analysis.domains.emplace_back(); + ar("domain", analysis.domains.back()); + } + ar("init", analysis.init); + ar("prior", analysis.prior); +} + +namespace { + +// The strings of the interpolation types and functions in the standard. +std::optional interpolationTypeFromString(std::string const &str) +{ + for (auto type : {InterpolationType::Add, InterpolationType::Mult}) { + if (str == toString(type)) { + return type; + } + } + return std::nullopt; +} + +std::optional interpolationFunctionFromString(std::string const &str) +{ + for (auto function : {InterpolationFunction::Poly1, InterpolationFunction::Poly2, InterpolationFunction::Poly6, + InterpolationFunction::Exp}) { + if (str == toString(function)) { + return function; + } + } + return std::nullopt; +} + +} // namespace + +char const *toString(InterpolationType type) +{ + return type == InterpolationType::Add ? "add" : "mult"; +} + +char const *toString(InterpolationFunction function) +{ + switch (function) { + case InterpolationFunction::Poly1: return "poly1"; + case InterpolationFunction::Poly2: return "poly2"; + case InterpolationFunction::Poly6: return "poly6"; + case InterpolationFunction::Exp: return "exp"; + } + return ""; +} + +char const *toString(ConstraintType type) +{ + switch (type) { + case ConstraintType::Gauss: return "Gauss"; + case ConstraintType::Poisson: return "Poisson"; + case ConstraintType::LogNormal: return "LogNormal"; + } + return ""; +} + +void readValue(Cursor const &c, Interpolation &out) +{ + JSONNode const &node = c.node(); + if (!node.is_map()) { + c.error("must be a struct with components 'type', 'in', and 'out'"); + return; + } + for (const char *component : {"type", "in", "out"}) { + if (!node.has_child(component)) { + c.error(std::string("does not define the required '") + component + "' component"); + return; + } + } + JSONNode const &typeNode = node["type"]; + JSONNode const &inNode = node["in"]; + JSONNode const &outNode = node["out"]; + if (typeNode.is_container() || typeNode.is_null() || inNode.is_container() || inNode.is_null()) { + c.error("components 'type' and 'in' must be strings"); + return; + } + + auto type = interpolationTypeFromString(typeNode.val()); + if (!type) { + c.error("has unknown interpolation type '" + typeNode.val() + "'"); + return; + } + auto in = interpolationFunctionFromString(inNode.val()); + if (!in) { + c.error("has unknown interpolation function '" + inNode.val() + "'"); + return; + } + out.type = *type; + out.in = *in; + out.out.reset(); + + if (!outNode.is_null()) { + if (outNode.is_container()) { + c.error("component 'out' must be a string or null"); + return; + } + out.out = interpolationFunctionFromString(outNode.val()); + if (!out.out) { + c.error("has unknown extrapolation function '" + outNode.val() + "'"); + return; + } + } + Reader reader{c}; + for (const char *component : {"type", "in", "out"}) { + reader.has(component); + } +} + +namespace { + +// Constraints: "constraints" names the constraint distribution of each parameter, or is null for an unconstrained one. +// Older documents give a type in "constraint", or a distribution in "constraint_name", to which upgradeDocument() +// translated the even older forms. +std::optional readConstraintType(Cursor const &c) +{ + std::string value; + readValue(c, value); + for (auto type : {ConstraintType::Gauss, ConstraintType::Poisson, ConstraintType::LogNormal}) { + if (value == toString(type)) { + return type; + } + } + c.error("unknown constraint type '" + value + "', supported are \"Gauss\", \"Poisson\" and \"LogNormal\""); + return std::nullopt; +} + +void readConstraint(Reader &ar, Constraint &out) +{ + if (ar.has("constraints")) { + Cursor c{ar.cursor().node()["constraints"], ar.cursor(), "constraints"}; + if (!c.node().is_seq() || c.node().num_children() != 1) { + c.error("must be an array with one entry"); + return; + } + JSONNode const &item = c.node().child(0); + if (item.is_null()) { + out = Unconstrained{}; + return; + } + ConstraintPdf pdf; + readValue(Cursor{item, c, std::size_t{0}}, pdf.pdf); + out = std::move(pdf); + } else if (ar.has("constraint_name")) { + ConstraintPdf pdf; + ar("constraint_name", pdf.pdf); + out = std::move(pdf); + } else if (ar.has("constraint")) { + if (auto type = readConstraintType(Cursor{ar.cursor().node()["constraint"], ar.cursor(), "constraint"})) { + out = *type; + } + } +} + +void readGammaConstraint(Reader &ar, GammaConstraint &out) +{ + if (ar.has("constraints")) { + GammaConstraintPdfs constraints; + Cursor c{ar.cursor().node()["constraints"], ar.cursor(), "constraints"}; + if (!c.node().is_seq()) { + c.error("must be an array"); + return; + } + std::size_t i = 0; + for (JSONNode const &item : c.node().children()) { + constraints.pdfs.emplace_back(); + if (!item.is_null()) { + constraints.pdfs.back().emplace(); + readValue(Cursor{item, c, i}, *constraints.pdfs.back()); + } + ++i; + } + out = std::move(constraints); + } else if (ar.has("constraint")) { + Cursor c{ar.cursor().node()["constraint"], ar.cursor(), "constraint"}; + // "Const" is a RooFit extension. + if (!c.node().is_container() && !c.node().is_null() && c.node().val() == "Const") { + out = ConstantGammas{}; + } else if (auto type = readConstraintType(c)) { + out = *type; + } + } +} + +} // namespace + +// The standard doesn't require a name for modifiers, but RooFit uses it to name the objects it creates. +void readModifierName(Reader &ar, std::string &name) +{ + std::optional optionalName; + ar("name", optionalName); + name = optionalName.value_or(""); +} + +// The parameter of a modifier with a single parameter is given in "parameters", or in "parameter" in older documents. +void readParameter(Reader &ar, ParameterRef ¶meter) +{ + if (!ar.has("parameters")) { + ar("parameter", parameter); + return; + } + Cursor c{ar.cursor().node()["parameters"], ar.cursor(), "parameters"}; + if (!c.node().is_seq() || c.node().num_children() != 1) { + c.error("must be an array with one entry"); + return; + } + readValue(Cursor{c.node().child(0), c, std::size_t{0}}, parameter); +} + +void describe(Reader &ar, NormFactor &m) +{ + readModifierName(ar, m.name); + readParameter(ar, m.parameter); + readConstraint(ar, m.constraint); +} + +void describe(Reader &ar, NormSys &m) +{ + readModifierName(ar, m.name); + readParameter(ar, m.parameter); + ar.nested("data", [&](Reader &data) { + data("hi", m.hi); + data("lo", m.lo); + }); + readConstraint(ar, m.constraint); + ar("interpolation", m.interpolation); +} + +void describe(Reader &ar, HistoSysVariation &variation) +{ + ar("contents", variation.contents); + ar("errors", variation.errors); +} + +void describe(Reader &ar, HistoSys &m) +{ + readModifierName(ar, m.name); + readParameter(ar, m.parameter); + ar.nested("data", [&](Reader &data) { + data("hi", m.hi); + data("lo", m.lo); + }); + readConstraint(ar, m.constraint); + ar("interpolation", m.interpolation); +} + +void describe(Reader &ar, ShapeSys &m) +{ + readModifierName(ar, m.name); + if (ar.has("parameter")) { + // A single name stands for one parameter per bin, which are named after it. + ar("parameter", m.name); + } + ar.list("parameters", m.parameters); + if (ar.has("data")) { + ar.nested("data", [&](Reader &data) { data.list("vals", m.vals); }); + } + readGammaConstraint(ar, m.constraint); +} + +void describe(Reader &ar, StatError &m) +{ + readModifierName(ar, m.name); + ar.list("parameters", m.parameters); + readGammaConstraint(ar, m.constraint); +} + +void describe(Reader &ar, CustomModifier &m) +{ + ar("name", m.name); +} + +void readValue(Cursor const &c, Modifier &out) +{ + if (!c.node().is_map()) { + c.error("must be a struct"); + return; + } + Reader ar{c}; + std::string type; + ar("type", type); + auto read = [&](auto modifier) { + describe(ar, modifier); + out = std::move(modifier); + }; + if (type == "normfactor") { + read(NormFactor{}); + } else if (type == "normsys") { + read(NormSys{}); + } else if (type == "histosys") { + read(HistoSys{}); + } else if (type == "shapesys" || type == "shapefactor") { + read(ShapeSys{}); + } else if (type == "staterror") { + read(StatError{}); + } else if (type == "custom") { + read(CustomModifier{}); + } else if (ar.has("type")) { + std::string name; + if (ar.has("name")) { + ar("name", name); + } + c.error("modifier '" + name + "' of unknown type '" + type + "'"); + } +} + +void describe(Reader &ar, Sample &sample) +{ + ar("name", sample.name); + ar.nested("data", [&](Reader &data) { + data("contents", sample.contents); + data("errors", sample.errors); + }); + ar.list("modifiers", sample.modifiers); +} + +void describe(Reader &ar, StatErrorConfig &config) +{ + std::optional relThreshold; + ar("relThreshold", relThreshold); + config.relThreshold = relThreshold.value_or(0.0); + readGammaConstraint(ar, config.constraint); + // Written by upgradeDocument() for older documents. + std::optional impliedPoissonErrors; + ar("implied_poisson_errors", impliedPoissonErrors); + config.impliedPoissonErrors = impliedPoissonErrors.value_or(false); +} + +void describe(Reader &ar, HistFactoryDist &dist) +{ + ar("axes", dist.axes); + ar("default_interpolation", dist.defaultInterpolation); + ar("samples", dist.samples); + if (ar.has("staterror")) { + readValue(Cursor{ar.cursor().node()["staterror"], ar.cursor(), "staterror"}, dist.statError); + } +} + +namespace { + +void readDistribution(Cursor const &c, Distribution &out, ParseOptions const &options) +{ + if (!c.node().is_map()) { + c.error("must be a struct"); + return; + } + Reader ar{c}; + ar("name", out.name); + std::string type; + ar("type", type); + if (options.typedHistFactory && type == "histfactory_dist") { + HistFactoryDist body; + describe(ar, body); + out.body = std::move(body); + } else { + out.body = GenericObject{type, &c.node()}; + // The importers read the other components. + for (JSONNode const &child : c.node().children()) { + ar.has(child.key().c_str()); + } + } +} + +} // namespace + +void readValue(Cursor const &c, Function &out) +{ + if (!c.node().is_map()) { + c.error("must be a struct"); + return; + } + Reader ar{c}; + ar("name", out.name); + ar("type", out.body.type); + out.body.node = &c.node(); + // The importers read the other components. + for (JSONNode const &child : c.node().children()) { + ar.has(child.key().c_str()); + } +} + +void describe(Reader &ar, Attributes &attributes) +{ + if (ar.has("dict")) { + Cursor dict{ar.cursor().node()["dict"], ar.cursor(), "dict"}; + if (!dict.node().is_map()) { + dict.error("must be a struct"); + return; + } + for (JSONNode const &item : dict.node().children()) { + readValue(Cursor{item, dict, item.key().c_str()}, attributes.dict[item.key()]); + } + } + ar.list("tags", attributes.tags); + if (ar.has("is_const_var")) { + int isConstVar = 0; + ar("is_const_var", isConstVar); + attributes.isConstVar = isConstVar == 1; + } +} + +void describe(Reader &ar, CombinedDistribution &info) +{ + ar("index_cat", info.indexCat); + ar("labels", info.labels); + ar("indices", info.indices); + ar("distributions", info.distributions); + if (info.labels.size() != info.indices.size() || info.labels.size() != info.distributions.size()) { + ar.cursor().error("mismatch in number of labels, indices and distributions!"); + } +} + +void describe(Reader &ar, CombinedDataset &info) +{ + ar("index_cat", info.indexCat); + ar("labels", info.labels); + ar("indices", info.indices); + if (info.labels.size() != info.indices.size()) { + ar.cursor().error("mismatch in number of indices and labels!"); + } +} + +void describe(Reader &ar, ModelConfigInfo &info) +{ + ar("mcName", info.mcName); + ar("pdfName", info.pdfName); + ar("combined_data_name", info.combinedDataName); +} + +namespace { + +template +void readMap(Reader &ar, const char *key, Map_t &out) +{ + if (!ar.has(key)) { + return; + } + Cursor c{ar.cursor().node()[key], ar.cursor(), key}; + if (c.node().is_null()) { + return; + } + if (!c.node().is_map()) { + c.error("must be a struct"); + return; + } + for (JSONNode const &item : c.node().children()) { + const std::string itemKey = item.key(); + readValue(Cursor{item, c, itemKey.c_str()}, out[itemKey]); + } +} + +// The content of "misc" is not supposed to change the result of evaluating the model, so problems with the step widths +// are only warnings. +void readStepWidths(Cursor const &misc, std::vector &out) +{ + JSONNode const *node = misc.node().find("minimization", "parameter_stepwidths"); + if (!node) { + return; + } + if (!node->is_seq()) { + misc.warning("minimization.parameter_stepwidths is not a sequence, skipping."); + return; + } + for (JSONNode const &item : node->children()) { + if (!item.is_map() || !item.has_child("name") || !item.has_child("step_width") || + !item["step_width"].is_number() || item["name"].is_container() || item["name"].is_null()) { + misc.warning("skipping malformed parameter_stepwidths entry."); + continue; + } + out.push_back({item["name"].val(), item["step_width"].val_double()}); + } +} + +void readMisc(Cursor const &root, RooFitInternal &out) +{ + if (!root.node().has_child("misc")) { + return; + } + Cursor misc{root.node()["misc"], root, "misc"}; + if (!misc.node().is_map()) { + return; + } + readStepWidths(misc, out.stepWidths); + if (!misc.node().has_child("ROOT_internal")) { + return; + } + Cursor internal{misc.node()["ROOT_internal"], misc, "ROOT_internal"}; + if (!internal.node().is_map()) { + return; + } + Reader ar{internal}; + readMap(ar, "attributes", out.attributes); + readMap(ar, "combined_distributions", out.combinedDistributions); + readMap(ar, "combined_datasets", out.combinedDatasets); + readMap(ar, "ModelConfigs", out.modelConfigs); + readMap(ar, "histfactory_constraints", out.channelConstraints); +} + +void readMetadata(Cursor const &root, Metadata &out, ParseOptions const &options) +{ + JSONNode const *metadata = root.node().find("metadata"); + JSONNode const *versionNode = metadata ? metadata->find("hs3_version") : nullptr; + if (!versionNode) { + if (options.requireMetadata) { + std::stringstream ss; + ss << "The HS3 version is missing in the JSON!\n" + << "Please include the HS3 version in the metadata field, e.g.:\n" + << " \"metadata\" :\n" + << " {\n" + << " \"hs3_version\" : \"" << supportedVersion << "\"\n" + << " }"; + root.error(ss.str()); + } + return; + } + Cursor metadataCursor{*metadata, root, "metadata"}; + Cursor versionCursor{*versionNode, metadataCursor, "hs3_version"}; + if (versionNode->is_container() || versionNode->is_null() || versionNode->is_number()) { + versionCursor.error(std::string("The HS3 version in the metadata must be a string, e.g. \"") + supportedVersion + + "\""); + return; + } + out.hs3Version = versionNode->val(); + + Version supported; + parseVersion(supportedVersion, supported); + if (!parseVersion(out.hs3Version, out.version)) { + versionCursor.warning("cannot parse hs3_version '" + out.hs3Version + + "', expected a string of the form \"MAJOR.MINOR\"."); + } else if (supported < Version{out.version.major, out.version.minor}) { + versionCursor.warning("the document has hs3_version '" + out.hs3Version + + "', which is newer than the version supported by this RooFit ('" + supportedVersion + + "'). Some content might not be imported correctly."); + } + + Reader ar{metadataCursor}; + ar.has("hs3_version"); + ar.list("packages", out.packages); + ar.list("authors", out.authors); + ar.list("publications", out.publications); + ar("description", out.description); +} + +} // namespace + +HS3Document parseDocument(JSONNode const &root, Diagnostics &diagnostics, ParseOptions const &options) +{ + HS3Document doc; + Cursor cursor{root, diagnostics}; + readMetadata(cursor, doc.metadata, options); + if (hasErrors(diagnostics)) { + return doc; + } + + Reader ar{cursor}; + ar.has("metadata"); + ar.has("misc"); + if (ar.has("distributions") && !root["distributions"].is_null()) { + Cursor distributions{root["distributions"], cursor, "distributions"}; + if (!distributions.node().is_seq()) { + distributions.error("must be an array"); + } else { + std::size_t i = 0; + doc.distributions.reserve(distributions.node().num_children()); + for (JSONNode const &item : distributions.node().children()) { + doc.distributions.emplace_back(); + readDistribution(Cursor{item, distributions, i}, doc.distributions.back(), options); + ++i; + } + } + } + ar.list("functions", doc.functions); + ar.list("data", doc.data); + ar.list("likelihoods", doc.likelihoods); + ar.list("domains", doc.domains); + ar.list("parameter_points", doc.parameterPoints); + ar.list("analyses", doc.analyses); + readMisc(cursor, doc.rooFit); + return doc; +} + +HistFactoryDist parseHistFactoryDist(JSONNode const &distribution, Diagnostics &diagnostics) +{ + HistFactoryDist out; + Cursor cursor{distribution, diagnostics}; + if (!distribution.is_map()) { + cursor.error("must be a struct"); + return out; + } + Reader ar{cursor}; + ar.has("name"); + ar.has("type"); + describe(ar, out); + return out; +} + +void readValue(Cursor const &c, NameOrNumber &out) +{ + if (c.node().is_number()) { + out = c.node().val_double(); + } else { + readValue(c, out.emplace()); + } +} + +std::optional readInterpolationCode(Cursor const &c, InterpolationClass interpolationClass) +{ + JSONNode const &node = c.node(); + if (node.is_container() || node.is_null() || !node.has_val()) { + c.error("must be a structured interpolation or a legacy integer code"); + return std::nullopt; + } + const std::string value = node.val(); + int code = 0; + const auto result = std::from_chars(value.data(), value.data() + value.size(), code); + if (result.ec != std::errc{} || result.ptr != value.data() + value.size()) { + c.error("has invalid legacy interpolation code '" + value + "'"); + return std::nullopt; + } + auto interpolation = interpolationFromCode(code, interpolationClass); + if (!interpolation) { + c.error(std::string("has unsupported ") + + (interpolationClass == InterpolationClass::Piecewise ? "PiecewiseInterpolation" : "FlexibleInterpVar") + + " code " + value); + } + return interpolation; +} + +namespace { + +// The "interpolations" of the functions "interpolation" and "interpolation0d": one for all parameters, or one per +// parameter. Older versions of RooFit wrote integer "interpolationCodes". Without both, the interpolation is linear. +void readInterpolations(Reader &ar, std::size_t nParameters, InterpolationClass interpolationClass, + std::vector &out) +{ + out.assign(nParameters, additivePiecewiseLinear); + if (ar.has("interpolations")) { + Cursor c{ar.cursor().node()["interpolations"], ar.cursor(), "interpolations"}; + const std::size_t errorsBefore = c.numErrors(); + std::vector interpolations; + readValue(c, interpolations); + const std::size_t size = interpolations.size(); + if (c.numErrors() != errorsBefore) { + return; + } + if (nParameters == 0 ? size != 0 : size != 1 && size != nParameters) { + c.error("component 'interpolations' must contain either one descriptor or one descriptor per parameter (got " + + std::to_string(size) + " for " + std::to_string(nParameters) + " parameters)"); + } else if (size == 1) { + out.assign(nParameters, interpolations.front()); + } else { + out = std::move(interpolations); + } + return; + } + if (!ar.has("interpolationCodes")) { + return; + } + Cursor c{ar.cursor().node()["interpolationCodes"], ar.cursor(), "interpolationCodes"}; + if (!c.node().is_seq() || c.node().num_children() != nParameters) { + c.error("legacy component 'interpolationCodes' must contain one code per parameter (got " + + std::to_string(c.node().is_seq() ? c.node().num_children() : 0) + " for " + std::to_string(nParameters) + + " parameters)"); + return; + } + std::size_t i = 0; + for (JSONNode const &child : c.node().children()) { + if (auto interpolation = readInterpolationCode(Cursor{child, c, i}, interpolationClass)) { + out[i] = *interpolation; + } + ++i; + } +} + +// The functions "interpolation" and "interpolation0d" have the variations "high" and "low" for each of the "vars". +template +Function_t parseInterpolated(JSONNode const &node, Diagnostics &diagnostics, InterpolationClass interpolationClass, + Fn &&readOthers) +{ + Function_t out; + Cursor cursor{node, diagnostics}; + if (!node.is_map()) { + cursor.error("must be a struct"); + return out; + } + const std::size_t errorsBefore = cursor.numErrors(); + Reader ar{cursor}; + ar.has("name"); + ar.has("type"); + ar("nom", out.nominal); + ar("vars", out.parameters); + ar("high", out.high); + ar("low", out.low); + readOthers(ar, out); + if (cursor.numErrors() == errorsBefore && + (out.high.size() != out.parameters.size() || out.low.size() != out.parameters.size())) { + cursor.error("has non-matching lengths of 'vars', 'high' and 'low'"); + } + readInterpolations(ar, out.parameters.size(), interpolationClass, out.interpolations); + return out; +} + +} // namespace + +std::vector parseBinnedAxes(JSONNode const &axes, Diagnostics &diagnostics) +{ + std::vector out; + readValue(Cursor{axes, diagnostics}, out); + return out; +} + +InterpolatedFunction parseInterpolatedFunction(JSONNode const &node, Diagnostics &diagnostics) +{ + return parseInterpolated( + node, diagnostics, InterpolationClass::Piecewise, + [](Reader &ar, InterpolatedFunction &out) { ar("positiveDefinite", out.positiveDefinite); }); +} + +InterpolatedNumber parseInterpolatedNumber(JSONNode const &node, Diagnostics &diagnostics) +{ + return parseInterpolated(node, diagnostics, InterpolationClass::Flexible, + [](Reader &, InterpolatedNumber &) {}); +} + +} // namespace HS3 +} // namespace RooFit diff --git a/roofit/hs3/src/HS3Schema.h b/roofit/hs3/src/HS3Schema.h new file mode 100644 index 0000000000000..6dfcebed75f60 --- /dev/null +++ b/roofit/hs3/src/HS3Schema.h @@ -0,0 +1,196 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef RooFit_HS3Schema_h +#define RooFit_HS3Schema_h + +#include "HS3Document.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace RooFit { +namespace HS3 { + +/// A position in the JSON tree. Keeps track of how it got there, in order to create the paths in error messages only +/// when they are needed. +class Cursor { +public: + Cursor(Detail::JSONNode const &node, Diagnostics &diagnostics) : _node{node}, _diagnostics{diagnostics} {} + Cursor(Detail::JSONNode const &node, Cursor const &parent, const char *key) + : _node{node}, _diagnostics{parent._diagnostics}, _parent{&parent}, _key{key} + { + } + Cursor(Detail::JSONNode const &node, Cursor const &parent, std::size_t index) + : _node{node}, _diagnostics{parent._diagnostics}, _parent{&parent}, _index{index} + { + } + + Detail::JSONNode const &node() const { return _node; } + std::string path() const; + void error(std::string const &message) const; + void warning(std::string const &message) const; + std::size_t numErrors() const; + +private: + Detail::JSONNode const &_node; + Diagnostics &_diagnostics; + Cursor const *_parent = nullptr; + const char *_key = nullptr; + std::size_t _index = 0; +}; + +/// Reads the components of a struct. +class Reader { +public: + explicit Reader(Cursor const &cursor); + /// Reports the components that were not read, unless reading the struct gave an error. + ~Reader(); + Reader(Reader const &) = delete; + Reader &operator=(Reader const &) = delete; + + /// Read a required component, or an optional one if the field is a std::optional. + template + void operator()(const char *key, T &field); + template + void operator()(const char *key, std::optional &field); + + /// Read an array that can be omitted, which is equivalent to an empty one. + template + void list(const char *key, std::vector &field); + + /// Read the components of a struct that is nested in a component. + template + void nested(const char *key, Fn &&fn); + + /// Whether the struct has a component. Checking for a component counts as reading it. + bool has(const char *key); + + Cursor const &cursor() const { return _cursor; } + +private: + Cursor const &_cursor; + std::vector _seen; + std::size_t _nErrorsBefore = 0; +}; + +void readValue(Cursor const &c, std::string &out); +void readValue(Cursor const &c, double &out); +void readValue(Cursor const &c, int &out); +void readValue(Cursor const &c, bool &out); +void readValue(Cursor const &c, ParameterValue &out); +void readValue(Cursor const &c, Data &out); +void readValue(Cursor const &c, Interpolation &out); +void readValue(Cursor const &c, NameOrNumber &out); + +/// Reads an integer interpolation code, as older versions of RooFit wrote them. +std::optional readInterpolationCode(Cursor const &c, InterpolationClass interpolationClass); + +template +void readValue(Cursor const &c, Ref &out) +{ + readValue(c, out.name); +} + +template +void readValue(Cursor const &c, std::vector &out) +{ + Detail::JSONNode const &node = c.node(); + if (!node.is_seq()) { + c.error("must be an array"); + return; + } + out.clear(); + out.reserve(node.num_children()); + std::size_t i = 0; + for (Detail::JSONNode const &child : node.children()) { + out.emplace_back(); + readValue(Cursor{child, c, i}, out.back()); + ++i; + } +} + +/// Everything that has a describe() function is read from a struct. +template +auto readValue(Cursor const &c, T &out) -> std::void_t(), out))> +{ + if (!c.node().is_map()) { + c.error("must be a struct"); + return; + } + Reader reader{c}; + describe(reader, out); +} + +template +void Reader::operator()(const char *key, T &field) +{ + if (!has(key)) { + _cursor.error(std::string("does not define the required component \"") + key + "\""); + return; + } + readValue(Cursor{_cursor.node()[key], _cursor, key}, field); +} + +template +void Reader::operator()(const char *key, std::optional &field) +{ + if (!has(key)) { + return; + } + Detail::JSONNode const &node = _cursor.node()[key]; + if (node.is_null()) { + return; + } + field.emplace(); + readValue(Cursor{node, _cursor, key}, *field); +} + +template +void Reader::list(const char *key, std::vector &field) +{ + if (!has(key)) { + return; + } + Detail::JSONNode const &node = _cursor.node()[key]; + if (node.is_null()) { + return; + } + readValue(Cursor{node, _cursor, key}, field); +} + +template +void Reader::nested(const char *key, Fn &&fn) +{ + if (!has(key)) { + _cursor.error(std::string("does not define the required component \"") + key + "\""); + return; + } + Cursor child{_cursor.node()[key], _cursor, key}; + if (!child.node().is_map()) { + child.error("must be a struct"); + return; + } + Reader reader{child}; + fn(reader); +} + +} // namespace HS3 +} // namespace RooFit + +#endif diff --git a/roofit/hs3/src/HS3Upgrade.cxx b/roofit/hs3/src/HS3Upgrade.cxx new file mode 100644 index 0000000000000..d5be0bcf733f9 --- /dev/null +++ b/roofit/hs3/src/HS3Upgrade.cxx @@ -0,0 +1,406 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +// Everything that older versions of RooFit wrote in a way that is not part of the HS3 standard is translated here into +// the form that the parser understands. The rewrites go by the shape of the JSON: the same version number was written +// by ROOT releases with different conventions. Only what a missing component means depends on the version. + +#include "HS3Schema.h" + +#include +#include +#include + +using RooFit::Detail::JSONNode; + +namespace RooFit { +namespace HS3 { +namespace { + +// The version that the standard requires the interpolation of modifiers from on. +constexpr Version explicitInterpolationVersion{0, 3, 0}; +// Before this version, normsys and histosys modifiers without constraint had an implicit Gaussian constraint. +constexpr Version explicitConstraintVersion{0, 2, 0}; +// From this version on, the gamma parameters of modifiers are free unless "constraints" names their constraint terms, +// "shapesys" is deprecated, and a sample without "errors" has no statistical uncertainty. +constexpr Version namedConstraintVersion{0, 3, 0}; + +using IsDistribution = std::function; + +bool isScalar(JSONNode const &node) +{ + return !node.is_container() && !node.is_null(); +} + +bool isConstraintType(std::string const &value) +{ + return value == "Gauss" || value == "Poisson" || value == "LogNormal"; +} + +// Older versions of RooFit spelled "LogNormal" differently. +std::string canonicalConstraintType(std::string const &value) +{ + return value == "Lognormal" ? "LogNormal" : value; +} + +// The "constraints" of the standard take precedence over the keys that older documents used. +bool eraseLegacyConstraints(JSONNode &mod) +{ + if (!mod.has_child("constraints")) { + return false; + } + mod.erase("constraint"); + mod.erase("constraint_name"); + mod.erase("constraint_type"); + return true; +} + +// Write a legacy constraint of a normsys, histosys or normfactor modifier as "constraint" if it is a type, or +// "constraint_name" if it refers to a distribution. +void upgradeConstraint(JSONNode &mod, IsDistribution const &isDistribution) +{ + if (eraseLegacyConstraints(mod)) { + return; + } + if (mod.has_child("constraint_name")) { + mod.erase("constraint"); + mod.erase("constraint_type"); + return; + } + if (mod.has_child("constraint")) { + mod.erase("constraint_type"); + JSONNode &constraint = mod["constraint"]; + if (!isScalar(constraint)) { + return; // reported by the parser + } + const std::string value = constraint.val(); + // In older documents, the "constraint" can be the name of a distribution, which has priority. + if (isDistribution(value)) { + mod["constraint_name"] << value; + mod.erase("constraint"); + } else { + constraint << canonicalConstraintType(value); + } + return; + } + if (mod.has_child("constraint_type")) { + JSONNode &constraintType = mod["constraint_type"]; + if (isScalar(constraintType)) { + mod["constraint"] << canonicalConstraintType(constraintType.val()); + } + mod.erase("constraint_type"); + } +} + +// The constraint of the gamma parameters of a shapesys modifier. In contrast to the other modifiers, it is not possible +// to name a distribution in this way, and "constraint_type" has priority. +void upgradeGammaConstraint(JSONNode &mod) +{ + if (eraseLegacyConstraints(mod)) { + return; + } + if (mod.has_child("constraint_type")) { + JSONNode &constraintType = mod["constraint_type"]; + if (isScalar(constraintType)) { + mod["constraint"] << canonicalConstraintType(constraintType.val()); + } + mod.erase("constraint_type"); + } else if (mod.has_child("constraint") && isScalar(mod["constraint"])) { + mod["constraint"] << canonicalConstraintType(mod["constraint"].val()); + } +} + +// An integer code that was used before the structured interpolations. +void upgradeInterpolationCode(JSONNode &mod, InterpolationClass interpolationClass, Cursor const &c) +{ + if (!mod.has_child("interpolation") || mod["interpolation"].is_map()) { + return; + } + JSONNode &node = mod["interpolation"]; + if (auto interpolation = readInterpolationCode(Cursor{node, c, "interpolation"}, interpolationClass)) { + node.set_null(); + writeInterpolation(node, *interpolation); + } +} + +struct ChannelState { + std::string channelName; + bool warnedInterpolation = false; + bool warnedUnconstrained = false; + bool warnedShapeSys = false; +}; + +// Before structured interpolations were introduced, both modifier classes used the integer code 4 as their implicit +// default. The meaning of the code is class-dependent. HS3 forbids implementation-specific defaults. +void upgradeMissingInterpolation(JSONNode &mod, InterpolationClass interpolationClass, Version const &version, + bool hasDefault, std::string const &context, ChannelState &state, Cursor const &c) +{ + if (mod.has_child("interpolation") || hasDefault || !(version < explicitInterpolationVersion)) { + return; + } + if (!state.warnedInterpolation) { + c.warning(context + " is not given, and the channel has no \"default_interpolation\". Using the implicit default " + "of older RooFit versions (code 4), which is deprecated."); + state.warnedInterpolation = true; + } + mod["interpolation"] << 4; + upgradeInterpolationCode(mod, interpolationClass, c); +} + +// Whether "constraints" leaves the single parameter of a modifier unconstrained. +bool isExplicitlyUnconstrained(JSONNode const &constraints) +{ + return constraints.is_seq() && constraints.num_children() == 1 && constraints.child(0).is_null(); +} + +void upgradeModifier(JSONNode &mod, std::size_t index, std::string const &sampleName, Version const &version, + bool hasDefaultInterpolation, IsDistribution const &isDistribution, ChannelState &state, + Cursor const &c) +{ + if (!mod.has_child("type") || !isScalar(mod["type"])) { + return; // reported by the parser + } + const std::string type = mod["type"].val(); + const bool hasSingleParameter = type == "normfactor" || type == "normsys" || type == "histosys"; + + // The "parameters" of the standard take precedence over the "parameter" of older documents. + std::string parameter; + if (hasSingleParameter && mod.has_child("parameters")) { + mod.erase("parameter"); + JSONNode const ¶meters = mod["parameters"]; + if (parameters.is_seq() && parameters.num_children() == 1 && isScalar(parameters.child(0))) { + parameter = parameters.child(0).val(); + } + } else if (mod.has_child("parameter") && isScalar(mod["parameter"])) { + parameter = mod["parameter"].val(); + } + + // The name of a modifier is not part of the standard. RooFit names objects after it, so a generated name must be + // unique, unlike the name of a parameter, which is shared on purpose. + if (!mod.has_child("name")) { + mod["name"] << (!parameter.empty() ? parameter + : state.channelName + "_" + sampleName + "_syst_" + std::to_string(index)); + } + const std::string name = isScalar(mod["name"]) ? mod["name"].val() : ""; + + if (hasSingleParameter && !mod.has_child("parameter") && !mod.has_child("parameters")) { + parameter = type == "normfactor" ? name : "alpha_" + name; + mod["parameter"] << parameter; + } + + if (type == "normfactor") { + upgradeConstraint(mod, isDistribution); + } else if (type == "normsys" || type == "histosys") { + upgradeConstraint(mod, isDistribution); + const bool constrained = mod.has_child("constraint") || mod.has_child("constraint_name") || + (mod.has_child("constraints") && !isExplicitlyUnconstrained(mod["constraints"])); + if (!constrained) { + if (version < explicitConstraintVersion) { + mod.erase("constraints"); + mod["constraint"] << "Gauss"; + } else if (!state.warnedUnconstrained) { + c.warning("modifier '" + name + "' of sample '" + sampleName + "' has no constraint, so its parameter '" + + parameter + + "' is unconstrained. Older RooFit versions implied a unit Gaussian. Other modifiers of this " + "channel are not reported."); + state.warnedUnconstrained = true; + } + } + const auto interpolationClass = type == "normsys" ? InterpolationClass::Flexible : InterpolationClass::Piecewise; + const std::string context = "interpolation for " + type + " modifier '" + name + "' in sample '" + sampleName + + "' of channel '" + state.channelName + "'"; + upgradeInterpolationCode(mod, interpolationClass, c); + upgradeMissingInterpolation(mod, interpolationClass, version, hasDefaultInterpolation, context, state, c); + } else if (type == "shapesys" || type == "shapefactor") { + if (type == "shapesys" && !(version < namedConstraintVersion) && !state.warnedShapeSys) { + c.warning("the modifier type \"shapesys\" is deprecated, please use \"shapefactor\" instead."); + state.warnedShapeSys = true; + } + upgradeGammaConstraint(mod); + if (!mod.has_child("constraint") && !mod.has_child("constraints")) { + if (version < namedConstraintVersion) { + // Without any information about the constraint, older versions created Gaussian constraints. + mod["constraint"] << "Gauss"; + } else if (mod.has_child("data") && mod["data"].is_map() && mod["data"].has_child("vals")) { + if (mod["data"]["vals"].is_seq() && mod["data"]["vals"].num_children() > 0) { + c.warning("the \"data.vals\" of modifier '" + name + "' of sample '" + sampleName + + "' are ignored, because it names no constraints, so its parameters are free"); + } + mod["data"].erase("vals"); + } + } + } else if (type == "staterror") { + if (eraseLegacyConstraints(mod)) { + return; + } + // ROOT 6.38 and 6.40.00 wrote "constraint_type". + if (mod.has_child("constraint_type")) { + if (!mod.has_child("constraint") && isScalar(mod["constraint_type"])) { + mod["constraint"] << mod["constraint_type"].val(); + } + mod.erase("constraint_type"); + } + } +} + +void upgradeHistFactory(JSONNode &dist, Version const &version, IsDistribution const &isDistribution, Cursor const &c) +{ + ChannelState state; + if (dist.has_child("name") && isScalar(dist["name"])) { + state.channelName = dist["name"].val(); + if (state.channelName.rfind("model_", 0) == 0) { + state.channelName.erase(0, 6); + } + } + + // Before HS3 0.3, the gamma parameters of the staterror modifiers had Poisson constraints unless the document said + // otherwise, which RooFit also creates if the document has the channel-level settings of RooFit. Samples without + // "errors" had Poisson uncertainties. + const bool legacyStatError = version < namedConstraintVersion; + if (legacyStatError && !dist.has_child("staterror")) { + dist["staterror"].set_map(); + } + if (dist.has_child("staterror") && dist["staterror"].is_map()) { + JSONNode &config = dist["staterror"]; + upgradeGammaConstraint(config); + if (!config.has_child("constraint") && !config.has_child("constraints")) { + config["constraint"] << "Poisson"; + } + if (legacyStatError) { + config["implied_poisson_errors"] << true; + } + } + + if (!dist.has_child("samples") || !dist["samples"].is_seq()) { + return; + } + const bool hasDefaultInterpolation = dist.has_child("default_interpolation"); + + Cursor samples{dist["samples"], c, "samples"}; + std::size_t iSample = 0; + for (JSONNode &sample : dist["samples"].children()) { + Cursor sampleCursor{sample, samples, iSample++}; + if (!sample.is_map() || !sample.has_child("modifiers") || !sample["modifiers"].is_seq()) { + continue; + } + const std::string sampleName = sample.has_child("name") && isScalar(sample["name"]) ? sample["name"].val() : ""; + Cursor modifiers{sample["modifiers"], sampleCursor, "modifiers"}; + std::size_t iModifier = 0; + for (JSONNode &mod : sample["modifiers"].children()) { + Cursor modCursor{mod, modifiers, iModifier}; + if (mod.is_map()) { + upgradeModifier(mod, iModifier, sampleName, version, hasDefaultInterpolation, isDistribution, state, + modCursor); + } + ++iModifier; + } + } +} + +// Older versions of RooFit wrote the binning of the observables also for unbinned data, which the standard forbids. +// The edges were written without "min" and "max". +void upgradeUnbinnedAxes(JSONNode &data) +{ + if (!data.has_child("axes") || !data["axes"].is_seq()) { + return; + } + for (JSONNode &axis : data["axes"].children()) { + if (!axis.is_map()) { + continue; + } + if (axis.has_child("edges") && axis["edges"].is_seq() && axis["edges"].num_children() > 0) { + JSONNode &edges = axis["edges"]; + JSONNode const &first = edges.child(0); + JSONNode const &last = edges.child(edges.num_children() - 1); + if (!axis.has_child("min") && first.is_number()) { + axis["min"] << first.val_double(); + } + if (!axis.has_child("max") && last.is_number()) { + axis["max"] << last.val_double(); + } + } + axis.erase("edges"); + axis.erase("nbins"); + } +} + +void upgradeData(JSONNode &root, Version const &version) +{ + if (!root.has_child("data") || !root["data"].is_seq() || !(version < explicitInterpolationVersion)) { + return; + } + for (JSONNode &data : root["data"].children()) { + if (data.is_map() && data.has_child("type") && isScalar(data["type"]) && data["type"].val() == "unbinned") { + upgradeUnbinnedAxes(data); + } + } +} + +Version documentVersion(JSONNode const &root) +{ + Version version; + if (JSONNode const *versionNode = root.find("metadata", "hs3_version")) { + if (isScalar(*versionNode)) { + parseVersion(versionNode->val(), version); + } + } + return version; +} + +} // namespace + +void upgradeDocument(JSONNode &root, Diagnostics &diagnostics, ParseOptions const &options) +{ + if (!root.is_map()) { + return; + } + const Version version = documentVersion(root); + upgradeData(root, version); + + if (!root.has_child("distributions") || !root["distributions"].is_seq() || !options.typedHistFactory) { + return; + } + + std::set distributionNames; + for (JSONNode const &dist : root["distributions"].children()) { + if (dist.is_map() && dist.has_child("name") && isScalar(dist["name"])) { + distributionNames.insert(dist["name"].val()); + } + } + // The version from which on "constraint" is only a type of the standard. + const bool constraintCanBeName = version < explicitInterpolationVersion; + IsDistribution isDistribution = [&](std::string const &name) { + return constraintCanBeName && distributionNames.count(name) > 0; + }; + + Cursor rootCursor{root, diagnostics}; + Cursor distributions{root["distributions"], rootCursor, "distributions"}; + std::size_t i = 0; + for (JSONNode &dist : root["distributions"].children()) { + Cursor distCursor{dist, distributions, i++}; + if (dist.is_map() && dist.has_child("type") && isScalar(dist["type"]) && + dist["type"].val() == "histfactory_dist") { + upgradeHistFactory(dist, version, isDistribution, distCursor); + } + } +} + +void upgradeHistFactoryDist(JSONNode &distribution, Version const &version, Diagnostics &diagnostics) +{ + IsDistribution isDistribution = [](std::string const &name) { + return !isConstraintType(name) && name != "Lognormal" && name != "Const"; + }; + upgradeHistFactory(distribution, version, isDistribution, Cursor{distribution, diagnostics}); +} + +} // namespace HS3 +} // namespace RooFit diff --git a/roofit/hs3/src/HS3Validate.cxx b/roofit/hs3/src/HS3Validate.cxx new file mode 100644 index 0000000000000..eaebe10785ca0 --- /dev/null +++ b/roofit/hs3/src/HS3Validate.cxx @@ -0,0 +1,606 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "HS3Document.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace RooFit { +namespace HS3 { +namespace { + +std::string at(const char *section, std::size_t index, std::string const &name) +{ + return std::string(section) + "[" + std::to_string(index) + "](" + name + ")"; +} + +class Validator { +public: + Validator(HS3Document const &doc, Diagnostics &diagnostics, ValidateOptions const &options) + : _doc{doc}, _diagnostics{diagnostics}, _options{options} + { + } + + void run() + { + collectNames(); + checkFixedAxes(); + checkLikelihoods(); + checkAnalyses(); + checkData(); + for (std::size_t i = 0; i < _doc.distributions.size(); ++i) { + auto const &dist = _doc.distributions[i]; + if (auto const *histFactory = std::get_if(&dist.body)) { + checkHistFactory(*histFactory, dist.name, at("distributions", i, dist.name)); + } + } + for (auto const &item : _doc.rooFit.combinedDistributions) { + for (auto const &ref : item.second.distributions) { + requireDistribution(ref.name, "misc.ROOT_internal.combined_distributions." + item.first, "distribution"); + } + } + for (auto const &item : _doc.rooFit.channelConstraints) { + requireDistribution(item.first, "misc.ROOT_internal.histfactory_constraints", "distribution"); + for (auto const &ref : item.second) { + requireDistribution(ref.name, "misc.ROOT_internal.histfactory_constraints." + item.first, "constraint"); + } + } + } + + void error(std::string const &path, std::string const &message) + { + _diagnostics.push_back({Diagnostic::Severity::Error, path, message}); + } + + void warning(std::string const &path, std::string const &message) + { + _diagnostics.push_back({Diagnostic::Severity::Warning, path, message}); + } + + // An error in documents that follow the current standard, which older versions of RooFit didn't care about. + void strictError(std::string const &path, std::string const &message) + { + _diagnostics.push_back( + {_options.strict ? Diagnostic::Severity::Error : Diagnostic::Severity::Warning, path, message}); + } + + void checkHistFactory(HistFactoryDist const &dist, std::string const &name, std::string const &path); + +private: + bool isExternal(std::string const &name) const { return _options.isExternal && _options.isExternal(name); } + + char const *sectionOf(std::string const &name) const + { + auto found = _sections.find(name); + return found != _sections.end() ? found->second.front() : nullptr; + } + + bool isDefinedIn(std::string const &name, char const *section) const + { + auto found = _sections.find(name); + if (found == _sections.end()) { + return false; + } + return std::any_of(found->second.begin(), found->second.end(), + [&](char const *s) { return std::string(s) == section; }); + } + + void requireDistribution(std::string const &name, std::string const &path, std::string const &what) + { + // RooFit refers to a RooSimultaneous by name, like the likelihood of a ModelConfig without data does. + if (!isDefinedIn(name, "distributions") && !_doc.rooFit.combinedDistributions.count(name) && !isExternal(name)) { + error(path, "the " + what + " '" + name + "' does not exist"); + } + } + + void addParameter(std::string const &name) { _parameters.insert(name); } + + // The number of bins of an axis, after reporting an invalid binning. + std::size_t checkAxis(BinnedAxis const &axis, std::string const &path); + std::size_t checkAxis(CategoricalAxis const &axis, std::string const &path); + template + std::size_t checkAxis(std::variant const &axis, std::string const &path) + { + return std::visit([&](auto const &a) { return checkAxis(a, path); }, axis); + } + // The number of bins of all axes, or nothing if it can't be stored. + template + std::optional numBins(Axes const &axes, std::string const &path); + + void collectNames(); + void checkFixedAxes(); + void checkLikelihoods(); + void checkAnalyses(); + void checkData(); + + HS3Document const &_doc; + Diagnostics &_diagnostics; + ValidateOptions const &_options; + std::map> _sections; ///< The sections where a name is defined. + std::set _parameters; +}; + +// HS3 requires the names of all objects to be unique within a document, including the parameters. RooFit writes such +// documents, e.g. for a range and a pdf of one name, so only ambiguous clashes in current documents are errors. +void Validator::collectNames() +{ + auto check = [&](char const *section, std::string const &name) { + auto §ions = _sections[name]; + sections.push_back(section); + if (sections.size() == 1) { + return; + } + std::string const firstSection = sections.front(); + // RooFit used to give analyses and likelihoods the name of the pdf or the analysis. Accept this for backwards + // compatibility: analyses are not referenced by name, and likelihoods only from analyses, so there is no + // ambiguity. + auto isLegacyNamed = [](std::string const &s) { return s == "likelihoods" || s == "analyses"; }; + // Within one section, and between distributions and functions (both can be referenced by name), a clash is + // ambiguous. Other clashes are produced by RooFit itself (a range, snapshot or data set named like a pdf). + auto isReferable = [](std::string const &s) { return s == "distributions" || s == "functions"; }; + const bool ambiguous = firstSection == section || (isReferable(firstSection) && isReferable(section)); + if (!_options.strict || !ambiguous || isLegacyNamed(firstSection) || isLegacyNamed(section)) { + warning("", "the name '" + name + "' is used in \"" + firstSection + "\" and in \"" + section + + "\", which is not allowed by the HS3 standard."); + return; + } + error("", "duplicate name '" + name + "': it is defined in \"" + firstSection + "\" and in \"" + section + + "\", but names must be unique within the document"); + }; + auto checkSection = [&](char const *section, auto const &objects) { + for (auto const &object : objects) { + check(section, object.name); + } + }; + checkSection("distributions", _doc.distributions); + checkSection("functions", _doc.functions); + checkSection("data", _doc.data); + checkSection("domains", _doc.domains); + checkSection("parameter_points", _doc.parameterPoints); + checkSection("likelihoods", _doc.likelihoods); + checkSection("analyses", _doc.analyses); + + // The parameters are the axes of the domains, the entries of the parameter points, and the parameters of + // HistFactory modifiers, which are created on the fly. + for (auto const &domain : _doc.domains) { + for (auto const &axis : domain.axes) { + addParameter(std::visit([](auto const &a) -> std::string const & { return a.name; }, axis)); + } + } + for (auto const &point : _doc.parameterPoints) { + for (auto const ¶meter : point.parameters) { + addParameter(parameter.name); + } + } + for (auto const &item : _doc.rooFit.combinedDistributions) { + addParameter(item.second.indexCat); + } + for (auto const &dist : _doc.distributions) { + auto const *histFactory = std::get_if(&dist.body); + if (!histFactory) { + continue; + } + for (auto const &sample : histFactory->samples) { + for (auto const &modifier : sample.modifiers) { + std::visit(overloaded{[&](NormFactor const &m) { addParameter(m.parameter.name); }, + [&](NormSys const &m) { addParameter(m.parameter.name); }, + [&](HistoSys const &m) { addParameter(m.parameter.name); }, + [&](ShapeSys const &m) { + for (auto const &p : m.parameters) { + addParameter(p.name); + } + }, + [&](StatError const &m) { + for (auto const &p : m.parameters) { + addParameter(p.name); + } + }, + [](CustomModifier const &) {}}, + modifier); + } + } + } + for (std::string const ¶meter : _parameters) { + if (char const *section = sectionOf(parameter)) { + strictError("", "the name '" + parameter + "' is used in \"" + section + + "\" and for a parameter, which is not allowed by the HS3 standard."); + } + } +} + +// A fixed axis says that the parameter only takes a single value. No other domain can give it a range or another +// value, and no parameter point another value. +void Validator::checkFixedAxes() +{ + struct FixedAxis { + double value = 0.0; + std::string const *domain = nullptr; + }; + std::map fixedAxes; + auto forEachAxis = [&](auto &&fn) { + for (std::size_t i = 0; i < _doc.domains.size(); ++i) { + for (auto const &axis : _doc.domains[i].axes) { + // A categorical axis is passed as null. + fn(i, std::visit([](auto const &a) -> std::string const & { return a.name; }, axis), + std::get_if(&axis)); + } + } + }; + forEachAxis([&](std::size_t i, std::string const &, DomainAxis const *continuous) { + if (!continuous || !continuous->fixedValue) { + return; + } + DomainAxis const &axis = *continuous; + Domain const &domain = _doc.domains[i]; + if (!std::isfinite(*axis.fixedValue)) { + error(at("domains", i, domain.name), + "the single value of the parameter '" + axis.name + "' is not finite, which JSON can't represent"); + } + auto inserted = fixedAxes.insert({axis.name, FixedAxis{*axis.fixedValue, &domain.name}}); + if (!inserted.second && inserted.first->second.value != *axis.fixedValue) { + error(at("domains", i, domain.name), + "the parameter '" + axis.name + "' has different single values in the domains '" + + *inserted.first->second.domain + "' and '" + domain.name + "'"); + } + }); + if (fixedAxes.empty()) { + return; + } + forEachAxis([&](std::size_t i, std::string const &name, DomainAxis const *continuous) { + auto found = fixedAxes.find(name); + if (found != fixedAxes.end() && !(continuous && continuous->fixedValue)) { + Domain const &domain = _doc.domains[i]; + error(at("domains", i, domain.name), + "the parameter '" + name + "' has the single value " + std::to_string(found->second.value) + + " in the domain '" + *found->second.domain + "', but " + (continuous ? "a range" : "categories") + + " in the domain '" + domain.name + "'"); + } + }); + for (std::size_t i = 0; i < _doc.parameterPoints.size(); ++i) { + ParameterPoint const &point = _doc.parameterPoints[i]; + for (ParameterValue const ¶meter : point.parameters) { + auto found = fixedAxes.find(parameter.name); + double const *value = parameter.value ? std::get_if(&*parameter.value) : nullptr; + if (value && found != fixedAxes.end() && *value != found->second.value) { + error(at("parameter_points", i, point.name), + "the parameter point '" + point.name + "' gives the parameter '" + parameter.name + "' the value " + + std::to_string(*value) + ", but its domain only has the value " + + std::to_string(found->second.value)); + } + } + } +} + +void Validator::checkLikelihoods() +{ + for (std::size_t i = 0; i < _doc.likelihoods.size(); ++i) { + auto const &likelihood = _doc.likelihoods[i]; + const std::string path = at("likelihoods", i, likelihood.name); + for (auto const &term : likelihood.terms) { + requireDistribution(term.distribution.name, path + ".distributions", "distribution"); + if (auto const *data = std::get_if(&term.data)) { + if (!isDefinedIn(data->name, "data")) { + error(path + ".data", "dataset '" + data->name + "' cannot be found!"); + } + } + } + for (auto const &aux : likelihood.auxDistributions) { + if (!isDefinedIn(aux.name, "distributions") && !isExternal(aux.name)) { + strictError(path + ".aux_distributions", "the distribution '" + aux.name + "' does not exist"); + } + } + } +} + +void Validator::checkAnalyses() +{ + for (std::size_t i = 0; i < _doc.analyses.size(); ++i) { + auto const &analysis = _doc.analyses[i]; + const std::string path = at("analyses", i, analysis.name); + if (!isDefinedIn(analysis.likelihood.name, "likelihoods")) { + error(path + ".likelihood", "likelihood node not found!"); + } + for (auto const &poi : analysis.parametersOfInterest) { + if (char const *section = sectionOf(poi.name)) { + error(path, "the parameter of interest '" + poi.name + "' of analysis '" + analysis.name + + "' is defined in \"" + section + "\", but it must be a parameter"); + } else if (!_parameters.count(poi.name) && !isExternal(poi.name)) { + error(path, "the object '" + poi.name + "' in \"parameters_of_interest\" of analysis '" + analysis.name + + "' does not exist"); + } + } + for (auto const &domain : analysis.domains) { + if (!isDefinedIn(domain.name, "domains")) { + strictError(path + ".domains", "the domain '" + domain.name + "' does not exist"); + } + } + if (analysis.init && !isDefinedIn(analysis.init->name, "parameter_points")) { + warning(path, "the parameter point '" + analysis.init->name + "' used as \"init\" of analysis '" + + analysis.name + "' does not exist."); + } + if (analysis.prior && !isDefinedIn(analysis.prior->name, "distributions") && !isExternal(analysis.prior->name)) { + strictError(path + ".prior", "the distribution '" + analysis.prior->name + "' does not exist"); + } + } +} + +std::size_t Validator::checkAxis(BinnedAxis const &axis, std::string const &path) +{ + if (auto const *regular = std::get_if(&axis.binning)) { + if (regular->nbins <= 0) { + error(path, "the axis '" + axis.name + "' needs a positive number of bins"); + return 0; + } + if (!(axis.min < axis.max)) { + error(path, "the axis '" + axis.name + "' needs \"min\" smaller than \"max\""); + } + return regular->nbins; + } + auto const &edges = std::get(axis.binning).edges; + return edges.empty() ? 0 : edges.size() - 1; +} + +std::size_t Validator::checkAxis(CategoricalAxis const &axis, std::string const &path) +{ + if (axis.categories.empty()) { + error(path, "the axis '" + axis.name + "' needs at least one category"); + } + return axis.categories.size(); +} + +template +std::optional Validator::numBins(Axes const &axes, std::string const &path) +{ + std::size_t nBins = 1; + bool valid = true; + for (std::size_t i = 0; i < axes.size(); ++i) { + const std::size_t n = checkAxis(axes[i], path + ".axes[" + std::to_string(i) + "]"); + if (n == 0) { + valid = false; // reported already + } else if (nBins > std::numeric_limits::max() / n) { + error(path + ".axes", "the axes have more bins than can be stored"); + return std::nullopt; + } + nBins *= n; + } + return valid ? std::optional{nBins} : std::nullopt; +} + +void Validator::checkData() +{ + for (std::size_t i = 0; i < _doc.data.size(); ++i) { + auto const &data = _doc.data[i]; + auto const *binned = std::get_if(&data.body); + if (!binned) { + continue; + } + const std::string path = at("data", i, data.name); + std::optional nBinsOrNone = numBins(binned->axes, path); + if (!nBinsOrNone) { + continue; + } + const std::size_t nBins = *nBinsOrNone; + if (binned->contents.size() != nBins) { + error(path, "inconsistent bin numbers: contents=" + std::to_string(binned->contents.size()) + + ", bins=" + std::to_string(nBins)); + } + if (binned->uncertainty) { + auto const &sigma = binned->uncertainty->sigma; + if (sigma.size() != binned->contents.size()) { + error(path, "inconsistent number of errors and contents in '" + data.name + + "': errors=" + std::to_string(sigma.size()) + + ", contents=" + std::to_string(binned->contents.size())); + } + auto const &correlation = binned->uncertainty->correlation; + if (!correlation.empty()) { + bool square = correlation.size() == binned->contents.size(); + for (auto const &row : correlation) { + square = square && row.size() == binned->contents.size(); + } + if (!square) { + error(path + ".uncertainty.correlation", "must be a square matrix of the size of \"contents\""); + } + } + } + } +} + +void Validator::checkHistFactory(HistFactoryDist const &dist, std::string const &name, std::string const &path) +{ + if (dist.axes.empty()) { + error(path + ".axes", "a HistFactory distribution needs at least one axis"); + return; + } + std::optional nBinsOrNone = numBins(dist.axes, path); + if (!nBinsOrNone) { + return; + } + const std::size_t nBins = *nBinsOrNone; + + std::string channelName = name; + if (channelName.rfind("model_", 0) == 0) { + channelName.erase(0, 6); + } + + auto checkSize = [&](std::string const &where, std::size_t size, char const *what) { + if (size != nBins) { + error(where, std::string(what) + " has " + std::to_string(size) + " entries, but the channel has " + + std::to_string(nBins) + " bins"); + } + }; + + // Two modifiers with the same parameter must have the same constraint. + std::map> constraintOfParameter; + auto checkCorrelated = [&](std::string const &where, std::string const ¶meter, Constraint const &constraint) { + auto inserted = constraintOfParameter.emplace(parameter, std::make_pair(constraint, where)); + if (inserted.second) { + return; + } + if (!(inserted.first->second.first == constraint)) { + warning(where, "the parameter '" + parameter + "' has a different constraint than in " + + inserted.first->second.second + + ", but modifiers with the same parameter must share the constraint"); + } + }; + + auto checkConstraint = [&](std::string const &where, Constraint const &constraint) { + if (auto const *pdf = std::get_if(&constraint)) { + requireDistribution(pdf->pdf.name, where, "constraint"); + } + }; + + // The constraint distributions of the gamma parameters of shape modifiers. + auto checkGammaConstraint = [&](std::string const &where, GammaConstraint const &constraint) { + auto const *pdfs = std::get_if(&constraint); + if (!pdfs) { + return; + } + checkSize(where + ".constraints", pdfs->pdfs.size(), "\"constraints\""); + for (auto const &pdf : pdfs->pdfs) { + if (pdf) { + requireDistribution(pdf->name, where + ".constraints", "constraint"); + } + } + }; + + auto checkParameter = [&](std::string const &where, std::string const ¶meter, std::string const &modifier) { + if (char const *section = sectionOf(parameter)) { + error(where, "the parameter '" + parameter + "' of the modifier '" + modifier + "' is defined in \"" + + section + "\", but it must be a parameter"); + } + }; + + checkGammaConstraint(path + ".staterror", dist.statError.constraint); + + std::set sampleNames; + for (std::size_t iSample = 0; iSample < dist.samples.size(); ++iSample) { + auto const &sample = dist.samples[iSample]; + const std::string samplePath = path + ".samples[" + std::to_string(iSample) + "](" + sample.name + ")"; + if (!sampleNames.insert(sample.name).second) { + error(samplePath, "the name of the sample is not unique in the channel"); + } + checkSize(samplePath + ".data.contents", sample.contents.size(), "\"contents\""); + // These modifiers create objects that are named after them. + std::set objectNames; + auto checkUniqueName = [&](std::string const &where, char const *type, std::string const &modifierName) { + if (!objectNames.insert(std::string(type) + " " + modifierName).second) { + error(where, + std::string("there is another ") + type + " modifier named '" + modifierName + "' in the sample"); + } + }; + if (sample.errors && sample.errors->size() != sample.contents.size()) { + error(samplePath + ".data", "inconsistent number of errors and contents in '" + sample.name + + "': errors=" + std::to_string(sample.errors->size()) + + ", contents=" + std::to_string(sample.contents.size())); + } + for (std::size_t iModifier = 0; iModifier < sample.modifiers.size(); ++iModifier) { + std::string modifierPath = samplePath + ".modifiers[" + std::to_string(iModifier) + "]"; + auto checkInterpolation = [&](std::optional const &interpolation, char const *type, + std::string const &modifierName) { + if (!interpolation && !dist.defaultInterpolation) { + error(modifierPath, std::string("interpolation for ") + type + " modifier '" + modifierName + + "' in sample '" + sample.name + "' of channel '" + channelName + + "' is not given, and the channel has no \"default_interpolation\""); + } + }; + std::visit(overloaded{ + [&](NormFactor const &m) { + modifierPath += "(" + m.name + ")"; + checkParameter(modifierPath, m.parameter.name, m.name); + checkConstraint(modifierPath, m.constraint); + checkCorrelated(modifierPath, m.parameter.name, m.constraint); + }, + [&](NormSys const &m) { + modifierPath += "(" + m.name + ")"; + checkParameter(modifierPath, m.parameter.name, m.name); + checkConstraint(modifierPath, m.constraint); + checkCorrelated(modifierPath, m.parameter.name, m.constraint); + checkInterpolation(m.interpolation, "normsys", m.name); + }, + [&](HistoSys const &m) { + modifierPath += "(" + m.name + ")"; + checkUniqueName(modifierPath, "histosys", m.name); + checkParameter(modifierPath, m.parameter.name, m.name); + checkConstraint(modifierPath, m.constraint); + checkCorrelated(modifierPath, m.parameter.name, m.constraint); + checkInterpolation(m.interpolation, "histosys", m.name); + checkSize(modifierPath + ".data.hi.contents", m.hi.contents.size(), "\"contents\""); + checkSize(modifierPath + ".data.lo.contents", m.lo.contents.size(), "\"contents\""); + }, + [&](ShapeSys const &m) { + modifierPath += "(" + m.name + ")"; + checkUniqueName(modifierPath, "shapesys", m.name); + if (!m.vals.empty()) { + checkSize(modifierPath + ".data.vals", m.vals.size(), "\"vals\""); + } + if (!m.parameters.empty()) { + checkSize(modifierPath + ".parameters", m.parameters.size(), "\"parameters\""); + } + for (auto const &p : m.parameters) { + checkParameter(modifierPath, p.name, m.name); + } + checkGammaConstraint(modifierPath, m.constraint); + }, + [&](StatError const &m) { + modifierPath += "(" + m.name + ")"; + if (!m.parameters.empty()) { + checkSize(modifierPath + ".parameters", m.parameters.size(), "\"parameters\""); + } + checkGammaConstraint(modifierPath, m.constraint); + }, + [&](CustomModifier const &m) { + modifierPath += "(" + m.name + ")"; + if (!isDefinedIn(m.name, "functions") && !isDefinedIn(m.name, "distributions") && + !isExternal(m.name)) { + error(modifierPath, "unable to find custom modifier '" + m.name + "'"); + } + }, + }, + sample.modifiers[iModifier]); + } + } +} + +} // namespace + +void checkDocument(HS3Document const &doc, Diagnostics &diagnostics, ValidateOptions const &options) +{ + Validator{doc, diagnostics, options}.run(); +} + +std::optional validate(HS3Document doc, Diagnostics &diagnostics, ValidateOptions const &options) +{ + checkDocument(doc, diagnostics, options); + if (hasErrors(diagnostics)) { + return std::nullopt; + } + return ValidatedHS3Document{std::move(doc)}; +} + +void validateHistFactoryDist(HistFactoryDist const &dist, std::string const &name, Diagnostics &diagnostics) +{ + // Without a document, all names are resolved later in the workspace. + HS3Document empty; + ValidateOptions options; + options.isExternal = [](std::string const &) { return true; }; + Validator{empty, diagnostics, options}.checkHistFactory(dist, name, ""); +} + +} // namespace HS3 +} // namespace RooFit diff --git a/roofit/hs3/src/HS3Writer.cxx b/roofit/hs3/src/HS3Writer.cxx new file mode 100644 index 0000000000000..7fb2c1f021a09 --- /dev/null +++ b/roofit/hs3/src/HS3Writer.cxx @@ -0,0 +1,521 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +// Writes the typed representation of HS3 documents. The keys are the same as in HS3Parser.cxx, which the round-trip +// tests in testHS3Document.cxx check. + +#include "HS3Document.h" +#include "JSONIOUtils.h" + +#include + +#include +#include +#include + +using RooFit::Detail::JSONNode; + +namespace RooFit { +namespace HS3 { +namespace { + +// Infinite bounds are written as null. +void writeBound(JSONNode &node, double value) +{ + if (std::isinf(value)) { + node.set_null(); + } else { + node << value; + } +} + +void writeStrings(JSONNode &node, std::vector const &values) +{ + node.fill_seq(values); +} + +template +void writeRefs(JSONNode &node, std::vector> const &refs) +{ + node.set_seq(); + for (auto const &ref : refs) { + node.append_child() << ref.name; + } +} + +JSONNode &appendNamed(JSONNode &array, std::string const &name) +{ + JSONNode &child = array.set_seq().append_child().set_map(); + child["name"] << name; + return child; +} + +void writeBinning(JSONNode &node, Binning const &binning) +{ + if (auto const *edges = std::get_if(&binning)) { + node["edges"].fill_seq(edges->edges); + } else { + node["nbins"] << std::get(binning).nbins; + } +} + +void writeAxis(JSONNode &node, BinnedAxis const &axis) +{ + node.set_map(); + node["name"] << axis.name; + // Binned axes always carry their range, also if it is redundant with the "edges" of an irregular binning. + node["min"] << axis.min; + node["max"] << axis.max; + writeBinning(node, axis.binning); +} + +void writeAxis(JSONNode &node, ContinuousAxis const &axis) +{ + node.set_map(); + node["name"] << axis.name; + if (axis.min) { + writeBound(node["min"], *axis.min); + } + if (axis.max) { + writeBound(node["max"], *axis.max); + } +} + +void writeAxis(JSONNode &node, CategoricalAxis const &axis) +{ + node.set_map(); + node["name"] << axis.name; + writeStrings(node["categories"], axis.categories); +} + +// An infinite or missing bound of a domain axis is omitted, which means that the range is unbounded. +void writeAxis(JSONNode &node, DomainAxis const &axis) +{ + node.set_map(); + node["name"] << axis.name; + if (axis.fixedValue) { + node[fixedValueKey] << *axis.fixedValue; + return; + } + if (axis.min && !std::isinf(*axis.min)) { + node["min"] << *axis.min; + } + if (axis.max && !std::isinf(*axis.max)) { + node["max"] << *axis.max; + } + if (axis.binning) { + writeBinning(node, *axis.binning); + } +} + +template +void writeAxes(JSONNode &node, std::vector> const &axes) +{ + node.set_seq(); + for (auto const &axis : axes) { + std::visit([&](auto const &a) { writeAxis(node.append_child(), a); }, axis); + } +} + +template +void writeAxes(JSONNode &node, std::vector const &axes) +{ + node.set_seq(); + for (auto const &axis : axes) { + writeAxis(node.append_child(), axis); + } +} + +// The parameter of a modifier with a single parameter, and the name of its constraint distribution. A constraint type +// is written in the form of older documents. +void writeParameter(JSONNode &mod, ParameterRef const ¶meter, Constraint const &constraint) +{ + mod["parameters"].set_seq().append_child() << parameter.name; + std::visit( + overloaded{[](Unconstrained) {}, [&](ConstraintType type) { mod["constraint"] << toString(type); }, + [&](ConstraintPdf const &pdf) { mod["constraints"].set_seq().append_child() << pdf.pdf.name; }}, + constraint); +} + +void writeGammaConstraint(JSONNode &node, GammaConstraint const &constraint, const char *typeKey) +{ + std::visit(overloaded{[](Unconstrained) {}, [&](ConstraintType type) { node[typeKey] << toString(type); }, + [&](ConstantGammas) { node[typeKey] << "Const"; }, + [&](GammaConstraintPdfs const &pdfs) { + JSONNode &names = node["constraints"].set_seq(); + for (auto const &pdf : pdfs.pdfs) { + if (pdf) { + names.append_child() << pdf->name; + } else { + names.append_child().set_null(); + } + } + }}, + constraint); +} + +JSONNode &appendModifier(JSONNode &modifiers, std::string const &name, const char *type) +{ + JSONNode &mod = modifiers.append_child().set_map(); + mod["name"] << name; + mod["type"] << type; + return mod; +} + +void writeModifier(JSONNode &modifiers, Modifier const &modifier) +{ + std::visit(overloaded{ + [&](NormFactor const &m) { + JSONNode &mod = appendModifier(modifiers, m.name, "normfactor"); + writeParameter(mod, m.parameter, m.constraint); + }, + [&](NormSys const &m) { + JSONNode &mod = appendModifier(modifiers, m.name, "normsys"); + writeParameter(mod, m.parameter, m.constraint); + if (m.interpolation) { + writeInterpolation(mod["interpolation"], *m.interpolation); + } + JSONNode &data = mod["data"].set_map(); + data["lo"] << m.lo; + data["hi"] << m.hi; + }, + [&](HistoSys const &m) { + JSONNode &mod = appendModifier(modifiers, m.name, "histosys"); + writeParameter(mod, m.parameter, m.constraint); + if (m.interpolation) { + writeInterpolation(mod["interpolation"], *m.interpolation); + } + JSONNode &data = mod["data"].set_map(); + writeContents(data["lo"].set_map()["contents"], m.lo.contents); + writeContents(data["hi"].set_map()["contents"], m.hi.contents); + }, + [&](ShapeSys const &m) { + JSONNode &mod = appendModifier(modifiers, m.name, "shapefactor"); + writeRefs(mod["parameters"], m.parameters); + writeGammaConstraint(mod, m.constraint, "constraint"); + // The values are only used by the constraint types of older documents. + if (!m.vals.empty()) { + mod["data"].set_map()["vals"].fill_seq(m.vals); + } + }, + [&](StatError const &m) { + JSONNode &mod = appendModifier(modifiers, m.name, "staterror"); + writeRefs(mod["parameters"], m.parameters); + writeGammaConstraint(mod, m.constraint, "constraint"); + }, + [&](CustomModifier const &m) { appendModifier(modifiers, m.name, "custom"); }, + }, + modifier); +} + +void writeParameterPoint(JSONNode &points, ParameterPoint const &point) +{ + JSONNode ¶meters = appendNamed(points, point.name)["parameters"].set_seq(); + for (auto const &p : point.parameters) { + JSONNode &node = appendNamed(parameters, p.name); + if (p.value) { + std::visit([&](auto const &value) { node["value"] << value; }, *p.value); + } + if (p.isConst) { + node["const"] << true; + } + } +} + +void writeData(JSONNode &dataArray, Data const &data) +{ + JSONNode &node = appendNamed(dataArray, data.name); + std::visit(overloaded{ + [&](BinnedData const &d) { + node["type"] << "binned"; + writeAxes(node["axes"], d.axes); + writeContents(node["contents"], d.contents); + if (d.uncertainty) { + JSONNode &uncertainty = node["uncertainty"].set_map(); + uncertainty["type"] << "gaussian_uncertainty"; + uncertainty["sigma"].fill_seq(d.uncertainty->sigma); + } + }, + [&](UnbinnedData const &d) { + node["type"] << "unbinned"; + writeAxes(node["axes"], d.axes); + JSONNode &entries = node["entries"].set_seq(); + const std::size_t nAxes = d.axes.size(); + for (std::size_t i = 0; i < d.numEntries(); ++i) { + JSONNode &row = entries.append_child().set_seq(); + for (std::size_t j = 0; j < nAxes; ++j) { + const double value = d.entries[i * nAxes + j]; + if (auto const *category = std::get_if(&d.axes[j])) { + row.append_child() << category->categories[static_cast(value)]; + } else { + row.append_child() << value; + } + } + } + if (d.weights) { + node["weights"].fill_seq(*d.weights); + } + }, + [&](PointData const &d) { + node["type"] << "point"; + node["value"] << d.value; + if (d.uncertainty) { + node["uncertainty"] << *d.uncertainty; + } + if (!d.axes.empty()) { + writeAxes(node["axes"], d.axes); + } + }, + }, + data.body); +} + +void writeLikelihood(JSONNode &likelihoods, Likelihood const &likelihood) +{ + JSONNode &node = appendNamed(likelihoods, likelihood.name); + JSONNode &distributions = node["distributions"].set_seq(); + JSONNode &data = node["data"].set_seq(); + for (auto const &term : likelihood.terms) { + distributions.append_child() << term.distribution.name; + std::visit(overloaded{[&](DataRef const &ref) { data.append_child() << ref.name; }, + [&](double value) { writeNumber(data.append_child(), value); }}, + term.data); + } + if (!likelihood.auxDistributions.empty()) { + writeRefs(node["aux_distributions"], likelihood.auxDistributions); + } +} + +void writeAnalysis(JSONNode &analyses, Analysis const &analysis) +{ + JSONNode &node = appendNamed(analyses, analysis.name); + writeRefs(node["domains"], analysis.domains); + node["likelihood"] << analysis.likelihood.name; + if (!analysis.parametersOfInterest.empty()) { + writeRefs(node["parameters_of_interest"], analysis.parametersOfInterest); + } + if (analysis.init) { + node["init"] << analysis.init->name; + } + if (analysis.prior) { + node["prior"] << analysis.prior->name; + } +} + +void writeRooFitInternal(JSONNode &root, RooFitInternal const &internal) +{ + auto rootInternal = [&]() -> JSONNode & { return root.get("misc", "ROOT_internal"); }; + if (!internal.attributes.empty()) { + JSONNode &attributes = rootInternal()["attributes"].set_map(); + for (auto const &item : internal.attributes) { + JSONNode &node = attributes[item.first].set_map(); + if (!item.second.dict.empty()) { + JSONNode &dict = node["dict"].set_map(); + for (auto const &entry : item.second.dict) { + dict[entry.first] << entry.second; + } + } + if (!item.second.tags.empty()) { + writeStrings(node["tags"], item.second.tags); + } + } + } + if (!internal.combinedDistributions.empty()) { + JSONNode &combined = rootInternal()["combined_distributions"].set_map(); + for (auto const &item : internal.combinedDistributions) { + JSONNode &node = combined[item.first].set_map(); + node["index_cat"] << item.second.indexCat; + writeStrings(node["labels"], item.second.labels); + node["indices"].fill_seq(item.second.indices); + writeRefs(node["distributions"], item.second.distributions); + } + } + if (!internal.combinedDatasets.empty()) { + JSONNode &combined = rootInternal()["combined_datasets"].set_map(); + for (auto const &item : internal.combinedDatasets) { + JSONNode &node = combined[item.first].set_map(); + node["index_cat"] << item.second.indexCat; + writeStrings(node["labels"], item.second.labels); + node["indices"].fill_seq(item.second.indices); + } + } + if (!internal.modelConfigs.empty()) { + JSONNode &modelConfigs = rootInternal()["ModelConfigs"].set_map(); + for (auto const &item : internal.modelConfigs) { + JSONNode &node = modelConfigs[item.first].set_map(); + if (item.second.mcName) { + node["mcName"] << *item.second.mcName; + } + if (item.second.pdfName) { + node["pdfName"] << *item.second.pdfName; + } + if (item.second.combinedDataName) { + node["combined_data_name"] << *item.second.combinedDataName; + } + } + } + if (!internal.channelConstraints.empty()) { + JSONNode &constraints = rootInternal()["histfactory_constraints"].set_map(); + for (auto const &item : internal.channelConstraints) { + writeRefs(constraints[item.first], item.second); + } + } + if (!internal.stepWidths.empty()) { + JSONNode &stepWidths = root.get("misc", "minimization")["parameter_stepwidths"].set_seq(); + for (auto const &stepWidth : internal.stepWidths) { + appendNamed(stepWidths, stepWidth.name)["step_width"] << stepWidth.width; + } + } +} + +} // namespace + +void writeNumber(JSONNode &node, double value) +{ + // To make sure there are no unnecessary floating points in the JSON + if (static_cast(value) == value) { + node << static_cast(value); + } else { + node << value; + } +} + +void writeContents(JSONNode &node, std::vector const &values) +{ + node.set_seq(); + for (double value : values) { + writeNumber(node.append_child(), value); + } +} + +void writeInterpolation(JSONNode &node, Interpolation const &interpolation) +{ + node.set_map(); + node["type"] << toString(interpolation.type); + node["in"] << toString(interpolation.in); + if (interpolation.out) { + node["out"] << toString(*interpolation.out); + } else { + node["out"].set_null(); + } +} + +namespace { + +void writeValue(JSONNode &node, NameOrNumber const &value) +{ + std::visit([&](auto const &v) { node << v; }, value); +} + +template +void writeValue(JSONNode &node, T const &value) +{ + node << value; +} + +template +void writeList(JSONNode &node, std::vector const &values) +{ + node.set_seq(); + for (auto const &value : values) { + writeValue(node.append_child(), value); + } +} + +template +void writeInterpolated(JSONNode &node, Function_t const &function, char const *type) +{ + node["type"] << type; + auto const &interpolations = function.interpolations; + const bool allEqual = std::all_of(interpolations.begin(), interpolations.end(), + [&](Interpolation const &i) { return i == interpolations.front(); }); + JSONNode &interpolationsNode = node["interpolations"].set_seq(); + for (std::size_t i = 0; i < (allEqual ? std::min(interpolations.size(), 1) : interpolations.size()); + ++i) { + writeInterpolation(interpolationsNode.append_child(), interpolations[i]); + } + writeList(node["vars"], function.parameters); + node["nom"] << function.nominal; + writeList(node["high"], function.high); + writeList(node["low"], function.low); +} + +} // namespace + +void writeInterpolatedFunction(JSONNode &node, InterpolatedFunction const &function) +{ + writeInterpolated(node, function, "interpolation"); + node["positiveDefinite"] << function.positiveDefinite; +} + +void writeInterpolatedNumber(JSONNode &node, InterpolatedNumber const &function) +{ + writeInterpolated(node, function, "interpolation0d"); +} + +void writeHistFactoryDist(JSONNode &node, HistFactoryDist const &dist) +{ + node.set_map(); + node["type"] << "histfactory_dist"; + if (!std::holds_alternative(dist.statError.constraint)) { + writeGammaConstraint(node["staterror"].set_map(), dist.statError.constraint, "constraint_type"); + } + if (dist.defaultInterpolation) { + writeInterpolation(node["default_interpolation"], *dist.defaultInterpolation); + } + writeAxes(node["axes"], dist.axes); + JSONNode &samples = node["samples"].set_seq(); + for (auto const &sample : dist.samples) { + JSONNode &s = appendNamed(samples, sample.name); + JSONNode &modifiers = s["modifiers"].set_seq(); + for (auto const &modifier : sample.modifiers) { + writeModifier(modifiers, modifier); + } + JSONNode &data = s["data"].set_map(); + writeContents(data["contents"], sample.contents); + if (sample.errors) { + writeContents(data["errors"], *sample.errors); + } + } +} + +void writeDocument(JSONNode &root, HS3Document const &doc) +{ + root.set_map(); + JSONNode &metadata = root["metadata"].set_map(); + metadata["hs3_version"] << doc.metadata.hs3Version; + for (auto const &package : doc.metadata.packages) { + appendNamed(metadata["packages"], package.name)["version"] << package.version; + } + + for (auto const &point : doc.parameterPoints) { + writeParameterPoint(root["parameter_points"], point); + } + for (auto const &domain : doc.domains) { + JSONNode &node = appendNamed(root["domains"], domain.name); + node["type"] << "product_domain"; + writeAxes(node["axes"], domain.axes); + } + for (auto const &data : doc.data) { + writeData(root["data"], data); + } + for (auto const &likelihood : doc.likelihoods) { + writeLikelihood(root["likelihoods"], likelihood); + } + for (auto const &analysis : doc.analyses) { + writeAnalysis(root["analyses"], analysis); + } + writeRooFitInternal(root, doc.rooFit); +} + +} // namespace HS3 +} // namespace RooFit diff --git a/roofit/hs3/src/JSONFactories_HistFactory.cxx b/roofit/hs3/src/JSONFactories_HistFactory.cxx index 25e4c9ce0cf94..d5414b2edbd03 100644 --- a/roofit/hs3/src/JSONFactories_HistFactory.cxx +++ b/roofit/hs3/src/JSONFactories_HistFactory.cxx @@ -33,19 +33,31 @@ #include #include +#include #include #include +#include #include +#include #include #include #include #include -#include +#include +#include #include "static_execute.h" +#include "HS3Builder.h" #include "JSONIOUtils.h" using RooFit::Detail::JSONNode; +using RooFit::HS3::additivePolynomialLinear; +using RooFit::HS3::Interpolation; +using RooFit::HS3::InterpolationClass; +using RooFit::HS3::multiplicativePiecewiseExponential; +using RooFit::HS3::multiplicativePolynomialExponential; + +namespace HS3 = RooFit::HS3; using namespace RooStats::HistFactory; using namespace RooStats::HistFactory::Detail; @@ -64,43 +76,12 @@ double round_prec(double d, int nSig) return sf * std::round(d / sf); } -// To avoid repeating the same string literals that can potentially get out of -// sync. -namespace Literals { -constexpr auto staterror = "staterror"; -} - -struct Interpolation { - std::string type; - std::string in; - std::optional out; - - bool operator==(const Interpolation &other) const - { - return std::tie(type, in, out) == std::tie(other.type, other.in, other.out); - } - - bool operator!=(const Interpolation &other) const { return !(*this == other); } - - bool operator<(const Interpolation &other) const - { - return std::tie(type, in, out) < std::tie(other.type, other.in, other.out); - } -}; - -const Interpolation additivePiecewiseLinear{"add", "poly1", std::nullopt}; -const Interpolation multiplicativePiecewiseExponential{"mult", "exp", std::nullopt}; -const Interpolation additiveQuadraticLinear{"add", "poly2", "poly1"}; -const Interpolation additivePolynomialLinear{"add", "poly6", "poly1"}; -const Interpolation multiplicativePolynomialExponential{"mult", "poly6", "exp"}; -const Interpolation multiplicativePolynomialLinear{"mult", "poly6", "poly1"}; - std::string interpolationString(const Interpolation &interpolation) { std::stringstream ss; - ss << R"({"type":")" << interpolation.type << R"(","in":")" << interpolation.in << R"(","out":)"; + ss << R"({"type":")" << toString(interpolation.type) << R"(","in":")" << toString(interpolation.in) << R"(","out":)"; if (interpolation.out) { - ss << '"' << *interpolation.out << '"'; + ss << '"' << toString(*interpolation.out) << '"'; } else { ss << "null"; } @@ -108,111 +89,6 @@ std::string interpolationString(const Interpolation &interpolation) return ss.str(); } -bool isInterpolationFunction(std::string_view function) -{ - return function == "poly1" || function == "poly2" || function == "poly6" || function == "exp"; -} - -Interpolation readInterpolation(const JSONNode &node, const std::string &context) -{ - if (!node.is_map()) { - RooJSONFactoryWSTool::error(context + " must be a struct with components 'type', 'in', and 'out'"); - } - for (const char *component : {"type", "in", "out"}) { - if (!node.has_child(component)) { - RooJSONFactoryWSTool::error(context + " does not define the required '" + component + "' component"); - } - } - - const auto &typeNode = node["type"]; - const auto &inNode = node["in"]; - const auto &outNode = node["out"]; - if (typeNode.is_container() || typeNode.is_null() || inNode.is_container() || inNode.is_null()) { - RooJSONFactoryWSTool::error(context + " components 'type' and 'in' must be strings"); - } - - Interpolation interpolation{typeNode.val(), inNode.val(), std::nullopt}; - if (interpolation.type != "add" && interpolation.type != "mult") { - RooJSONFactoryWSTool::error(context + " has unknown interpolation type '" + interpolation.type + "'"); - } - if (!isInterpolationFunction(interpolation.in)) { - RooJSONFactoryWSTool::error(context + " has unknown interpolation function '" + interpolation.in + "'"); - } - - if (!outNode.is_null()) { - if (outNode.is_container()) { - RooJSONFactoryWSTool::error(context + " component 'out' must be a string or null"); - } - interpolation.out = outNode.val(); - if (!isInterpolationFunction(*interpolation.out)) { - RooJSONFactoryWSTool::error(context + " has unknown extrapolation function '" + *interpolation.out + "'"); - } - } - - return interpolation; -} - -void writeInterpolation(JSONNode &node, const Interpolation &interpolation) -{ - node.set_map(); - node["type"] << interpolation.type; - node["in"] << interpolation.in; - if (interpolation.out) { - node["out"] << *interpolation.out; - } else { - node["out"].set_null(); - } -} - -int readLegacyInterpolationCode(const JSONNode &node, const std::string &context) -{ - if (node.is_container() || node.is_null() || !node.has_val()) { - RooJSONFactoryWSTool::error(context + " must be a structured interpolation or a legacy integer code"); - } - - const std::string value = node.val(); - int code = 0; - const auto result = std::from_chars(value.data(), value.data() + value.size(), code); - if (result.ec != std::errc{} || result.ptr != value.data() + value.size()) { - RooJSONFactoryWSTool::error(context + " has invalid legacy interpolation code '" + value + "'"); - } - return code; -} - -enum class InterpolationClass { - Piecewise, - Flexible -}; - -// Single source of truth for the mapping between the structured HS3 interpolation -// descriptors and the RooFit integer codes of PiecewiseInterpolation and -// FlexibleInterpVar. A code of `kUnrepresentable` means the descriptor cannot be -// expressed by that class: FlexibleInterpVar internally remaps code 4 to code 5, -// so it has no additive-linear or multiplicative-linear poly6 variant. -struct InterpolationCodes { - const Interpolation &descriptor; - int piecewise; - int flexible; -}; - -constexpr int kUnrepresentable = -1; - -// clang-format off -const std::vector interpolationTable{ - {additivePiecewiseLinear, 0, 0}, - {multiplicativePiecewiseExponential, 1, 1}, - {additiveQuadraticLinear, 2, 2}, - {additivePolynomialLinear, 4, kUnrepresentable}, - {multiplicativePolynomialExponential, 5, 4}, - {multiplicativePolynomialLinear, 6, kUnrepresentable}, -}; -// clang-format on - -int codeForClass(const InterpolationCodes &row, InterpolationClass interpolationClass) -{ - return interpolationClass == InterpolationClass::Piecewise ? row.piecewise : row.flexible; -} - const char *interpolationClassName(InterpolationClass interpolationClass) { return interpolationClass == InterpolationClass::Piecewise ? "PiecewiseInterpolation" : "FlexibleInterpVar"; @@ -220,19 +96,8 @@ const char *interpolationClassName(InterpolationClass interpolationClass) Interpolation interpolationFromCode(int code, InterpolationClass interpolationClass, const std::string &context) { - // Code 3 was historically an unimplemented alias of code 2 for both classes, - // and FlexibleInterpVar treats code 5 identically to its canonical code 4. - if (code == 3) { - code = 2; - } - if (interpolationClass == InterpolationClass::Flexible && code == 5) { - code = 4; - } - for (const auto &row : interpolationTable) { - const int rowCode = codeForClass(row, interpolationClass); - if (rowCode != kUnrepresentable && rowCode == code) { - return row.descriptor; - } + if (auto interpolation = HS3::interpolationFromCode(code, interpolationClass)) { + return *interpolation; } RooJSONFactoryWSTool::error(context + " has unsupported " + interpolationClassName(interpolationClass) + " code " + std::to_string(code)); @@ -241,145 +106,57 @@ Interpolation interpolationFromCode(int code, InterpolationClass interpolationCl int codeFromInterpolation(const Interpolation &interpolation, InterpolationClass interpolationClass, const std::string &context) { - for (const auto &row : interpolationTable) { - const int rowCode = codeForClass(row, interpolationClass); - if (rowCode != kUnrepresentable && row.descriptor == interpolation) { - return rowCode; - } + if (auto code = HS3::codeFromInterpolation(interpolation, interpolationClass)) { + return *code; } RooJSONFactoryWSTool::error(context + " " + interpolationString(interpolation) + " cannot be represented by " + interpolationClassName(interpolationClass)); } -void writeInterpolations(JSONNode &node, const std::vector &codes, InterpolationClass interpolationClass, - const std::string &context) +// The interpolations of the parameters of a PiecewiseInterpolation or FlexibleInterpVar. +std::vector +toInterpolations(std::vector const &codes, InterpolationClass interpolationClass, std::string const &context) { - auto &interpolations = node.set_seq(); - if (codes.empty()) { - return; - } - - std::vector descriptors; - descriptors.reserve(codes.size()); + std::vector out; for (std::size_t i = 0; i < codes.size(); ++i) { - descriptors.push_back( + out.push_back( interpolationFromCode(codes[i], interpolationClass, context + " at parameter index " + std::to_string(i))); } - - bool allEqual = true; - for (std::size_t i = 1; i < descriptors.size(); ++i) { - if (descriptors[i] != descriptors.front()) { - allEqual = false; - break; - } - } - - const std::size_t outputSize = allEqual ? 1 : descriptors.size(); - for (std::size_t i = 0; i < outputSize; ++i) { - writeInterpolation(interpolations.append_child(), descriptors[i]); - } + return out; } -std::vector readInterpolations(const JSONNode &object, std::size_t nParameters, - InterpolationClass interpolationClass, const std::string &context) +std::vector toCodes(std::vector const &interpolations, InterpolationClass interpolationClass, + std::string const &context) { - if (const auto *interpolations = object.find("interpolations")) { - if (!interpolations->is_seq()) { - RooJSONFactoryWSTool::error(context + " component 'interpolations' must be an array"); - } - - const std::size_t size = interpolations->num_children(); - const bool validSize = nParameters == 0 ? size == 0 : size == 1 || size == nParameters; - if (!validSize) { - RooJSONFactoryWSTool::error(context + - " component 'interpolations' must contain either one descriptor or one " - "descriptor per parameter (got " + - std::to_string(size) + " for " + std::to_string(nParameters) + " parameters)"); - } - - std::vector codes; - codes.reserve(size); - std::size_t i = 0; - for (const auto &node : interpolations->children()) { - const std::string entryContext = context + " component 'interpolations' at index " + std::to_string(i); - codes.push_back( - codeFromInterpolation(readInterpolation(node, entryContext), interpolationClass, entryContext)); - ++i; - } - if (size == 1) { - codes.resize(nParameters, codes.front()); - } - return codes; + std::vector out; + for (std::size_t i = 0; i < interpolations.size(); ++i) { + out.push_back(codeFromInterpolation(interpolations[i], interpolationClass, + context + " component 'interpolations' at index " + std::to_string(i))); } + return out; +} - std::vector codes(nParameters, 0); - if (const auto *legacyCodes = object.find("interpolationCodes")) { - if (!legacyCodes->is_seq()) { - RooJSONFactoryWSTool::error(context + " legacy component 'interpolationCodes' must be an array"); - } - if (legacyCodes->num_children() != nParameters) { - RooJSONFactoryWSTool::error(context + - " legacy component 'interpolationCodes' must contain one code per " - "parameter (got " + - std::to_string(legacyCodes->num_children()) + " for " + - std::to_string(nParameters) + " parameters)"); - } - - std::size_t i = 0; - for (const auto &node : legacyCodes->children()) { - const std::string entryContext = - context + " legacy component 'interpolationCodes' at index " + std::to_string(i); - const Interpolation interpolation = - interpolationFromCode(readLegacyInterpolationCode(node, entryContext), interpolationClass, entryContext); - codes[i] = codeFromInterpolation(interpolation, interpolationClass, entryContext); - ++i; - } +// Reports the problems with an object that doesn't come with a document. +void report(std::string const &name, HS3::Diagnostics &diagnostics) +{ + for (auto &diagnostic : diagnostics) { + diagnostic.path = name + (diagnostic.path.empty() ? "" : "." + diagnostic.path); } - return codes; + HS3::report(diagnostics); } -// The fallback for modifiers without interpolation in documents that predate HS3 v0.3, see interpolationCode(). Before -// that, RooFit did not write the interpolation of modifiers that used its implicit default. -struct LegacyInterpolationDefault { - bool allowed = false; - bool warned = false; -}; - -int interpolationCode(const JSONNode &modifier, const std::optional &defaultInterpolation, - InterpolationClass interpolationClass, const std::string &context, - LegacyInterpolationDefault &legacyDefault) +int interpolationCode(std::optional const &interpolation, + std::optional const &defaultInterpolation, InterpolationClass interpolationClass, + std::string const &context) { - const auto toCode = [&](const Interpolation &interpolation) { - return codeFromInterpolation(interpolation, interpolationClass, context); - }; - - if (const auto *interpolationNode = modifier.find("interpolation")) { - if (interpolationNode->is_map()) { - return toCode(readInterpolation(*interpolationNode, context)); - } - const int legacyCode = readLegacyInterpolationCode(*interpolationNode, context); - const Interpolation interpolation = interpolationFromCode(legacyCode, interpolationClass, context); - return toCode(interpolation); + if (interpolation) { + return codeFromInterpolation(*interpolation, interpolationClass, context); } - if (defaultInterpolation) { - return toCode(*defaultInterpolation); - } - // HS3 forbids implementation-specific default interpolations. - if (!legacyDefault.allowed) { + if (!defaultInterpolation) { RooJSONFactoryWSTool::error(context + " is not given, and the channel has no \"default_interpolation\""); } - - // Before structured interpolation was introduced, both modifier classes - // used the integer code 4 as their implicit default. The meaning of code 4 - // is class-dependent. Files written by these ROOT versions are still read. - if (!legacyDefault.warned) { - RooJSONFactoryWSTool::warning("RooFitHS3: " + context + - " is not given, and the channel has no \"default_interpolation\". Using the " - "implicit default of older RooFit versions (code 4), which is deprecated."); - legacyDefault.warned = true; - } - return 4; + return codeFromInterpolation(*defaultInterpolation, interpolationClass, context); } void erasePrefix(std::string &str, std::string_view prefix) @@ -436,33 +213,6 @@ inline std::string defaultGammaName(std::string const &sysname, std::size_t i) return "gamma_" + sysname + "_bin_" + std::to_string(i); } -/// Export the names of the gamma parameters to the modifier struct -void exportGammaParameters(JSONNode &mod, std::vector const ¶ms) -{ - std::vector paramNames; - for (RooAbsReal *param : params) { - paramNames.emplace_back(param->GetName()); - } - mod["parameters"].fill_seq(paramNames); -} - -/// Export the names of the constraint distributions of a modifier, with null for unconstrained parameters. Nothing is -/// written if all parameters are unconstrained. -void exportConstraintNames(JSONNode &mod, std::vector const &constraints) -{ - if (std::none_of(constraints.begin(), constraints.end(), [](auto *pdf) { return pdf != nullptr; })) { - return; - } - auto &names = mod["constraints"].set_seq(); - for (auto *constraint : constraints) { - if (constraint) { - names.append_child() << constraint->GetName(); - } else { - names.append_child().set_null(); - } - } -} - RooRealVar &createNominal(RooWorkspace &ws, std::string const &parname, double val, double min, double max) { RooRealVar &nom = getOrCreate(ws, "nom_" + parname, val, min, max); @@ -477,12 +227,6 @@ std::string constraintName(std::string const ¶mName) return paramName + "Constraint"; } -bool isLegacyConstraintType(std::string const &value) -{ - // HS3 spells "LogNormal", older RooFit versions used "Lognormal". - return value == "Gauss" || value == "Poisson" || value == "Const" || value == "LogNormal" || value == "Lognormal"; -} - RooAbsPdf *findNamedConstraint(RooJSONFactoryWSTool &tool, std::string const &constraintName, std::string const &sample) { if (auto *constraint = tool.workspace()->pdf(constraintName)) { @@ -500,210 +244,146 @@ RooAbsPdf *findNamedConstraint(RooJSONFactoryWSTool &tool, std::string const &co return nullptr; } -RooAbsPdf &createLegacyConstraint(RooJSONFactoryWSTool &tool, const JSONNode &mod, RooRealVar ¶m, - std::string const &constraintType) -{ - if (constraintType == "Gauss") { - param.setError(1.0); - RooRealVar &nominal = createNominal(*tool.workspace(), param.GetName(), 0.0, -10, 10); - return getOrCreate(*tool.workspace(), constraintName(param.GetName()), param, nominal, 1.); - } - - RooJSONFactoryWSTool::error("legacy constraint value '" + constraintType + "' for modifier '" + - RooJSONFactoryWSTool::name(mod) + - "' is a known constraint type, but it cannot be resolved in this context"); -} - -ParamHistFunc &createPHF(const std::string &phfname, std::string const &sysname, - const std::vector &parnames, const std::vector &vals, - RooJSONFactoryWSTool &tool, RooAbsCollection &constraints, const RooArgSet &observables, - const std::string &constraintType, double gammaMin, double gammaMax, double minSigma, - bool createConstraints = true) +// The parameter of a modifier, which is created if the document doesn't define it. It's a variable, or a constant if +// the document doesn't give it a domain. +RooAbsReal &getOrCreateParameter(RooJSONFactoryWSTool &tool, std::string const &name, double val, double min, + double max, std::string const &modifier) { RooWorkspace &ws = *tool.workspace(); - - size_t n = std::max(vals.size(), parnames.size()); - RooArgList gammas; - for (std::size_t i = 0; i < n; ++i) { - const std::string name = parnames.empty() ? defaultGammaName(sysname, i) : parnames[i]; - auto *e = dynamic_cast(ws.obj(name.c_str())); - if (e) - gammas.add(*e); - else - gammas.add(getOrCreate(ws, name, 1., gammaMin, gammaMax)); + TObject *existing = ws.obj(name); + if (!existing) { + return getOrCreate(ws, name, val, min, max); } - - auto &phf = tool.wsEmplace(phfname, observables, gammas); - - if (vals.size() > 0) { - if (!createConstraints) { - configureConstrainedGammas(gammas, vals, minSigma); - } else if (constraintType != "Const") { - if (constraintType == "LogNormal" || constraintType == "Lognormal") { - RooJSONFactoryWSTool::error("unsupported constraint '" + constraintType + - "' for the gamma parameters of '" + sysname + - "', supported are \"Poisson\" and \"Gauss\""); - } - auto constraintsInfo = createGammaConstraints( - gammas, vals, minSigma, constraintType == "Poisson" ? Constraint::Poisson : Constraint::Gaussian); - for (auto const &term : constraintsInfo.constraints) { - ws.import(*term, RooFit::RecycleConflictNodes(), RooFit::Silence()); - constraints.add(*ws.pdf(term->GetName())); - } - } else { - for (auto *gamma : static_range_cast(gammas)) { - gamma->setConstant(true); - } - } - } - - return phf; -} - -/// Find the staterror modifier of a sample, or return nullptr if there is none. -const JSONNode *findStaterror(const JSONNode &comp) -{ - if (comp.has_child("modifiers")) { - for (const auto &mod : comp["modifiers"].children()) { - if (mod["type"].val() == ::Literals::staterror) - return &mod; - } + if (dynamic_cast(existing) || dynamic_cast(existing)) { + return *static_cast(existing); } - return nullptr; + RooJSONFactoryWSTool::error("the parameter '" + name + "' of the modifier '" + modifier + + "' is neither a variable nor a constant"); } -// The entries of the HS3 "constraints" of a modifier: the names of the constraint distributions, or std::nullopt for -// unconstrained parameters (null entries). -std::vector> constraintNames(const JSONNode &mod) +void setErrorIfVariable(RooAbsReal ¶m, double error) { - std::vector> out; - auto *constraints = mod.find("constraints"); - if (!constraints) { - return out; - } - if (!constraints->is_seq()) { - RooJSONFactoryWSTool::error("\"constraints\" of modifier '" + RooJSONFactoryWSTool::name(mod) + - "' must be a list"); + if (auto *var = dynamic_cast(¶m)) { + var->setError(error); } - for (const auto &entry : constraints->children()) { - out.push_back(entry.is_null() ? std::nullopt : std::make_optional(entry.val())); - } - return out; } -// The parameter of a modifier with a single parameter, given in the HS3 "parameters" or in the legacy "parameter". -std::optional modifierParameter(const JSONNode &mod) +// All constraints of parameters that control a whole sample are created here. Following the HS3 standard, the nominal +// value is defined by the type of the modifier, and the standard deviation is one. +void addConstraint(RooJSONFactoryWSTool &tool, HS3::Constraint const &constraint, RooAbsReal ¶m, double nominal, + std::string const &modifier, std::string const &sample, RooArgSet &constraints) { - if (auto *parameters = mod.find("parameters")) { - if (!parameters->is_seq() || parameters->num_children() != 1) { - RooJSONFactoryWSTool::error("\"parameters\" of the " + mod["type"].val() + - " modifier must be a list with one entry"); - } - return parameters->child(0).val(); - } - if (auto *parameter = mod.find("parameter")) { - return parameter->val(); - } - return std::nullopt; + RooWorkspace &ws = *tool.workspace(); + std::visit(HS3::overloaded{ + [](HS3::Unconstrained) {}, + [&](HS3::ConstraintType type) { + if (type != HS3::ConstraintType::Gauss) { + RooJSONFactoryWSTool::error(std::string("legacy constraint value '") + toString(type) + + "' for modifier '" + modifier + + "' is a known constraint type, but it cannot be resolved in this " + "context"); + } + RooRealVar &nom = createNominal(ws, param.GetName(), nominal, -10, 10); + setErrorIfVariable(param, 1.0); + const std::string name = constraintName(param.GetName()); + RooAbsPdf *pdf = ws.pdf(name); + constraints.add(pdf ? *pdf : tool.wsEmplace(name, param, nom, 1.)); + }, + [&](HS3::ConstraintPdf const &constraintPdf) { + RooAbsPdf *pdf = findNamedConstraint(tool, constraintPdf.pdf.name, sample); + if (!pdf) { + RooJSONFactoryWSTool::error("unable to find definition of of constraint '" + + constraintPdf.pdf.name + "' for modifier '" + modifier + "'"); + } + if (auto gauss = dynamic_cast(pdf)) { + setErrorIfVariable(param, gauss->getSigma().getVal()); + } + constraints.add(*pdf); + }, + }, + constraint); } -// The constraint distribution named in the HS3 "constraints" of a modifier with a single parameter. Returns an empty -// string if the parameter is explicitly unconstrained, and std::nullopt if the modifier has no "constraints". -std::optional singleConstraintName(const JSONNode &mod) +// Create the ParamHistFunc of the gamma parameters, together with their constraints. +ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, std::string const &sysname, + std::vector const &parnames, std::vector const &vals, + HS3::GammaConstraint const &constraint, RooArgSet &constraints, RooArgSet const &observables, + double gammaMax, double minSigma, std::size_t nBins) { - if (!mod.has_child("constraints")) { - return std::nullopt; - } - const auto names = constraintNames(mod); - if (names.size() != 1) { - RooJSONFactoryWSTool::error("\"constraints\" of the " + mod["type"].val() + - " modifier must have one entry per parameter"); - } - return names.front().value_or(""); -} + RooWorkspace &ws = *tool.workspace(); -RooAbsPdf & -getOrCreateConstraint(RooJSONFactoryWSTool &tool, const JSONNode &mod, RooRealVar ¶m, const std::string &sample) -{ - // HS3 references the constraint distributions by name. - if (auto name = singleConstraintName(mod); name && !name->empty()) { - auto *constraint = findNamedConstraint(tool, *name, sample); - if (!constraint) { - RooJSONFactoryWSTool::error("unable to find the constraint distribution '" + *name + "' of the modifier of '" + - param.GetName() + "'"); - } - if (auto gauss = dynamic_cast(constraint)) { - param.setError(gauss->getSigma().getVal()); + // The validation made sure that there is a value, parameter and constraint for each bin, if they are given. + RooArgList gammas; + for (std::size_t i = 0; i < nBins; ++i) { + const std::string name = parnames.empty() ? defaultGammaName(sysname, i) : parnames[i].name; + if (auto *existing = dynamic_cast(ws.obj(name))) { + gammas.add(*existing); + } else { + gammas.add(getOrCreate(ws, name, 1., defaultGammaMin, gammaMax)); } - return *constraint; } - JSONNode const *constrName = mod.find("constraint_name"); - if (constrName) { - auto constraint_name = constrName->val(); - auto constraint = findNamedConstraint(tool, constraint_name, sample); - if (!constraint) { - RooJSONFactoryWSTool::error("unable to find definition of of constraint '" + constraint_name + - "' for modifier '" + RooJSONFactoryWSTool::name(mod) + "'"); + // Shape modifiers of the same name in a channel share the function, so they need the same parameters. + if (RooAbsArg *existing = ws.arg(phfname.c_str())) { + auto *existingPHF = dynamic_cast(existing); + bool same = existingPHF && existingPHF->paramList().size() == gammas.size(); + for (std::size_t i = 0; same && i < gammas.size(); ++i) { + same = &existingPHF->paramList()[i] == &gammas[i]; } - if (auto gauss = dynamic_cast(constraint)) { - param.setError(gauss->getSigma().getVal()); + if (!same) { + RooJSONFactoryWSTool::error("the modifier '" + sysname + "' has other parameters than another modifier " + + "of this name in the channel"); } - return *constraint; } + auto &phf = tool.wsEmplace(phfname, observables, gammas); - if (auto constr = mod.find("constraint")) { - std::string constraintValue = constr->val(); - if (auto *constraint = findNamedConstraint(tool, constraintValue, sample)) { - if (auto gauss = dynamic_cast(constraint)) { - param.setError(gauss->getSigma().getVal()); - } - return *constraint; - } - - if (isLegacyConstraintType(constraintValue)) { - return createLegacyConstraint(tool, mod, param, constraintValue); + auto addGammaConstraints = [&](Constraint::Type type) { + for (auto const &term : createGammaConstraints(gammas, vals, minSigma, type).constraints) { + ws.import(*term, RooFit::RecycleConflictNodes(), RooFit::Silence()); + constraints.add(*ws.pdf(term->GetName())); } + }; - RooJSONFactoryWSTool::error("unable to resolve constraint value '" + constraintValue + "' for modifier '" + - RooJSONFactoryWSTool::name(mod) + - "': this looks like a legacy workspace where the 'constraint' field is neither a " - "constraint pdf name nor a supported legacy constraint type"); - } - - std::string constraint_type = "Gauss"; - if (auto constrType = mod.find("constraint_type")) { - constraint_type = constrType->val(); - } - if (isLegacyConstraintType(constraint_type)) { - return createLegacyConstraint(tool, mod, param, constraint_type); - } - RooJSONFactoryWSTool::error("unknown or invalid constraint for modifier '" + RooJSONFactoryWSTool::name(mod) + "'"); -} -bool hasConstraint(const JSONNode &mod) -{ - if (auto name = singleConstraintName(mod)) { - return !name->empty(); - } - return mod.has_child("constraint") || mod.has_child("constraint_name") || mod.has_child("constraint_type"); -} + if (auto const *pdfs = std::get_if(&constraint)) { + for (auto const &pdfName : pdfs->pdfs) { + if (!pdfName) { + continue; + } + auto *pdf = findNamedConstraint(tool, pdfName->name, sysname); + if (!pdf) { + RooJSONFactoryWSTool::error("unable to find definition of constraint '" + pdfName->name + + "' for modifier '" + sysname + "'"); + } + constraints.add(*pdf); + } + } + + if (vals.empty()) { + return phf; + } + + std::visit(HS3::overloaded{ + [&](HS3::Unconstrained) { configureConstrainedGammas(gammas, vals, minSigma); }, + [&](HS3::GammaConstraintPdfs const &) { configureConstrainedGammas(gammas, vals, minSigma); }, + [&](HS3::ConstantGammas) { + for (auto *gamma : static_range_cast(gammas)) { + gamma->setConstant(true); + } + }, + [&](HS3::ConstraintType type) { + switch (type) { + case HS3::ConstraintType::Gauss: addGammaConstraints(Constraint::Gaussian); break; + case HS3::ConstraintType::Poisson: addGammaConstraints(Constraint::Poisson); break; + case HS3::ConstraintType::LogNormal: + RooJSONFactoryWSTool::error(std::string("unsupported constraint '") + toString(type) + + "' for the gamma parameters of '" + sysname + + "', supported are \"Poisson\" and \"Gauss\""); + } + }, + }, + constraint); -// Create the constraint of a normsys or histosys modifier. Without a constraint, the parameter is unconstrained. Only -// documents that predate HS3 v0.2 imply a unit Gaussian, like the ones written by the former HistFactory::JSONTool. -void addSysConstraint(RooJSONFactoryWSTool &tool, const JSONNode &mod, RooRealVar ¶m, std::string const &sysname, - std::string const &sample, bool &warnedUnconstrained, RooArgSet &constraints) -{ - if (!hasConstraint(mod) && !isHS3VersionBefore(tool.hs3VersionInput(), 0, 2)) { - if (!warnedUnconstrained) { - RooJSONFactoryWSTool::warning("RooFitHS3: modifier '" + sysname + "' of sample '" + sample + - "' has no constraint, so its parameter '" + param.GetName() + - "' is unconstrained. Older RooFit versions implied a unit Gaussian. Other " - "modifiers of this channel are not reported."); - warnedUnconstrained = true; - } - return; - } - constraints.add(getOrCreateConstraint(tool, mod, param, sample)); + return phf; } double poissonTau(RooPoisson const &constraint, RooAbsArg const &gamma) @@ -751,401 +431,217 @@ double constraintRelError(RooAbsPdf const &constraint, RooAbsArg const &gamma) return std::numeric_limits::quiet_NaN(); } -bool importHistSample(RooJSONFactoryWSTool &tool, RooDataHist &dh, RooArgSet const &varlist, - RooAbsArg const *mcStatObject, const std::string &fprefix, const JSONNode &p, - const std::optional &defaultInterpolation, - LegacyInterpolationDefault &legacyInterpolationDefault, bool &warnedUnconstrained, - bool &warnedShapeSys, RooArgSet &constraints) +struct ChannelContext { + RooJSONFactoryWSTool &tool; + HS3::HistFactoryDist const &dist; + RooArgSet const &observables; + std::string fprefix; + std::string channelName; + std::size_t nBins; + RooAbsArg const *mcStatObject; + RooArgSet &constraints; +}; + +// Collects what the modifiers of a sample contribute to the RooFit objects. +struct SampleElements { + RooArgList shapeElems; + RooArgList normElems; + + RooArgList overallNps; + std::vector overallLow; + std::vector overallHigh; + std::vector overallInterp; + + RooArgList histNps; + RooArgList histoLo; + RooArgList histoHi; + std::vector histoInterp; +}; + +void buildSample(ChannelContext const &ctx, HS3::Sample const &sample, RooDataHist &dh) { + RooJSONFactoryWSTool &tool = ctx.tool; RooWorkspace &ws = *tool.workspace(); - std::string sampleName = RooJSONFactoryWSTool::name(p); - std::string prefixedName = fprefix + "_" + sampleName; + std::string const &sampleName = sample.name; + std::string prefixedName = ctx.fprefix + "_" + sampleName; - std::string channelName = fprefix; - erasePrefix(channelName, "model_"); + // The objects of a sample are not shared, so an existing object of the same name is a name clash. + auto newName = [&](std::string name) { + if (ws.arg(name.c_str())) { + RooJSONFactoryWSTool::error("the object '" + name + "' for sample '" + sampleName + "' of channel '" + + ctx.channelName + "' has the name of an existing object"); + } + return name; + }; - if (!p.has_child("data")) { - RooJSONFactoryWSTool::error("sample '" + sampleName + "' does not define a 'data' key"); - } + auto &hf = tool.wsEmplace(newName("hist_" + prefixedName), ctx.observables, dh); + hf.SetTitle(sampleName.c_str()); - auto &hf = tool.wsEmplace("hist_" + prefixedName, varlist, dh); - hf.SetTitle(RooJSONFactoryWSTool::name(p).c_str()); + SampleElements elems; + elems.shapeElems.add(tool.wsEmplace(newName(prefixedName + "_binWidth"), hf, true)); - RooArgList shapeElems; - RooArgList normElems; + const bool hasStatError = std::any_of(sample.modifiers.begin(), sample.modifiers.end(), + [](auto const &m) { return std::holds_alternative(m); }); + if (hasStatError) { + elems.shapeElems.add(*ctx.mcStatObject); + } - shapeElems.add(tool.wsEmplace(prefixedName + "_binWidth", hf, true)); - - if (findStaterror(p)) { - shapeElems.add(*mcStatObject); - } - - if (p.has_child("modifiers")) { - RooArgList overall_nps; - std::vector overall_low; - std::vector overall_high; - std::vector overall_interp; - - RooArgList histNps; - RooArgList histoLo; - RooArgList histoHi; - std::vector histoInterp; - - int idx = 0; - for (const auto &mod : p["modifiers"].children()) { - std::string const &modtype = mod["type"].val(); - const std::optional parameter = - modtype == "normfactor" || modtype == "normsys" || modtype == "histosys" ? modifierParameter(mod) - : std::nullopt; - std::string const &sysname = - mod.has_child("name") ? mod["name"].val() : parameter.value_or("syst_" + std::to_string(idx)); - ++idx; - if (modtype == "staterror") { - // this is dealt with at a different place, ignore it for now - } else if (modtype == "normfactor") { - // The "name" is only a label of the modifier, the parameter is given by "parameters" if present. - RooRealVar &constrParam = getOrCreate(ws, parameter.value_or(sysname), 1., -3, 5); - constrParam.setError(0.0); - normElems.add(constrParam); - if (hasConstraint(mod)) { + auto context = [&](const char *type, std::string const &name) { + return std::string("interpolation for ") + type + " modifier '" + name + "' in sample '" + sampleName + + "' of channel '" + ctx.channelName + "'"; + }; + + for (HS3::Modifier const &modifier : sample.modifiers) { + std::visit( + HS3::overloaded{ + [&](HS3::StatError const &) { + // this is dealt with at a different place, ignore it for now + }, + [&](HS3::NormFactor const &mod) { + RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 1., -3, 5, mod.name); + setErrorIfVariable(param, 0.0); + elems.normElems.add(param); // for norm factors, constraints are optional - constraints.add(getOrCreateConstraint(tool, mod, constrParam, sampleName)); - } - } else if (modtype == "normsys") { - std::string parname(parameter.value_or("alpha_" + sysname)); - auto &par = getOrCreate(ws, parname, 0., -5, 5); - overall_nps.add(par); - auto &data = mod["data"]; - const std::string context = "interpolation for normsys modifier '" + sysname + "' in sample '" + - sampleName + "' of channel '" + channelName + "'"; - const int interp = interpolationCode(mod, defaultInterpolation, InterpolationClass::Flexible, context, - legacyInterpolationDefault); - double low = data["lo"].val_double(); - double high = data["hi"].val_double(); - - // the below contains a a hack to cut off variations that go below 0 - // This is needed because FlexibleInterpVar code 4 interpolates in log-space. Hence, values <= 0 result in - // NaN, which propagates throughout the model and causes evaluations to fail. If you know a nicer way to - // solve this, please go ahead and fix the lines below. - if (interp == 4 && low <= 0) - low = std::numeric_limits::epsilon(); - if (interp == 4 && high <= 0) - high = std::numeric_limits::epsilon(); - - overall_low.push_back(low); - overall_high.push_back(high); - overall_interp.push_back(interp); - - addSysConstraint(tool, mod, par, sysname, sampleName, warnedUnconstrained, constraints); - } else if (modtype == "histosys") { - std::string parname(parameter.value_or("alpha_" + sysname)); - auto &par = getOrCreate(ws, parname, 0., -5, 5); - histNps.add(par); - auto &data = mod["data"]; - histoLo.add(tool.wsEmplace( - sysname + "Low_" + prefixedName, varlist, - RooJSONFactoryWSTool::readBinnedData(data["lo"], sysname + "Low_" + prefixedName, varlist))); - histoHi.add(tool.wsEmplace( - sysname + "High_" + prefixedName, varlist, - RooJSONFactoryWSTool::readBinnedData(data["hi"], sysname + "High_" + prefixedName, varlist))); - const std::string context = "interpolation for histosys modifier '" + sysname + "' in sample '" + - sampleName + "' of channel '" + channelName + "'"; - histoInterp.push_back(interpolationCode(mod, defaultInterpolation, InterpolationClass::Piecewise, context, - legacyInterpolationDefault)); - addSysConstraint(tool, mod, par, sysname, sampleName, warnedUnconstrained, constraints); - } else if (modtype == "shapesys" || modtype == "shapefactor") { - // HS3 has a single per-bin modifier type, "shapefactor", whose parameters are constrained by the - // distributions in "constraints", if given. "shapesys" is a deprecated alias. - if (modtype == "shapesys" && !isHS3VersionBefore(tool.hs3VersionInput(), 0, 3) && !warnedShapeSys) { - RooJSONFactoryWSTool::warning("RooFitHS3: the modifier type \"shapesys\" is deprecated, please use " - "\"shapefactor\" instead."); - warnedShapeSys = true; - } - std::vector vals; - if (mod["data"].has_child("vals")) { - for (const auto &v : mod["data"]["vals"].children()) { - vals.push_back(v.val_double()); - } - } - std::vector parnames; - for (const auto &v : mod["parameters"].children()) { - parnames.push_back(v.val()); - } - if (vals.empty() && parnames.empty()) { - RooJSONFactoryWSTool::error("unable to instantiate shapesys '" + sysname + - "' with neither values nor parameters!"); - } - std::string constraint = "unknown"; - std::vector constraintPdfs; - bool const hasConstraintList = mod.has_child("constraints"); - if (hasConstraintList) { - for (const auto &v : mod["constraints"].children()) { - if (v.is_null()) { - constraintPdfs.push_back(nullptr); - } else { - std::string constraintName = v.val(); - auto *constraintPdf = findNamedConstraint(tool, constraintName, sampleName); - if (!constraintPdf) { - RooJSONFactoryWSTool::error("unable to find definition of constraint '" + constraintName + - "' for modifier '" + RooJSONFactoryWSTool::name(mod) + "'"); - } - constraintPdfs.push_back(constraintPdf); - } - } - std::size_t const nGammas = std::max(vals.size(), parnames.size()); - if (constraintPdfs.size() != nGammas) { - std::stringstream ss; - ss << "modifier '" << RooJSONFactoryWSTool::name(mod) << "' has " << constraintPdfs.size() - << " constraints, but " << nGammas << " parameters"; - RooJSONFactoryWSTool::error(ss.str()); - } - } else if (mod.has_child("constraint_type")) { - constraint = mod["constraint_type"].val(); - } else if (mod.has_child("constraint")) { - std::string constraintValue = mod["constraint"].val(); - if (isLegacyConstraintType(constraintValue)) { - constraint = constraintValue; + addConstraint(tool, mod.constraint, param, 1.0, mod.name, sampleName, ctx.constraints); + }, + [&](HS3::NormSys const &mod) { + RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 0., -5, 5, mod.name); + elems.overallNps.add(param); + const int interp = interpolationCode(mod.interpolation, ctx.dist.defaultInterpolation, + InterpolationClass::Flexible, context("normsys", mod.name)); + double low = mod.lo; + double high = mod.hi; + + // the below contains a a hack to cut off variations that go below 0 + // This is needed because FlexibleInterpVar code 4 interpolates in log-space. Hence, values <= 0 result + // in NaN, which propagates throughout the model and causes evaluations to fail. If you know a nicer way + // to solve this, please go ahead and fix the lines below. + if (interp == 4 && low <= 0) + low = std::numeric_limits::epsilon(); + if (interp == 4 && high <= 0) + high = std::numeric_limits::epsilon(); + + elems.overallLow.push_back(low); + elems.overallHigh.push_back(high); + elems.overallInterp.push_back(interp); + + addConstraint(tool, mod.constraint, param, 0.0, mod.name, sampleName, ctx.constraints); + }, + [&](HS3::HistoSys const &mod) { + RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 0., -5, 5, mod.name); + elems.histNps.add(param); + auto addVariation = [&](RooArgList &list, HS3::HistoSysVariation const &variation, const char *suffix) { + const std::string name = newName(mod.name + suffix + prefixedName); + list.add( + tool.wsEmplace(name, ctx.observables, + HS3::createDataHist(name, ctx.observables, variation.contents, + variation.errors ? &*variation.errors : nullptr))); + }; + addVariation(elems.histoLo, mod.lo, "Low_"); + addVariation(elems.histoHi, mod.hi, "High_"); + elems.histoInterp.push_back(interpolationCode(mod.interpolation, ctx.dist.defaultInterpolation, + InterpolationClass::Piecewise, + context("histosys", mod.name))); + addConstraint(tool, mod.constraint, param, 0.0, mod.name, sampleName, ctx.constraints); + }, + [&](HS3::ShapeSys const &mod) { + const bool constrained = std::visit( + HS3::overloaded{[](HS3::Unconstrained) { return false; }, [](HS3::ConstantGammas) { return false; }, + [&](HS3::ConstraintType) { return !mod.vals.empty(); }, + [](HS3::GammaConstraintPdfs const &constraints) { + return std::any_of(constraints.pdfs.begin(), constraints.pdfs.end(), + [](auto const &pdf) { return pdf.has_value(); }); + }}, + mod.constraint); + // Follow the naming of HistFactory: "__ShapeSys" for constrained and + // "__shapeFactor" for free parameters. + const std::string funcName = + ctx.channelName + "_" + mod.name + (constrained ? "_ShapeSys" : "_shapeFactor"); + elems.shapeElems.add(createPHF(tool, funcName, mod.name, mod.parameters, mod.vals, mod.constraint, + ctx.constraints, ctx.observables, defaultShapeSysGammaMax, + minShapeUncertainty, ctx.nBins)); + }, + [&](HS3::CustomModifier const &mod) { + // The function can come later in the document. + RooAbsReal *obj = tool.request(mod.name, ctx.channelName); + if (obj->dependsOn(ctx.observables)) { + elems.shapeElems.add(*obj); } else { - RooJSONFactoryWSTool::error("unable to resolve constraint value '" + constraintValue + - "' for modifier '" + RooJSONFactoryWSTool::name(mod) + - "': this looks like a legacy workspace where the 'constraint' field is " - "not a supported legacy constraint type"); + elems.normElems.add(*obj); } - } - // Without "constraints", only the legacy constraint keys, or documents before HS3 0.3, make RooFit - // create constraints from "data.vals". Otherwise, the parameters are free. - const bool legacyConstraint = mod.has_child("constraint_type") || mod.has_child("constraint"); - const bool createConstraints = - !hasConstraintList && (legacyConstraint || isHS3VersionBefore(tool.hs3VersionInput(), 0, 3)); - if (!hasConstraintList && !createConstraints && !vals.empty()) { - oocoutW(nullptr, ObjectHandling) - << "RooFitHS3: the \"data.vals\" of modifier '" << sysname << "' of sample '" << sampleName - << "' are ignored, because it names no constraints, so its " - << "parameters are free" << std::endl; - vals.clear(); - } - const bool constrained = createConstraints ? !vals.empty() && constraint != "Const" - : std::any_of(constraintPdfs.begin(), constraintPdfs.end(), - [](RooAbsPdf *pdf) { return pdf != nullptr; }); - // Follow the naming of HistFactory: "__ShapeSys" for constrained and - // "__shapeFactor" for free parameters. - const std::string funcName = channelName + "_" + sysname + (constrained ? "_ShapeSys" : "_shapeFactor"); - shapeElems.add(createPHF(funcName, sysname, parnames, vals, tool, constraints, varlist, constraint, - defaultGammaMin, defaultShapeSysGammaMax, minShapeUncertainty, createConstraints)); - for (auto *constraintPdf : constraintPdfs) { - if (constraintPdf) { - constraints.add(*constraintPdf); - } - } - } else if (modtype == "custom") { - RooAbsReal *obj = ws.function(sysname); - if (!obj) { - RooJSONFactoryWSTool::error("unable to find custom modifier '" + sysname + "'"); - } - if (obj->dependsOn(varlist)) { - shapeElems.add(*obj); - } else { - normElems.add(*obj); - } - } else { - RooJSONFactoryWSTool::error("modifier '" + sysname + "' of unknown type '" + modtype + "'"); - } - } - - std::string interpName = sampleName + "_" + channelName + "_epsilon"; - if (!overall_nps.empty()) { - auto &v = tool.wsEmplace(interpName, overall_nps, 1., overall_low, - overall_high, overall_interp); - normElems.add(v); - } - if (!histNps.empty()) { - auto &v = tool.wsEmplace("histoSys_" + prefixedName, hf, histoLo, histoHi, histNps, - histoInterp); - v.setPositiveDefinite(); - shapeElems.add(v); - } else { - shapeElems.add(hf); - } + }, + }, + modifier); } - tool.wsEmplace(prefixedName + "_shapes", shapeElems); - if (!normElems.empty()) { - tool.wsEmplace(prefixedName + "_scaleFactors", normElems); + if (!elems.overallNps.empty()) { + std::string interpName = newName(sampleName + "_" + ctx.channelName + "_epsilon"); + elems.normElems.add(tool.wsEmplace( + interpName, elems.overallNps, 1., elems.overallLow, elems.overallHigh, elems.overallInterp)); + } + if (!elems.histNps.empty()) { + auto &v = tool.wsEmplace(newName("histoSys_" + prefixedName), hf, elems.histoLo, + elems.histoHi, elems.histNps, elems.histoInterp); + v.setPositiveDefinite(); + elems.shapeElems.add(v); } else { - ws.factory("RooConstVar::" + prefixedName + "_scaleFactors(1.)"); + elems.shapeElems.add(hf); } - return true; + tool.wsEmplace(newName(prefixedName + "_shapes"), elems.shapeElems); + if (!elems.normElems.empty()) { + tool.wsEmplace(newName(prefixedName + "_scaleFactors"), elems.normElems); + } else { + ws.factory("RooConstVar::" + newName(prefixedName + "_scaleFactors") + "(1.)"); + } } -class HistFactoryImporter : public RooFit::JSONIO::Importer { -public: - bool importArg(RooJSONFactoryWSTool *tool, const JSONNode &p) const override - { - std::string name = RooJSONFactoryWSTool::name(p); - if (!p.has_child("samples")) { - RooJSONFactoryWSTool::error("no samples in '" + name + "', skipping."); - } - double statErrThresh = 0; - std::string statErrType = "Poisson"; - std::optional defaultInterpolation; - if (p.has_child("default_interpolation")) { - defaultInterpolation = - readInterpolation(p["default_interpolation"], "default_interpolation of channel '" + name + "'"); - } - if (p.has_child(::Literals::staterror)) { - auto &staterr = p[::Literals::staterror]; - if (staterr.has_child("relThreshold")) - statErrThresh = staterr["relThreshold"].val_double(); - if (staterr.has_child("constraint_type")) - statErrType = staterr["constraint_type"].val(); - } - std::vector sumW; - std::vector sumW2; - std::vector gammaParnames; - std::string modifierStatErrType; - std::optional>> statErrConstraintNames; - RooArgSet observables = RooJSONFactoryWSTool::readAxes(p); - - std::string fprefix = name; - - std::vector> data; - for (const auto &comp : p["samples"].children()) { - std::unique_ptr dh = RooJSONFactoryWSTool::readBinnedData( - comp["data"], fprefix + "_" + RooJSONFactoryWSTool::name(comp) + "_dataHist", observables); - size_t nbins = dh->numEntries(); - - if (const JSONNode *staterror = findStaterror(comp)) { - if (sumW.empty()) { - sumW.resize(nbins); - sumW2.resize(nbins); - } - // A sample without "errors" contributes no statistical uncertainty in HS3. Before HS3 0.3, RooFit - // assumed Poisson uncertainties for it. - const bool useWeightSquared = - comp["data"].has_child("errors") || isHS3VersionBefore(tool->hs3VersionInput(), 0, 3); - for (size_t i = 0; i < nbins; ++i) { - sumW[i] += dh->weight(i); - sumW2[i] += useWeightSquared ? dh->weightSquared(i) : 0.; - } - if (gammaParnames.empty()) { - if (auto staterrorParams = staterror->find("parameters")) { - for (const auto &v : staterrorParams->children()) { - gammaParnames.push_back(v.val()); - } - } - } - // HS3 names the constraint distributions of the gammas in "constraints". - if (!statErrConstraintNames && staterror->has_child("constraints")) { - statErrConstraintNames = constraintNames(*staterror); - } - // The legacy constraint type given on the staterror modifier takes precedence over the channel-level - // setting. - if (modifierStatErrType.empty()) { - // ROOT 6.38 and 6.40.00 wrote "constraint_type". - auto constraint = staterror->find("constraint"); - if (!constraint) { - constraint = staterror->find("constraint_type"); - } - if (constraint) { - modifierStatErrType = constraint->val(); - if (modifierStatErrType != "Poisson" && modifierStatErrType != "Gauss") { - RooJSONFactoryWSTool::error("unsupported constraint '" + modifierStatErrType + - "' of the staterror modifier in channel '" + name + - "', supported are \"Poisson\" and \"Gauss\""); - } - } - } +// The constraint of the gamma parameters of the staterror modifiers. Without one, the gamma parameters are free. +HS3::GammaConstraint statErrorConstraint(HS3::HistFactoryDist const &dist) +{ + // The constraint distributions named by a staterror modifier take precedence over a constraint type, and a type + // given on a staterror modifier over the channel-level setting. + HS3::GammaConstraint const *modifierConstraint = nullptr; + for (auto const &sample : dist.samples) { + for (auto const &modifier : sample.modifiers) { + auto const *statError = std::get_if(&modifier); + if (!statError) { + continue; } - data.emplace_back(std::move(dh)); - } - - if (!modifierStatErrType.empty()) { - statErrType = modifierStatErrType; - } - // Without any constraint information, the gammas are unconstrained in HS3. Older documents relied on RooFit - // creating Poisson constraints. - const bool legacyStatErrConstraints = statErrConstraintNames || !modifierStatErrType.empty() || - p.has_child(::Literals::staterror) || - isHS3VersionBefore(tool->hs3VersionInput(), 0, 3); - - RooAbsArg *mcStatObject = nullptr; - RooArgSet constraints; - if (!sumW.empty()) { - std::string channelName = name; - erasePrefix(channelName, "model_"); - - std::vector errs(sumW.size()); - for (size_t i = 0; i < sumW.size(); ++i) { - if (sumW[i] == 0.) { - errs[i] = 0.; - continue; - } - errs[i] = std::sqrt(sumW2[i]) / sumW[i]; - // avoid negative sigma. This NP will be set constant anyway later - errs[i] = std::max(errs[i], 0.); + if (std::holds_alternative(statError->constraint)) { + return statError->constraint; } - - const bool createConstraints = !statErrConstraintNames && legacyStatErrConstraints; - mcStatObject = &createPHF("mc_stat_" + channelName, "stat_" + channelName, gammaParnames, errs, *tool, - constraints, observables, statErrType, defaultGammaMin, defaultStatErrorGammaMax, - statErrThresh, createConstraints); - if (statErrConstraintNames) { - if (statErrConstraintNames->size() != errs.size()) { - RooJSONFactoryWSTool::error("the staterror modifier of channel '" + name + "' has " + - std::to_string(statErrConstraintNames->size()) + " constraints, but " + - std::to_string(errs.size()) + " parameters"); - } - for (auto const &constraintName : *statErrConstraintNames) { - if (!constraintName) { - continue; - } - auto *constraint = findNamedConstraint(*tool, *constraintName, name); - if (!constraint) { - RooJSONFactoryWSTool::error("unable to find the constraint distribution '" + *constraintName + - "' of the staterror modifier of channel '" + name + "'"); - } - constraints.add(*constraint); - } + if (!modifierConstraint && !std::holds_alternative(statError->constraint)) { + modifierConstraint = &statError->constraint; } } + } + return modifierConstraint ? *modifierConstraint : dist.statError.constraint; +} - LegacyInterpolationDefault legacyInterpolationDefault; - legacyInterpolationDefault.allowed = isHS3VersionBefore(tool->hs3VersionInput(), 0, 3); - bool warnedUnconstrained = false; - bool warnedShapeSys = false; - - int idx = 0; - RooArgList funcs; - RooArgList coefs; - for (const auto &comp : p["samples"].children()) { - importHistSample(*tool, *data[idx], observables, mcStatObject, fprefix, comp, defaultInterpolation, - legacyInterpolationDefault, warnedUnconstrained, warnedShapeSys, constraints); - ++idx; - - std::string const &compName = RooJSONFactoryWSTool::name(comp); - funcs.add(*tool->request(fprefix + "_" + compName + "_shapes", name)); - coefs.add(*tool->request(fprefix + "_" + compName + "_scaleFactors", name)); +// Reject constraints of staterror modifiers that make no sense for gamma parameters. +void checkStatErrorConstraints(HS3::HistFactoryDist const &dist, std::string const &name) +{ + for (auto const &sample : dist.samples) { + for (auto const &modifier : sample.modifiers) { + auto const *statError = std::get_if(&modifier); + auto const *type = statError ? std::get_if(&statError->constraint) : nullptr; + if (type && *type == HS3::ConstraintType::LogNormal) { + RooJSONFactoryWSTool::error(std::string("unsupported constraint '") + toString(*type) + + "' of the staterror modifier in channel '" + name + + "', supported are \"Poisson\" and \"Gauss\""); + } } + } +} - // A histfactory_dist is a product of Poisson distributions of the bin counts. RooFit's binned likelihood - // computes exactly that (it differs from the event-wise likelihood only by a constant), and binned generation - // generates bin counts, so both are enabled, as HistFactory does by default. - auto enableBinned = [](RooRealSumPdf &sum) { - sum.setAttribute("BinnedLikelihood"); - sum.setAttribute("GenerateBinned"); - }; - if (constraints.empty()) { - enableBinned(tool->wsEmplace(name, funcs, coefs, true)); - } else { - std::string sumName = name + "_model"; - erasePrefix(sumName, "model_"); - auto &sum = tool->wsEmplace(sumName, funcs, coefs, true); - sum.SetTitle(name.c_str()); - enableBinned(sum); - tool->wsEmplace(name, constraints, RooFit::Conditional(sum, observables)); - } +class HistFactoryImporter : public RooFit::JSONIO::Importer { +public: + // This is only used for HistFactory distributions that don't come with a document. + bool importArg(RooJSONFactoryWSTool *tool, const JSONNode &p) const override + { + RooFit::HS3::importHistFactoryNode(*tool, p, nullptr); return true; } }; @@ -1160,19 +656,21 @@ class FlexibleInterpVarStreamer : public RooFit::JSONIO::Exporter { bool exportObject(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem) const override { auto fip = static_cast(func); + const std::string context = "FlexibleInterpVar '" + std::string{fip->GetName()} + "'"; const std::size_t nParameters = fip->variables().size(); if (fip->low().size() != nParameters || fip->high().size() != nParameters || fip->interpolationCodes().size() != nParameters) { - RooJSONFactoryWSTool::error("FlexibleInterpVar '" + std::string{fip->GetName()} + - "' has non-matching parameter, variation, and interpolation lengths"); + RooJSONFactoryWSTool::error(context + " has non-matching parameter, variation, and interpolation lengths"); } - elem["type"] << key(); - writeInterpolations(elem["interpolations"], fip->interpolationCodes(), InterpolationClass::Flexible, - "FlexibleInterpVar '" + std::string{fip->GetName()} + "'"); - RooJSONFactoryWSTool::fillSeq(elem["vars"], fip->variables()); - elem["nom"] << fip->nominal(); - elem["high"].fill_seq(fip->high(), fip->variables().size()); - elem["low"].fill_seq(fip->low(), fip->variables().size()); + HS3::InterpolatedNumber out; + for (RooAbsArg const *var : fip->variables()) { + out.parameters.push_back(var->GetName()); + } + out.nominal = fip->nominal(); + out.high = fip->high(); + out.low = fip->low(); + out.interpolations = toInterpolations(fip->interpolationCodes(), InterpolationClass::Flexible, context); + HS3::writeInterpolatedNumber(elem, out); return true; } }; @@ -1187,20 +685,22 @@ class PiecewiseInterpolationStreamer : public RooFit::JSONIO::Exporter { bool exportObject(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem) const override { const PiecewiseInterpolation *pip = static_cast(func); + const std::string context = "PiecewiseInterpolation '" + std::string{pip->GetName()} + "'"; const std::size_t nParameters = pip->paramList().size(); if (pip->lowList().size() != nParameters || pip->highList().size() != nParameters || pip->interpolationCodes().size() != nParameters) { - RooJSONFactoryWSTool::error("PiecewiseInterpolation '" + std::string{pip->GetName()} + - "' has non-matching parameter, variation, and interpolation lengths"); - } - elem["type"] << key(); - writeInterpolations(elem["interpolations"], pip->interpolationCodes(), InterpolationClass::Piecewise, - "PiecewiseInterpolation '" + std::string{pip->GetName()} + "'"); - elem["positiveDefinite"] << pip->positiveDefinite(); - RooJSONFactoryWSTool::fillSeq(elem["vars"], pip->paramList()); - elem["nom"] << pip->nominalHist()->GetName(); - RooJSONFactoryWSTool::fillSeq(elem["high"], pip->highList(), pip->paramList().size()); - RooJSONFactoryWSTool::fillSeq(elem["low"], pip->lowList(), pip->paramList().size()); + RooJSONFactoryWSTool::error(context + " has non-matching parameter, variation, and interpolation lengths"); + } + HS3::InterpolatedFunction out; + out.nominal = pip->nominalHist()->GetName(); + for (std::size_t i = 0; i < nParameters; ++i) { + out.parameters.push_back(HS3::toNameOrNumber(pip->paramList()[i])); + out.high.push_back(HS3::toNameOrNumber(pip->highList()[i])); + out.low.push_back(HS3::toNameOrNumber(pip->lowList()[i])); + } + out.interpolations = toInterpolations(pip->interpolationCodes(), InterpolationClass::Piecewise, context); + out.positiveDefinite = pip->positiveDefinite(); + HS3::writeInterpolatedFunction(elem, out); return true; } }; @@ -1210,22 +710,25 @@ class PiecewiseInterpolationFactory : public RooFit::JSONIO::Importer { bool importArg(RooJSONFactoryWSTool *tool, const JSONNode &p) const override { std::string name(RooJSONFactoryWSTool::name(p)); - - RooArgList vars{tool->requestArgList(p, "vars")}; - RooArgList low{tool->requestArgList(p, "low")}; - RooArgList high{tool->requestArgList(p, "high")}; - if (vars.size() != low.size() || vars.size() != high.size()) { - RooJSONFactoryWSTool::error("PiecewiseInterpolation '" + name + - "' has non-matching lengths of 'vars', 'high' and 'low'"); - } - const std::vector codes = - readInterpolations(p, vars.size(), InterpolationClass::Piecewise, "PiecewiseInterpolation '" + name + "'"); - - auto &pip = - tool->wsEmplace(name, *tool->requestArg(p, "nom"), low, high, vars, codes); - - pip.setPositiveDefinite(p["positiveDefinite"].val_bool()); - + HS3::Diagnostics diagnostics; + const HS3::InterpolatedFunction in = HS3::parseInterpolatedFunction(p, diagnostics); + report(name, diagnostics); + + auto request = [&](std::vector const &refs) { + RooArgList out; + for (auto const &ref : refs) { + auto const *value = std::get_if(&ref); + out.add(value ? RooFit::RooConst(*value) : *tool->request(std::get(ref), name)); + } + return out; + }; + RooArgList vars{request(in.parameters)}; + RooArgList low{request(in.low)}; + RooArgList high{request(in.high)}; + auto &pip = tool->wsEmplace( + name, *tool->request(in.nominal, name), low, high, vars, + toCodes(in.interpolations, InterpolationClass::Piecewise, "PiecewiseInterpolation '" + name + "'")); + pip.setPositiveDefinite(in.positiveDefinite); return true; } }; @@ -1235,92 +738,21 @@ class FlexibleInterpVarFactory : public RooFit::JSONIO::Importer { bool importArg(RooJSONFactoryWSTool *tool, const JSONNode &p) const override { std::string name(RooJSONFactoryWSTool::name(p)); - if (!p.has_child("high")) { - RooJSONFactoryWSTool::error("no high variations of '" + name + "'"); - } - if (!p.has_child("low")) { - RooJSONFactoryWSTool::error("no low variations of '" + name + "'"); - } - if (!p.has_child("nom")) { - RooJSONFactoryWSTool::error("no nominal variation of '" + name + "'"); - } - - double nom(p["nom"].val_double()); - - RooArgList vars{tool->requestArgList(p, "vars")}; + HS3::Diagnostics diagnostics; + const HS3::InterpolatedNumber in = HS3::parseInterpolatedNumber(p, diagnostics); + report(name, diagnostics); - std::vector high; - high << p["high"]; - - std::vector low; - low << p["low"]; - - if (vars.size() != low.size() || vars.size() != high.size()) { - RooJSONFactoryWSTool::error("FlexibleInterpVar '" + name + - "' has non-matching lengths of 'vars', 'high' and 'low'!"); + RooArgList vars; + for (std::string const &var : in.parameters) { + vars.add(*tool->request(var, name)); } - const std::vector codes = - readInterpolations(p, vars.size(), InterpolationClass::Flexible, "FlexibleInterpVar '" + name + "'"); - - tool->wsEmplace(name, vars, nom, low, high, codes); - + tool->wsEmplace( + name, vars, in.nominal, in.low, in.high, + toCodes(in.interpolations, InterpolationClass::Flexible, "FlexibleInterpVar '" + name + "'")); return true; } }; -struct NormFactor { - std::string name; - RooAbsReal const *param = nullptr; - RooAbsPdf const *constraint = nullptr; - NormFactor(RooAbsReal const &par, const RooAbsPdf *constr = nullptr) - : name{par.GetName()}, param{&par}, constraint{constr} - { - } -}; - -struct NormSys { - std::string name = ""; - RooAbsReal const *param = nullptr; - double low = 1.; - double high = 1.; - Interpolation interpolation = multiplicativePolynomialExponential; - RooAbsPdf const *constraint = nullptr; - NormSys() {}; - NormSys(const std::string &n, RooAbsReal *const p, double h, double l, Interpolation i, const RooAbsPdf *c) - : name(n), param(p), low(l), high(h), interpolation(std::move(i)), constraint(c) - { - } -}; - -struct HistoSys { - std::string name; - RooAbsReal const *param = nullptr; - std::vector low; - std::vector high; - Interpolation interpolation = additivePolynomialLinear; - RooAbsPdf const *constraint = nullptr; - HistoSys(const std::string &n, RooAbsReal *const p, RooHistFunc *l, RooHistFunc *h, Interpolation i, - const RooAbsPdf *c) - : name(n), param(p), interpolation(std::move(i)), constraint(c) - { - low.assign(l->dataHist().weightArray(), l->dataHist().weightArray() + l->dataHist().numEntries()); - high.assign(h->dataHist().weightArray(), h->dataHist().weightArray() + h->dataHist().numEntries()); - } -}; -// A per-bin modifier ("shapefactor"): HistFactory's ShapeSys (constrained) or ShapeFactor (free). -struct ShapeSys { - std::string name; - std::vector constraintPdfs; - std::vector parameters; - ShapeSys(const std::string &n) : name{n} {} -}; - -struct GenericElement { - std::string name; - RooAbsReal *function = nullptr; - GenericElement(RooAbsReal *e) : name(e->GetName()), function(e) {}; -}; - std::string stripOuterParens(const std::string &s) { size_t start = 0; @@ -1407,19 +839,22 @@ std::vector splitTopLevelProduct(const std::string &expr) return parts; } -NormSys parseOverallModifierFormula(const std::string &s, RooFormulaVar *formula) +// A normsys that is written as the formula "1 +- a * b", where one of a and b is a constrained parameter. +struct FormulaNormSys { + RooAbsReal *param = nullptr; + RooAbsPdf *constraint = nullptr; + double variation = 0.0; +}; + +FormulaNormSys parseOverallModifierFormula(const std::string &s, RooFormulaVar *formula) { static const std::regex pattern( R"(^\s*1(?:\.0)?\s*([\+\-])\s*([a-zA-Z_][a-zA-Z0-9_]*|[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)\s*\*\s*([a-zA-Z_][a-zA-Z0-9_]*|[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)\s*$)"); - NormSys sys; - double sign = 1.0; - + FormulaNormSys sys; std::smatch match; if (std::regex_match(s, match, pattern)) { - if (match[1].str() == "-") { - sign = -1.0; - } + const double sign = match[1].str() == "-" ? -1.0 : 1.0; std::string token2 = match[2].str(); std::string token3 = match[4].str(); @@ -1433,34 +868,21 @@ NormSys parseOverallModifierFormula(const std::string &s, RooFormulaVar *formula auto *constr3 = findConstraint(v3); if (constr2 && !p3) { - sys.name = p2->GetName(); sys.param = p2; - sys.high = 1.0 + sign * toDouble(token3); - sys.low = 1.0 - sign * toDouble(token3); + sys.variation = sign * toDouble(token3); } else if (!p2 && constr3) { - sys.name = p3->GetName(); sys.param = p3; - sys.high = 1.0 + sign * toDouble(token2); - sys.low = 1.0 - sign * toDouble(token2); + sys.variation = sign * toDouble(token2); } else if (constr2 && p3 && !constr3) { - sys.name = v2->GetName(); sys.param = v2; - sys.high = 1.0 + sign * p3->getVal(); - sys.low = 1.0 - sign * p3->getVal(); + sys.variation = sign * p3->getVal(); } else if (p2 && !constr2 && constr3) { - sys.name = v3->GetName(); sys.param = v3; - sys.high = 1.0 + sign * p2->getVal(); - sys.low = 1.0 - sign * p2->getVal(); + sys.variation = sign * p2->getVal(); } // A recognized formula has exactly one constrained parameter. sys.constraint = constr2 ? constr2 : constr3; - - // Preserve the legacy export behaviour for recognized explicit formulae. - sys.interpolation = multiplicativePiecewiseExponential; - - erasePrefix(sys.name, "alpha_"); } return sys; } @@ -1486,46 +908,34 @@ bool allRooRealVar(const RooAbsCollection &list) return true; } -struct Sample { - std::string name; - std::vector hist; - std::vector histError; - std::vector normfactors; - std::vector normsys; - std::vector histosys; - std::vector shapesys; - std::vector tmpElements; - std::vector otherElements; - bool useBarlowBeestonLight = false; - std::vector staterrorParameters; - std::vector staterrorConstraints; - Sample(const std::string &n) : name{n} {} -}; - -void addNormFactor(RooRealVar const *par, Sample &sample, RooWorkspace *ws) +// The parameters of normsys and histosys modifiers must be variables. +void checkModifierParameter(RooAbsReal const ¶m, RooAbsPdf const *constraint) { - std::string parname = par->GetName(); - bool isConstrained = false; - for (RooAbsArg const *pdf : ws->allPdfs()) { - if (auto gauss = dynamic_cast(pdf)) { - if (parname == gauss->getX().GetName()) { - sample.normfactors.emplace_back(*par, gauss); - isConstrained = true; - } - } + if (!constraint && !param.isConstant() && !dynamic_cast(¶m)) { + RooJSONFactoryWSTool::error("cannot find constraint for " + std::string(param.GetName()) + + ": functions are not supported as parameters of this modifier"); } - if (!isConstrained) - sample.normfactors.emplace_back(*par); } -struct Channel { +std::vector contents(RooDataHist const &dataHist) +{ + return {dataHist.weightArray(), dataHist.weightArray() + dataHist.numEntries()}; +} + +// A channel as it is exported, and what the exporter needs to know about its RooFit objects. +struct ExportedChannel { std::string name; - std::vector samples; - std::map tot_yield; - std::map tot_yield2; - std::map rel_errors; - RooArgSet const *varSet = nullptr; - long unsigned int nBins = 0; + HS3::HistFactoryDist dist; + RooArgSet const *observables = nullptr; + /// The pdfs that the modifiers name as constraints, by name. + std::map constraints; + std::vector customModifiers; + /// Parts of formulas that are exported as custom modifiers. They are created for the export. + std::vector> temporaries; + // To reconstruct the statistical uncertainties, by bin starting at one. + std::map totYield; + std::map totYield2; + std::map relErrors; // Constant inverse bin widths of HistFactory before ROOT 6.26, which histfactory_dist implies. std::set legacyBinWidthFactors; }; @@ -1590,9 +1000,9 @@ bool isLegacyBinWidthFactor(RooAbsArg const &factor, RooRealSumPdf const &sumpdf [&](double volume) { return volume > 0. && std::abs(value * volume - 1.) < 1e-6; }); } -Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, const RooRealSumPdf *sumpdf) +ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, const RooRealSumPdf *sumpdf) { - Channel channel; + ExportedChannel channel; RooWorkspace *ws = tool->workspace(); @@ -1604,25 +1014,56 @@ Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, cons sumpdf->treeNodeServerList(&channelNodes); const std::vector volumes = binVolumes(*sumpdf); + auto constraint = [&](RooAbsPdf const *pdf) -> HS3::Constraint { + if (!pdf) { + return HS3::Unconstrained{}; + } + channel.constraints.emplace(pdf->GetName(), pdf); + return HS3::ConstraintPdf{{pdf->GetName()}}; + }; + auto gammaConstraint = [&](RooAbsPdf const *pdf) -> std::optional { + if (!pdf) { + return std::nullopt; + } + channel.constraints.emplace(pdf->GetName(), pdf); + return HS3::DistributionRef{pdf->GetName()}; + }; + for (size_t sampleidx = 0; sampleidx < sumpdf->funcList().size(); ++sampleidx) { PiecewiseInterpolation *pip = nullptr; std::vector phfs; const auto func = sumpdf->funcList().at(sampleidx); - Sample sample(func->GetName()); + HS3::Sample sample; + sample.name = func->GetName(); erasePrefix(sample.name, "L_x_"); eraseSuffix(sample.name, "_shapes"); eraseSuffix(sample.name, "_" + channel.name); erasePrefix(sample.name, pdfname + "_"); + // The modifiers by type, which are written in this order. + std::vector normFactors; + std::vector normSys; + std::vector histoSys; + std::vector shapeSys; + std::vector custom; + std::vector temporaries; + std::optional statError; + + auto addNormSys = [&](RooAbsReal ¶m, double hi, double lo, HS3::Interpolation interpolation, + RooAbsPdf const *pdf) { + checkModifierParameter(param, pdf); + std::string sysname(param.GetName()); + erasePrefix(sysname, "alpha_"); + normSys.push_back({sysname, {param.GetName()}, hi, lo, constraint(pdf), interpolation}); + }; + auto updateObservables = [&](RooDataHist const &dataHist) { - if (channel.varSet == nullptr) { - channel.varSet = dataHist.get(); - channel.nBins = dataHist.numEntries(); + if (channel.observables == nullptr) { + channel.observables = dataHist.get(); } - if (sample.hist.empty()) { - auto *w = dataHist.weightArray(); - sample.hist.assign(w, w + dataHist.numEntries()); + if (sample.contents.empty()) { + sample.contents = contents(dataHist); } }; auto processElements = [&](const auto &elements, auto &&self) -> void { @@ -1644,10 +1085,22 @@ Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, cons } } else if (auto constVar = dynamic_cast(e)) { if (constVar->getVal() != 1.) { - sample.normfactors.emplace_back(*constVar); + normFactors.push_back({constVar->GetName(), {constVar->GetName()}, HS3::Unconstrained{}}); } } else if (auto par = dynamic_cast(e)) { - addNormFactor(par, sample, ws); + // Every Gaussian of the parameter is a candidate for its constraint. + bool isConstrained = false; + for (RooAbsArg const *pdf : ws->allPdfs()) { + if (auto gauss = dynamic_cast(pdf)) { + if (std::string(par->GetName()) == gauss->getX().GetName()) { + normFactors.push_back({par->GetName(), {par->GetName()}, constraint(gauss)}); + isConstrained = true; + } + } + } + if (!isConstrained) { + normFactors.push_back({par->GetName(), {par->GetName()}, HS3::Unconstrained{}}); + } } else if (auto hf = dynamic_cast(e)) { updateObservables(hf->dataHist()); } else if (ParamHistFunc *phf = dynamic_cast(e); phf && allRooRealVar(phf->paramList())) { @@ -1659,14 +1112,12 @@ Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, cons RooAbsReal *var = static_cast(fip->variables().at(i)); std::string sysname(var->GetName()); erasePrefix(sysname, "alpha_"); - // A parameter without constraint is unconstrained, see checkUnconstrainedModifiers(). - const auto *constraint = findConstraint(var); const std::string context = "normsys modifier '" + sysname + "' in sample '" + sample.name + "' of channel '" + channel.name + "'"; - sample.normsys.emplace_back( - sysname, var, fip->high()[i], fip->low()[i], - interpolationFromCode(fip->interpolationCodes()[i], InterpolationClass::Flexible, context), - constraint); + // A parameter without constraint is unconstrained, see checkUnconstrainedModifiers(). + addNormSys(*var, fip->high()[i], fip->low()[i], + interpolationFromCode(fip->interpolationCodes()[i], InterpolationClass::Flexible, context), + findConstraint(var)); } } else if (!pip && (pip = dynamic_cast(e))) { // nothing to do here, already assigned @@ -1682,7 +1133,8 @@ Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, cons auto components = splitTopLevelProduct(expression.Data()); if (components.size() == 0) { // it's not a product, let's just treat it as an unknown element - sample.otherElements.push_back(formula); + custom.push_back({formula->GetName()}); + channel.customModifiers.push_back(formula); } else { // it is a prododuct, we can try to handle the elements separately std::vector realComponents; @@ -1695,22 +1147,26 @@ Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, cons continue; } // check if this is an attempt at explicitly encoding an overallSys - auto normsys = parseOverallModifierFormula(comp, formula); - if (normsys.param) { - sample.normsys.emplace_back(std::move(normsys)); + FormulaNormSys sys = parseOverallModifierFormula(comp, formula); + if (sys.param) { + // Preserve the legacy export behaviour for recognized explicit formulae. + addNormSys(*sys.param, 1.0 + sys.variation, 1.0 - sys.variation, + multiplicativePiecewiseExponential, sys.constraint); continue; } // this is something non-trivial, let's deal with it separately std::string name = std::string(formula->GetName()) + "_part" + std::to_string(idx); ++idx; - auto *var = new RooFormulaVar(name.c_str(), name.c_str(), comp.c_str(), formula->dependents()); - sample.tmpElements.push_back({var}); + temporaries.push_back({name}); + channel.temporaries.push_back(std::make_unique( + name.c_str(), name.c_str(), comp.c_str(), formula->dependents())); } self(realComponents, self); } } else if (auto real = dynamic_cast(e)) { - sample.otherElements.push_back(real); + custom.push_back({real->GetName()}); + channel.customModifiers.push_back(real); } } }; @@ -1727,11 +1183,6 @@ Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, cons } } - // sort and configure norms - sortByName(sample.normfactors); - sortByName(sample.normsys); - - // sort and configure the histosys if (pip) { for (size_t i = 0; i < pip->paramList().size(); ++i) { RooAbsReal *var = static_cast(pip->paramList().at(i)); @@ -1740,36 +1191,43 @@ Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, cons if (auto lo = dynamic_cast(pip->lowList().at(i))) { if (auto hi = dynamic_cast(pip->highList().at(i))) { // A parameter without constraint is unconstrained, see checkUnconstrainedModifiers(). - const auto *constraint = findConstraint(var); + const auto *pdf = findConstraint(var); + checkModifierParameter(*var, pdf); const std::string context = "histosys modifier '" + sysname + "' in sample '" + sample.name + "' of channel '" + channel.name + "'"; - sample.histosys.emplace_back( - sysname, var, lo, hi, - interpolationFromCode(pip->interpolationCodes()[i], InterpolationClass::Piecewise, context), - constraint); + histoSys.push_back( + {sysname, + {var->GetName()}, + {contents(hi->dataHist()), std::nullopt}, + {contents(lo->dataHist()), std::nullopt}, + constraint(pdf), + interpolationFromCode(pip->interpolationCodes()[i], InterpolationClass::Piecewise, context)}); } } } - sortByName(sample.histosys); } for (ParamHistFunc *phf : phfs) { if (startsWith(std::string(phf->GetName()), "mc_stat_")) { // MC stat uncertainty + statError.emplace(); + statError->name = "staterror"; + HS3::GammaConstraintPdfs pdfs; int idx = 0; for (const auto &g : phf->paramList()) { - sample.staterrorParameters.push_back(static_cast(g)); + statError->parameters.push_back({g->GetName()}); ++idx; - RooAbsPdf *constraint = findConstraint(g); - sample.staterrorConstraints.push_back(constraint); - channel.tot_yield[idx] += sample.hist[idx - 1]; - channel.tot_yield2[idx] += (sample.hist[idx - 1] * sample.hist[idx - 1]); - if (constraint) { - channel.rel_errors[idx] = constraintRelError(*constraint, *g); + RooAbsPdf *pdf = findConstraint(g); + pdfs.pdfs.push_back(gammaConstraint(pdf)); + channel.totYield[idx] += sample.contents[idx - 1]; + channel.totYield2[idx] += (sample.contents[idx - 1] * sample.contents[idx - 1]); + if (pdf) { + channel.relErrors[idx] = constraintRelError(*pdf, *g); } } - sample.useBarlowBeestonLight = true; + statError->constraint = std::move(pdfs); } else { // ShapeSys or ShapeFactor - ShapeSys sys(phf->GetName()); + HS3::ShapeSys &sys = shapeSys.emplace_back(); + sys.name = phf->GetName(); erasePrefix(sys.name, channel.name + "_"); for (std::string_view suffix : {"_ShapeSys", "_shapeSys", "_ShapeFactor", "_shapeFactor"}) { if (eraseSuffix(sys.name, suffix)) @@ -1778,403 +1236,116 @@ Channel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, cons // Both are written as "shapefactor": the constrained parameters name their constraint distribution in // "constraints", the others are free. + HS3::GammaConstraintPdfs pdfs; for (const auto &g : phf->paramList()) { - sys.parameters.push_back(static_cast(g)); - RooAbsPdf *constraint = findConstraint(g); - if (!constraint) - constraint = ws->pdf(constraintName(g->GetName())); - sys.constraintPdfs.push_back(constraint); + sys.parameters.push_back({g->GetName()}); + RooAbsPdf *pdf = findConstraint(g); + if (!pdf) + pdf = ws->pdf(constraintName(g->GetName())); + pdfs.pdfs.push_back(gammaConstraint(pdf)); } - sample.shapesys.emplace_back(std::move(sys)); + sys.constraint = std::move(pdfs); } } - sortByName(sample.shapesys); - - // add the sample - channel.samples.emplace_back(std::move(sample)); - } - sortByName(channel.samples); - return channel; -} - -bool hasSameMetadata(const RooAbsArg *lhs, const RooAbsArg *rhs) -{ - if (!lhs || !rhs) { - return lhs == rhs; - } - return std::string{lhs->GetName()} == rhs->GetName() && lhs->IsA() == rhs->IsA(); -} - -[[noreturn]] void duplicateModifierError(const Channel &channel, const Sample &sample, std::string_view type, - std::string_view name, std::string_view reason) -{ - std::stringstream ss; - ss << "cannot combine duplicate modifier '" << name << "' of type '" << type << "' in sample '" << sample.name - << "' of channel '" << channel.name << "': " << reason; - RooJSONFactoryWSTool::error(ss.str().c_str()); -} - -void warnDuplicateModifiersCombined(const Channel &channel, const Sample &sample, std::string_view type, - std::string_view name, std::size_t count) -{ - std::stringstream ss; - ss << "combined " << count << " duplicate modifiers named '" << name << "' of type '" << type << "' in sample '" - << sample.name << "' of channel '" << channel.name << "'"; - RooJSONFactoryWSTool::warning(ss.str()); -} - -// Multiplicatively combining two normsys is only faithful when the interpolation is done in log-space, so that -// f1(alpha) * f2(alpha) is again representable by a single normsys with the multiplied lo/hi factors. This holds for -// the piecewise-exponential code 1 (exact everywhere) and for the default code 4 (exact at the +-1 sigma anchors and in -// the exponential extrapolation region). The linear-space codes (e.g. 0 and 2) would turn the product into a shape that -// cannot be represented by a single normsys, so those must not be merged. -bool normSysSupportsMultiplicativeMerge(const Interpolation &interpolation) -{ - return interpolation == multiplicativePiecewiseExponential || interpolation == multiplicativePolynomialExponential; -} - -// Combines runs of adjacent modifiers that share the same name (the container is sorted by name beforehand) into a -// single modifier. The shared metadata (constraint, parameter and interpolation behaviour) must be identical across the -// duplicates; the type-specific `combine` callable performs the actual merge and any additional validation. -template -void mergeDuplicateModifiers(const Channel &channel, const Sample &sample, Modifiers &modifiers, std::string_view type, - CombineFn combine) -{ - Modifiers mergedModifiers; - mergedModifiers.reserve(modifiers.size()); - - for (std::size_t begin = 0; begin < modifiers.size();) { - std::size_t end = begin + 1; - while (end < modifiers.size() && modifiers[end].name == modifiers[begin].name) { - ++end; - } - - auto merged = modifiers[begin]; - for (std::size_t i = begin + 1; i < end; ++i) { - const auto &modifier = modifiers[i]; - if (!hasSameMetadata(merged.constraint, modifier.constraint)) { - duplicateModifierError(channel, sample, type, merged.name, "constraint metadata differs"); - } - if (!hasSameMetadata(merged.param, modifier.param)) { - duplicateModifierError(channel, sample, type, merged.name, "parameter metadata differs"); - } - if (merged.interpolation != modifier.interpolation) { - duplicateModifierError(channel, sample, type, merged.name, "interpolation behaviours differ"); - } - combine(merged, modifier); + sortByName(normFactors); + sortByName(normSys); + sortByName(histoSys); + sortByName(shapeSys); + auto append = [&](auto &modifiers) { + sample.modifiers.insert(sample.modifiers.end(), std::make_move_iterator(modifiers.begin()), + std::make_move_iterator(modifiers.end())); + }; + append(normFactors); + append(normSys); + append(histoSys); + append(shapeSys); + append(custom); + append(temporaries); + if (statError) { + sample.modifiers.push_back(std::move(*statError)); } - if (end - begin > 1) { - warnDuplicateModifiersCombined(channel, sample, type, merged.name, end - begin); - } - mergedModifiers.emplace_back(std::move(merged)); - begin = end; + channel.dist.samples.push_back(std::move(sample)); } - modifiers = std::move(mergedModifiers); -} - -void mergeDuplicateNormSys(const Channel &channel, Sample &sample) -{ - mergeDuplicateModifiers(channel, sample, sample.normsys, "normsys", [&](NormSys &merged, const NormSys &modifier) { - if (!normSysSupportsMultiplicativeMerge(merged.interpolation)) { - duplicateModifierError(channel, sample, "normsys", merged.name, - "multiplicative combination is only valid for log-space interpolation"); - } - merged.low *= modifier.low; - merged.high *= modifier.high; - }); -} - -void mergeDuplicateHistoSys(const Channel &channel, Sample &sample) -{ - const std::size_t nBins = sample.hist.size(); - mergeDuplicateModifiers( - channel, sample, sample.histosys, "histosys", [&](HistoSys &merged, const HistoSys &modifier) { - if (merged.interpolation != additivePolynomialLinear) { - duplicateModifierError(channel, sample, "histosys", merged.name, - "this interpolation cannot currently be combined for duplicate histosys " - "modifiers"); - } - if (merged.low.size() != nBins || merged.high.size() != nBins || modifier.low.size() != nBins || - modifier.high.size() != nBins) { - duplicateModifierError(channel, sample, "histosys", merged.name, "histogram binning differs"); - } - for (std::size_t bin = 0; bin < nBins; ++bin) { - merged.low[bin] += modifier.low[bin] - sample.hist[bin]; - merged.high[bin] += modifier.high[bin] - sample.hist[bin]; - } - }); + sortByName(channel.dist.samples); + return channel; } -void ensureUniqueModifiers(const Channel &channel, const Sample &sample) +// The statistical uncertainties of the samples are reconstructed from the constraints of the gamma parameters, with +// the (arbitrary) convention that all samples have the same relative uncertainty. +void configureStatError(ExportedChannel &channel) { - std::set> seen; - auto add = [&](std::string type, const std::string &name) { - if (!seen.emplace(type, name).second) { - duplicateModifierError(channel, sample, type, name, - "this modifier type cannot be combined without changing its meaning"); + for (HS3::Sample &sample : channel.dist.samples) { + if (std::none_of(sample.modifiers.begin(), sample.modifiers.end(), + [](auto const &m) { return std::holds_alternative(m); })) { + continue; } - }; - - for (const auto &modifier : sample.normfactors) - add("normfactor", modifier.name); - for (const auto &modifier : sample.normsys) - add("normsys", modifier.name); - for (const auto &modifier : sample.histosys) - add("histosys", modifier.name); - for (const auto &modifier : sample.shapesys) - add("shapefactor", modifier.name); - for (const auto &modifier : sample.otherElements) - add("custom", modifier.name); - for (const auto &modifier : sample.tmpElements) - add("custom", modifier.name); - if (sample.useBarlowBeestonLight) - add(::Literals::staterror, ::Literals::staterror); -} - -void canonicalizeModifiers(Channel &channel) -{ - for (auto &sample : channel.samples) { - mergeDuplicateNormSys(channel, sample); - mergeDuplicateHistoSys(channel, sample); - ensureUniqueModifiers(channel, sample); - } -} - -void configureStatError(Channel &channel) -{ - for (auto &sample : channel.samples) { - if (sample.useBarlowBeestonLight) { - sample.histError.resize(sample.hist.size()); - for (auto bin : channel.rel_errors) { - // reverse engineering the correct partial error - // the (arbitrary) convention used here is that all samples should have the same relative error - const int i = bin.first; - const double relerr_tot = bin.second; - const double count = sample.hist[i - 1]; - // this reconstruction is inherently imprecise, so we truncate it at some decimal places to make sure that - // we don't carry around too many useless digits - sample.histError[i - 1] = - round_prec(relerr_tot * channel.tot_yield[i] / std::sqrt(channel.tot_yield2[i]) * count, 7); - } + std::vector errors(sample.contents.size()); + for (auto const &bin : channel.relErrors) { + const int i = bin.first; + // this reconstruction is inherently imprecise, so we truncate it at some decimal places to make sure that + // we don't carry around too many useless digits + errors[i - 1] = + round_prec(bin.second * channel.totYield[i] / std::sqrt(channel.totYield2[i]) * sample.contents[i - 1], 7); } + sample.errors = std::move(errors); } } -std::optional defaultInterpolation(const Channel &channel) +// The modifiers name only the constraint terms of the channel. A pdf of a parameter that is not, like an external +// constraint or a prior, stays where it is. Returns the named constraint terms. +std::set +nameChannelConstraints(ExportedChannel &channel, std::vector const &constraints) { - std::map counts; - for (const auto &sample : channel.samples) { - for (const auto &modifier : sample.normsys) { - ++counts[modifier.interpolation]; - } - for (const auto &modifier : sample.histosys) { - ++counts[modifier.interpolation]; - } - } - if (counts.empty()) { - return std::nullopt; - } - - auto best = counts.begin(); - for (auto current = std::next(counts.begin()); current != counts.end(); ++current) { - if (current->second > best->second || - (current->second == best->second && current->first == multiplicativePolynomialExponential && - best->first != multiplicativePolynomialExponential)) { - best = current; + std::set named; + auto isNamed = [&](std::string const &name) { + RooAbsPdf const *pdf = channel.constraints.at(name); + if (std::find(constraints.begin(), constraints.end(), pdf) == constraints.end()) { + return false; } - } - return best->first; -} - -bool exportChannel(RooJSONFactoryWSTool *tool, const Channel &channel, JSONNode &elem) -{ - // Write the parameter and the name of the constraint distribution of a modifier with a single parameter. - auto writeParameter = [](JSONNode &mod, RooAbsReal const ¶m, RooAbsPdf const *constraint) { - mod["parameters"].set_seq().append_child() << param.GetName(); - exportConstraintNames(mod, {constraint}); - }; - auto addModifier = [](JSONNode &modifiers, std::string const &name, const char *type) -> JSONNode & { - auto &mod = modifiers.append_child(); - mod.set_map(); - mod["name"] << name; - mod["type"] << type; - return mod; + named.insert(pdf); + return true; }; - - elem["type"] << "histfactory_dist"; - const auto channelDefaultInterpolation = defaultInterpolation(channel); - if (channelDefaultInterpolation) { - writeInterpolation(elem["default_interpolation"], *channelDefaultInterpolation); - } - - bool observablesWritten = false; - for (const auto &sample : channel.samples) { - - auto &s = RooJSONFactoryWSTool::appendNamedChild(elem["samples"], sample.name); - - auto &modifiers = s["modifiers"]; - modifiers.set_seq(); - - for (const auto &nf : sample.normfactors) { - auto &mod = modifiers.append_child(); - mod.set_map(); - mod["name"] << nf.name; - writeParameter(mod, *nf.param, nf.constraint); - mod["type"] << "normfactor"; - if (nf.constraint) { - tool->queueExport(*nf.constraint); - } + auto filter = [&](HS3::Constraint &constraint) { + auto const *pdf = std::get_if(&constraint); + if (pdf && !isNamed(pdf->pdf.name)) { + constraint = HS3::Unconstrained{}; } - - for (const auto &sys : sample.normsys) { - auto &mod = addModifier(modifiers, sys.name, "normsys"); - writeParameter(mod, *sys.param, sys.constraint); - if (!channelDefaultInterpolation || sys.interpolation != *channelDefaultInterpolation) { - writeInterpolation(mod["interpolation"], sys.interpolation); - } - auto &data = mod["data"].set_map(); - data["lo"] << sys.low; - data["hi"] << sys.high; - } - - for (const auto &sys : sample.histosys) { - auto &mod = addModifier(modifiers, sys.name, "histosys"); - writeParameter(mod, *sys.param, sys.constraint); - if (!channelDefaultInterpolation || sys.interpolation != *channelDefaultInterpolation) { - writeInterpolation(mod["interpolation"], sys.interpolation); - } - auto &data = mod["data"].set_map(); - if (channel.nBins != sys.low.size() || channel.nBins != sys.high.size()) { - RooJSONFactoryWSTool::error("inconsistent binning: " + std::to_string(channel.nBins) + - " bins expected, but " + std::to_string(sys.low.size()) + "/" + - std::to_string(sys.high.size()) + " found in nominal histogram errors!"); - } - RooJSONFactoryWSTool::exportArray(channel.nBins, sys.low.data(), data["lo"].set_map()["contents"]); - RooJSONFactoryWSTool::exportArray(channel.nBins, sys.high.data(), data["hi"].set_map()["contents"]); - } - - for (const auto &sys : sample.shapesys) { - auto &mod = addModifier(modifiers, sys.name, "shapefactor"); - exportGammaParameters(mod, sys.parameters); - // The relative uncertainties are encoded in the constraint distributions. - exportConstraintNames(mod, sys.constraintPdfs); - } - - for (const auto &other : sample.otherElements) { - addModifier(modifiers, other.name, "custom"); - } - for (const auto &other : sample.tmpElements) { - addModifier(modifiers, other.name, "custom"); - } - - if (sample.useBarlowBeestonLight) { - auto &mod = addModifier(modifiers, ::Literals::staterror, ::Literals::staterror); - exportGammaParameters(mod, sample.staterrorParameters); - exportConstraintNames(mod, sample.staterrorConstraints); - } - - if (!observablesWritten) { - auto &output = elem["axes"].set_seq(); - for (auto *obs : static_range_cast(*channel.varSet)) { - RooJSONFactoryWSTool::exportAxis(output.append_child().set_map(), *obs); - } - observablesWritten = true; - } - auto &dataNode = s["data"].set_map(); - if (channel.nBins != sample.hist.size()) { - RooJSONFactoryWSTool::error("inconsistent binning: " + std::to_string(channel.nBins) + " bins expected, but " + - std::to_string(sample.hist.size()) + " found in nominal histogram!"); - } - RooJSONFactoryWSTool::exportArray(channel.nBins, sample.hist.data(), dataNode["contents"]); - if (!sample.histError.empty()) { - if (channel.nBins != sample.histError.size()) { - RooJSONFactoryWSTool::error("inconsistent binning: " + std::to_string(channel.nBins) + - " bins expected, but " + std::to_string(sample.histError.size()) + - " found in nominal histogram errors!"); - } - RooJSONFactoryWSTool::exportArray(channel.nBins, sample.histError.data(), dataNode["errors"]); - } - } - - return true; -} - -std::vector findLostConstraints(const Channel &channel, const std::vector &constraints) -{ - // collect all the vars that are used by the model - std::set vars; - for (const auto &sample : channel.samples) { - for (const auto &nf : sample.normfactors) { - vars.insert(nf.param); - } - for (const auto &sys : sample.normsys) { - vars.insert(sys.param); - } - - for (const auto &sys : sample.histosys) { - vars.insert(sys.param); - } - for (const auto &sys : sample.shapesys) { - for (const auto &par : sys.parameters) { - vars.insert(par); - } - } - if (sample.useBarlowBeestonLight) { - for (const auto &par : sample.staterrorParameters) { - vars.insert(par); - } - } - } - - // check if there is any constraint present that is unrelated to these vars - std::vector lostConstraints; - for (auto *pdf : constraints) { - bool related = false; - for (const auto *var : vars) { - if (pdf->dependsOn(*var)) { - related = true; + }; + auto filterGammas = [&](HS3::GammaConstraint &constraint) { + auto *pdfs = std::get_if(&constraint); + if (!pdfs) { + return; + } + bool any = false; + for (auto &pdf : pdfs->pdfs) { + if (pdf && !isNamed(pdf->name)) { + pdf.reset(); } + any = any || pdf; } - if (!related) { - lostConstraints.push_back(pdf); - } - } - // return the constraints that would be "lost" when exporting the model - return lostConstraints; -} - -// A normsys or histosys modifier without a recognized constraint is exported as unconstrained. That is only correct if -// its parameter is a variable, and none of the other constraint terms of the channel depends on it. -void checkUnconstrainedModifiers(const Channel &channel, const std::vector &constraints) -{ - auto check = [&](auto const &modifiers) { - for (auto const &sys : modifiers) { - if (sys.constraint || sys.param->isConstant()) { - continue; - } - if (!dynamic_cast(sys.param)) { - RooJSONFactoryWSTool::error("cannot find constraint for " + std::string(sys.param->GetName()) + - ": functions are not supported as parameters of this modifier"); - } - for (RooAbsPdf const *pdf : constraints) { - if (pdf->dependsOn(*sys.param)) { - RooJSONFactoryWSTool::error("cannot find constraint for " + std::string(sys.param->GetName()) + - ": the constraint term '" + pdf->GetName() + "' is not supported"); - } - } + // Nothing is written if all parameters are unconstrained. + if (!any) { + constraint = HS3::Unconstrained{}; } }; - for (const auto &sample : channel.samples) { - check(sample.normsys); - check(sample.histosys); + for (HS3::Sample &sample : channel.dist.samples) { + for (HS3::Modifier &modifier : sample.modifiers) { + std::visit(HS3::overloaded{ + [&](HS3::NormFactor &m) { filter(m.constraint); }, + [&](HS3::NormSys &m) { filter(m.constraint); }, [&](HS3::HistoSys &m) { filter(m.constraint); }, + [&](HS3::ShapeSys &m) { filterGammas(m.constraint); }, + [&](HS3::StatError &m) { filterGammas(m.constraint); }, [](HS3::CustomModifier &) {}}, + modifier); + } } + return named; } bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname, const RooRealSumPdf *sumpdf, - std::vector constraints, JSONNode &elem) + std::vector constraints) { // some preliminary checks if (!sumpdf) { @@ -2187,72 +1358,43 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname } } - auto channel = readChannel(tool, pdfname, sumpdf); + ExportedChannel channel = readChannel(tool, pdfname, sumpdf); + HS3::HistFactoryDist &dist = channel.dist; // sanity checks - if (channel.samples.size() == 0) + if (dist.samples.empty()) return false; - for (auto &sample : channel.samples) { - if (sample.hist.empty()) { + for (auto &sample : dist.samples) { + if (sample.contents.empty()) { return false; } } - canonicalizeModifiers(channel); - checkUnconstrainedModifiers(channel, constraints); - - // stat error handling + HS3::Diagnostics diagnostics; + HS3::mergeDuplicateModifiers(dist, channel.name, diagnostics); + HS3::report(diagnostics); configureStatError(channel); + HS3::factorOutDefaultInterpolation(dist); - auto lostConstraints = findLostConstraints(channel, constraints); - // Export all the lost constraints - for (const auto *constraint : lostConstraints) { - RooJSONFactoryWSTool::warning( - "losing constraint term '" + std::string(constraint->GetName()) + - "', implicit constraints are not supported by HS3 yet! The term will appear in the HS3 file, but will not be " - "picked up when creating a likelihood from it! You will have to add it manually as an external constraint."); - tool->queueExport(*constraint); - } - - // Export all the regular modifiers - auto queueConstraints = [&](auto const &modifiers) { - for (auto &modifier : modifiers) { - if (modifier.constraint) { - tool->queueExport(*modifier.constraint); - } - } - }; - for (const auto &sample : channel.samples) { - queueConstraints(sample.normfactors); - queueConstraints(sample.normsys); - queueConstraints(sample.histosys); - for (auto &modifier : sample.shapesys) { - for (auto *constraint : modifier.constraintPdfs) { - if (constraint) { - tool->queueExport(*constraint); - } - } - } - for (auto *constraint : sample.staterrorConstraints) { - if (constraint) { - tool->queueExport(*constraint); - } - } + for (auto *obs : static_range_cast(*channel.observables)) { + dist.axes.push_back(HS3::toBinnedAxis(*obs)); } - // Export all the custom modifiers - for (const auto &sample : channel.samples) { - for (auto &modifier : sample.otherElements) { - tool->queueExport(*modifier.function); - } - for (auto &modifier : sample.tmpElements) { - tool->queueExportTemporary(modifier.function); - } + std::set namedConstraints = nameChannelConstraints(channel, constraints); + for (RooAbsPdf const *pdf : namedConstraints) { + tool->queueExport(*pdf); + } + + for (RooAbsReal *modifier : channel.customModifiers) { + tool->queueExport(*modifier); + } + for (auto &modifier : channel.temporaries) { + tool->queueExportTemporary(modifier.release()); } // Export all model parameters RooArgSet parameters; - sumpdf->getParameters(channel.varSet, parameters); + sumpdf->getParameters(channel.observables, parameters); for (RooAbsArg *param : parameters) { // This should exclude the global observables. The legacy inverse bin widths are not written at all. if (!startsWith(std::string{param->GetName()}, "nom_") && !channel.legacyBinWidthFactors.count(param)) { @@ -2260,13 +1402,23 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname } } - return exportChannel(tool, channel, elem); + // All other constraint terms of the channel are written separately. + std::vector channelConstraintNames; + for (RooAbsPdf const *pdf : constraints) { + if (!namedConstraints.count(pdf)) { + tool->queueExport(*pdf); + channelConstraintNames.push_back({pdf->GetName()}); + } + } + + HS3::setExportedObject(*tool, std::move(dist), std::move(channelConstraintNames)); + return true; } class HistFactoryStreamer_ProdPdf : public RooFit::JSONIO::Exporter { public: bool autoExportDependants() const override { return false; } - bool tryExport(RooJSONFactoryWSTool *tool, const RooProdPdf *prodpdf, JSONNode &elem) const + bool tryExport(RooJSONFactoryWSTool *tool, const RooProdPdf *prodpdf) const { std::vector constraints; RooRealSumPdf *sumpdf = nullptr; @@ -2284,36 +1436,35 @@ class HistFactoryStreamer_ProdPdf : public RooFit::JSONIO::Exporter { if (!sumpdf) return false; - bool ok = tryExportHistFactory(tool, prodpdf->GetName(), sumpdf, constraints, elem); - return ok; + return tryExportHistFactory(tool, prodpdf->GetName(), sumpdf, constraints); } std::string const &key() const override { static const std::string keystring = "histfactory_dist"; return keystring; } - bool exportObject(RooJSONFactoryWSTool *tool, const RooAbsArg *p, JSONNode &elem) const override + bool exportObject(RooJSONFactoryWSTool *tool, const RooAbsArg *p, JSONNode &) const override { - return tryExport(tool, static_cast(p), elem); + return tryExport(tool, static_cast(p)); } }; class HistFactoryStreamer_SumPdf : public RooFit::JSONIO::Exporter { public: bool autoExportDependants() const override { return false; } - bool tryExport(RooJSONFactoryWSTool *tool, const RooRealSumPdf *sumpdf, JSONNode &elem) const + bool tryExport(RooJSONFactoryWSTool *tool, const RooRealSumPdf *sumpdf) const { std::vector constraints; - return tryExportHistFactory(tool, sumpdf->GetName(), sumpdf, constraints, elem); + return tryExportHistFactory(tool, sumpdf->GetName(), sumpdf, constraints); } std::string const &key() const override { static const std::string keystring = "histfactory_dist"; return keystring; } - bool exportObject(RooJSONFactoryWSTool *tool, const RooAbsArg *p, JSONNode &elem) const override + bool exportObject(RooJSONFactoryWSTool *tool, const RooAbsArg *p, JSONNode &) const override { - return tryExport(tool, static_cast(p), elem); + return tryExport(tool, static_cast(p)); } }; @@ -2330,3 +1481,139 @@ STATIC_EXECUTE([]() { }); } // namespace + +void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string const &name, HistFactoryDist const &dist, + std::vector const *channelConstraints) +{ + checkStatErrorConstraints(dist, name); + + RooArgSet observables = createObservables(dist.axes); + + std::vector> data; + data.reserve(dist.samples.size()); + std::vector sumW; + std::vector sumW2; + std::vector gammaParnames; + for (auto const &sample : dist.samples) { + auto dh = createDataHist(name + "_" + sample.name + "_dataHist", observables, sample.contents, + sample.errors ? &*sample.errors : nullptr); + const std::size_t nbins = dh->numEntries(); + + auto found = std::find_if(sample.modifiers.begin(), sample.modifiers.end(), + [](auto const &m) { return std::holds_alternative(m); }); + if (found != sample.modifiers.end()) { + if (sumW.empty()) { + sumW.resize(nbins); + sumW2.resize(nbins); + } + // A sample without "errors" contributes no statistical uncertainty, unless Poisson uncertainties are implied. + const bool hasErrors = sample.errors || dist.statError.impliedPoissonErrors; + for (std::size_t i = 0; i < nbins; ++i) { + sumW[i] += dh->weight(i); + sumW2[i] += hasErrors ? dh->weightSquared(i) : 0.; + } + if (gammaParnames.empty()) { + gammaParnames = std::get(*found).parameters; + } + } + data.emplace_back(std::move(dh)); + } + + std::string channelName = name; + erasePrefix(channelName, "model_"); + + std::size_t nBins = 1; + for (auto const &axis : dist.axes) { + nBins *= std::visit(overloaded{[](RegularBinning const &b) { return static_cast(b.nbins); }, + [](EdgesBinning const &b) { return b.edges.size() - 1; }}, + axis.binning); + } + + RooAbsArg *mcStatObject = nullptr; + RooArgSet constraints; + if (!sumW.empty()) { + std::vector errs(sumW.size()); + for (std::size_t i = 0; i < sumW.size(); ++i) { + if (sumW[i] == 0.) { + errs[i] = 0.; + continue; + } + errs[i] = std::sqrt(sumW2[i]) / sumW[i]; + // avoid negative sigma. This NP will be set constant anyway later + errs[i] = std::max(errs[i], 0.); + } + + mcStatObject = &createPHF(tool, "mc_stat_" + channelName, "stat_" + channelName, gammaParnames, errs, + statErrorConstraint(dist), constraints, observables, defaultStatErrorGammaMax, + dist.statError.relThreshold, nBins); + } + + ChannelContext ctx{tool, dist, observables, name, channelName, nBins, mcStatObject, constraints}; + + RooArgList funcs; + RooArgList coefs; + for (std::size_t i = 0; i < dist.samples.size(); ++i) { + HS3::Sample const &sample = dist.samples[i]; + buildSample(ctx, sample, *data[i]); + funcs.add(*tool.request(name + "_" + sample.name + "_shapes", name)); + coefs.add(*tool.request(name + "_" + sample.name + "_scaleFactors", name)); + } + + if (channelConstraints) { + for (auto const &ref : *channelConstraints) { + RooAbsPdf *pdf = findNamedConstraint(tool, ref.name, name); + if (!pdf) { + RooJSONFactoryWSTool::error("unable to find the constraint '" + ref.name + "' of '" + name + "'"); + } + constraints.add(*pdf); + } + } + + // A histfactory_dist is a product of Poisson distributions of the bin counts. RooFit's binned likelihood + // computes exactly that (it differs from the event-wise likelihood only by a constant), and binned generation + // generates bin counts, so both are enabled, as HistFactory does by default. + auto enableBinned = [](RooRealSumPdf &sum) { + sum.setAttribute("BinnedLikelihood"); + sum.setAttribute("GenerateBinned"); + }; + if (constraints.empty()) { + enableBinned(tool.wsEmplace(name, funcs, coefs, true)); + } else { + std::string sumName = name + "_model"; + erasePrefix(sumName, "model_"); + auto &sum = tool.wsEmplace(sumName, funcs, coefs, true); + sum.SetTitle(name.c_str()); + enableBinned(sum); + tool.wsEmplace(name, constraints, RooFit::Conditional(sum, observables)); + } +} + +bool RooFit::HS3::isBuiltinHistFactoryImporter(RooFit::JSONIO::Importer const &importer) +{ + return dynamic_cast(&importer) != nullptr; +} + +// The upgrade rewrites the JSON, so it has to work on a copy. +void RooFit::HS3::importHistFactoryNode(RooJSONFactoryWSTool &tool, JSONNode const &p, + std::vector const *channelConstraints) +{ + std::string name = RooJSONFactoryWSTool::name(p); + if (!p.has_child("samples")) { + RooJSONFactoryWSTool::error("no samples in '" + name + "', skipping."); + } + std::stringstream ss; + p.writeJSON(ss); + std::unique_ptr tree = RooFit::Detail::JSONTree::create(ss.str()); + + HS3::Version version; + HS3::parseVersion(tool.hs3VersionInput(), version); + + HS3::Diagnostics diagnostics; + HS3::upgradeHistFactoryDist(tree->rootnode(), version, diagnostics); + HS3::HistFactoryDist dist = HS3::parseHistFactoryDist(tree->rootnode(), diagnostics); + if (!HS3::hasErrors(diagnostics)) { + HS3::validateHistFactoryDist(dist, name, diagnostics); + } + HS3::report(diagnostics); + HS3::buildHistFactoryDist(tool, name, dist, channelConstraints); +} diff --git a/roofit/hs3/src/JSONFactories_RooFitCore.cxx b/roofit/hs3/src/JSONFactories_RooFitCore.cxx index 16ae38d8569d5..baf0ac79acc38 100644 --- a/roofit/hs3/src/JSONFactories_RooFitCore.cxx +++ b/roofit/hs3/src/JSONFactories_RooFitCore.cxx @@ -285,8 +285,8 @@ readFormulaAxisBinning(const JSONNode &axis, const std::string &axisName, const const bool hasMax = axis.has_child("max"); const bool hasNBins = axis.has_child("nbins"); - if (hasEdges && (hasMin || hasMax || hasNBins)) { - RooJSONFactoryWSTool::error(context + " must use either \"edges\" or \"min\"/\"max\"/\"nbins\""); + if (hasEdges && hasNBins) { + RooJSONFactoryWSTool::error(context + " must use either \"edges\" or \"nbins\""); } if (hasEdges) { @@ -313,6 +313,12 @@ readFormulaAxisBinning(const JSONNode &axis, const std::string &axisName, const if (edges.size() < 2) { RooJSONFactoryWSTool::error("\"edges\" in " + context + " must contain at least two values"); } + // The HS3 standard requires "min" and "max" also for irregular binnings, which older versions of RooFit didn't + // write. + if ((hasMin && (!axis["min"].is_number() || axis["min"].val_double() != edges.front())) || + (hasMax && (!axis["max"].is_number() || axis["max"].val_double() != edges.back()))) { + RooJSONFactoryWSTool::error("\"min\" and \"max\" in " + context + " must agree with the \"edges\""); + } return std::make_unique(static_cast(edges.size() - 1), edges.data()); } @@ -548,7 +554,8 @@ bool importDecay(RooJSONFactoryWSTool *tool, const JSONNode &p) std::string name(RooJSONFactoryWSTool::name(p)); RooRealVar *t = tool->requestArg(p, "t"); RooAbsReal *tau = tool->requestArg(p, "tau"); - RooResolutionModel *model = dynamic_cast(tool->requestArg(p, "resolutionModel")); + // HS3 lists some resolution models as functions. + RooResolutionModel *model = dynamic_cast(tool->requestArg(p, "resolutionModel")); if (!model) { RooJSONFactoryWSTool::error("resolutionModel of '" + name + "' is not a RooResolutionModel"); } @@ -1611,10 +1618,17 @@ STATIC_EXECUTE([]() { RooFit::JSONIO::registerImporter("ARGUS_dist", std::make_unique("ARGUS_dist", "argus_dist")); RooFit::JSONIO::registerImporter("binsampling", std::make_unique("binsampling", "binsampling_dist")); + RooFit::JSONIO::registerImporter( + "mixture_resolution_model", std::make_unique("mixture_resolution_model", "mixture_model")); + RooFit::JSONIO::registerImporter("delta_resolution_model", std::make_unique( + "delta_resolution_model", "truth_model_function")); + RooFit::JSONIO::registerImporter("gauss_resolution_model", std::make_unique( + "gauss_resolution_model", "gauss_model_function")); registerImporter("product", false); registerImporter("product_dist", false); registerImporter("sum", false); registerImporter("mixture_dist", false); + registerImporter("mixture_model", false); registerImporter("mixture_resolution_model", false); registerImporter("binsampling_dist", false); registerImporter("binsampling", false); @@ -1632,7 +1646,9 @@ STATIC_EXECUTE([]() { registerImporter("multivariate_normal_dist", false); registerImporter("poisson_dist", false); registerImporter("decay_dist", false); + registerImporter("truth_model_function", false); registerImporter("delta_resolution_model", false); + registerImporter("gauss_model_function", false); registerImporter("gauss_resolution_model", false); registerImporter>("polynomial_dist", false); registerImporter>("polynomial", false); @@ -1647,7 +1663,7 @@ STATIC_EXECUTE([]() { registerExporter(RooWrapperPdf::Class(), "density_function_dist"); registerExporter>(RooAddPdf::Class(), "mixture_dist", false); - registerExporter>(RooAddModel::Class(), "mixture_resolution_model", false); + registerExporter>(RooAddModel::Class(), "mixture_model", false); registerExporter(RooBinSamplingPdf::Class(), "binsampling_dist", false); registerExporter(RooCrystalBall::Class(), "crystalball_doublesided_dist", false); registerExporter(RooBinWidthFunction::Class(), "binvolume", false); @@ -1661,8 +1677,8 @@ STATIC_EXECUTE([]() { registerExporter(RooMultiVarGaussian::Class(), "multivariate_normal_dist", false); registerExporter(RooPoisson::Class(), "poisson_dist", false); registerExporter(RooDecay::Class(), "decay_dist", false, /*autoExportDependants=*/false); - registerExporter(RooTruthModel::Class(), "delta_resolution_model", false); - registerExporter(RooGaussModel::Class(), "gauss_resolution_model", false); + registerExporter(RooTruthModel::Class(), "truth_model_function", false); + registerExporter(RooGaussModel::Class(), "gauss_model_function", false); registerExporter>(RooPolynomial::Class(), "polynomial_dist", false); registerExporter>(RooPolyVar::Class(), "polynomial", false); registerExporter>(RooRealSumFunc::Class(), "weighted_sum", false); diff --git a/roofit/hs3/src/JSONIOUtils.cxx b/roofit/hs3/src/JSONIOUtils.cxx index dd634f469444e..51dc6e597c3fb 100644 --- a/roofit/hs3/src/JSONIOUtils.cxx +++ b/roofit/hs3/src/JSONIOUtils.cxx @@ -1,14 +1,12 @@ #include "JSONIOUtils.h" #include -#include #include #include #include using RooFit::Detail::JSONNode; -using RooFit::Detail::JSONTree; bool startsWith(std::string_view str, std::string_view prefix) { @@ -28,71 +26,11 @@ std::string removeSuffix(std::string_view str, std::string_view suffix) return out; } -std::unique_ptr varJSONString(const JSONNode &treeRoot) -{ - std::string varName = treeRoot.find("name")->val(); - double val = 0; - double maxVal = 0; - double minVal = 0; - bool isConstant = false; - bool isRange = false; - - if (auto n = treeRoot.find("value")) { - val = n->val_double(); - isConstant = true; - } - - auto maxNode = treeRoot.find("max"); - auto minNode = treeRoot.find("min"); - if (maxNode && minNode) { - maxVal = maxNode->val_double(); - minVal = minNode->val_double(); - isRange = true; - } - if (!isConstant) { - val = (maxVal + minVal) / 2; - } - // Check if variable is at least a range or constant else throw error - if (!isConstant && !isRange) { - throw std::invalid_argument("Invalid Syntax: Please provide either 'value' or 'min' and 'max' or both"); - } - - std::unique_ptr jsonDict = RooFit::Detail::JSONTree::create(); - JSONNode &n = jsonDict->rootnode().set_map(); - JSONNode &_domains = n["domains"].set_seq().append_child().set_map(); - JSONNode &_parameterPoints = n["parameter_points"].set_seq().append_child().set_map(); - - _domains["name"] << "default_domain"; - _domains["type"] << "product_domain"; - JSONNode &_axes = _domains["axes"].set_seq().append_child().set_map(); - _axes["name"] << varName; - - _parameterPoints["name"] << "default_values"; - JSONNode &_parameters = _parameterPoints["parameters"].set_seq().append_child().set_map(); - _parameters["name"] << varName; - _parameters["value"] << val; - - if (isRange) { - _axes["max"] << maxVal; - _axes["min"] << minVal; - } - - if (isConstant && !isRange) { - _parameters["const"] << true; - JSONNode &_misc = n["misc"].set_map(); - JSONNode &rootInternal = _misc["ROOT_internal"].set_map(); - JSONNode &_var = rootInternal[varName].set_map(); - _var["tags"] << "Constant"; - } - - return jsonDict; -} - void writeAxisBinning(JSONNode &node, const RooAbsBinning &binning) { + node["min"] << binning.lowBound(); + node["max"] << binning.highBound(); if (binning.isUniform()) { - node["min"] << binning.lowBound(); - node["max"] << binning.highBound(); node["nbins"] << binning.numBins(); return; } @@ -104,16 +42,6 @@ void writeAxisBinning(JSONNode &node, const RooAbsBinning &binning) } } -// Read a bound of an axis. Infinite bounds are written as null. -double readBound(JSONNode const &node, const char *key, double defaultValue) -{ - if (!node.has_child(key)) { - return defaultValue; - } - auto const &bound = node[key]; - return bound.is_null() ? defaultValue : bound.val_double(); -} - // Parse the leading "MAJOR.MINOR" of an HS3 version string like "0.2", "0.2.9" or "0.3". bool parseHS3Version(std::string const &version, int &major, int &minor) { @@ -148,22 +76,3 @@ bool isHS3VersionBefore(std::string const &version, int major, int minor) } return std::make_pair(versionMajor, versionMinor) < std::make_pair(major, minor); } - -// Read the bin edges of an axis, which must be at least two strictly ascending numbers. -std::vector readEdges(const JSONNode &node, std::string const &context) -{ - if (!node.is_seq()) { - RooJSONFactoryWSTool::error("\"edges\" of " + context + " must be a list"); - } - std::vector edges; - for (auto const &edge : node.children()) { - edges.push_back(edge.val_double()); - if (edges.size() > 1 && !(edges.back() > edges[edges.size() - 2])) { - RooJSONFactoryWSTool::error("\"edges\" of " + context + " must be strictly ascending"); - } - } - if (edges.size() < 2) { - RooJSONFactoryWSTool::error("\"edges\" of " + context + " must contain at least two values"); - } - return edges; -} diff --git a/roofit/hs3/src/JSONIOUtils.h b/roofit/hs3/src/JSONIOUtils.h index 2e8dd007c30ae..ac2f807345680 100644 --- a/roofit/hs3/src/JSONIOUtils.h +++ b/roofit/hs3/src/JSONIOUtils.h @@ -10,15 +10,12 @@ class RooAbsBinning; bool startsWith(std::string_view str, std::string_view prefix); bool endsWith(std::string_view str, std::string_view suffix); std::string removeSuffix(std::string_view str, std::string_view suffix); -std::unique_ptr varJSONString(const RooFit::Detail::JSONNode &treeRoot); void writeAxisBinning(RooFit::Detail::JSONNode &node, const RooAbsBinning &binning); -double readBound(RooFit::Detail::JSONNode const &node, const char *key, double defaultValue); // The key of a fixed domain axis (HS3 #131): the parameter only takes this value, and RooFit reads it as a // RooConstVar. inline constexpr const char *fixedValueKey = "value"; bool parseHS3Version(std::string const &version, int &major, int &minor); bool isHS3VersionBefore(std::string const &version, int major, int minor); -std::vector readEdges(const RooFit::Detail::JSONNode &node, std::string const &context); #endif diff --git a/roofit/hs3/src/JSONInterface.cxx b/roofit/hs3/src/JSONInterface.cxx index f4a2dbf504eab..3ac9637b25e5c 100644 --- a/roofit/hs3/src/JSONInterface.cxx +++ b/roofit/hs3/src/JSONInterface.cxx @@ -12,8 +12,11 @@ #include +#include "JSONReadRecorder.h" + #include +#include #include #include #include @@ -88,10 +91,19 @@ class JSONNode::Storage { return list; } + // Tells the recorders that the component `key` of `value` is read, where nullptr means all components. + void recordRead(void const *value, std::string const *key) + { + for (JSONReadRecorder *recorder : recorders) { + recorder->record(value, key); + } + } + nlohmann::json root; JSONNode rootNode; std::vector> nodes; std::deque> childLists; + std::vector recorders; }; // JSONNode @@ -103,6 +115,7 @@ JSONNode::JSONNode(Storage &storage, void *value, std::string key) void JSONNode::writeJSON(std::ostream &os) const { + _storage->recordRead(_value, nullptr); os << json(_value); } @@ -143,6 +156,7 @@ JSONNode &JSONNode::operator[](std::string const &k) const JSONNode &JSONNode::operator[](std::string const &k) const { + _storage->recordRead(_value, &k); return _storage->makeNode(json(_value)[k], k); } @@ -210,6 +224,11 @@ void JSONNode::clear() json(_value).clear(); } +void JSONNode::erase(std::string const &key) +{ + json(_value).erase(key); +} + std::string JSONNode::key() const { return _key; @@ -270,6 +289,7 @@ bool JSONNode::has_val() const bool JSONNode::has_child(std::string const &c) const { + _storage->recordRead(_value, &c); return json(_value).find(c) != json(_value).end(); } @@ -291,6 +311,7 @@ JSONNode &JSONNode::child(size_t pos) const JSONNode &JSONNode::child(size_t pos) const { + _storage->recordRead(_value, nullptr); return _storage->makeNode(json(_value).at(pos), ""); } @@ -302,6 +323,7 @@ JSONNode::children_view JSONNode::children() JSONNode::const_children_view JSONNode::children() const { + _storage->recordRead(_value, nullptr); std::vector const &list = _storage->makeChildren(json(_value)); return {const_child_iterator{list.data()}, const_child_iterator{list.data() + list.size()}}; } @@ -339,5 +361,45 @@ std::unique_ptr JSONTree::create(std::string const &str) return JSONTree::create(ss); } +// JSONReadRecorder + +JSONReadRecorder::JSONReadRecorder(JSONNode const &node) : _node{node} +{ + _node._storage->recorders.push_back(this); +} + +JSONReadRecorder::~JSONReadRecorder() +{ + auto &recorders = _node._storage->recorders; + recorders.erase(std::find(recorders.begin(), recorders.end(), this)); +} + +void JSONReadRecorder::record(void const *value, std::string const *key) +{ + if (value != _node._value) { + return; + } + if (key) { + _read.insert(*key); + } else { + _readAll = true; + } +} + +std::vector JSONReadRecorder::unreadComponents() const +{ + std::vector out; + nlohmann::json const &value = json(_node._value); + if (!value.is_object() || _readAll) { + return out; + } + for (auto const &item : value.items()) { + if (_read.count(item.key()) == 0) { + out.push_back(item.key()); + } + } + return out; +} + } // namespace Detail } // namespace RooFit diff --git a/roofit/hs3/src/JSONReadRecorder.h b/roofit/hs3/src/JSONReadRecorder.h new file mode 100644 index 0000000000000..77ac8b45cedca --- /dev/null +++ b/roofit/hs3/src/JSONReadRecorder.h @@ -0,0 +1,50 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef RooFit_Detail_JSONReadRecorder_h +#define RooFit_Detail_JSONReadRecorder_h + +#include + +#include +#include +#include + +namespace RooFit { +namespace Detail { + +/// Records which components of a JSON map are read while it is alive, through any node of the tree. Like that, the +/// components that no importer understands can be reported without any change to the importers. Iterating over the +/// components or writing the map counts as reading all of them. +class JSONReadRecorder { +public: + explicit JSONReadRecorder(JSONNode const &node); + ~JSONReadRecorder(); + JSONReadRecorder(JSONReadRecorder const &) = delete; + JSONReadRecorder &operator=(JSONReadRecorder const &) = delete; + + /// The components of the map that were not read. + std::vector unreadComponents() const; + + /// Called by the tree for every read of the component `key` of `value`, where nullptr means all components. + void record(void const *value, std::string const *key); + +private: + JSONNode const &_node; + std::set _read; + bool _readAll = false; +}; + +} // namespace Detail +} // namespace RooFit + +#endif diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index e951197158419..1a3bcd8514465 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -29,13 +29,16 @@ #include #include "JSONIOUtils.h" -#include "Domains.h" +#include "HS3Builder.h" +#include "HS3Schema.h" +#include "JSONReadRecorder.h" #include #include #include #include +#include #include #include @@ -108,21 +111,240 @@ tool.writedoc("hs3.tex") constexpr auto hs3VersionTag = "0.3"; +using RooFit::HS3Document; using RooFit::Detail::JSONNode; using RooFit::Detail::JSONTree; +namespace HS3 = RooFit::HS3; + +/// What the tool needs to know about the document that it is importing. +struct RooFit::HS3::ToolImportState { + explicit ToolImportState(HS3Document const &doc) : document{doc} + { + // Without these indices, every cross-reference resolved during import (e.g. dependencies of a + // PiecewiseInterpolation, or factory-expression arguments) would need a linear scan over all sibling objects, + // which becomes O(N^2) on workspaces with thousands of entries. + for (auto const &function : doc.functions) { + functions.emplace(function.name, &function); + } + for (auto const &distribution : doc.distributions) { + distributions.emplace(distribution.name, &distribution); + } + for (auto const &point : doc.parameterPoints) { + if (point.name == "default_values") { + defaultValues = &point; + for (auto const &p : point.parameters) { + defaultValuesByName.emplace(p.name, &p); + } + break; + } + } + } + + HS3Document const &document; + std::unordered_map functions; + std::unordered_map distributions; + HS3::ParameterPoint const *defaultValues = nullptr; + std::unordered_map defaultValuesByName; + HS3::DomainRanges ranges{document.domains}; +}; + +/// The document that the tool is exporting. +struct RooFit::HS3::ToolExportState { + HS3Document doc; + JSONNode *root = nullptr; + /// The ranges and binnings of the exported variables, by domain and variable name. + std::map> domains; + /// Set by the exporter of an object that is interpreted by the document. + std::optional typedObject; + std::vector channelConstraints; +}; + +void RooFit::HS3::setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist dist, + std::vector channelConstraints) +{ + tool._export->typedObject = std::move(dist); + tool._export->channelConstraints = std::move(channelConstraints); +} + namespace { -std::vector valsToStringVec(JSONNode const &node) +HS3::Binning toBinning(RooAbsBinning const &binning) { - std::vector out; - out.reserve(node.num_children()); - for (JSONNode const &elem : node.children()) { - out.push_back(elem.val()); + if (binning.isUniform()) { + return HS3::RegularBinning{binning.numBins()}; + } + HS3::EdgesBinning edges; + edges.edges.push_back(binning.binLow(0)); + for (int i = 0; i < binning.numBins(); ++i) { + edges.edges.push_back(binning.binHigh(i)); + } + return edges; +} + +// The axis of a domain for a range and binning. Infinite bounds are omitted, and only a bounded range has a binning. +HS3::DomainAxis toDomainAxis(std::string const &name, RooAbsBinning const &binning) +{ + HS3::DomainAxis axis; + axis.name = name; + if (!RooNumber::isInfinite(binning.lowBound())) { + axis.min = binning.lowBound(); + } + if (!RooNumber::isInfinite(binning.highBound())) { + axis.max = binning.highBound(); + } + if (axis.min && axis.max && binning.numBins() > 0) { + axis.binning = toBinning(binning); + } + return axis; +} + +// Record the range and binning of a variable in the "default_domain", and its named ranges in the domains of their +// names. +void recordDomains(std::map> &domains, RooRealVar const &var) +{ + domains[HS3::defaultDomainName][var.GetName()] = toDomainAxis(var.GetName(), var.getBinning()); + for (auto const &name : var.getBinningNames()) { + if (!name.empty()) { + domains[name][var.GetName()] = toDomainAxis(var.GetName(), var.getBinning(name.c_str())); + } + } +} + +void appendDomain(std::vector &domains, std::string const &name, + std::map axes) +{ + HS3::Domain &domain = domains.emplace_back(); + domain.name = name; + for (auto &item : axes) { + domain.axes.emplace_back(std::move(item.second)); + } +} + +// Set the binning of a variable, or the binning of one of its named ranges. +void applyBinning(RooRealVar &var, HS3::Binning const &binning, const char *name = nullptr) +{ + if (auto const *edges = std::get_if(&binning)) { + if (edges->edges.empty()) { + return; + } + RooBinning bins(edges->edges.front(), edges->edges.back()); + for (double edge : edges->edges) { + bins.addBoundary(edge); + } + var.setBinning(bins, name); + } else if (int nbins = std::get(binning).nbins) { + var.setBins(nbins, name); + } +} + +void applyRange(RooRealVar &var, HS3::DomainAxis const &axis) +{ + if (axis.min) { + if (RooNumber::isInfinite(*axis.min)) { + var.removeMin(); + } else { + var.setMin(*axis.min); + } + } + if (axis.max) { + if (RooNumber::isInfinite(*axis.max)) { + var.removeMax(); + } else { + var.setMax(*axis.max); + } + } + if (axis.binning) { + applyBinning(var, *axis.binning); } - return out; } +// Create the variables of the domains that don't exist yet, and the domains other than the "default_domain" as named +// ranges. +void createVariables(RooWorkspace &ws, std::vector const &domains, HS3::DomainRanges const &ranges) +{ + auto forEachAxis = [&](auto &&fn) { + for (HS3::Domain const &domain : domains) { + for (auto const &axis : domain.axes) { + if (auto const *continuous = std::get_if(&axis)) { + fn(domain.name, *continuous); + } + } + } + }; + const double inf = RooNumber::infinity(); + // Parameters with a single value are RooConstVars, whichever domain gives the value. + forEachAxis([&](std::string const &, HS3::DomainAxis const &axis) { + if (axis.fixedValue && !ws.arg(axis.name)) { + ws.import(RooConstVar{axis.name.c_str(), axis.name.c_str(), *axis.fixedValue}, RooFit::Silence()); + } + }); + forEachAxis([&](std::string const &, HS3::DomainAxis const &axis) { + if (!axis.fixedValue && ranges.find(axis.name) == &axis && !ws.arg(axis.name)) { + RooRealVar var{axis.name.c_str(), axis.name.c_str(), axis.min.value_or(-inf), axis.max.value_or(inf)}; + if (axis.binning) { + applyBinning(var, *axis.binning); + } + ws.import(var, RooFit::Silence()); + } + }); + forEachAxis([&](std::string const &domain, HS3::DomainAxis const &axis) { + RooRealVar *var = ws.var(axis.name); + if (var && domain != HS3::defaultDomainName) { + var->setRange(domain.c_str(), axis.min.value_or(-inf), axis.max.value_or(inf)); + if (axis.binning) { + applyBinning(*var, *axis.binning, domain.c_str()); + } + } + }); +} + +} // namespace + +namespace { + +// Keeps track of the objects that are being imported. Objects that depend on each other would otherwise recurse until +// the stack overflows. +class ImportGuard { +public: + ImportGuard(std::vector &stack, std::string const &name) : _stack{stack} + { + if (std::find(_stack.begin(), _stack.end(), name) != _stack.end()) { + std::string chain; + for (std::string const &ancestor : _stack) { + chain += ancestor + " -> "; + } + RooJSONFactoryWSTool::error("circular dependency in the document: " + chain + name); + } + _stack.push_back(name); + } + ~ImportGuard() { _stack.pop_back(); } + +private: + std::vector &_stack; +}; + +} // namespace + +void RooFit::HS3::report(Diagnostics const &diagnostics) +{ + std::string errors; + for (Diagnostic const &diagnostic : diagnostics) { + const std::string text = + diagnostic.path.empty() ? diagnostic.message : diagnostic.path + ": " + diagnostic.message; + if (diagnostic.severity == Diagnostic::Severity::Error) { + errors += (errors.empty() ? "" : "\n") + text; + } else { + RooJSONFactoryWSTool::warning("RooFitHS3: " + text); + } + } + if (!errors.empty()) { + RooJSONFactoryWSTool::error(errors); + } +} + +namespace { + // True if the number of components in `data` matches the number of categories in `pdf`. bool matches(const RooJSONFactoryWSTool::CombinedData &data, const RooSimultaneous *pdf) { @@ -140,37 +362,24 @@ bool isNumber(const std::string &str) return (stream >> value) && stream.eof(); } -// Configure `v` (value, error, binning, constness) from the JSON node `p`. -void configureVariable(RooFit::JSONIO::Detail::Domains &domains, const JSONNode &p, RooRealVar &v) +// Configure `v` (value, error, binning, constness) from the parameter `p`. +void configureVariable(HS3::DomainRanges const &ranges, HS3::ParameterValue const &p, RooRealVar &v) { - if (!p.has_child("name")) { - RooJSONFactoryWSTool::error("cannot instantiate variable without \"name\"!"); + if (p.value) { + if (auto const *value = std::get_if(&*p.value)) { + v.setVal(*value); + } } - if (auto n = p.find("value")) - v.setVal(n->val_double()); - domains.writeVariable(v); - if (auto n = p.find("nbins")) - v.setBins(n->val_int()); - if (auto n = p.find("relErr")) - v.setError(v.getVal() * n->val_double()); - if (auto n = p.find("err")) - v.setError(n->val_double()); - if (auto n = p.find("const")) { - v.setConstant(n->val_bool()); - } else { - v.setConstant(false); + if (HS3::DomainAxis const *range = ranges.find(v.GetName())) { + applyRange(v, *range); } -} - -JSONNode const *getVariablesNode(JSONNode const &rootNode) -{ - auto paramPointsNode = rootNode.find("parameter_points"); - if (!paramPointsNode) - return nullptr; - auto out = RooJSONFactoryWSTool::findNamedChild(*paramPointsNode, "default_values"); - if (out == nullptr) - return nullptr; - return &((*out)["parameters"]); + if (p.nbins) + v.setBins(*p.nbins); + if (p.relError) + v.setError(v.getVal() * *p.relError); + if (p.error) + v.setError(*p.error); + v.setConstant(p.isConst); } std::string genPrefix(const JSONNode &p, bool trailing_underscore) @@ -205,18 +414,14 @@ void genIndicesHelper(std::vector> &combinations, std::vectorchildren()) { - arg->setStringAttribute(attr.key().c_str(), attr.val().c_str()); - } + for (auto const &attr : attributes.dict) { + arg->setStringAttribute(attr.first.c_str(), attr.second.c_str()); } - if (auto seq = node.find("tags")) { - for (const auto &attr : seq->children()) { - arg->setAttribute(attr.val().c_str()); - } + for (auto const &tag : attributes.tags) { + arg->setAttribute(tag.c_str()); } } @@ -268,7 +473,7 @@ void collectParameterStepWidthCandidatesFromPdfs(std::vector const } void exportParameterStepWidths(RooWorkspace const &workspace, std::vector const &pdfs, - std::vector const &data, JSONNode &rootnode) + std::vector const &data, std::vector &out) { RooArgSet candidates; RooArgSet excluded; @@ -278,7 +483,6 @@ void exportParameterStepWidths(RooWorkspace const &workspace, std::vectorGetName()); - stepWidthNode["step_width"] << var->getError(); + out.push_back({var->GetName(), var->getError()}); } } -void importParameterStepWidths(RooWorkspace &workspace, JSONNode const &rootnode) +void importParameterStepWidths(RooWorkspace &workspace, HS3Document const &doc) { - auto const *parameterStepWidthsNode = rootnode.find("misc", "minimization", "parameter_stepwidths"); - if (!parameterStepWidthsNode) { - return; - } - if (!parameterStepWidthsNode->is_seq()) { - RooJSONFactoryWSTool::warning("RooFitHS3: misc.minimization.parameter_stepwidths is not a sequence, skipping."); - return; - } - - for (JSONNode const &stepWidthNode : parameterStepWidthsNode->children()) { - if (!stepWidthNode.is_map() || !stepWidthNode.has_child("name") || !stepWidthNode.has_child("step_width")) { - RooJSONFactoryWSTool::warning("RooFitHS3: skipping malformed parameter_stepwidths entry."); - continue; - } - - const std::string name = RooJSONFactoryWSTool::name(stepWidthNode); - RooAbsArg *arg = workspace.arg(name); - auto *var = dynamic_cast(arg); + for (HS3::StepWidth const &stepWidth : doc.rooFit.stepWidths) { + auto *var = dynamic_cast(workspace.arg(stepWidth.name)); if (!var) { - RooJSONFactoryWSTool::warning( - "RooFitHS3: skipping parameter_stepwidths entry for unknown or non-real variable '" + name + "'."); + RooJSONFactoryWSTool::warning("RooFitHS3: skipping parameter_stepwidths entry for unknown or non-real " + "variable '" + + stepWidth.name + "'."); continue; } - - var->setError(stepWidthNode.find("step_width")->val_double()); + var->setError(stepWidth.width); } } @@ -388,159 +571,182 @@ bool isLiteralConstVar(RooAbsArg const &arg) return isRooConstVar && isNumber(arg.GetName()); } -// Export the string attributes and tags of `arg` into the ROOT-internal attributes node. -void exportAttributes(const RooAbsArg *arg, JSONNode &rootnode) +// Export the string attributes and tags of `arg` into the ROOT-internal attributes. +void exportAttributes(const RooAbsArg *arg, std::map &attributes) { // If this RooConst is a literal number, we don't need to export the attributes. - if (isLiteralConstVar(*arg)) { + if (isLiteralConstVar(*arg) || dynamic_cast(arg)) { return; } - JSONNode *node = nullptr; - - auto initializeNode = [&]() { - if (node) - return; - - node = &RooJSONFactoryWSTool::getRooFitInternal(rootnode, "attributes").set_map()[arg->GetName()].set_map(); + HS3::Attributes *out = nullptr; + auto initialize = [&]() { + if (!out) { + out = &attributes[arg->GetName()]; + } }; - if (dynamic_cast(arg)) { - return; + // export all string attributes of an object + for (const auto &it : arg->stringAttributes()) { + // Skip some RooFit internals + if (it.first == "factory_tag" || it.first == "PROD_TERM_TYPE") + continue; + initialize(); + out->dict[it.first] = it.second; } + for (auto const &attr : arg->attributes()) { + // Skip some RooFit internals + if (attr == "SnapShot_ExtRefClone" || attr == "RooRealConstant_Factory_Object") + continue; + initialize(); + out->tags.push_back(attr); + } +} - // export all string attributes of an object - if (!arg->stringAttributes().empty()) { - for (const auto &it : arg->stringAttributes()) { - // Skip some RooFit internals - if (it.first == "factory_tag" || it.first == "PROD_TERM_TYPE") - continue; - initializeNode(); - (*node)["dict"].set_map()[it.first] << it.second; - } +} // namespace + +HS3::BinnedAxis RooFit::HS3::toBinnedAxis(RooRealVar const &var) +{ + RooJSONFactoryWSTool::testValidName(var.GetName(), false); + RooAbsBinning const &binning = var.getBinning(); + HS3::BinnedAxis axis; + axis.name = var.GetName(); + axis.min = binning.lowBound(); + axis.max = binning.highBound(); + axis.binning = toBinning(binning); + return axis; +} + +HS3::NameOrNumber RooFit::HS3::toNameOrNumber(RooAbsArg const &arg) +{ + if (isLiteralConstVar(arg)) { + return static_cast(arg).getVal(); } - if (!arg->attributes().empty()) { - for (auto const &attr : arg->attributes()) { - // Skip some RooFit internals - if (attr == "SnapShot_ExtRefClone" || attr == "RooRealConstant_Factory_Object") - continue; - initializeNode(); - (*node)["tags"].set_seq().append_child() << attr; + return std::string{arg.GetName()}; +} + +namespace { + +std::unique_ptr createObservable(HS3::BinnedAxis const &axis) +{ + auto obs = std::make_unique(axis.name.c_str(), axis.name.c_str(), axis.min, axis.max); + applyBinning(*obs, axis.binning); + return obs; +} + +// Create the variables for the axes of binned data. +RooArgSet createBinnedObservables(std::vector> const &axes) +{ + RooArgSet vars; + for (auto const &axisVariant : axes) { + auto const *axis = std::get_if(&axisVariant); + if (!axis) { + RooJSONFactoryWSTool::error("categorical axes of binned data are not supported yet"); } + vars.addOwned(createObservable(*axis)); } + return vars; } -// Collect the observables named in the "axes" field of `node` from the workspace into `out`. -void getObservables(RooWorkspace const &ws, const JSONNode &node, RooAbsCollection &out) +// Make the range of an observable of unbinned data consistent with the "min" and "max" of the axis. Variables that +// didn't get their range from a domain take it from the axis. If `ranges` is null, all ranges are considered explicit. +void applyUnbinnedAxisRange(HS3::ContinuousAxis const &axis, std::string const &dataName, RooRealVar &var, + HS3::DomainRanges const *ranges) { - for (const auto &p : node["axes"].children()) { - std::string name(RooJSONFactoryWSTool::name(p)); - if (ws.var(name)) { - out.add(*ws.var(name)); - } else { - RooJSONFactoryWSTool::error("The observable \"" + name + "\" could not be found in the workspace!"); - } + if (!axis.min || !axis.max) { + return; + } + if (ranges && !ranges->contains(axis.name)) { + var.setRange(*axis.min, *axis.max); + } else if (var.getMin() != *axis.min || var.getMax() != *axis.max) { + std::stringstream ss; + ss << "RooFitHS3: the range [" << *axis.min << ", " << *axis.max << "] of the axis '" << axis.name + << "' of the dataset '" << dataName << "' differs from the range [" << var.getMin() << ", " << var.getMax() + << "] of the variable."; + RooJSONFactoryWSTool::warning(ss.str()); } } -// Make the ranges of the observables of unbinned data consistent with the "min" and "max" of the axes. Variables that -// didn't get their range from a domain take it from the axis. If `domains` is null, all ranges are considered explicit. -void applyUnbinnedAxisRanges(const JSONNode &p, RooWorkspace &workspace, RooFit::JSONIO::Detail::Domains const *domains) +std::unique_ptr createUnbinnedData(std::string const &name, HS3::UnbinnedData const &input, + RooWorkspace &workspace, HS3::DomainRanges const *ranges) { - for (const auto &axis : p["axes"].children()) { - if (!axis.has_child("min") || !axis.has_child("max")) { - continue; + RooArgList varlist; + for (auto const &axisVariant : input.axes) { + auto const *axis = std::get_if(&axisVariant); + if (!axis) { + RooJSONFactoryWSTool::error("categorical axes of unbinned data are not supported yet"); } - const std::string name = RooJSONFactoryWSTool::name(axis); - RooRealVar *var = workspace.var(name); + RooRealVar *var = workspace.var(axis->name); if (!var) { - continue; // reported by getObservables() + RooJSONFactoryWSTool::error("The observable \"" + axis->name + "\" could not be found in the workspace!"); } - const double min = readBound(axis, "min", -RooNumber::infinity()); - const double max = readBound(axis, "max", RooNumber::infinity()); - if (domains && !domains->hasVariable(name.c_str())) { - var->setRange(min, max); - } else if (var->getMin() != min || var->getMax() != max) { - std::stringstream ss; - ss << "RooFitHS3: the range [" << min << ", " << max << "] of the axis '" << name << "' of the dataset '" - << RooJSONFactoryWSTool::name(p) << "' differs from the range [" << var->getMin() << ", " << var->getMax() - << "] of the variable."; - RooJSONFactoryWSTool::warning(ss.str()); + varlist.add(*var); + applyUnbinnedAxisRange(*axis, name, *var, ranges); + } + RooArgSet vars(varlist); + auto data = std::make_unique(name, name, vars, RooFit::WeightVar()); + const std::size_t nAxes = input.axes.size(); + for (std::size_t i = 0; i < input.numEntries(); ++i) { + for (std::size_t j = 0; j < nAxes; ++j) { + static_cast(varlist.at(j))->setVal(input.entries[i * nAxes + j]); } + data->add(vars, input.weights ? (*input.weights)[i] : 1.); } + return data; } -// Create a RooAbsData (binned or unbinned) from the JSON node `p`. +// Create a RooAbsData (binned or unbinned) from its description. std::unique_ptr -loadData(const JSONNode &p, RooWorkspace &workspace, RooFit::JSONIO::Detail::Domains const *domains) +loadData(HS3::Data const &input, RooWorkspace &workspace, HS3::DomainRanges const *ranges) { - std::string name(RooJSONFactoryWSTool::name(p)); + RooJSONFactoryWSTool::testValidName(input.name, true); - RooJSONFactoryWSTool::testValidName(name, true); - - std::string const &type = p["type"].val(); - if (type == "binned") { - // binned - return RooJSONFactoryWSTool::readBinnedData(p, name, RooJSONFactoryWSTool::readAxes(p)); - } else if (type == "unbinned") { - // unbinned - RooArgList varlist; - getObservables(workspace, p, varlist); - applyUnbinnedAxisRanges(p, workspace, domains); - RooArgSet vars(varlist); - auto data = std::make_unique(name, name, vars, RooFit::WeightVar()); - auto &coords = p["entries"]; - if (!coords.is_seq()) { - RooJSONFactoryWSTool::error("key 'entries' is not a list!"); - } - std::vector weightVals; - if (p.has_child("weights")) { - auto &weights = p["weights"]; - if (coords.num_children() != weights.num_children()) { - RooJSONFactoryWSTool::error("inconsistent number of entries and weights!"); - } - for (auto const &weight : weights.children()) { - weightVals.push_back(weight.val_double()); - } + // What RooFit can't store is reported, instead of being dropped silently. + if (auto const *binned = std::get_if(&input.body)) { + if (binned->uncertainty && !binned->uncertainty->correlation.empty()) { + RooJSONFactoryWSTool::warning("RooFitHS3: the correlations of the uncertainties of the dataset '" + + input.name + "' are ignored, because RooDataHist can't store them."); } - std::size_t i = 0; - for (auto const &point : coords.children()) { - if (!point.is_seq()) { - RooJSONFactoryWSTool::error("coordinate point '" + std::to_string(i) + "' is not a list!"); - } - if (point.num_children() != varlist.size()) { - RooJSONFactoryWSTool::error("inconsistent number of entries and observables!"); - } - std::size_t j = 0; - for (auto const &pointj : point.children()) { - auto *v = static_cast(varlist.at(j)); - v->setVal(pointj.val_double()); - ++j; - } - if (weightVals.size() > 0) { - data->add(vars, weightVals[i]); - } else { - data->add(vars, 1.); - } - ++i; + return HS3::createDataHist(input.name, createBinnedObservables(binned->axes), binned->contents, + binned->uncertainty ? &binned->uncertainty->sigma : nullptr); + } + if (auto const *unbinned = std::get_if(&input.body)) { + if (unbinned->hasEntriesUncertainties) { + RooJSONFactoryWSTool::warning("RooFitHS3: the \"entries_uncertainties\" of the dataset '" + input.name + + "' are ignored, because RooDataSet can't store them."); } - return data; + return createUnbinnedData(input.name, *unbinned, workspace, ranges); } + RooJSONFactoryWSTool::error("datasets of type \"point\" like '" + input.name + "' are not supported yet"); +} + +// The version of ROOT that writes the document, with dots only such that it can be read automatically. +std::string rootVersion() +{ + std::string versionName = gROOT->GetVersion(); + std::replace(versionName.begin(), versionName.end(), '/', '.'); + return versionName; +} - RooJSONFactoryWSTool::error("RooJSONFactoryWSTool() failed to create dataset " + name); +// A distribution of the document can be a function in RooFit, or it can be skipped. +void requirePdf(RooAbsArg const *arg, std::string const &what) +{ + if (!dynamic_cast(arg)) { + RooJSONFactoryWSTool::error(what + " is " + + (arg ? std::string("a ") + arg->ClassName() + ", not a pdf" : "not imported")); + } } // Import an analysis (likelihood + domains) as one or more ModelConfig objects into the workspace. -void importAnalysis(const JSONNode &rootnode, const JSONNode &analysisNode, const JSONNode &likelihoodsNode, - const JSONNode &domainsNode, RooWorkspace &workspace, +void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWorkspace &workspace, const std::vector> &datasets) { // if this is a toplevel pdf, also create a modelConfig for it - std::string const &analysisName = RooJSONFactoryWSTool::name(analysisNode); - JSONNode const *mcAuxNode = findRooFitInternal(rootnode, "ModelConfigs", analysisName); + std::string const &analysisName = analysis.name; + auto foundInfo = doc.rooFit.modelConfigs.find(analysisName); + HS3::ModelConfigInfo const *mcInfo = foundInfo != doc.rooFit.modelConfigs.end() ? &foundInfo->second : nullptr; - JSONNode const *mcNameNode = mcAuxNode ? mcAuxNode->find("mcName") : nullptr; - std::string mcname = mcNameNode ? mcNameNode->val() : analysisName; + std::string mcname = mcInfo && mcInfo->mcName ? *mcInfo->mcName : analysisName; if (workspace.obj(mcname)) return; @@ -548,38 +754,66 @@ void importAnalysis(const JSONNode &rootnode, const JSONNode &analysisNode, cons auto *mc = static_cast(workspace.obj(mcname)); mc->SetWS(workspace); - auto *nllNode = RooJSONFactoryWSTool::findNamedChild(likelihoodsNode, analysisNode["likelihood"].val()); - if (!nllNode) { + auto foundLikelihood = std::find_if(doc.likelihoods.begin(), doc.likelihoods.end(), + [&](HS3::Likelihood const &l) { return l.name == analysis.likelihood.name; }); + if (foundLikelihood == doc.likelihoods.end()) { throw std::runtime_error("likelihood node not found!"); } - if (!nllNode->has_child("distributions")) { - throw std::runtime_error("likelihood node has no distributions attached!"); + HS3::Likelihood const &likelihood = *foundLikelihood; + + auto checkPdf = [&](RooAbsArg const *arg, std::string const &name, std::string const &what) { + requirePdf(arg, "the " + what + " '" + name + "' of analysis '" + analysisName + "'"); + }; + + std::vector nllDistNames; + nllDistNames.reserve(likelihood.terms.size()); + for (auto const &term : likelihood.terms) { + checkPdf(workspace.arg(term.distribution.name), term.distribution.name, "distribution"); + nllDistNames.push_back(term.distribution.name); } - if (!nllNode->has_child("data")) { - throw std::runtime_error("likelihood node has no data attached!"); + // The constraint terms of HistFactory distributions are listed as auxiliary distributions, but they are part of the + // distributions in RooFit. + std::set channelConstraints; + for (auto const &item : doc.rooFit.channelConstraints) { + for (auto const &ref : item.second) { + channelConstraints.insert(ref.name); + } } - std::vector nllDistNames = valsToStringVec((*nllNode)["distributions"]); RooArgSet extConstraints; - for (auto &nameNode : (*nllNode)["aux_distributions"].children()) { - if (RooAbsArg *extConstraint = workspace.arg(nameNode.val())) { - extConstraints.add(*extConstraint); + for (auto const &aux : likelihood.auxDistributions) { + RooAbsArg *extConstraint = workspace.arg(aux.name); + if (!extConstraint) { + continue; + } + checkPdf(extConstraint, aux.name, "auxiliary distribution"); + // Only if the constraint really ended up in a distribution of the likelihood. + if (channelConstraints.count(aux.name) && + std::any_of(nllDistNames.begin(), nllDistNames.end(), [&](std::string const &distName) { + RooAbsArg const *dist = workspace.arg(distName); + return dist && dist->dependsOn(*extConstraint); + })) { + continue; } + extConstraints.add(*extConstraint); } RooArgSet observables; - for (auto &nameNode : (*nllNode)["data"].children()) { + for (auto const &term : likelihood.terms) { + auto const *dataRef = std::get_if(&term.data); + if (!dataRef) { + continue; + } bool found = false; for (const auto &d : datasets) { - if (d->GetName() == nameNode.val()) { + if (d->GetName() == dataRef->name) { found = true; observables.add(*d->get(), true); } } - if (nameNode.val() != "0" && !found) - throw std::runtime_error("dataset '" + nameNode.val() + "' cannot be found!"); + if (!found) + throw std::runtime_error("dataset '" + dataRef->name + "' cannot be found!"); } - JSONNode const *pdfNameNode = mcAuxNode ? mcAuxNode->find("pdfName") : nullptr; - std::string const pdfName = pdfNameNode ? pdfNameNode->val() : "simPdf"; + std::string const pdfName = mcInfo && mcInfo->pdfName ? *mcInfo->pdfName : "simPdf"; RooAbsPdf *pdf = workspace.pdf(pdfName); @@ -609,35 +843,29 @@ void importAnalysis(const JSONNode &rootnode, const JSONNode &analysisNode, cons if (!extConstraints.empty()) mc->SetExternalConstraints(extConstraints); - // HS3 v0.2.9 used the singular keys "parameter_of_interest" (a list) and "domain" (a single name). They are still - // accepted, but the plural keys take precedence. - auto findAnalysisKey = [&](const char *key, const char *legacyKey) -> JSONNode const * { - if (auto node = analysisNode.find(key)) { - return node; - } - if (auto node = analysisNode.find(legacyKey)) { - RooJSONFactoryWSTool::warning(std::string("RooFitHS3: the key \"") + legacyKey + "\" of analysis '" + - analysisName + "' is deprecated, please use \"" + key + "\" instead."); - return node; + if (analysis.prior) { + if (RooAbsArg *prior = workspace.arg(analysis.prior->name)) { + checkPdf(prior, analysis.prior->name, "prior"); + mc->SetPriorPdf(static_cast(*prior)); } - return nullptr; - }; - JSONNode const *poiNode = findAnalysisKey("parameters_of_interest", "parameter_of_interest"); - JSONNode const *domainsNodeOfAnalysis = findAnalysisKey("domains", "domain"); - std::vector analysisDomains; - if (domainsNodeOfAnalysis && domainsNodeOfAnalysis->is_seq()) { - analysisDomains = valsToStringVec(*domainsNodeOfAnalysis); - } else if (domainsNodeOfAnalysis) { - analysisDomains.push_back(domainsNodeOfAnalysis->val()); } - RooArgSet pois; - if (poiNode) { - for (auto const &child : poiNode->children()) { - pois.add(*workspace.arg(child.val())); + RooArgSet parametersOfInterest; + for (auto const &poi : analysis.parametersOfInterest) { + RooAbsArg *arg = workspace.arg(poi.name); + if (!arg) { + RooJSONFactoryWSTool::error("the object '" + poi.name + "' in \"parameters_of_interest\" of analysis '" + + analysisName + "' does not exist"); + } + if (!dynamic_cast(arg)) { + RooJSONFactoryWSTool::error("the parameter of interest '" + poi.name + "' of analysis '" + analysisName + + "' is a " + arg->ClassName() + ", not a variable" + + (dynamic_cast(arg) ? ", because no domain lists it" : "")); } + parametersOfInterest.add(*arg); } - mc->SetParametersOfInterest(pois); + + mc->SetParametersOfInterest(parametersOfInterest); mc->SetObservables(observables); RooArgSet pars; pdf->getParameters(&observables, pars); @@ -649,12 +877,13 @@ void importAnalysis(const JSONNode &rootnode, const JSONNode &analysisNode, cons pdf->getAllConstraints(observables, mainPars, /*stripDisconnected*/ true); RooArgSet domainPars; - for (std::string const &domain : analysisDomains) { - const auto &thisDomain = RooJSONFactoryWSTool::findNamedChild(domainsNode, domain); - if (!thisDomain || !thisDomain->has_child("axes")) + for (auto const &domainRef : analysis.domains) { + auto thisDomain = std::find_if(doc.domains.begin(), doc.domains.end(), + [&](HS3::Domain const &d) { return d.name == domainRef.name; }); + if (thisDomain == doc.domains.end()) continue; - for (auto &var : (*thisDomain)["axes"].children()) { - auto *wsvar = workspace.var(RooJSONFactoryWSTool::name(var)); + for (auto const &axis : thisDomain->axes) { + auto *wsvar = workspace.var(std::visit([](auto const &a) -> std::string const & { return a.name; }, axis)); if (wsvar) domainPars.add(*wsvar); } @@ -675,20 +904,14 @@ void importAnalysis(const JSONNode &rootnode, const JSONNode &analysisNode, cons mc->SetGlobalObservables(globs); mc->SetNuisanceParameters(nps); - if (mcAuxNode) { - if (auto found = mcAuxNode->find("combined_data_name")) { - pdf->setStringAttribute("combined_data_name", found->val().c_str()); - } + if (mcInfo && mcInfo->combinedDataName) { + pdf->setStringAttribute("combined_data_name", mcInfo->combinedDataName->c_str()); } - if (analysisNode.has_child("init")) { - const std::string initName = analysisNode["init"].val(); - RooArgSet const *init = workspace.getSnapshot(initName.c_str()); - if (!init) { - RooJSONFactoryWSTool::warning("RooFitHS3: the parameter point '" + initName + - "' used as \"init\" of analysis '" + analysisName + "' does not exist."); - return; - } + // A missing parameter point was reported by HS3::validate(). + RooArgSet const *init = analysis.init ? workspace.getSnapshot(analysis.init->name.c_str()) : nullptr; + if (init) { + const std::string &initName = analysis.init->name; // ModelConfig::SetSnapshot() saves the snapshot as "__snapshot". If "init" already follows // this pattern, which is the case for files written by RooFit, pick the set name such that it is preserved. std::string setName = initName; @@ -706,33 +929,21 @@ void importAnalysis(const JSONNode &rootnode, const JSONNode &analysisNode, cons } } -void combinePdfs(const JSONNode &rootnode, RooWorkspace &ws) +void combinePdfs(HS3Document const &doc, RooWorkspace &ws) { - auto *combinedPdfInfoNode = findRooFitInternal(rootnode, "combined_distributions"); - - // If there is no info on combining pdfs - if (combinedPdfInfoNode == nullptr) { - return; - } + for (auto const &item : doc.rooFit.combinedDistributions) { + std::string const &combinedName = item.first; + HS3::CombinedDistribution const &info = item.second; - for (auto &info : combinedPdfInfoNode->children()) { - - // parse the information - std::string combinedName = info.key(); - std::string indexCatName = info["index_cat"].val(); - std::vector labels = valsToStringVec(info["labels"]); - std::vector indices; - std::vector pdfNames = valsToStringVec(info["distributions"]); - for (auto &n : info["indices"].children()) { - indices.push_back(n.val_int()); - } - - RooCategory indexCat{indexCatName.c_str(), indexCatName.c_str()}; + RooCategory indexCat{info.indexCat.c_str(), info.indexCat.c_str()}; std::map pdfMap; - for (std::size_t iChannel = 0; iChannel < labels.size(); ++iChannel) { - indexCat.defineType(labels[iChannel], indices[iChannel]); - pdfMap[labels[iChannel]] = ws.pdf(pdfNames[iChannel]); + for (std::size_t iChannel = 0; iChannel < info.labels.size(); ++iChannel) { + indexCat.defineType(info.labels[iChannel], info.indices[iChannel]); + std::string const &name = info.distributions[iChannel].name; + requirePdf(ws.arg(name), + "the distribution '" + name + "' of the combined distribution '" + combinedName + "'"); + pdfMap[info.labels[iChannel]] = ws.pdf(name); } RooSimultaneous simPdf{combinedName.c_str(), combinedName.c_str(), pdfMap, indexCat}; @@ -740,44 +951,28 @@ void combinePdfs(const JSONNode &rootnode, RooWorkspace &ws) } } -void combineDatasets(const JSONNode &rootnode, std::vector> &datasets) +void combineDatasets(HS3Document const &doc, std::vector> &datasets) { - auto *combinedDataInfoNode = findRooFitInternal(rootnode, "combined_datasets"); - - // If there is no info on combining datasets - if (combinedDataInfoNode == nullptr) { - return; - } - - for (auto &info : combinedDataInfoNode->children()) { - - // parse the information - std::string combinedName = info.key(); - std::string indexCatName = info["index_cat"].val(); - std::vector labels = valsToStringVec(info["labels"]); - std::vector indices; - for (auto &n : info["indices"].children()) { - indices.push_back(n.val_int()); - } - if (indices.size() != labels.size()) { - RooJSONFactoryWSTool::error("mismatch in number of indices and labels!"); - } + for (auto const &item : doc.rooFit.combinedDatasets) { + std::string const &combinedName = item.first; + HS3::CombinedDataset const &info = item.second; // Create the combined dataset for RooFit std::map> dsMap; - RooCategory indexCat{indexCatName.c_str(), indexCatName.c_str()}; + RooCategory indexCat{info.indexCat.c_str(), info.indexCat.c_str()}; RooArgSet allVars{indexCat}; - for (std::size_t iChannel = 0; iChannel < labels.size(); ++iChannel) { - auto componentName = combinedName + "_" + labels[iChannel]; + for (std::size_t iChannel = 0; iChannel < info.labels.size(); ++iChannel) { + auto componentName = combinedName + "_" + info.labels[iChannel]; // We move the found channel data out of the "datasets" vector, such that // the data components don't get imported anymore. - std::unique_ptr &component = *std::find_if( - datasets.begin(), datasets.end(), [&](auto &d) { return d && d->GetName() == componentName; }); - if (!component) + auto found = + std::find_if(datasets.begin(), datasets.end(), [&](auto &d) { return d && d->GetName() == componentName; }); + if (found == datasets.end()) { RooJSONFactoryWSTool::error("unable to obtain component matching component name '" + componentName + "'"); - allVars.add(*component->get(), true); - dsMap.insert({labels[iChannel], std::move(component)}); - indexCat.defineType(labels[iChannel], indices[iChannel]); + } + allVars.add(*(*found)->get(), true); + dsMap.insert({info.labels[iChannel], std::move(*found)}); + indexCat.defineType(info.labels[iChannel], info.indices[iChannel]); } auto combined = std::make_unique(combinedName, combinedName, allVars, RooFit::Import(dsMap), @@ -825,73 +1020,45 @@ RooAbsCategory *findCategoryObservable(RooAbsData const &data) return cat; } -/// Warn if the HS3 version of an imported document is not understood or newer than what RooFit writes. -void checkHS3Version(std::string const &version) +} // namespace + +RooArgSet RooFit::HS3::createObservables(std::vector const &axes) { - int major = 0; - int minor = 0; - int supportedMajor = 0; - int supportedMinor = 0; - parseHS3Version(hs3VersionTag, supportedMajor, supportedMinor); - if (!parseHS3Version(version, major, minor)) { - RooJSONFactoryWSTool::warning("RooFitHS3: cannot parse hs3_version '" + version + - "', expected a string of the form \"MAJOR.MINOR\"."); - return; - } - if (std::make_pair(major, minor) > std::make_pair(supportedMajor, supportedMinor)) { - RooJSONFactoryWSTool::warning("RooFitHS3: the document has hs3_version '" + version + - "', which is newer than the version supported by this RooFit ('" + hs3VersionTag + - "'). Some content might not be imported correctly."); + RooArgSet vars; + for (BinnedAxis const &axis : axes) { + vars.addOwned(createObservable(axis)); } + return vars; } -/// HS3 requires the names of all top-level objects to be unique within a document. If `strict`, throw if a name is -/// used twice within one section or by a distribution and a function. Other clashes, and all clashes if not -/// `strict`, only warn: RooFit's own export still produces them, e.g. for a range and a pdf of one name, and it -/// must be able to read its own output. -void checkUniqueNames(JSONNode const &rootnode, bool strict) +std::unique_ptr RooFit::HS3::createDataHist(std::string const &name, RooArgSet const &vars, + std::vector const &contents, + std::vector const *errors) { - std::map sectionOfName; - for (const char *section : - {"distributions", "functions", "data", "domains", "parameter_points", "likelihoods", "analyses"}) { - auto seq = rootnode.find(section); - if (!seq || !seq->is_seq()) { - continue; - } - for (JSONNode const &child : seq->children()) { - if (!child.is_map() || !child.has_child("name")) { - continue; // malformed entries are reported when they are imported - } - std::string const name = child["name"].val(); - auto inserted = sectionOfName.emplace(name, section); - if (inserted.second) { - continue; - } - std::string const &firstSection = inserted.first->second; - // RooFit used to give analyses and likelihoods the name of the pdf or the analysis. Accept this for backwards - // compatibility: analyses are not referenced by name, and likelihoods only from analyses, so there is no - // ambiguity. - auto isLegacyNamed = [](std::string const &s) { return s == "likelihoods" || s == "analyses"; }; - // Within one section, and between distributions and functions (both can be referenced by name), a clash is - // ambiguous. Other clashes are produced by RooFit itself (a range, snapshot or data set named like a pdf). - auto isReferable = [](std::string const &s) { return s == "distributions" || s == "functions"; }; - const bool ambiguous = firstSection == section || (isReferable(firstSection) && isReferable(section)); - if (!strict || !ambiguous || isLegacyNamed(firstSection) || isLegacyNamed(section)) { - RooJSONFactoryWSTool::warning("RooFitHS3: the name '" + name + "' is used in \"" + firstSection + - "\" and in \"" + section + "\", which is not allowed by the HS3 standard."); - continue; - } - RooJSONFactoryWSTool::error("duplicate name '" + name + "': it is defined in \"" + firstSection + - "\" and in \"" + section + "\", but names must be unique within the document"); - } + if (errors && errors->size() != contents.size()) { + RooJSONFactoryWSTool::error("inconsistent number of errors and contents in '" + name + "': errors=" + + std::to_string(errors->size()) + ", contents=" + std::to_string(contents.size())); + } + auto bins = generateBinIndices(vars); + if (contents.size() != bins.size()) { + RooJSONFactoryWSTool::error("inconsistent bin numbers: contents=" + std::to_string(contents.size()) + + ", bins=" + std::to_string(bins.size())); + } + auto dh = std::make_unique(name, name, vars); + // RooDataHist only starts to store squared weights when it sees a positive error, and then initializes them from + // the weights so far. Trigger that before setting any weight, so that errors of zero stay zero. + if (errors && !errors->empty()) { + dh->set(0, 0., 1.); } + for (std::size_t ibin = 0; ibin < bins.size(); ++ibin) { + dh->set(ibin, contents[ibin], errors ? (*errors)[ibin] : -1); + } + return dh; } -} // namespace - RooJSONFactoryWSTool::RooJSONFactoryWSTool(RooWorkspace &ws) : _workspace{ws} {} -RooJSONFactoryWSTool::~RooJSONFactoryWSTool() {} +RooJSONFactoryWSTool::~RooJSONFactoryWSTool() = default; void RooJSONFactoryWSTool::fillSeq(JSONNode &node, RooAbsCollection const &coll, size_t nMax) { @@ -915,9 +1082,6 @@ void RooJSONFactoryWSTool::fillSeq(JSONNode &node, RooAbsCollection const &coll, JSONNode &RooJSONFactoryWSTool::appendNamedChild(JSONNode &node, std::string const &name) { - if (!useListsInsteadOfDicts) { - return node.set_map()[name].set_map(); - } JSONNode &child = node.set_seq().append_child().set_map(); child["name"] << name; return child; @@ -925,11 +1089,6 @@ JSONNode &RooJSONFactoryWSTool::appendNamedChild(JSONNode &node, std::string con JSONNode const *RooJSONFactoryWSTool::findNamedChild(JSONNode const &node, std::string const &name) { - if (!useListsInsteadOfDicts) { - if (!node.is_map()) - return nullptr; - return node.find(name); - } if (!node.is_seq()) return nullptr; for (JSONNode const &child : node.children()) { @@ -986,7 +1145,7 @@ bool RooJSONFactoryWSTool::testValidName(const std::string &name, bool forceErro std::string RooJSONFactoryWSTool::name(const JSONNode &n) { - return useListsInsteadOfDicts ? n["name"].val() : n.key(); + return n["name"].val(); } JSONNode &RooJSONFactoryWSTool::makeVariablesNode(JSONNode &rootNode) @@ -999,9 +1158,10 @@ RooRealVar *RooJSONFactoryWSTool::requestImpl(const std::string &obj { if (RooRealVar *retval = _workspace.var(objname)) return retval; - if (const auto *vars = getVariablesNode(*_rootnodeInput)) { - if (const auto &node = vars->find(objname)) { - this->importVariable(*node); + if (_import) { + auto it = _import->defaultValuesByName.find(objname); + if (it != _import->defaultValuesByName.end()) { + this->importVariable(*it->second); if (RooRealVar *retval = _workspace.var(objname)) return retval; } @@ -1014,11 +1174,20 @@ RooAbsPdf *RooJSONFactoryWSTool::requestImpl(const std::string &objna { if (RooAbsPdf *retval = _workspace.pdf(objname)) return retval; - auto it = _distributionsByName.find(objname); - if (it != _distributionsByName.end()) { - this->importFunction(*it->second, true); - if (RooAbsPdf *retval = _workspace.pdf(objname)) - return retval; + if (_import) { + auto it = _import->distributions.find(objname); + if (it != _import->distributions.end()) { + this->importDistribution(*it->second); + if (RooAbsPdf *retval = _workspace.pdf(objname)) + return retval; + } + // HS3 lists some resolution models, which are pdfs in RooFit, as functions. + auto function = _import->functions.find(objname); + if (function != _import->functions.end()) { + this->importFunction(*function->second->body.node, true); + if (RooAbsPdf *retval = _workspace.pdf(objname)) + return retval; + } } return nullptr; } @@ -1036,17 +1205,19 @@ RooAbsReal *RooJSONFactoryWSTool::requestImpl(const std::string &obj return var; if (RooAbsReal *retval = _workspace.function(objname)) return retval; - auto it = _functionsByName.find(objname); - if (it != _functionsByName.end()) { - this->importFunction(*it->second, true); - if (RooAbsReal *retval = _workspace.function(objname)) - return retval; + if (_import) { + auto it = _import->functions.find(objname); + if (it != _import->functions.end()) { + this->importFunction(*it->second->body.node, true); + if (RooAbsReal *retval = _workspace.function(objname)) + return retval; + } } return nullptr; } -// Export a single variable (RooRealVar or RooConstVar) `v` as a named child of `node`. -void RooJSONFactoryWSTool::exportVariable(const RooAbsArg *v, JSONNode &node, bool storeConstant, bool storeBins) +// Export a single variable (RooRealVar or RooConstVar) `v` to the parameter point. +void RooJSONFactoryWSTool::exportVariable(const RooAbsArg *v, HS3::ParameterPoint &point, bool storeConstant) { auto *cv = dynamic_cast(v); auto *rrv = dynamic_cast(v); @@ -1061,34 +1232,26 @@ void RooJSONFactoryWSTool::exportVariable(const RooAbsArg *v, JSONNode &node, bo // A named RooConstVar only takes its value: it is written as a domain axis with that single value, not in the // parameter points. if (cv) { - _domains->readFixedValue(cv->GetName(), cv->getVal()); + HS3::DomainAxis axis; + axis.name = cv->GetName(); + axis.fixedValue = cv->getVal(); + _export->domains[HS3::defaultDomainName][axis.name] = std::move(axis); return; } - JSONNode &var = appendNamedChild(node, v->GetName()); - - if (rrv) { - var["value"] << rrv->getVal(); - if (storeConstant && (rrv->isConstant() || rrv->getMin() >= rrv->getMax())) { - var["const"] << true; - } else if (storeBins) { - var["min"] << rrv->getMin(); - var["max"] << rrv->getMax(); - } - if (rrv->getBins() != 0 && storeBins) { - var["nbins"] << rrv->getBins(); - } - _domains->readVariable(*rrv); - } + HS3::ParameterValue var; + var.name = v->GetName(); + var.value = rrv->getVal(); + var.isConst = storeConstant && (rrv->isConstant() || rrv->getMin() >= rrv->getMax()); + recordDomains(_export->domains, *rrv); + point.parameters.push_back(std::move(var)); } -// Export all variables in `allElems` as a sequence under `n`. -void RooJSONFactoryWSTool::exportVariables(const RooArgSet &allElems, JSONNode &n, bool storeConstant, bool storeBins) +// Export all variables in `allElems` to the parameter point. +void RooJSONFactoryWSTool::exportVariables(const RooArgSet &allElems, HS3::ParameterPoint &point, bool storeConstant) { - // export a list of RooRealVar objects - n.set_seq(); for (RooAbsArg *arg : allElems) { - exportVariable(arg, n, storeConstant, storeBins); + exportVariable(arg, point, storeConstant); } } @@ -1097,16 +1260,52 @@ std::string RooJSONFactoryWSTool::exportTransformed(const RooAbsReal *original, { std::string newname = std::string(original->GetName()) + suffix; // Another object that shares the parameter might have exported the helper already. - if (findNamedChild((*_rootnodeOutput)["functions"], newname)) { + auto &functions = _export->doc.functions; + if (std::find_if(functions.begin(), functions.end(), [&](auto const &f) { return f.name == newname; }) != + functions.end()) { return newname; } - RooFit::Detail::JSONNode &trafo_node = appendNamedChild((*_rootnodeOutput)["functions"], newname); + RooFit::Detail::JSONNode &trafo_node = appendNamedChild((*_export->root)["functions"], newname); trafo_node["type"] << "generic"; trafo_node["expression"] << TString::Format(formula.c_str(), original->GetName()).Data(); + functions.push_back({newname, {"generic", nullptr}}); this->setAttribute(newname, "roofit_skip"); // this function should not be imported back in return newname; } +namespace { + +// Turn an arbitrary string into a valid variable name, but refuse to change the +// first character (which would silently rename the object). +std::string makeValidNameOrError(std::string const &in) +{ + if (!std::isalpha(in[0])) { + RooJSONFactoryWSTool::error("refusing to change first character of string '" + in + "' to make a valid name!"); + } + std::string out = RooFit::Detail::makeValidVarName(in); + if (out != in) { + oocoutW(nullptr, ObjectHandling) << "RooFitHS3: changed '" << in << "' to '" << out << "' to become a valid name"; + } + return out; +} + +} // namespace + +namespace { + +// Record an object that was written by an exporter in the document. +void recordGenericObject(HS3Document &doc, bool isPdf, std::string const &name, JSONNode const &elem) +{ + std::string type = elem.has_child("type") ? elem["type"].val() : ""; + if (isPdf) { + doc.distributions.push_back({name, HS3::GenericObject{std::move(type), nullptr}}); + } else { + doc.functions.push_back({name, {std::move(type), nullptr}}); + } +} + +} // namespace + // Export a single object `func` (pdf, function, variable or category) to the output JSON, recording its name in // `exportedObjectNames` to avoid exporting it twice. void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func, std::set &exportedObjectNames) @@ -1124,18 +1323,12 @@ void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func, std::set channelNames; + HS3::CombinedDistribution &info = _export->doc.rooFit.combinedDistributions[simPdf->GetName()]; + info.indexCat = simPdf->indexCat().GetName(); for (auto const &item : simPdf->indexCat()) { - channelNames.push_back(item.first); - } - - auto &infoNode = getRooFitInternal(*_rootnodeOutput, "combined_distributions").set_map(); - auto &child = infoNode[simPdf->GetName()].set_map(); - child["index_cat"] << simPdf->indexCat().GetName(); - exportCategory(simPdf->indexCat(), child); - child["distributions"].set_seq(); - for (auto const &item : simPdf->indexCat()) { - child["distributions"].append_child() << simPdf->getPdf(item.first.c_str())->GetName(); + info.labels.push_back(makeValidNameOrError(item.first)); + info.indices.push_back(item.second); + info.distributions.push_back({simPdf->getPdf(item.first.c_str())->GetName()}); } return; @@ -1143,11 +1336,14 @@ void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func, std::set(&func) || dynamic_cast(&func)) { - exportVariable(&func, *_varsNode, true, false); + exportVariable(&func, _export->doc.parameterPoints.front(), true); return; } - auto &collectionNode = (*_rootnodeOutput)[dynamic_cast(&func) ? "distributions" : "functions"]; + // HS3 lists some resolution models as functions. + const bool isPdf = dynamic_cast(&func) && !func.InheritsFrom("RooGaussModel") && + !func.InheritsFrom("RooTruthModel"); + auto &collectionNode = (*_export->root)[isPdf ? "distributions" : "functions"]; auto const &exporters = RooFit::JSONIO::exporters(); auto const &exportKeys = RooFit::JSONIO::exportKeys(); @@ -1166,11 +1362,22 @@ void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func, std::settypedObject) { + HS3::writeHistFactoryDist(elem, *_export->typedObject); + elem["name"] << name; + _export->doc.distributions.push_back({name, std::move(*_export->typedObject)}); + _export->typedObject.reset(); + if (!_export->channelConstraints.empty()) { + _export->doc.rooFit.channelConstraints[name] = std::move(_export->channelConstraints); + } + _export->channelConstraints.clear(); + } else { + recordGenericObject(_export->doc, isPdf, name, elem); + } if (exp->autoExportDependants()) { exportObjects(func.servers(), exportedObjectNames); } else { @@ -1204,6 +1411,7 @@ void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func, std::setsecond.type; + recordGenericObject(_export->doc, isPdf, name, elem); size_t nprox = func.numProxies(); @@ -1262,6 +1470,8 @@ void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func, std::setsecond) { try { - ok = imp->importArg(this, p); + // The built-in importer doesn't see the constraint terms that the document adds to the channel. + if (_import && HS3::isBuiltinHistFactoryImporter(*imp)) { + auto const &channelConstraints = _import->document.rooFit.channelConstraints; + auto found = channelConstraints.find(name); + HS3::importHistFactoryNode(*this, p, found != channelConstraints.end() ? &found->second : nullptr); + ok = true; + } else { + ok = imp->importArg(this, p); + } } catch (const std::exception &e) { std::stringstream ss; const auto *ptr = imp.get(); @@ -1353,6 +1572,31 @@ void RooJSONFactoryWSTool::importFunction(const JSONNode &p, bool importAllDepen if (!func) { RooJSONFactoryWSTool::error("something went wrong importing function '" + name + "'."); } + for (std::string const &component : recorder.unreadComponents()) { + warning("RooFitHS3: " + name + ": unknown component \"" + component + "\" is ignored"); + } +} + +// Import a distribution of the document, which is interpreted by the document or by the registered importers. +void RooJSONFactoryWSTool::importDistribution(HS3::Distribution const &distribution) +{ + if (auto const *generic = std::get_if(&distribution.body)) { + this->importFunction(*generic->node, true); + return; + } + std::string const &name = distribution.name; + if (hasAttribute(name, "roofit_skip")) { + return; + } + RooJSONFactoryWSTool::testValidName(name, true); + if (_workspace.arg(name)) { + return; + } + ImportGuard importGuard{_importStack, name}; + auto const &channelConstraints = _import->document.rooFit.channelConstraints; + auto found = channelConstraints.find(name); + HS3::buildHistFactoryDist(*this, name, std::get(distribution.body), + found != channelConstraints.end() ? &found->second : nullptr); } /** @@ -1384,11 +1628,6 @@ void RooJSONFactoryWSTool::exportAxis(JSONNode &obsNode, RooRealVar const &var) obsNode["name"] << name; writeAxisBinning(obsNode, var.getBinning()); - // Binned axes always carry their range, also if it is redundant with the "edges" of an irregular binning. - if (!obsNode.has_child("min")) { - obsNode["min"] << var.getBinning().lowBound(); - obsNode["max"] << var.getBinning().highBound(); - } } /** @@ -1439,24 +1678,6 @@ void RooJSONFactoryWSTool::exportArray(std::size_t n, double const *contents, JS } } -namespace { - -// Turn an arbitrary string into a valid variable name, but refuse to change the -// first character (which would silently rename the object). -std::string makeValidNameOrError(std::string const &in) -{ - if (!std::isalpha(in[0])) { - RooJSONFactoryWSTool::error("refusing to change first character of string '" + in + "' to make a valid name!"); - } - std::string out = RooFit::Detail::makeValidVarName(in); - if (out != in) { - oocoutW(nullptr, ObjectHandling) << "RooFitHS3: changed '" << in << "' to '" << out << "' to become a valid name"; - } - return out; -} - -} // namespace - /** * @brief Export a RooAbsCategory object to a JSONNode. * @@ -1494,18 +1715,18 @@ RooJSONFactoryWSTool::CombinedData RooJSONFactoryWSTool::exportCombinedData(RooA datamap.name = data.GetName(); // Write information necessary to reconstruct the combined dataset upon import - auto &child = getRooFitInternal(*_rootnodeOutput, "combined_datasets").set_map()[data.GetName()].set_map(); - child["index_cat"] << cat->GetName(); - exportCategory(*cat, child); + HS3::CombinedDataset &info = _export->doc.rooFit.combinedDatasets[data.GetName()]; + info.indexCat = cat->GetName(); + for (auto const &item : *cat) { + info.labels.push_back(makeValidNameOrError(item.first)); + info.indices.push_back(item.second); + } // Find a RooSimultaneous model that would fit to this dataset RooSimultaneous const *simPdf = nullptr; - auto *combinedPdfInfoNode = findRooFitInternal(*_rootnodeOutput, "combined_distributions"); - if (combinedPdfInfoNode) { - for (auto &info : combinedPdfInfoNode->children()) { - if (info["index_cat"].val() == cat->GetName()) { - simPdf = static_cast(_workspace.pdf(info.key())); - } + for (auto const &item : _export->doc.rooFit.combinedDistributions) { + if (item.second.indexCat == cat->GetName()) { + simPdf = static_cast(_workspace.pdf(item.first)); } } @@ -1525,7 +1746,7 @@ RooJSONFactoryWSTool::CombinedData RooJSONFactoryWSTool::exportCombinedData(RooA return datamap; } -// Export a single dataset `data` (binned or unbinned) to the output JSON. +// Export a single dataset `data` (binned or unbinned) to the output document. void RooJSONFactoryWSTool::exportData(RooAbsData const &data) { RooAbsCategory *cat = findCategoryObservable(data); @@ -1533,7 +1754,8 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) if (cat) return; - JSONNode &output = appendNamedChild((*_rootnodeOutput)["data"], data.GetName()); + HS3::Data output; + output.name = data.GetName(); // This works around a problem in RooStats/HistFactory that was only fixed // in ROOT 6.30: until then, the weight variable of the observed dataset, @@ -1546,13 +1768,20 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) variables.remove(*weightVar); } - // this is a regular binned dataset - if (auto dh = dynamic_cast(&data)) { - output["type"] << "binned"; + auto exportBinned = [&](std::size_t n, double const *contents) { + HS3::BinnedData binned; for (auto *var : static_range_cast(variables)) { - _domains->readVariable(*var); + recordDomains(_export->domains, *var); + binned.axes.emplace_back(HS3::toBinnedAxis(*var)); } - return exportHisto(variables, dh->numEntries(), dh->weightArray(), output); + binned.contents.assign(contents, contents + n); + output.body = std::move(binned); + _export->doc.data.push_back(std::move(output)); + }; + + // this is a regular binned dataset + if (auto dh = dynamic_cast(&data)) { + return exportBinned(dh->numEntries(), dh->weightArray()); } // Check if this actually represents a binned dataset, and then import it @@ -1575,32 +1804,25 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) if (i == x.getBins()) isBinnedData = true; if (isBinnedData) { - output["type"] << "binned"; - for (auto *var : static_range_cast(variables)) { - _domains->readVariable(*var); - } - return exportHisto(variables, data.numEntries(), contents.data(), output); + return exportBinned(data.numEntries(), contents.data()); } } // this really is an unbinned dataset - output["type"] << "unbinned"; - auto &observablesNode = output["axes"].set_seq(); + HS3::UnbinnedData unbinned; for (auto *var : static_range_cast(variables)) { - _domains->readVariable(*var); + recordDomains(_export->domains, *var); // HS3 forbids binning information on the axes of unbinned data. - auto &axis = observablesNode.append_child().set_map(); testValidName(var->GetName(), false); - axis["name"] << var->GetName(); - axis["min"] << var->getMin(); - axis["max"] << var->getMax(); + unbinned.axes.emplace_back(HS3::ContinuousAxis{var->GetName(), var->getMin(), var->getMax()}); } - auto &coords = output["entries"].set_seq(); std::vector weightVals; bool hasNonUnityWeights = false; for (int i = 0; i < data.numEntries(); ++i) { data.get(i); - coords.append_child().fill_seq(variables, [](auto x) { return static_cast(x)->getVal(); }); + for (auto *var : static_range_cast(variables)) { + unbinned.entries.push_back(var->getVal()); + } if (data.isWeighted()) { weightVals.push_back(data.weight()); if (data.weight() != 1.) @@ -1608,8 +1830,10 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) } } if (data.isWeighted() && hasNonUnityWeights) { - output["weights"].fill_seq(weightVals); + unbinned.weights = std::move(weightVals); } + output.body = std::move(unbinned); + _export->doc.data.push_back(std::move(output)); } /** @@ -1623,28 +1847,10 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) */ RooArgSet RooJSONFactoryWSTool::readAxes(const JSONNode &topNode) { - RooArgSet vars; - - for (JSONNode const &node : topNode["axes"].children()) { - if (node.has_child("edges")) { - std::vector edges = readEdges(node["edges"], "axis '" + node["name"].val() + "'"); - auto obs = std::make_unique(node["name"].val().c_str(), node["name"].val().c_str(), edges[0], - edges[edges.size() - 1]); - RooBinning bins(obs->getMin(), obs->getMax()); - for (auto b : edges) { - bins.addBoundary(b); - } - obs->setBinning(bins); - vars.addOwned(std::move(obs)); - } else { - auto obs = std::make_unique(node["name"].val().c_str(), node["name"].val().c_str(), - node["min"].val_double(), node["max"].val_double()); - obs->setBins(node["nbins"].val_int()); - vars.addOwned(std::move(obs)); - } - } - - return vars; + HS3::Diagnostics diagnostics; + std::vector axes = HS3::parseBinnedAxes(topNode["axes"], diagnostics); + HS3::report(diagnostics); + return HS3::createObservables(axes); } /** @@ -1669,123 +1875,70 @@ RooJSONFactoryWSTool::readBinnedData(const JSONNode &n, const std::string &name, if (!contents.is_seq()) RooJSONFactoryWSTool::error("contents are not in list form"); - JSONNode const *errors = nullptr; - if (n.has_child("errors")) { - errors = &n["errors"]; - if (!errors->is_seq()) - RooJSONFactoryWSTool::error("errors are not in list form"); - if (errors->num_children() != contents.num_children()) { - RooJSONFactoryWSTool::error("inconsistent number of errors and contents in '" + name + - "': errors=" + std::to_string(errors->num_children()) + - ", contents=" + std::to_string(contents.num_children())); - } - } - - auto bins = generateBinIndices(vars); - if (contents.num_children() != bins.size()) { - RooJSONFactoryWSTool::error("inconsistent bin numbers: contents=" + std::to_string(contents.num_children()) + - ", bins=" + std::to_string(bins.size())); - } - auto dh = std::make_unique(name, name, vars); std::vector contentVals; - contentVals.reserve(contents.num_children()); - for (auto const &cont : contents.children()) { - contentVals.push_back(cont.val_double()); + contentVals << contents; + + if (!n.has_child("errors")) { + return HS3::createDataHist(name, vars, contentVals, nullptr); } + if (!n["errors"].is_seq()) + RooJSONFactoryWSTool::error("errors are not in list form"); std::vector errorVals; - if (errors) { - errorVals.reserve(errors->num_children()); - for (auto const &err : errors->children()) { - errorVals.push_back(err.val_double()); - } - } - // RooDataHist only starts to store squared weights when it sees a positive error, and then initializes them from - // the weights so far. Trigger that before setting any weight, so that errors of zero stay zero. - if (!errorVals.empty()) { - dh->set(0, 0., 1.); - } - for (size_t ibin = 0; ibin < bins.size(); ++ibin) { - const double err = errors ? errorVals[ibin] : -1; - dh->set(ibin, contentVals[ibin], err); - } - return dh; + errorVals << n["errors"]; + return HS3::createDataHist(name, vars, contentVals, &errorVals); } -// Import a single variable (RooRealVar or RooConstVar) from the JSON node `p` into the workspace. -void RooJSONFactoryWSTool::importVariable(const JSONNode &p) +// Import a single variable (RooRealVar or RooConstVar) from the parameter `p` into the workspace. +void RooJSONFactoryWSTool::importVariable(HS3::ParameterValue const &p) { // import a RooRealVar object - std::string name(RooJSONFactoryWSTool::name(p)); - RooJSONFactoryWSTool::testValidName(name, true); + RooJSONFactoryWSTool::testValidName(p.name, true); - if (!p.is_map()) { - oocoutE(nullptr, InputArguments) << "RooJSONFactoryWSTool() node '" << name << "' is not a map, skipping." - << std::endl; + if (_workspace.arg(p.name)) return; - } - checkFixedValue(name, p, "default_values"); - if (_workspace.arg(name)) + // A string selects the state of a category, which is done together with the other parameter points. + if (p.value && std::holds_alternative(*p.value)) return; - if (!_domains->hasVariable(name.c_str())) { + if (!_import->ranges.contains(p.name)) { // ROOT versions before HS3 0.3 marked named RooConstVars with an attribute. - JSONNode const *attrNode = _attributesNode ? _attributesNode->find(name) : nullptr; - if (attrNode && attrNode->has_child("is_const_var") && attrNode->find("is_const_var")->val_int() == 1) { - if (!p.has_child("value")) { - RooJSONFactoryWSTool::error("cannot instantiate RooConstVar '" + name + "' without \"value\"!"); + auto const &attributes = _import->document.rooFit.attributes; + auto found = attributes.find(p.name); + if (found != attributes.end() && found->second.isConstVar) { + if (!p.value) { + RooJSONFactoryWSTool::error("cannot instantiate RooConstVar '" + p.name + "' without \"value\"!"); } - wsEmplace(name, p["value"].val_double()); + wsEmplace(p.name, std::get(*p.value)); return; } } - RooRealVar &var = wsEmplace(name, 1.); - configureVariable(*_domains, p, var); + RooRealVar &var = wsEmplace(p.name, 1.); + configureVariable(_import->ranges, p, var); warnIfFreeWithoutDomain(var); } -// A parameter with a single value in its domain only takes that value: a parameter point must not give it another. -void RooJSONFactoryWSTool::checkFixedValue(std::string const &name, JSONNode const &p, std::string const &pointName) -{ - double const *fixed = _domains ? _domains->fixedValue(name.c_str()) : nullptr; - JSONNode const *valueNode = p.find("value"); - if (fixed && valueNode && valueNode->is_number() && valueNode->val_double() != *fixed) { - RooJSONFactoryWSTool::error("the parameter point '" + pointName + "' gives the parameter '" + name + - "' the value " + valueNode->val() + ", but its domain only has the value " + - std::to_string(*fixed)); - } -} - // HS3 requires every parameter to be listed in a domain or constant ("const": true) in the parameter point. void RooJSONFactoryWSTool::warnIfFreeWithoutDomain(RooRealVar const &var) { - if (!var.isConstant() && _domains && !_domains->hasVariable(var.GetName())) { + if (!var.isConstant() && !_import->ranges.contains(var.GetName())) { warning(std::string("RooFitHS3: the parameter '") + var.GetName() + "' is not listed in any domain and is not constant: HS3 requires a domain, or \"const\": true. It " "is read as a free parameter without range."); } } -// Import all dependants (variables, functions and distributions) of node `n` into the workspace. +// Import the objects of a fragment of a document, for which the node `n` plays the role of the document. void RooJSONFactoryWSTool::importDependants(const JSONNode &n) { - // import all the dependants of an object - if (JSONNode const *varsNode = getVariablesNode(n)) { - for (const auto &p : varsNode->children()) { - importVariable(p); - } - } - if (auto seq = n.find("functions")) { - for (const auto &p : seq->children()) { - this->importFunction(p, true); - } - } - if (auto seq = n.find("distributions")) { - for (const auto &p : seq->children()) { - this->importFunction(p, true); + for (const char *section : {"functions", "distributions"}) { + if (auto seq = n.find(section)) { + for (const auto &p : seq->children()) { + this->importFunction(p, true); + } } } } -void RooJSONFactoryWSTool::exportModelConfig(JSONNode &rootnode, RooFit::ModelConfig const &mc, +void RooJSONFactoryWSTool::exportModelConfig(RooFit::ModelConfig const &mc, const std::vector &combDataSets, const std::vector &singleDataSets) { @@ -1801,7 +1954,7 @@ void RooJSONFactoryWSTool::exportModelConfig(JSONNode &rootnode, RooFit::ModelCo if (hasdata) analysisName += "_" + combDataSets[i].name; - exportSingleModelConfig(rootnode, mc, analysisName, hasdata ? &combDataSets[i].components : nullptr); + exportSingleModelConfig(mc, analysisName, hasdata ? &combDataSets[i].components : nullptr); } } else { RooArgSet observables(*mc.GetObservables()); @@ -1810,87 +1963,89 @@ void RooJSONFactoryWSTool::exportModelConfig(JSONNode &rootnode, RooFit::ModelCo if (observables.equals(*(data->get()))) { std::map mapping; mapping[pdf->GetName()] = data->GetName(); - exportSingleModelConfig(rootnode, mc, std::string(mc.GetName()) + "_" + data->GetName(), &mapping); + exportSingleModelConfig(mc, std::string(mc.GetName()) + "_" + data->GetName(), &mapping); ++founddata; } } if (founddata == 0) { - exportSingleModelConfig(rootnode, mc, mc.GetName(), nullptr); + exportSingleModelConfig(mc, mc.GetName(), nullptr); } } } -void RooJSONFactoryWSTool::exportSingleModelConfig(JSONNode &rootnode, RooFit::ModelConfig const &mc, - std::string const &analysisName, +void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc, std::string const &analysisName, std::map const *dataComponents) { + HS3Document &doc = _export->doc; auto pdf = mc.GetPdf(); - JSONNode &analysisNode = appendNamedChild(rootnode["analyses"], analysisName); - - auto &domains = analysisNode["domains"].set_seq(); + HS3::Analysis analysis; + analysis.name = analysisName; // HS3 names must be unique across all top-level components, so the likelihood can't have the analysis name. - const std::string likelihoodName = analysisName + "_likelihood"; - analysisNode["likelihood"] << likelihoodName; - - auto &nllNode = appendNamedChild(rootnode["likelihoods"], likelihoodName); - nllNode["distributions"].set_seq(); - nllNode["data"].set_seq(); + HS3::Likelihood likelihood; + likelihood.name = analysisName + "_likelihood"; + analysis.likelihood.name = likelihood.name; if (dataComponents) { auto simPdf = dynamic_cast(pdf); if (simPdf) { for (auto const &item : simPdf->indexCat()) { const auto &dataComp = dataComponents->find(item.first); - nllNode["distributions"].append_child() << simPdf->getPdf(item.first)->GetName(); - nllNode["data"].append_child() << dataComp->second; + likelihood.terms.push_back({{simPdf->getPdf(item.first)->GetName()}, HS3::DataRef{dataComp->second}}); } } else { for (auto it : *dataComponents) { - nllNode["distributions"].append_child() << it.first; - nllNode["data"].append_child() << it.second; + likelihood.terms.push_back({{it.first}, HS3::DataRef{it.second}}); } } } else { - nllNode["distributions"].append_child() << pdf->GetName(); - nllNode["data"].append_child() << 0; + likelihood.terms.push_back({{pdf->GetName()}, 0.0}); } + auto addAux = [&](std::string const &name) { + auto &aux = likelihood.auxDistributions; + if (std::find_if(aux.begin(), aux.end(), [&](auto const &ref) { return ref.name == name; }) == aux.end()) { + aux.push_back({name}); + } + }; + // The constraint terms of HistFactory channels that no modifier names are part of the likelihood. + for (auto const &term : likelihood.terms) { + auto found = doc.rooFit.channelConstraints.find(term.distribution.name); + if (found != doc.rooFit.channelConstraints.end()) { + for (auto const &ref : found->second) { + addAux(ref.name); + } + } + } if (mc.GetExternalConstraints()) { - auto &extConstrNode = nllNode["aux_distributions"]; - extConstrNode.set_seq(); for (const auto &constr : *mc.GetExternalConstraints()) { - extConstrNode.append_child() << constr->GetName(); + addAux(constr->GetName()); } } + doc.likelihoods.push_back(std::move(likelihood)); - auto writeList = [&](const char *name, RooArgSet const *args) { - if (!args || !args->size()) - return; - + if (RooArgSet const *pois = mc.GetParametersOfInterest()) { std::vector names; - names.reserve(args->size()); - for (RooAbsArg const *arg : *args) + names.reserve(pois->size()); + for (RooAbsArg const *arg : *pois) names.push_back(arg->GetName()); std::sort(names.begin(), names.end()); - analysisNode[name].fill_seq(names); - }; - - writeList("parameters_of_interest", mc.GetParametersOfInterest()); - - auto &domainsNode = rootnode["domains"]; + for (auto const &name : names) { + analysis.parametersOfInterest.push_back({name}); + } + } auto writeProductDomain = [&](const char *suffix, RooArgSet const *args) { if (!args || args->empty()) return; const std::string domainName = analysisName + suffix; - domains.append_child() << domainName; - RooFit::JSONIO::Detail::Domains::ProductDomain domain; + analysis.domains.push_back({domainName}); + std::map axes; for (auto *var : static_range_cast(*args)) { - domain.readVariable(*var); + axes[var->GetName()] = toDomainAxis(var->GetName(), var->getBinning()); } - domain.writeJSON(appendNamedChild(domainsNode, domainName)); + appendDomain(doc.domains, domainName, std::move(axes)); }; writeProductDomain("_nuisance_parameters", mc.GetNuisanceParameters()); @@ -1899,21 +2054,34 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(JSONNode &rootnode, RooFit::M // The snapshots are exported as parameter points. Reference the one of the ModelConfig as the initial values. if (_workspace.getSnapshot(mc.GetSnapshotName().c_str())) { - analysisNode["init"] << mc.GetSnapshotName(); + analysis.init = HS3::ParameterPointRef{mc.GetSnapshotName()}; + } + // The prior is a top-level pdf of the workspace or a component of one, so it is exported already. + if (RooAbsPdf const *prior = mc.GetPriorPdf()) { + analysis.prior = HS3::DistributionRef{prior->GetName()}; } + doc.analyses.push_back(std::move(analysis)); - auto &modelConfigAux = getRooFitInternal(rootnode, "ModelConfigs", analysisName); - modelConfigAux.set_map(); - modelConfigAux["pdfName"] << pdf->GetName(); - modelConfigAux["mcName"] << mc.GetName(); + HS3::ModelConfigInfo &info = doc.rooFit.modelConfigs[analysisName]; + info.pdfName = pdf->GetName(); + info.mcName = mc.GetName(); } // Export all top-level pdfs, functions, datasets and ModelConfigs of the workspace into `n`. void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) { - _domains = std::make_unique(); - _varsNode = &makeVariablesNode(n); - _rootnodeOutput = &n; + _export = std::make_unique(); + _export->root = &n; + // The state of the export must not survive it, not even if there is an exception. + struct ExportScope { + RooJSONFactoryWSTool &tool; + ~ExportScope() { tool._export.reset(); } + } exportScope{*this}; + + HS3Document &doc = _export->doc; + doc.metadata.hs3Version = hs3VersionTag; + doc.metadata.packages.push_back({"ROOT", rootVersion()}); + doc.parameterPoints.push_back({"default_values", {}}); // export all toplevel pdfs std::vector allpdfs; @@ -1942,7 +2110,7 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) // export attributes of all objects for (RooAbsArg *arg : _workspace.components()) { - exportAttributes(arg, n); + exportAttributes(arg, doc.rooFit.attributes); } // collect all datasets @@ -1969,11 +2137,11 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) // export all ModelConfig objects and attached Pdfs for (TObject *obj : _workspace.allGenericObjects()) { if (auto mc = dynamic_cast(obj)) { - exportModelConfig(n, *mc, combData, singleData); + exportModelConfig(*mc, combData, singleData); } } - exportParameterStepWidths(_workspace, allpdfs, alldata, n); + exportParameterStepWidths(_workspace, allpdfs, alldata, doc.rooFit.stepWidths); for (auto *snsh : static_range_cast(_workspace.getSnapshots())) { RooArgSet snapshotSorted; @@ -1995,14 +2163,31 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) snapshotSorted.sort(); std::string name(snsh->GetName()); if (name != "default_values") { - this->exportVariables(snapshotSorted, appendNamedChild(n["parameter_points"], name)["parameters"], true, - false); + doc.parameterPoints.push_back({name, {}}); + this->exportVariables(snapshotSorted, doc.parameterPoints.back(), true); } } - _varsNode = nullptr; - _domains->writeJSON(n["domains"]); - _domains.reset(); - _rootnodeOutput = nullptr; + // The domains of the analyses take precedence. + for (auto &item : _export->domains) { + if (std::none_of(doc.domains.begin(), doc.domains.end(), + [&](HS3::Domain const &domain) { return domain.name == item.first; })) { + appendDomain(doc.domains, item.first, std::move(item.second)); + } + } + + // The document that RooFit writes has to pass the same checks as the ones that it reads. Until RooFit writes + // documents that follow the current standard, problems are only reported. + HS3::Diagnostics diagnostics; + HS3::ValidateOptions validateOptions; + validateOptions.isExternal = [](std::string const &) { return false; }; + doc.metadata.version = HS3::Version{}; + HS3::checkDocument(doc, diagnostics, validateOptions); + for (auto &diagnostic : diagnostics) { + diagnostic.severity = HS3::Diagnostic::Severity::Warning; + diagnostic.message = "the exported document is not valid: " + diagnostic.message; + } + HS3::report(diagnostics); + HS3::writeDocument(n, doc); } /** @@ -2044,13 +2229,7 @@ std::unique_ptr RooJSONFactoryWSTool::createNewJSONTree() // add the mandatory hs3 version number metadata["hs3_version"] << hs3VersionTag; - // Add information about the ROOT version that was used to generate this file - auto &rootInfo = appendNamedChild(metadata["packages"], "ROOT"); - std::string versionName = gROOT->GetVersion(); - // We want to consistently use dots such that the version name can be easily - // digested automatically. - std::replace(versionName.begin(), versionName.end(), '/', '.'); - rootInfo["version"] << versionName; + appendNamedChild(metadata["packages"], "ROOT")["version"] << rootVersion(); return tree; } @@ -2063,8 +2242,8 @@ std::unique_ptr RooJSONFactoryWSTool::createNewJSONTree() */ bool RooJSONFactoryWSTool::exportJSON(std::ostream &os) { - std::unique_ptr tree = createNewJSONTree(); - JSONNode &n = tree->rootnode(); + std::unique_ptr tree = JSONTree::create(); + JSONNode &n = tree->rootnode().set_map(); this->exportAllObjects(n); n.writeJSON(os); return true; @@ -2086,181 +2265,151 @@ bool RooJSONFactoryWSTool::exportJSON(std::string const &filename) bool RooJSONFactoryWSTool::hasAttribute(const std::string &obj, const std::string &attrib) { - if (!_attributesNode) + if (!_import) return false; - if (auto attrNode = _attributesNode->find(obj)) { - if (auto seq = attrNode->find("tags")) { - for (auto &a : seq->children()) { - if (a.val() == attrib) - return true; - } - } - } - return false; + auto const &attributes = _import->document.rooFit.attributes; + auto found = attributes.find(obj); + if (found == attributes.end()) + return false; + auto const &tags = found->second.tags; + return std::find(tags.begin(), tags.end(), attrib) != tags.end(); } void RooJSONFactoryWSTool::setAttribute(const std::string &obj, const std::string &attrib) { - auto node = &RooJSONFactoryWSTool::getRooFitInternal(*_rootnodeOutput, "attributes").set_map()[obj].set_map(); - auto &tags = (*node)["tags"]; - tags.set_seq(); - tags.append_child() << attrib; + _export->doc.rooFit.attributes[obj].tags.push_back(attrib); } std::string RooJSONFactoryWSTool::getStringAttribute(const std::string &obj, const std::string &attrib) { - if (!_attributesNode) + if (!_import) return ""; - if (auto attrNode = _attributesNode->find(obj)) { - if (auto dict = attrNode->find("dict")) { - if (auto *a = dict->find(attrib)) { - return a->val(); - } - } - } - return ""; + auto const &attributes = _import->document.rooFit.attributes; + auto found = attributes.find(obj); + if (found == attributes.end()) + return ""; + auto value = found->second.dict.find(attrib); + return value != found->second.dict.end() ? value->second : ""; } void RooJSONFactoryWSTool::setStringAttribute(const std::string &obj, const std::string &attrib, const std::string &value) { - auto node = &RooJSONFactoryWSTool::getRooFitInternal(*_rootnodeOutput, "attributes").set_map()[obj].set_map(); - auto &dict = (*node)["dict"]; - dict.set_map(); - dict[attrib] << value; + _export->doc.rooFit.attributes[obj].dict[attrib] = value; } -// Import all nodes of the JSON document rooted at `n` into the workspace. -void RooJSONFactoryWSTool::importAllNodes(const JSONNode &n) +// Import the JSON document rooted at `n` into the workspace. +void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) { - // Per HS3 standard, the hs3_version in the metadata is required. So we - // error out if it is missing or not a string, and warn if the document is - // newer than what this version of RooFit was written for. - auto metadata = n.find("metadata"); - auto versionNode = metadata ? metadata->find("hs3_version") : nullptr; - if (!versionNode) { - std::stringstream ss; - ss << "The HS3 version is missing in the JSON!\n" - << "Please include the HS3 version in the metadata field, e.g.:\n" - << " \"metadata\" :\n" - << " {\n" - << " \"hs3_version\" : \"" << hs3VersionTag << "\"\n" - << " }"; - error(ss.str()); - } - if (versionNode->is_container() || versionNode->is_null() || versionNode->is_number()) { - error(std::string("The HS3 version in the metadata must be a string, e.g. \"") + hs3VersionTag + "\""); - } - _hs3VersionInput = versionNode->val(); - checkHS3Version(_hs3VersionInput); - checkUniqueNames(n, !isHS3VersionBefore(_hs3VersionInput, 0, 3)); - - _rootnodeInput = &n; - - _attributesNode = findRooFitInternal(*_rootnodeInput, "attributes"); - - _domains = std::make_unique(); - if (auto domains = n.find("domains")) { - _domains->readJSON(*domains); - } - _domains->populate(_workspace); - - // Build name-keyed indices over the "functions" and "distributions" - // sequences. Without these, every cross-reference resolved during import - // (e.g. dependencies of a PiecewiseInterpolation, or factory-expression - // arguments) triggers a linear scan over all sibling nodes via - // findNamedChild(), which becomes O(N^2) on workspaces with thousands of - // entries. Populating the maps up-front turns each lookup into O(1). - auto buildIndex = [&n](const char *key, auto &index) { - index.clear(); - auto seq = n.find(key); - if (!seq || !seq->is_seq()) - return; - index.reserve(seq->num_children()); - for (const auto &p : seq->children()) { - index.emplace(RooJSONFactoryWSTool::name(p), &p); + HS3::ParseOptions options; + // A user can replace how HistFactory distributions are imported by registering their own importer. + auto const &importers = RooFit::JSONIO::importers(); + auto histFactoryImporters = importers.find("histfactory_dist"); + options.typedHistFactory = histFactoryImporters != importers.end() && !histFactoryImporters->second.empty() && + HS3::isBuiltinHistFactoryImporter(*histFactoryImporters->second.front()); + + HS3::Diagnostics diagnostics; + HS3::upgradeDocument(n, diagnostics, options); + HS3Document parsed = HS3::parseDocument(n, diagnostics, options); + std::optional validated; + if (!HS3::hasErrors(diagnostics)) { + HS3::ValidateOptions validateOptions; + validateOptions.strict = !isHS3VersionBefore(parsed.metadata.hs3Version, 0, 3); + validateOptions.isExternal = [this](std::string const &name) { return _workspace.obj(name) != nullptr; }; + validated = HS3::validate(std::move(parsed), diagnostics, validateOptions); + } + HS3::report(diagnostics); + HS3Document const &doc = validated->doc(); + + // The state of the import refers to the document, so it must not survive it, not even if there is an exception. + struct ImportScope { + RooJSONFactoryWSTool &tool; + ~ImportScope() + { + tool._import.reset(); + tool._hs3VersionInput.clear(); } - }; - buildIndex("functions", _functionsByName); - buildIndex("distributions", _distributionsByName); + } importScope{*this}; - this->importDependants(n); + _hs3VersionInput = doc.metadata.hs3Version; + + _import = std::make_unique(doc); + + createVariables(_workspace, _import->document.domains, _import->ranges); + + if (_import->defaultValues) { + for (auto const &p : _import->defaultValues->parameters) { + importVariable(p); + } + } + for (auto const &function : doc.functions) { + this->importFunction(*function.body.node, true); + } + for (auto const &distribution : doc.distributions) { + this->importDistribution(distribution); + } // This creates the index categories, which the parameter points can refer to. - combinePdfs(*_rootnodeInput, _workspace); - - if (auto paramPointsNode = n.find("parameter_points")) { - for (const auto &snsh : paramPointsNode->children()) { - std::string name = RooJSONFactoryWSTool::name(snsh); - RooJSONFactoryWSTool::testValidName(name, true); - - RooArgSet vars; - for (const auto &var : snsh["parameters"].children()) { - const std::string varName = RooJSONFactoryWSTool::name(var); - JSONNode const *valueNode = var.find("value"); - const bool hasStringValue = valueNode && !valueNode->is_number(); - if (RooCategory *cat = _workspace.cat(varName)) { - if (hasStringValue) { - cat->setLabel(valueNode->val().c_str()); - } else if (valueNode) { - cat->setIndex(valueNode->val_int()); - } - vars.add(*cat); - continue; - } - if (hasStringValue) { - warning("RooFitHS3: ignoring the string value of parameter '" + varName + "' in parameter point '" + - name + "', because there is no category with that name."); - continue; - } - checkFixedValue(varName, var, name); - RooRealVar *rrv = _workspace.var(varName); - if (!rrv && !_workspace.arg(varName)) { - // The parameter is not used by any other component: create it. "const": false is the same as no - // "const" at all. - testValidName(varName, true); - rrv = &wsEmplace(varName, 1.); - configureVariable(*_domains, var, *rrv); - warnIfFreeWithoutDomain(*rrv); - } else if (rrv) { - configureVariable(*_domains, var, *rrv); - } - if (rrv) { - vars.add(*rrv); + combinePdfs(doc, _workspace); + + for (auto const &point : doc.parameterPoints) { + RooJSONFactoryWSTool::testValidName(point.name, true); + + RooArgSet vars; + for (auto const &var : point.parameters) { + std::string const *label = var.value ? std::get_if(&*var.value) : nullptr; + if (RooCategory *cat = _workspace.cat(var.name)) { + if (label) { + cat->setLabel(label->c_str()); + } else if (var.value) { + cat->setIndex(static_cast(std::get(*var.value))); } + vars.add(*cat); + continue; + } + if (label) { + warning("RooFitHS3: ignoring the string value of parameter '" + var.name + "' in parameter point '" + + point.name + "', because there is no category with that name."); + continue; + } + RooRealVar *rrv = _workspace.var(var.name); + if (!rrv && !_workspace.arg(var.name)) { + // The parameter is not used by any other component: create it. "const": false is the same as no + // "const" at all. + testValidName(var.name, true); + rrv = &wsEmplace(var.name, 1.); + configureVariable(_import->ranges, var, *rrv); + warnIfFreeWithoutDomain(*rrv); + } else if (rrv) { + configureVariable(_import->ranges, var, *rrv); + } + if (rrv) { + vars.add(*rrv); } - _workspace.saveSnapshot(name, vars); } + _workspace.saveSnapshot(point.name, vars); } // Import attributes - if (_attributesNode) { - for (const auto &elem : _attributesNode->children()) { - if (RooAbsArg *arg = _workspace.arg(elem.key())) - importAttributes(arg, elem); - } + for (auto const &item : doc.rooFit.attributes) { + if (RooAbsArg *arg = _workspace.arg(item.first)) + importAttributes(arg, item.second); } - _attributesNode = nullptr; - // We delay the import of the data to after combineDatasets(), because it // might be that some datasets are merged to combined datasets there. In // that case, we will remove the components from the "datasets" vector so they // don't get imported. std::vector> datasets; - if (auto dataNode = n.find("data")) { - for (const auto &p : dataNode->children()) { - datasets.push_back(loadData(p, _workspace, _domains.get())); - } + for (auto const &data : doc.data) { + datasets.push_back(loadData(data, _workspace, &_import->ranges)); } // Now, read in analyses and likelihoods if there are any - if (auto analysesNode = n.find("analyses")) { - for (JSONNode const &analysisNode : analysesNode->children()) { - importAnalysis(*_rootnodeInput, analysisNode, n["likelihoods"], n["domains"], _workspace, datasets); - } + for (auto const &analysis : doc.analyses) { + importAnalysis(doc, analysis, _workspace, datasets); } - combineDatasets(*_rootnodeInput, datasets); + combineDatasets(doc, datasets); for (auto const &d : datasets) { if (d) { @@ -2273,11 +2422,10 @@ void RooJSONFactoryWSTool::importAllNodes(const JSONNode &n) } } - _rootnodeInput = nullptr; - _hs3VersionInput.clear(); - _domains.reset(); - _functionsByName.clear(); - _distributionsByName.clear(); + if (_workspace.getSnapshot("default_values")) { + _workspace.loadSnapshot("default_values"); + } + importParameterStepWidths(_workspace, doc); } /** @@ -2290,12 +2438,7 @@ bool RooJSONFactoryWSTool::importJSON(std::istream &is) { // import a JSON file to the workspace std::unique_ptr tree = JSONTree::create(is); - JSONNode const &rootnode = tree->rootnode(); - this->importAllNodes(rootnode); - if (this->workspace()->getSnapshot("default_values")) { - this->workspace()->loadSnapshot("default_values"); - } - importParameterStepWidths(*this->workspace(), rootnode); + this->importAllNodes(tree->rootnode()); return true; } @@ -2351,8 +2494,11 @@ void RooJSONFactoryWSTool::importJSONElement(const std::string &name, const std: if (isVariable) { this->importVariableElement(n); } else if (isData) { - auto absData = loadData(n, _workspace, nullptr); - _workspace.import(*absData, RooFit::Silence()); + HS3::Diagnostics diagnostics; + HS3::Data data; + readValue(HS3::Cursor{n, diagnostics}, data); + HS3::report(diagnostics); + _workspace.import(*loadData(data, _workspace, nullptr), RooFit::Silence()); } else { this->importFunction(n, false); } @@ -2360,36 +2506,37 @@ void RooJSONFactoryWSTool::importJSONElement(const std::string &name, const std: void RooJSONFactoryWSTool::importVariableElement(const JSONNode &elementNode) { - std::unique_ptr tree = varJSONString(elementNode); - JSONNode &n = tree->rootnode(); - _domains = std::make_unique(); - if (auto domains = n.find("domains")) - _domains->readJSON(*domains); - - _rootnodeInput = &n; - _attributesNode = findRooFitInternal(*_rootnodeInput, "attributes"); - - JSONNode const *varsNode = getVariablesNode(n); - const auto &p = varsNode->child(0); - importVariable(p); - - auto paramPointsNode = n.find("parameter_points"); - const auto &var = paramPointsNode->child(0)["parameters"].child(0); - if (RooRealVar *rrv = _workspace.var(RooJSONFactoryWSTool::name(var))) { - configureVariable(*_domains, var, *rrv); - } - - // Import attributes - if (_attributesNode) { - for (const auto &elem : _attributesNode->children()) { - if (RooAbsArg *arg = _workspace.arg(elem.key())) - importAttributes(arg, elem); - } + // A variable is given by its "value", by its range "min" and "max", or by both. With only a value, it is constant. + JSONNode const *value = elementNode.find("value"); + JSONNode const *min = elementNode.find("min"); + JSONNode const *max = elementNode.find("max"); + if (!value && !(min && max)) { + throw std::invalid_argument("Invalid Syntax: Please provide either 'value' or 'min' and 'max' or both"); + } + HS3::DomainAxis axis; + axis.name = elementNode["name"].val(); + HS3::ParameterValue parameter; + parameter.name = axis.name; + if (min && max) { + axis.min = min->val_double(); + axis.max = max->val_double(); + } + parameter.value = value ? value->val_double() : (*axis.max + *axis.min) / 2; + parameter.isConst = value && !(min && max); + HS3Document doc; + doc.domains.push_back({HS3::defaultDomainName, {std::move(axis)}}); + doc.parameterPoints.push_back({"default_values", {parameter}}); + + _import = std::make_unique(doc); + struct ImportScope { + RooJSONFactoryWSTool &tool; + ~ImportScope() { tool._import.reset(); } + } importScope{*this}; + + importVariable(parameter); + if (RooRealVar *rrv = _workspace.var(parameter.name)) { + configureVariable(_import->ranges, parameter, *rrv); } - - _attributesNode = nullptr; - _rootnodeInput = nullptr; - _domains.reset(); } /** diff --git a/roofit/hs3/test/testRooFitHS3.cxx b/roofit/hs3/test/testRooFitHS3.cxx index 07da2618ccf5c..15648932e00ee 100644 --- a/roofit/hs3/test/testRooFitHS3.cxx +++ b/roofit/hs3/test/testRooFitHS3.cxx @@ -49,6 +49,7 @@ #include #include +#include #include #include #include @@ -4019,7 +4020,7 @@ TEST(RooFitHS3, HistFactoryLogNormalGammaConstraint) for (std::string key : {"constraint", "constraint_type"}) { expectImportThrowsWithError(makeHistFactoryJSON(R"({"name": "shape", "type": "shapesys", ")" + key + R"(": ")" + type + R"(", "data": {"vals": [0.1, 0.2]}})"), - "unsupported constraint '" + type + "'"); + "unsupported constraint 'LogNormal'"); } } @@ -4986,6 +4987,63 @@ TEST(RooFitHS3, HistFactoryLegacyBinWidthFactor) EXPECT_EQ(normfactorsOf(jsonWithFactors), renamedFactors.size()) << jsonWithFactors; } +// If the exported document is not valid, this is reported, but everything that can be exported is still written. Here +// the pdf of the ModelConfig has no exporter, so the likelihood refers to a distribution that doesn't exist. +TEST(RooFitHS3, InvalidExportIsStillWritten) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + + RooWorkspace ws{"ws"}; + ws.factory("Voigtian::voigt(x[0, -10, 10], mu[1, -5, 5], width[1, 0.1, 5], sigma[1, 0.1, 5])"); + ws.factory("Gaussian::gauss(x, mu, sigma)"); + RooDataSet data{"data", "data", *ws.var("x")}; + data.add(*ws.var("x")); + ws.import(data); + ws.saveSnapshot("snap", *ws.var("mu")); + RooFit::ModelConfig mc{"mc", &ws}; + mc.SetPdf("voigt"); + mc.SetParametersOfInterest("mu"); + mc.SetObservables("x"); + ws.import(mc); + + std::string json; + const std::string warnings = captureMessages(RooFit::WARNING, RooFit::ObjectHandling, + [&] { json = RooJSONFactoryWSTool{ws}.exportJSONtoString(); }); + EXPECT_NE(warnings.find("the exported document is not valid: the distribution 'voigt' does not exist"), + std::string::npos) + << warnings; + + auto tree = RooFit::Detail::JSONTree::create(json); + auto const &root = tree->rootnode(); + EXPECT_EQ(root["metadata"]["hs3_version"].val(), "0.3") << json; + for (const char *section : {"domains", "parameter_points", "data", "likelihoods", "analyses"}) { + ASSERT_TRUE(root.has_child(section)) << section << ": " << json; + EXPECT_GT(root[section].num_children(), 0u) << section << ": " << json; + } + EXPECT_NE(RooJSONFactoryWSTool::findNamedChild(root["parameter_points"], "snap"), nullptr) << json; + EXPECT_NE(RooJSONFactoryWSTool::findNamedChild(root["distributions"], "gauss"), nullptr) << json; +} + +// A named RooConstVar is exported as a fixed domain axis. JSON can't represent a value that is not finite, which is +// reported on export, and such an axis is not read back as a parameter without range. +TEST(RooFitHS3, NonFiniteRooConstVar) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + + RooWorkspace ws{"ws"}; + RooRealVar x{"x", "x", 0.0, -10.0, 10.0}; + RooConstVar cinf{"cinf", "cinf", std::numeric_limits::infinity()}; + RooFormulaVar formula{"formula", "x[0] + x[1]", {x, cinf}}; + ws.import(formula, RooFit::Silence()); + + std::string json; + const std::string warnings = captureMessages(RooFit::WARNING, RooFit::ObjectHandling, + [&] { json = RooJSONFactoryWSTool{ws}.exportJSONtoString(); }); + EXPECT_NE(warnings.find("the single value of the parameter 'cinf' is not finite"), std::string::npos) << warnings; + + expectImportThrowsWithError(json, "must be a number"); +} + // A fixed domain axis (HS3 #131): the parameter only takes the given "value", and RooFit reads it as a RooConstVar. // It needs no entry in a parameter point. A parameter point must not give it another value, the axis has no range, // binning or categories, and all domains that list the parameter give it the same value. @@ -5048,6 +5106,7 @@ TEST(RooFitHS3, FixedDomainAxis) "has \"value\" and"); } expectImportThrowsWithError(document(R"({"name": "mZ", "value": "heavy"})"), "must be a number"); + expectImportThrowsWithError(document(R"({"name": "mZ", "value": null})"), "must be a number"); // All domains that list the parameter give it the same single value. expectImportThrowsWithError( @@ -5058,6 +5117,10 @@ TEST(RooFitHS3, FixedDomainAxis) document(R"({"name": "mZ", "value": 91.188})", R"(, {"name": "other", "type": "product_domain", "axes": [{"name": "mZ", "min": 80.0, "max": 100.0}]})"), "but a range in the domain 'other'"); + expectImportThrowsWithError( + document(R"({"name": "mZ", "value": 91.188})", + R"(, {"name": "other", "type": "product_domain", "axes": [{"name": "mZ", "categories": ["a", "b"]}]})"), + "but categories in the domain 'other'"); // A parameter without domain is a RooRealVar: constant with "const": true, otherwise free, with a warning. {