[QC-1020] Custom Parameters for Post-Processing (#2041)

Barthelemy · knopers8 · web-flow · commit 650bc5d94377 · 2023-12-20T17:30:24.000+01:00
* [QC-1020] CP for PP

* format

* remove cout

* Update Framework/src/PostProcessingConfig.cxx

Co-authored-by: Piotr Konopka &lt;piotr.jan.konopka@cern.ch&gt;

* Update PostProcessingConfig.cxx

* Extract the parsing of the config for the Custom Parameters

* fix test

* rename

---------

Co-authored-by: Piotr Konopka &lt;piotr.jan.konopka@cern.ch&gt;
diff --git a/Framework/CMakeLists.txt b/Framework/CMakeLists.txt
@@ -323,7 +323,7 @@ endforeach()
 
 foreach(t testWorkflow testTaskRunner testCheckWorkflow
         testPostProcessingConfig testPostProcessingInterface
-        testCheck testTrendingTask)
+        testCheck testTrendingTask testCustomParameters)
   target_sources(${t} PRIVATE
                  ${CMAKE_BINARY_DIR}/getTestDataDirectory.cxx)
   target_include_directories(${t} PRIVATE ${CMAKE_SOURCE_DIR})
diff --git a/Framework/include/QualityControl/CustomParameters.h b/Framework/include/QualityControl/CustomParameters.h
@@ -22,6 +22,7 @@
 #include <string>
 #include <unordered_map>
 #include <optional>
+#include <boost/property_tree/ptree_fwd.hpp>
 
 namespace o2::quality_control::core
 {
@@ -167,6 +168,12 @@ class CustomParameters
    */
   friend std::ostream& operator<<(std::ostream& out, const CustomParameters& customParameters);
 
+  /**
+   * \brief Provided the config subtree of the custom parameters, load its content and populate this CustomParameters.
+   * \param paramsTree The subtree corresponding to extendedTaskParameters, extendedCheckParameters, etc...
+   */
+  void populateCustomParameters(const boost::property_tree::ptree& paramsTree);
+
  private:
   CustomParametersType mCustomParameters;
 };
diff --git a/Framework/include/QualityControl/PostProcessingConfig.h b/Framework/include/QualityControl/PostProcessingConfig.h
@@ -17,6 +17,8 @@
 #ifndef QUALITYCONTROL_POSTPROCESSINGCONFIG_H
 #define QUALITYCONTROL_POSTPROCESSINGCONFIG_H
 
+#include "QualityControl/CustomParameters.h"
+
 #include <vector>
 #include <string>
 #include <boost/property_tree/ptree_fwd.hpp>
@@ -45,6 +47,7 @@ struct PostProcessingConfig {
   std::string consulUrl;
   core::Activity activity;
   bool matchAnyRunNumber = false;
+  core::CustomParameters customParameters;
 };
 
 } // namespace o2::quality_control::postprocessing
diff --git a/Framework/include/QualityControl/PostProcessingInterface.h b/Framework/include/QualityControl/PostProcessingInterface.h
@@ -17,6 +17,8 @@
 #ifndef QUALITYCONTROL_POSTPROCESSINTERFACE_H
 #define QUALITYCONTROL_POSTPROCESSINTERFACE_H
 
+#include "QualityControl/CustomParameters.h"
+
 #include <string>
 #include <boost/property_tree/ptree_fwd.hpp>
 #include "QualityControl/Triggers.h"
@@ -65,6 +67,7 @@ class PostProcessingInterface
   /// \param services Interface containing optional interfaces, for example DatabaseInterface
   virtual void finalize(Trigger trigger, framework::ServiceRegistryRef services) = 0;
 
+  void setCustomParameters(const core::CustomParameters& parameters);
   void setObjectsManager(std::shared_ptr<core::ObjectsManager> objectsManager);
   void setID(const std::string& id);
   [[nodiscard]] const std::string& getID() const;
@@ -73,6 +76,7 @@ class PostProcessingInterface
 
  protected:
   std::shared_ptr<core::ObjectsManager> getObjectsManager();
+  core::CustomParameters mCustomParameters;
 
  private:
   std::string mID;
diff --git a/Framework/include/QualityControl/PostProcessingTaskSpec.h b/Framework/include/QualityControl/PostProcessingTaskSpec.h
@@ -43,6 +43,7 @@ struct PostProcessingTaskSpec {
   bool active = true;
   std::string detectorName = "Invalid";
   boost::property_tree::ptree tree = {};
+  core::CustomParameters customParameters;
 };
 
 } // namespace o2::quality_control::core
diff --git a/Framework/src/CustomParameters.cxx b/Framework/src/CustomParameters.cxx
@@ -12,6 +12,7 @@
 #include "QualityControl/CustomParameters.h"
 #include <DataFormatsParameters/ECSDataAdapters.h>
 #include <iostream>
+#include <boost/property_tree/ptree.hpp>
 #include <string_view>
 #include <vector>
 
@@ -33,7 +34,7 @@ std::ostream& operator<<(std::ostream& out, const CustomParameters& customParame
 
 CustomParameters::CustomParameters()
 {
-  mCustomParameters["default"]["default"] = {};
+  mCustomParameters["null"]["null"] = {};
 }
 
 void CustomParameters::set(const std::string& key, const std::string& value, const std::string& runType, const std::string& beamType)
@@ -154,7 +155,7 @@ std::unordered_map<std::string, std::string>::const_iterator CustomParameters::f
 
 std::unordered_map<std::string, std::string>::const_iterator CustomParameters::end() const
 {
-  return mCustomParameters.at("default").at("default").end();
+  return mCustomParameters.at("null").at("null").end();
 }
 
 std::string CustomParameters::operator[](const std::string& key) const
@@ -170,4 +171,15 @@ std::string& CustomParameters::operator[](const std::string& key)
   return mCustomParameters.at("default").at("default").at(key);
 }
 
+void CustomParameters::populateCustomParameters(const boost::property_tree::ptree& tree)
+{
+  for (const auto& [runtype, subTreeRunType] : tree) {
+    for (const auto& [beamtype, subTreeBeamType] : subTreeRunType) {
+      for (const auto& [key, value] : subTreeBeamType) {
+        set(key, value.get_value<std::string>(), runtype, beamtype);
+      }
+    }
+  }
+}
+
 } // namespace o2::quality_control::core
diff --git a/Framework/src/InfrastructureSpecReader.cxx b/Framework/src/InfrastructureSpecReader.cxx
@@ -117,13 +117,7 @@ TaskSpec InfrastructureSpecReader::readSpecEntry<TaskSpec>(const std::string& ta
     ILOG(Warning, Devel) << "Both taskParameters and extendedTaskParameters are defined in the QC config file. We will use only extendedTaskParameters. " << ENDM;
   }
   if (taskTree.count("extendedTaskParameters") > 0) {
-    for (const auto& [runtype, subTreeRunType] : taskTree.get_child("extendedTaskParameters")) {
-      for (const auto& [beamtype, subTreeBeamType] : subTreeRunType) {
-        for (const auto& [key, value] : subTreeBeamType) {
-          ts.customParameters.set(key, value.get_value<std::string>(), runtype, beamtype);
-        }
-      }
-    }
+    ts.customParameters.populateCustomParameters(taskTree.get_child("extendedTaskParameters"));
   } else if (taskTree.count("taskParameters") > 0) {
     for (const auto& [key, value] : taskTree.get_child("taskParameters")) {
       ts.customParameters.set(key, value.get_value<std::string>());
@@ -316,13 +310,7 @@ CheckSpec InfrastructureSpecReader::readSpecEntry<CheckSpec>(const std::string&
 
   cs.active = checkTree.get<bool>("active", cs.active);
   if (checkTree.count("extendedCheckParameters") > 0) {
-    for (const auto& [runtype, subTreeRunType] : checkTree.get_child("extendedCheckParameters")) {
-      for (const auto& [beamtype, subTreeBeamType] : subTreeRunType) {
-        for (const auto& [key, value] : subTreeBeamType) {
-          cs.customParameters.set(key, value.get_value<std::string>(), runtype, beamtype);
-        }
-      }
-    }
+    cs.customParameters.populateCustomParameters(checkTree.get_child("extendedCheckParameters"));
   }
   if (checkTree.count("checkParameters") > 0) {
     for (const auto& [key, value] : checkTree.get_child("checkParameters")) {
@@ -356,13 +344,7 @@ AggregatorSpec InfrastructureSpecReader::readSpecEntry<AggregatorSpec>(const std
 
   as.active = aggregatorTree.get<bool>("active", as.active);
   if (aggregatorTree.count("extendedAggregatorParameters") > 0) {
-    for (const auto& [runtype, subTreeRunType] : aggregatorTree.get_child("extendedAggregatorParameters")) {
-      for (const auto& [beamtype, subTreeBeamType] : subTreeRunType) {
-        for (const auto& [key, value] : subTreeBeamType) {
-          as.customParameters.set(key, value.get_value<std::string>(), runtype, beamtype);
-        }
-      }
-    }
+    as.customParameters.populateCustomParameters(aggregatorTree.get_child("extendedAggregatorParameters"));
   }
   if (aggregatorTree.count("aggregatorParameters") > 0) {
     for (const auto& [key, value] : aggregatorTree.get_child("aggregatorParameters")) {
diff --git a/Framework/src/PostProcessingConfig.cxx b/Framework/src/PostProcessingConfig.cxx
@@ -15,6 +15,7 @@
 ///
 
 #include "QualityControl/PostProcessingConfig.h"
+
 #include <boost/property_tree/ptree.hpp>
 
 namespace o2::quality_control::postprocessing
@@ -47,6 +48,10 @@ PostProcessingConfig::PostProcessingConfig(const std::string& id, const boost::p
   for (const auto& stopTrigger : config.get_child("qc.postprocessing." + id + ".stopTrigger")) {
     stopTriggers.push_back(stopTrigger.second.get_value<std::string>());
   }
+  auto ppTree = config.get_child("qc.postprocessing." + id);
+  if (ppTree.count("extendedTaskParameters")) {
+    customParameters.populateCustomParameters(ppTree.get_child("extendedTaskParameters"));
+  }
 }
 
-} // namespace o2::quality_control::postprocessing
+} // namespace o2::quality_control::postprocessing
diff --git a/Framework/src/PostProcessingFactory.cxx b/Framework/src/PostProcessingFactory.cxx
@@ -27,7 +27,9 @@ namespace o2::quality_control::postprocessing
 // todo: consider having a common helper for each class which loads tasks like below (QC tasks, checks, pp)
 PostProcessingInterface* PostProcessingFactory::create(const PostProcessingConfig& config)
 {
-  return root_class_factory::create<PostProcessingInterface>(config.moduleName, config.className);
+  auto* result = root_class_factory::create<PostProcessingInterface>(config.moduleName, config.className);
+  result->setCustomParameters(config.customParameters);
+  return result;
 }
 
 } // namespace o2::quality_control::postprocessing
diff --git a/Framework/src/PostProcessingInterface.cxx b/Framework/src/PostProcessingInterface.cxx
@@ -50,4 +50,9 @@ std::shared_ptr<core::ObjectsManager> PostProcessingInterface::getObjectsManager
   return mObjectsManager;
 }
 
+void PostProcessingInterface::setCustomParameters(const core::CustomParameters& parameters)
+{
+  mCustomParameters = parameters;
+}
+
 } // namespace o2::quality_control::postprocessing
diff --git a/Framework/src/PostProcessingRunner.cxx b/Framework/src/PostProcessingRunner.cxx
@@ -106,6 +106,7 @@ void PostProcessingRunner::init(const PostProcessingRunnerConfig& runnerConfig,
     mTaskState = TaskState::Created;
     mTask->setID(mTaskConfig.id);
     mTask->setName(mTaskConfig.taskName);
+    mTask->setCustomParameters(mTaskConfig.customParameters);
     mTask->configure(mRunnerConfig.configTree);
   } else {
     throw std::runtime_error("Failed to create the task '" + mTaskConfig.taskName + "' (det " + mTaskConfig.detectorName + ")");
diff --git a/Framework/test/testCustomParameters.cxx b/Framework/test/testCustomParameters.cxx
@@ -21,6 +21,9 @@
 #define BOOST_TEST_DYN_LINK
 
 #include <boost/test/unit_test.hpp>
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/json_parser.hpp>
+#include "getTestDataDirectory.h"
 
 using namespace o2::quality_control::core;
 using namespace std;
@@ -183,6 +186,27 @@ BOOST_AUTO_TEST_CASE(test_cp_new_access_pattern)
   }
 }
 
+BOOST_AUTO_TEST_CASE(test_load_from_ptree)
+{
+  boost::property_tree::ptree jsontree;
+  std::string configFilePath = std::string(getTestDataDirectory()) + "testWorkflow.json";
+
+  boost::property_tree::read_json(configFilePath, jsontree);
+
+  string v0 = jsontree.get<string>("qc.tasks.skeletonTask.extendedTaskParameters.default.default.myOwnKey");
+
+  boost::property_tree::ptree params = jsontree.get_child("qc.tasks.skeletonTask.extendedTaskParameters");
+
+  CustomParameters cp;
+  cp.populateCustomParameters(params);
+
+  cout << cp << endl;
+
+  BOOST_CHECK_EQUAL(cp.at("myOwnKey"), "myOwnValue");
+  BOOST_CHECK_EQUAL(cp.at("myOwnKey1", "PHYSICS"), "myOwnValue1b");
+  BOOST_CHECK_EQUAL(cp.atOptional("asdf").has_value(), false);
+}
+
 BOOST_AUTO_TEST_CASE(test_default_if_not_found_at_optional)
 {
   CustomParameters cp;
diff --git a/Framework/test/testWorkflow.json b/Framework/test/testWorkflow.json
@@ -25,7 +25,20 @@
           "type": "dataSamplingPolicy",
           "name": "test-policy"
         },
-        "taskParameters": {},
+        "extendedTaskParameters": {
+          "default": {
+            "default": {
+              "myOwnKey": "myOwnValue",
+              "myOwnKey2": "myOwnValue2"
+            }
+          },
+          "PHYSICS": {
+            "default": {
+              "myOwnKey1": "myOwnValue1b",
+              "myOwnKey2": "myOwnValue2b"
+            }
+          }
+        },
         "location": "remote",
         "localMachines": []
       }
diff --git a/doc/PostProcessing.md b/doc/PostProcessing.md
@@ -142,6 +142,12 @@ Checks can be applied to the results of Post-processing Tasks just as for normal
 ...
 ```
 
+## Definition and access of user-specific configuration
+
+A postprocessing task can access custom parameters declared in the configuration file at `qc.postprocessing.<task_id>.extendedTaskParameters`. They are stored inside an object of type `CustomParameters` named `mCustomParameters`, which is a protected member of `TaskInterface`.
+
+[More details](Advanced.md#definition-and-access-of-user-specific-configuration) can be found about this feature in the Tasks (same behaviour). 
+
 #### Triggers configuration
 
 Each of the three methods can be invoked by one or more triggers. Below are listed the possible options (case insensitive).

Original file line number	Diff line number	Diff line change
`@@ -27,7 +27,9 @@ namespace o2::quality_control::postprocessing`
`27`	`27`	`// todo: consider having a common helper for each class which loads tasks like below (QC tasks, checks, pp)`
`28`	`28`	`PostProcessingInterface* PostProcessingFactory::create(const PostProcessingConfig& config)`
`29`	`29`	`{`
`30`		`- return root_class_factory::create<PostProcessingInterface>(config.moduleName, config.className);`
	`30`	`+ auto* result = root_class_factory::create<PostProcessingInterface>(config.moduleName, config.className);`
	`31`	`+ result->setCustomParameters(config.customParameters);`
	`32`	`+ return result;`
`31`	`33`	`}`
`32`	`34`
`33`	`35`	`} // namespace o2::quality_control::postprocessing`
Original file line number	Diff line number	Diff line change
`@@ -50,4 +50,9 @@ std::shared_ptr<core::ObjectsManager> PostProcessingInterface::getObjectsManager`
`50`	`50`	`return mObjectsManager;`
`51`	`51`	`}`
`52`	`52`
	`53`	`+void PostProcessingInterface::setCustomParameters(const core::CustomParameters& parameters)`
	`54`	`+{`
	`55`	`+ mCustomParameters = parameters;`
	`56`	`+}`
	`57`	`+`
`53`	`58`	`} // namespace o2::quality_control::postprocessing`