[QC-1020] Custom Parameters for Post-Processing

Framework/CMakeLists.txt

@@ -323,7 +323,7 @@ endforeach()
 
 foreach(t testWorkflow testTaskRunner testCheckWorkflow
         testPostProcessingConfig testPostProcessingInterface
-        testCheck testTrendingTask)
+        testCheck testTrendingTask testCustomParameters)
   target_sources(${t} PRIVATE
                  ${CMAKE_BINARY_DIR}/getTestDataDirectory.cxx)
   target_include_directories(${t} PRIVATE ${CMAKE_SOURCE_DIR})

Framework/include/QualityControl/CustomParameters.h

@@ -22,6 +22,7 @@
 #include <string>
 #include <unordered_map>
 #include <optional>
+#include <boost/property_tree/ptree_fwd.hpp>
 
 namespace o2::quality_control::core
 {
@@ -167,6 +168,12 @@ class CustomParameters
    */
   friend std::ostream& operator<<(std::ostream& out, const CustomParameters& customParameters);
 
+  /**
+   * \brief Provided the config subtree of the custom parameters, load its content and populate this CustomParameters.
+   * \param paramsTree The subtree corresponding to extendedTaskParameters, extendedCheckParameters, etc...
+   */
+  void populateCustomParameters(const boost::property_tree::ptree& paramsTree);
+
  private:
   CustomParametersType mCustomParameters;
 };

Framework/include/QualityControl/PostProcessingConfig.h

@@ -17,6 +17,8 @@
 #ifndef QUALITYCONTROL_POSTPROCESSINGCONFIG_H
 #define QUALITYCONTROL_POSTPROCESSINGCONFIG_H
 
+#include "QualityControl/CustomParameters.h"
+
 #include <vector>
 #include <string>
 #include <boost/property_tree/ptree_fwd.hpp>
@@ -45,6 +47,7 @@ struct PostProcessingConfig {
   std::string consulUrl;
   core::Activity activity;
   bool matchAnyRunNumber = false;
+  core::CustomParameters customParameters;
 };
 
 } // namespace o2::quality_control::postprocessing

Framework/include/QualityControl/PostProcessingInterface.h

@@ -17,6 +17,8 @@
 #ifndef QUALITYCONTROL_POSTPROCESSINTERFACE_H
 #define QUALITYCONTROL_POSTPROCESSINTERFACE_H
 
+#include "QualityControl/CustomParameters.h"
+
 #include <string>
 #include <boost/property_tree/ptree_fwd.hpp>
 #include "QualityControl/Triggers.h"
@@ -65,6 +67,7 @@ class PostProcessingInterface
   /// \param services Interface containing optional interfaces, for example DatabaseInterface
   virtual void finalize(Trigger trigger, framework::ServiceRegistryRef services) = 0;
 
+  void setCustomParameters(const core::CustomParameters& parameters);
   void setObjectsManager(std::shared_ptr<core::ObjectsManager> objectsManager);
   void setID(const std::string& id);
   [[nodiscard]] const std::string& getID() const;
@@ -73,6 +76,7 @@ class PostProcessingInterface
 
  protected:
   std::shared_ptr<core::ObjectsManager> getObjectsManager();
+  core::CustomParameters mCustomParameters;
 
  private:
   std::string mID;

Framework/include/QualityControl/PostProcessingTaskSpec.h

@@ -43,6 +43,7 @@ struct PostProcessingTaskSpec {
   bool active = true;
   std::string detectorName = "Invalid";
   boost::property_tree::ptree tree = {};
+  core::CustomParameters customParameters;
 };
 
 } // namespace o2::quality_control::core

Framework/src/CustomParameters.cxx

@@ -12,6 +12,7 @@
 #include "QualityControl/CustomParameters.h"
 #include <DataFormatsParameters/ECSDataAdapters.h>
 #include <iostream>
+#include <boost/property_tree/ptree.hpp>
 #include <string_view>
 #include <vector>
 
@@ -33,7 +34,7 @@ std::ostream& operator<<(std::ostream& out, const CustomParameters& customParame
 
 CustomParameters::CustomParameters()
 {
-  mCustomParameters["default"]["default"] = {};
+  mCustomParameters["null"]["null"] = {};
 }
 
 void CustomParameters::set(const std::string& key, const std::string& value, const std::string& runType, const std::string& beamType)
@@ -154,7 +155,7 @@ std::unordered_map<std::string, std::string>::const_iterator CustomParameters::f
 
 std::unordered_map<std::string, std::string>::const_iterator CustomParameters::end() const
 {
-  return mCustomParameters.at("default").at("default").end();
+  return mCustomParameters.at("null").at("null").end();
 }
 
 std::string CustomParameters::operator[](const std::string& key) const
@@ -170,4 +171,15 @@ std::string& CustomParameters::operator[](const std::string& key)
   return mCustomParameters.at("default").at("default").at(key);
 }
 
+void CustomParameters::populateCustomParameters(const boost::property_tree::ptree& tree)
+{
+  for (const auto& [runtype, subTreeRunType] : tree) {
+    for (const auto& [beamtype, subTreeBeamType] : subTreeRunType) {
+      for (const auto& [key, value] : subTreeBeamType) {
+        set(key, value.get_value<std::string>(), runtype, beamtype);
+      }
+    }
+  }
+}
+
 } // namespace o2::quality_control::core

Framework/src/InfrastructureSpecReader.cxx

@@ -117,13 +117,7 @@ TaskSpec InfrastructureSpecReader::readSpecEntry<TaskSpec>(const std::string& ta
     ILOG(Warning, Devel) << "Both taskParameters and extendedTaskParameters are defined in the QC config file. We will use only extendedTaskParameters. " << ENDM;
   }
   if (taskTree.count("extendedTaskParameters") > 0) {
-    for (const auto& [runtype, subTreeRunType] : taskTree.get_child("extendedTaskParameters")) {
-      for (const auto& [beamtype, subTreeBeamType] : subTreeRunType) {
-        for (const auto& [key, value] : subTreeBeamType) {
-          ts.customParameters.set(key, value.get_value<std::string>(), runtype, beamtype);
-        }
-      }
-    }
+    ts.customParameters.populateCustomParameters(taskTree.get_child("extendedTaskParameters"));
   } else if (taskTree.count("taskParameters") > 0) {
     for (const auto& [key, value] : taskTree.get_child("taskParameters")) {
       ts.customParameters.set(key, value.get_value<std::string>());
@@ -316,13 +310,7 @@ CheckSpec InfrastructureSpecReader::readSpecEntry<CheckSpec>(const std::string&
 
   cs.active = checkTree.get<bool>("active", cs.active);
   if (checkTree.count("extendedCheckParameters") > 0) {
-    for (const auto& [runtype, subTreeRunType] : checkTree.get_child("extendedCheckParameters")) {
-      for (const auto& [beamtype, subTreeBeamType] : subTreeRunType) {
-        for (const auto& [key, value] : subTreeBeamType) {
-          cs.customParameters.set(key, value.get_value<std::string>(), runtype, beamtype);
-        }
-      }
-    }
+    cs.customParameters.populateCustomParameters(checkTree.get_child("extendedCheckParameters"));
   }
   if (checkTree.count("checkParameters") > 0) {
     for (const auto& [key, value] : checkTree.get_child("checkParameters")) {
@@ -356,13 +344,7 @@ AggregatorSpec InfrastructureSpecReader::readSpecEntry<AggregatorSpec>(const std
 
   as.active = aggregatorTree.get<bool>("active", as.active);
   if (aggregatorTree.count("extendedAggregatorParameters") > 0) {
-    for (const auto& [runtype, subTreeRunType] : aggregatorTree.get_child("extendedAggregatorParameters")) {
-      for (const auto& [beamtype, subTreeBeamType] : subTreeRunType) {
-        for (const auto& [key, value] : subTreeBeamType) {
-          as.customParameters.set(key, value.get_value<std::string>(), runtype, beamtype);
-        }
-      }
-    }
+    as.customParameters.populateCustomParameters(aggregatorTree.get_child("extendedAggregatorParameters"));
   }
   if (aggregatorTree.count("aggregatorParameters") > 0) {
     for (const auto& [key, value] : aggregatorTree.get_child("aggregatorParameters")) {

Framework/src/PostProcessingConfig.cxx

@@ -15,6 +15,7 @@
 ///
 
 #include "QualityControl/PostProcessingConfig.h"
+
 #include <boost/property_tree/ptree.hpp>
 
 namespace o2::quality_control::postprocessing
@@ -47,6 +48,10 @@ PostProcessingConfig::PostProcessingConfig(const std::string& id, const boost::p
   for (const auto& stopTrigger : config.get_child("qc.postprocessing." + id + ".stopTrigger")) {
     stopTriggers.push_back(stopTrigger.second.get_value<std::string>());
   }
+  auto ppTree = config.get_child("qc.postprocessing." + id);
+  if (ppTree.count("extendedTaskParameters")) {
+    customParameters.populateCustomParameters(ppTree.get_child("extendedTaskParameters"));
+  }
 }
 
-} // namespace o2::quality_control::postprocessing
+} // namespace o2::quality_control::postprocessing

Framework/src/PostProcessingFactory.cxx

@@ -27,7 +27,9 @@ namespace o2::quality_control::postprocessing
 // todo: consider having a common helper for each class which loads tasks like below (QC tasks, checks, pp)
 PostProcessingInterface* PostProcessingFactory::create(const PostProcessingConfig& config)
 {
-  return root_class_factory::create<PostProcessingInterface>(config.moduleName, config.className);
+  auto* result = root_class_factory::create<PostProcessingInterface>(config.moduleName, config.className);
+  result->setCustomParameters(config.customParameters);
+  return result;
 }
 
 } // namespace o2::quality_control::postprocessing

Framework/src/PostProcessingInterface.cxx

@@ -50,4 +50,9 @@ std::shared_ptr<core::ObjectsManager> PostProcessingInterface::getObjectsManager
   return mObjectsManager;
 }
 
+void PostProcessingInterface::setCustomParameters(const core::CustomParameters& parameters)
+{
+  mCustomParameters = parameters;
+}
+
 } // namespace o2::quality_control::postprocessing

Framework/src/PostProcessingRunner.cxx

@@ -106,6 +106,7 @@ void PostProcessingRunner::init(const PostProcessingRunnerConfig& runnerConfig,
     mTaskState = TaskState::Created;
     mTask->setID(mTaskConfig.id);
     mTask->setName(mTaskConfig.taskName);
+    mTask->setCustomParameters(mTaskConfig.customParameters);
     mTask->configure(mRunnerConfig.configTree);
   } else {
     throw std::runtime_error("Failed to create the task '" + mTaskConfig.taskName + "' (det " + mTaskConfig.detectorName + ")");

Framework/test/testCustomParameters.cxx

@@ -21,6 +21,9 @@
 #define BOOST_TEST_DYN_LINK
 
 #include <boost/test/unit_test.hpp>
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/json_parser.hpp>
+#include "getTestDataDirectory.h"
 
 using namespace o2::quality_control::core;
 using namespace std;
@@ -183,6 +186,27 @@ BOOST_AUTO_TEST_CASE(test_cp_new_access_pattern)
   }
 }
 
+BOOST_AUTO_TEST_CASE(test_load_from_ptree)
+{
+  boost::property_tree::ptree jsontree;
+  std::string configFilePath = std::string(getTestDataDirectory()) + "testWorkflow.json";
+
+  boost::property_tree::read_json(configFilePath, jsontree);
+
+  string v0 = jsontree.get<string>("qc.tasks.skeletonTask.extendedTaskParameters.default.default.myOwnKey");
+
+  boost::property_tree::ptree params = jsontree.get_child("qc.tasks.skeletonTask.extendedTaskParameters");
+
+  CustomParameters cp;
+  cp.populateCustomParameters(params);
+
+  cout << cp << endl;
+
+  BOOST_CHECK_EQUAL(cp.at("myOwnKey"), "myOwnValue");
+  BOOST_CHECK_EQUAL(cp.at("myOwnKey1", "PHYSICS"), "myOwnValue1b");
+  BOOST_CHECK_EQUAL(cp.atOptional("asdf").has_value(), false);
+}
+
 BOOST_AUTO_TEST_CASE(test_default_if_not_found_at_optional)
 {
   CustomParameters cp;

Framework/test/testWorkflow.json

@@ -25,7 +25,20 @@
           "type": "dataSamplingPolicy",
           "name": "test-policy"
         },
-        "taskParameters": {},
+        "extendedTaskParameters": {
+          "default": {
+            "default": {
+              "myOwnKey": "myOwnValue",
+              "myOwnKey2": "myOwnValue2"
+            }
+          },
+          "PHYSICS": {
+            "default": {
+              "myOwnKey1": "myOwnValue1b",
+              "myOwnKey2": "myOwnValue2b"
+            }
+          }
+        },
         "location": "remote",
         "localMachines": []
       }

doc/PostProcessing.md

@@ -142,6 +142,12 @@ Checks can be applied to the results of Post-processing Tasks just as for normal
 ...
 ```
 
+## Definition and access of user-specific configuration
+
+A postprocessing task can access custom parameters declared in the configuration file at `qc.postprocessing.<task_id>.extendedTaskParameters`. They are stored inside an object of type `CustomParameters` named `mCustomParameters`, which is a protected member of `TaskInterface`.
+
+[More details](Advanced.md#definition-and-access-of-user-specific-configuration) can be found about this feature in the Tasks (same behaviour). 
+
 #### Triggers configuration
 
 Each of the three methods can be invoked by one or more triggers. Below are listed the possible options (case insensitive).