GPU: Clean up deletion of old calib objects

Author: davidrohr

GPU/Workflow/include/GPUWorkflow/GPUWorkflowSpec.h

@@ -28,6 +28,7 @@
 #include <vector>
 #include <mutex>
 #include <functional>
+#include <queue>
 
 class TStopwatch;
 namespace fair::mq
@@ -141,6 +142,14 @@ class GPURecoWorkflowSpec : public o2::framework::Task
   void deinitialize();
 
  private:
+  struct calibObjectStruct {
+    std::unique_ptr<TPCFastTransform> mFastTransform;
+    std::unique_ptr<TPCFastTransform> mFastTransformRef;
+    std::unique_ptr<o2::tpc::CorrectionMapsLoader> mFastTransformHelper;
+    std::unique_ptr<TPCPadGainCalib> mTPCPadGainCalib;
+    std::unique_ptr<o2::tpc::CalibdEdxContainer> mdEdxCalibContainer;
+  };
+
   /// initialize TPC options from command line
   void initFunctionTPCCalib(o2::framework::InitContext& ic);
   void initFunctionITS(o2::framework::InitContext& ic);
@@ -149,12 +158,12 @@ class GPURecoWorkflowSpec : public o2::framework::Task
   void finaliseCCDBITS(o2::framework::ConcreteDataMatcher& matcher, void* obj);
   /// asking for newer calib objects
   template <class T>
-  bool fetchCalibsCCDBTPC(o2::framework::ProcessingContext& pc, T& newCalibObjects);
+  bool fetchCalibsCCDBTPC(o2::framework::ProcessingContext& pc, T& newCalibObjects, calibObjectStruct& oldCalibObjects);
   bool fetchCalibsCCDBITS(o2::framework::ProcessingContext& pc);
-  /// storing the new calib objects by overwritting the old calibs
-  void cleanOldCalibsTPCPtrs();
+  /// delete old calib objects no longer needed
+  void cleanOldCalibsTPCPtrs(calibObjectStruct& oldCalibObjects);
 
-  void doCalibUpdates(o2::framework::ProcessingContext& pc);
+  void doCalibUpdates(o2::framework::ProcessingContext& pc, calibObjectStruct& oldCalibObjects);
 
   void doTrackTuneTPC(GPUTrackingInOutPointers& ptrs, char* buffout);
 
@@ -178,18 +187,12 @@ class GPURecoWorkflowSpec : public o2::framework::Task
   std::function<bool(o2::framework::DataProcessingHeader::StartTime)> mPolicyOrder;
   std::unique_ptr<GPUO2Interface> mGPUReco;
   std::unique_ptr<GPUDisplayFrontendInterface> mDisplayFrontend;
-  std::unique_ptr<TPCFastTransform> mFastTransform;
-  std::unique_ptr<TPCFastTransform> mFastTransformRef;
-  std::unique_ptr<TPCFastTransform> mFastTransformNew;
-  std::unique_ptr<TPCFastTransform> mFastTransformRefNew;
-  std::unique_ptr<o2::tpc::CorrectionMapsLoader> mFastTransformHelper;
-  std::unique_ptr<o2::tpc::CorrectionMapsLoader> mFastTransformHelperNew;
-
-  std::unique_ptr<TPCPadGainCalib> mTPCPadGainCalib;
+
+  calibObjectStruct mCalibObjects;
+  std::unique_ptr<o2::tpc::CalibdEdxContainer> mdEdxCalibContainerBufferNew;
   std::unique_ptr<TPCPadGainCalib> mTPCPadGainCalibBufferNew;
+  std::queue<calibObjectStruct> mOldCalibObjects;
   std::unique_ptr<TPCZSLinkMapping> mTPCZSLinkMapping;
-  std::unique_ptr<o2::tpc::CalibdEdxContainer> mdEdxCalibContainer;
-  std::unique_ptr<o2::tpc::CalibdEdxContainer> mdEdxCalibContainerBufferNew;
   std::unique_ptr<o2::tpc::VDriftHelper> mTPCVDriftHelper;
   std::unique_ptr<o2::trd::GeometryFlat> mTRDGeometry;
   std::unique_ptr<GPUO2InterfaceConfiguration> mConfig;
@@ -223,7 +226,6 @@ class GPURecoWorkflowSpec : public o2::framework::Task
   bool mITSGeometryCreated = false;
   bool mTRDGeometryCreated = false;
   bool mPropagatorInstanceCreated = false;
-  bool mMustUpdateFastTransform = false;
   bool mITSRunVertexer = false;
   bool mITSCosmicsProcessing = false;
   std::string mITSMode = "sync";

GPU/Workflow/src/GPUWorkflowSpec.cxx

@@ -247,9 +247,9 @@ void GPURecoWorkflowSpec::init(InitContext& ic)
     // initialize TPC calib objects
     initFunctionTPCCalib(ic);
 
-    mConfig->configCalib.fastTransform = mFastTransformHelper->getCorrMap();
-    mConfig->configCalib.fastTransformRef = mFastTransformHelper->getCorrMapRef();
-    mConfig->configCalib.fastTransformHelper = mFastTransformHelper.get();
+    mConfig->configCalib.fastTransform = mCalibObjects.mFastTransformHelper->getCorrMap();
+    mConfig->configCalib.fastTransformRef = mCalibObjects.mFastTransformHelper->getCorrMapRef();
+    mConfig->configCalib.fastTransformHelper = mCalibObjects.mFastTransformHelper.get();
     if (mConfig->configCalib.fastTransform == nullptr) {
       throw std::invalid_argument("GPU workflow: initialization of the TPC transformation failed");
     }
@@ -501,14 +501,20 @@ int GPURecoWorkflowSpec::runMain(o2::framework::ProcessingContext* pc, GPUTracki
     }
   }
 
-  cleanOldCalibsTPCPtrs(); // setting TPC calibration objects
-
   if (!mSpecConfig.enableDoublePipeline) { // TODO: Why is this needed for double-pipeline?
     mGPUReco->Clear(false, threadIndex);   // clean non-output memory used by GPU Reconstruction
   }
   return retVal;
 }
 
+void GPURecoWorkflowSpec::cleanOldCalibsTPCPtrs(calibObjectStruct& oldCalibObjects)
+{
+  if (mOldCalibObjects.size() > 0) {
+    mOldCalibObjects.pop();
+  }
+  mOldCalibObjects.emplace(std::move(oldCalibObjects));
+}
+
 void GPURecoWorkflowSpec::run(ProcessingContext& pc)
 {
   constexpr static size_t NSectors = o2::tpc::Sector::MAXSECTOR;
@@ -746,7 +752,8 @@ void GPURecoWorkflowSpec::run(ProcessingContext& pc)
 
   const auto& holdData = o2::tpc::TPCTrackingDigitsPreCheck::runPrecheck(&ptrs, mConfig.get());
 
-  doCalibUpdates(pc);
+  calibObjectStruct oldCalibObjects;
+  doCalibUpdates(pc, oldCalibObjects);
 
   lockDecodeInput.reset();
 
@@ -784,6 +791,7 @@ void GPURecoWorkflowSpec::run(ProcessingContext& pc)
   if (retVal != 0) {
     debugTFDump = true;
   }
+  cleanOldCalibsTPCPtrs(oldCalibObjects);
 
   o2::utils::DebugStreamer::instance()->flush(); // flushing debug output to file
 
@@ -959,7 +967,7 @@ void GPURecoWorkflowSpec::run(ProcessingContext& pc)
   LOG(info) << "GPU Reoncstruction time for this TF " << mTimer->CpuTime() - cput << " s (cpu), " << mTimer->RealTime() - realt << " s (wall)";
 }
 
-void GPURecoWorkflowSpec::doCalibUpdates(o2::framework::ProcessingContext& pc)
+void GPURecoWorkflowSpec::doCalibUpdates(o2::framework::ProcessingContext& pc, calibObjectStruct& oldCalibObjects)
 {
   GPUCalibObjectsConst newCalibObjects;
   GPUNewCalibValues newCalibValues;
@@ -1011,7 +1019,7 @@ void GPURecoWorkflowSpec::doCalibUpdates(o2::framework::ProcessingContext& pc)
       mTRDGeometryCreated = true;
     }
   }
-  needCalibUpdate = fetchCalibsCCDBTPC(pc, newCalibObjects) || needCalibUpdate;
+  needCalibUpdate = fetchCalibsCCDBTPC(pc, newCalibObjects, oldCalibObjects) || needCalibUpdate;
   if (mSpecConfig.runITSTracking) {
     needCalibUpdate = fetchCalibsCCDBITS(pc) || needCalibUpdate;
   }
@@ -1069,7 +1077,7 @@ Inputs GPURecoWorkflowSpec::inputs()
     inputs.emplace_back("tpcthreshold", gDataOriginTPC, "PADTHRESHOLD", 0, Lifetime::Condition, ccdbParamSpec("TPC/Config/FEEPad"));
     o2::tpc::VDriftHelper::requestCCDBInputs(inputs);
     Options optsDummy;
-    mFastTransformHelper->requestCCDBInputs(inputs, optsDummy, mSpecConfig.requireCTPLumi, mSpecConfig.lumiScaleMode); // option filled here is lost
+    mCalibObjects.mFastTransformHelper->requestCCDBInputs(inputs, optsDummy, mSpecConfig.requireCTPLumi, mSpecConfig.lumiScaleMode); // option filled here is lost
   }
   if (mSpecConfig.decompressTPC) {
     inputs.emplace_back(InputSpec{"input", ConcreteDataTypeMatcher{gDataOriginTPC, mSpecConfig.decompressTPCFromROOT ? o2::header::DataDescription("COMPCLUSTERS") : o2::header::DataDescription("COMPCLUSTERSFLAT")}, Lifetime::Timeframe});