From 45c1b8a56ed8165cb469c3a108e1c8a1467501fd Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 19 Oct 2023 13:35:44 +0200
Subject: [PATCH 001/726] Port the same fix we have in the Run3 code

---
 Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx b/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx
index f66454ebad52e..652bef585c98c 100644
--- a/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx
+++ b/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx
@@ -189,7 +189,6 @@ void TrackerDPL::run(ProcessingContext& pc)
     LOG(info) << labels->getIndexedSize() << " MC label objects , in " << mc2rofs.size() << " MC events";
   }
 
-  std::vector<TrackITSExt> tracks;
   auto& allClusIdx = pc.outputs().make<std::vector<int>>(Output{"IT3", "TRACKCLSID", 0, Lifetime::Timeframe});
   std::vector<o2::MCCompLabel> trackLabels;
   std::vector<MCCompLabel> verticesLabels;
@@ -281,7 +280,7 @@ void TrackerDPL::run(ProcessingContext& pc)
 
     for (unsigned int iROF{0}; iROF < rofs.size(); ++iROF) {
       auto& rof{rofs[iROF]};
-      tracks = timeFrame->getTracks(iROF);
+      auto &tracks = timeFrame->getTracks(iROF);
       trackLabels = timeFrame->getTracksLabel(iROF);
       auto number{tracks.size()};
       auto first{allTracks.size()};

From ef510cc22bb818c0bcd95e942fb926c801e44af4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 19 Oct 2023 13:45:57 +0200
Subject: [PATCH 002/726] Drop obsolete flag

-undefined error is the default now.
---
 dependencies/O2CompileFlags.cmake | 1 -
 1 file changed, 1 deletion(-)

diff --git a/dependencies/O2CompileFlags.cmake b/dependencies/O2CompileFlags.cmake
index e52260ed74455..7fe9019701efe 100644
--- a/dependencies/O2CompileFlags.cmake
+++ b/dependencies/O2CompileFlags.cmake
@@ -126,7 +126,6 @@ set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}}
 set(CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_C_WARNINGS}")
 
 if(APPLE)
-  set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-undefined,error") # avoid undefined in our libs
 elseif(UNIX)
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined") # avoid undefined in our libs
 endif()

From a367067ddaa6c5da2bc85b4ed48def2165fb729e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 19 Oct 2023 15:54:20 +0200
Subject: [PATCH 003/726] DPL: change InjectorFunction to return a bool

The result is now wether or not something was send out of band
---
 .../CTP/workflowScalers/src/ctp-proxy.cxx     |  3 ++-
 .../CTP/workflowScalers/src/ctp-qc-proxy.cxx  |  5 ++--
 .../DCS/testWorkflow/src/DCStoDPLconverter.h  |  7 +++--
 .../DCS/testWorkflow/src/dcs-config-proxy.cxx | 11 ++++----
 .../Framework/ExternalFairMQDeviceProxy.h     | 11 +++++++-
 .../Core/src/ExternalFairMQDeviceProxy.cxx    |  6 +++--
 .../benchmark_ExternalFairMQDeviceProxies.cxx | 11 ++++----
 .../test_ExternalFairMQDeviceWorkflow.cxx     | 26 +++++++++----------
 .../src/DataSamplingReadoutAdapter.cxx        |  1 +
 run/o2sim_mctracks_proxy.cxx                  |  8 ++++--
 10 files changed, 56 insertions(+), 33 deletions(-)

diff --git a/Detectors/CTP/workflowScalers/src/ctp-proxy.cxx b/Detectors/CTP/workflowScalers/src/ctp-proxy.cxx
index 449cd0903c882..3634df6d30bda 100644
--- a/Detectors/CTP/workflowScalers/src/ctp-proxy.cxx
+++ b/Detectors/CTP/workflowScalers/src/ctp-proxy.cxx
@@ -52,7 +52,7 @@ InjectorFunction dcs2dpl(std::string& ccdbhost)
   auto runMgr = std::make_shared<o2::ctp::CTPRunManager>();
   runMgr->setCCDBHost(ccdbhost);
   runMgr->init();
-  return [runMgr](TimingInfo&, ServiceRegistryRef const& services, fair::mq::Parts& parts, ChannelRetriever channelRetriever, size_t newTimesliceId, bool& stop) {
+  return [runMgr](TimingInfo&, ServiceRegistryRef const& services, fair::mq::Parts& parts, ChannelRetriever channelRetriever, size_t newTimesliceId, bool& stop) -> bool {
     // FIXME: Why isn't this function using the timeslice index?
     // make sure just 2 messages received
     // if (parts.Size() != 2) {
@@ -64,6 +64,7 @@ InjectorFunction dcs2dpl(std::string& ccdbhost)
     std::string messageData{static_cast<const char*>(parts.At(1)->GetData()), parts.At(1)->GetSize()};
     LOG(info) << "received message " << messageHeader << " of size " << dataSize << " # parts:" << parts.Size(); // << " Payload:" << messageData;
     runMgr->processMessage(messageHeader, messageData);
+    return true;
   };
 }
 
diff --git a/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx b/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx
index 0502bbae86493..2372ba07a28f7 100644
--- a/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx
+++ b/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx
@@ -46,7 +46,7 @@ using DetID = o2::detectors::DetID;
 InjectorFunction dcs2dpl()
 // InjectorFunction dcs2dpl()
 {
-  return [](TimingInfo&, ServiceRegistryRef const& services, fair::mq::Parts& parts, ChannelRetriever channelRetriever, size_t newTimesliceId, bool&) {
+  return [](TimingInfo&, ServiceRegistryRef const& services, fair::mq::Parts& parts, ChannelRetriever channelRetriever, size_t newTimesliceId, bool&) -> bool {
     auto *device = services.get<RawDeviceService>().device();
     std::string messageHeader{static_cast<const char*>(parts.At(0)->GetData()), parts.At(0)->GetSize()};
     size_t dataSize = parts.At(1)->GetSize();
@@ -57,7 +57,7 @@ InjectorFunction dcs2dpl()
     auto channel = channelRetriever(outsp, newTimesliceId);
     if (channel.empty()) {
       LOG(error) << "No output channel found for OutputSpec " << outsp;
-      return;
+      return false;
     }
     hdrF.tfCounter = newTimesliceId; // this also
     hdrF.payloadSerializationMethod = o2::header::gSerializationMethodNone;
@@ -91,6 +91,7 @@ InjectorFunction dcs2dpl()
     outParts.AddPart(std::move(plMessageF));
     sendOnChannel(*device, outParts, channel, newTimesliceId);
     LOG(info) << "Sent CTP counters DPL message" << std::flush;
+    return true;
   };
 }
 
diff --git a/Detectors/DCS/testWorkflow/src/DCStoDPLconverter.h b/Detectors/DCS/testWorkflow/src/DCStoDPLconverter.h
index dab2fd730b360..4384628d01f4c 100644
--- a/Detectors/DCS/testWorkflow/src/DCStoDPLconverter.h
+++ b/Detectors/DCS/testWorkflow/src/DCStoDPLconverter.h
@@ -53,7 +53,7 @@ using DPCOM = o2::dcs::DataPointCompositeObject;
 o2f::InjectorFunction dcs2dpl(std::unordered_map<DPID, o2h::DataDescription>& dpid2group, bool fbiFirst, bool verbose = false, int FBIPerInterval = 1)
 {
 
-  return [dpid2group, fbiFirst, verbose, FBIPerInterval](o2::framework::TimingInfo& tinfo, framework::ServiceRegistryRef const& services, fair::mq::Parts& parts, o2f::ChannelRetriever channelRetriever, size_t newTimesliceId, bool& stop) {
+  return [dpid2group, fbiFirst, verbose, FBIPerInterval](o2::framework::TimingInfo& tinfo, framework::ServiceRegistryRef const& services, fair::mq::Parts& parts, o2f::ChannelRetriever channelRetriever, size_t newTimesliceId, bool& stop) -> bool {
     auto *device = services.get<framework::RawDeviceService>().device();
     static std::unordered_map<DPID, DPCOM> cache; // will keep only the latest measurement in the 1-second wide window for each DPID
     static std::unordered_map<std::string, int> sentToChannel;
@@ -69,7 +69,7 @@ o2f::InjectorFunction dcs2dpl(std::unordered_map<DPID, o2h::DataDescription>& dp
     // check if we got FBI (Master) or delta (MasterDelta)
     if (!parts.Size()) {
       LOGP(warn, "Empty input recieved at timeslice {}", tinfo.timeslice);
-      return;
+      return false;
     }
     std::string firstName = std::string((char*)&(reinterpret_cast<const DPCOM*>(parts.At(0)->GetData()))->id);
 
@@ -124,6 +124,7 @@ o2f::InjectorFunction dcs2dpl(std::unordered_map<DPID, o2h::DataDescription>& dp
     }
 
     std::chrono::duration<double, std::ratio<1>> duration = timerNow - timer;
+    bool didSendMessages = false;
     if (duration.count() > 1 && (seenFBI || !fbiFirst)) { // did we accumulate for 1 sec and have we seen FBI if it was requested?
       std::unordered_map<o2h::DataDescription, pmr::vector<DPCOM>, std::hash<o2h::DataDescription>> outputs;
       // in the cache we have the final values of the DPs that we should put in the output
@@ -183,6 +184,7 @@ o2f::InjectorFunction dcs2dpl(std::unordered_map<DPID, o2h::DataDescription>& dp
         }
         o2f::sendOnChannel(*device, *msgIt.second.get(), msgIt.first, tinfo.timeslice);
         sentToChannel[msgIt.first]++;
+        didSendMessages |= msgIt.second->Size() > 0;
       }
       timer = timerNow;
       cache.clear();
@@ -200,6 +202,7 @@ o2f::InjectorFunction dcs2dpl(std::unordered_map<DPID, o2h::DataDescription>& dp
       }
       LOGP(info, "{} inputs ({} bytes) of which {} FBI ({} bytes) seen in {:.3f} s | {}", nInp, fmt::group_digits(szInp), nInpFBI, fmt::group_digits(szInpFBI), runtime, sent);
     }
+    return didSendMessages;
   };
 }
 
diff --git a/Detectors/DCS/testWorkflow/src/dcs-config-proxy.cxx b/Detectors/DCS/testWorkflow/src/dcs-config-proxy.cxx
index 936ceebf1e0f2..9b697da428ad2 100644
--- a/Detectors/DCS/testWorkflow/src/dcs-config-proxy.cxx
+++ b/Detectors/DCS/testWorkflow/src/dcs-config-proxy.cxx
@@ -67,17 +67,17 @@ auto getDataOriginFromFilename(const std::string& filename)
 
 InjectorFunction dcs2dpl(const std::string& acknowledge)
 {
-  return [acknowledge](TimingInfo&, ServiceRegistryRef const& services, fair::mq::Parts& parts, ChannelRetriever channelRetriever, size_t newTimesliceId, bool&) {
+  return [acknowledge](TimingInfo&, ServiceRegistryRef const& services, fair::mq::Parts& parts, ChannelRetriever channelRetriever, size_t newTimesliceId, bool&) -> bool {
     auto *device = services.get<RawDeviceService>().device();
     if (parts.Size() == 0) { // received at ^c, ignore
       LOG(info) << "ignoring empty message";
-      return;
+      return false;
     }
     // make sure just 2 messages received
     if (parts.Size() != 2) {
       LOG(error) << "received " << parts.Size() << " instead of 2 expected";
       sendAnswer("error0: wrong number of messages", acknowledge, *device);
-      return;
+      return false;
     }
     std::string filename{static_cast<const char*>(parts.At(0)->GetData()), parts.At(0)->GetSize()};
     size_t filesize = parts.At(1)->GetSize();
@@ -86,7 +86,7 @@ InjectorFunction dcs2dpl(const std::string& acknowledge)
     if (dataOrigin == o2::header::gDataOriginInvalid) {
       LOG(error) << "unknown detector for " << filename;
       sendAnswer(fmt::format("{}:error1: unrecognized filename", filename), acknowledge, *device);
-      return;
+      return false;
     }
 
     o2::header::DataHeader hdrF("DCS_CONFIG_FILE", dataOrigin, 0);
@@ -96,7 +96,7 @@ InjectorFunction dcs2dpl(const std::string& acknowledge)
     if (channel.empty()) {
       LOG(error) << "No output channel found for OutputSpec " << outsp;
       sendAnswer(fmt::format("{}:error2: no channel to send", filename), acknowledge, *device);
-      return;
+      return false;
     }
 
     hdrF.tfCounter = newTimesliceId;
@@ -140,6 +140,7 @@ InjectorFunction dcs2dpl(const std::string& acknowledge)
 
     sendAnswer(fmt::format("{}:ok", filename), acknowledge, *device);
     LOG(info) << "Sent DPL message and acknowledgment for file " << filename;
+    return true;
   };
 }
 
diff --git a/Framework/Core/include/Framework/ExternalFairMQDeviceProxy.h b/Framework/Core/include/Framework/ExternalFairMQDeviceProxy.h
index cfa05cd07d1d2..3ff77ae703b9b 100644
--- a/Framework/Core/include/Framework/ExternalFairMQDeviceProxy.h
+++ b/Framework/Core/include/Framework/ExternalFairMQDeviceProxy.h
@@ -23,7 +23,16 @@ namespace o2::framework
 /// A callback function to retrieve the fair::mq::Channel name to be used for sending
 /// messages of the specified OutputSpec
 using ChannelRetriever = std::function<std::string(OutputSpec const&, DataProcessingHeader::StartTime)>;
-using InjectorFunction = std::function<void(TimingInfo&, ServiceRegistryRef const& services, fair::mq::Parts& inputs, ChannelRetriever, size_t newTimesliceId, bool& stop)>;
+/// The callback which actually does the heavy lifting of converting the input data into
+/// DPL messages. The callback is invoked with the following parameters:
+/// @param timingInfo is the timing information of the current timeslice
+/// @param services is the service registry
+/// @param inputs is the list of input messages
+/// @param channelRetriever is a callback to retrieve the fair::mq::Channel name to be used for
+///        sending the messages
+/// @param newTimesliceId is the timeslice ID of the current timeslice
+/// @return true if any message were sent, false otherwise
+using InjectorFunction = std::function<bool(TimingInfo&, ServiceRegistryRef const& services, fair::mq::Parts& inputs, ChannelRetriever, size_t newTimesliceId, bool& stop)>;
 using ChannelSelector = std::function<std::string(InputSpec const& input, const std::unordered_map<std::string, std::vector<fair::mq::Channel>>& channels)>;
 
 struct InputChannelSpec;
diff --git a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
index d1a45a95c6d98..44a9b15d684b3 100644
--- a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
+++ b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
@@ -203,7 +203,7 @@ void appendForSending(fair::mq::Device& device, o2::header::Stack&& headerStack,
 
 InjectorFunction o2DataModelAdaptor(OutputSpec const& spec, uint64_t startTime, uint64_t /*step*/)
 {
-  return [spec](TimingInfo&, ServiceRegistryRef const& ref, fair::mq::Parts& parts, ChannelRetriever channelRetriever, size_t newTimesliceId, bool& stop) {
+  return [spec](TimingInfo&, ServiceRegistryRef const& ref, fair::mq::Parts& parts, ChannelRetriever channelRetriever, size_t newTimesliceId, bool& stop) -> bool {
     auto* device = ref.get<RawDeviceService>().device();
     for (int i = 0; i < parts.Size() / 2; ++i) {
       auto dh = o2::header::get<DataHeader*>(parts.At(i * 2)->GetData());
@@ -212,6 +212,7 @@ InjectorFunction o2DataModelAdaptor(OutputSpec const& spec, uint64_t startTime,
       o2::header::Stack headerStack{*dh, dph};
       sendOnChannel(*device, std::move(headerStack), std::move(parts.At(i * 2 + 1)), spec, channelRetriever);
     }
+    return parts.Size() > 0;
   };
 }
 
@@ -589,7 +590,7 @@ InjectorFunction dplModelAdaptor(std::vector<OutputSpec> const& filterSpecs, DPL
         }
       }
     }
-    return;
+    return didSendParts;
   };
 }
 
@@ -621,6 +622,7 @@ InjectorFunction incrementalConverter(OutputSpec const& spec, o2::header::Serial
 
       sendOnChannel(*device, std::move(headerStack), std::move(parts.At(i)), spec, channelRetriever);
     }
+    return parts.Size();
   };
 }
 
diff --git a/Framework/Core/test/benchmark_ExternalFairMQDeviceProxies.cxx b/Framework/Core/test/benchmark_ExternalFairMQDeviceProxies.cxx
index 55e0b4cec89db..bdb4deb443021 100644
--- a/Framework/Core/test/benchmark_ExternalFairMQDeviceProxies.cxx
+++ b/Framework/Core/test/benchmark_ExternalFairMQDeviceProxies.cxx
@@ -469,22 +469,22 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
   // reads the messages from the output proxy via the out-of-band channel
 
   // converter callback for the external FairMQ device proxy ProcessorSpec generator
-  auto converter = [](TimingInfo&, ServiceRegistryRef const& ref, fair::mq::Parts& inputs, ChannelRetriever channelRetriever, size_t newTimesliceId, bool&) {
+  InjectorFunction converter = [](TimingInfo&, ServiceRegistryRef const& ref, fair::mq::Parts& inputs, ChannelRetriever channelRetriever, size_t newTimesliceId, bool&) -> bool {
     auto* device = ref.get<RawDeviceService>().device();
     ASSERT_ERROR(inputs.Size() >= 2);
     if (inputs.Size() < 2) {
-      return;
+      return false;
     }
     int msgidx = 0;
     auto dh = o2::header::get<o2::header::DataHeader*>(inputs.At(msgidx)->GetData());
     if (!dh) {
       LOG(error) << "data on input " << msgidx << " does not follow the O2 data model, DataHeader missing";
-      return;
+      return false;
     }
     auto dph = o2::header::get<DataProcessingHeader*>(inputs.At(msgidx)->GetData());
     if (!dph) {
       LOG(error) << "data on input " << msgidx << " does not follow the O2 data model, DataProcessingHeader missing";
-      return;
+      return false;
     }
     // Note: we want to run both the output and input proxy in the same workflow and thus we need
     // different data identifiers and change the data origin in the forwarding
@@ -497,7 +497,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
     ASSERT_ERROR(!isData || !channelName.empty());
     LOG(debug) << "using channel '" << channelName << "' for " << DataSpecUtils::describe(OutputSpec{dh->dataOrigin, dh->dataDescription, dh->subSpecification});
     if (channelName.empty()) {
-      return;
+      return false;
     }
     // make a copy of the header message, get the data header and change origin
     auto outHeaderMessage = device->NewMessageFor(channelName, 0, inputs.At(msgidx)->GetSize());
@@ -511,6 +511,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
     output.AddPart(std::move(inputs.At(msgidx + 1)));
     LOG(debug) << "sending " << DataSpecUtils::describe(OutputSpec{odh->dataOrigin, odh->dataDescription, odh->subSpecification});
     o2::framework::sendOnChannel(*device, output, channelName, (size_t)-1);
+    return output.Size() > 0;
   };
 
   // we use the same spec to build the configuration string, ideally we would have some helpers
diff --git a/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx b/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx
index 98f3139fe7b86..c105a9d0f662c 100644
--- a/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx
+++ b/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx
@@ -176,7 +176,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
     // the compute callback of the producer
     auto producerCallback = [nRolls, channelName, proxyMode, counter = std::make_shared<size_t>()](DataAllocator& outputs, ControlService& control, RawDeviceService& rds) {
       int data = *counter;
-      //outputs.make<int>(OutputRef{"data", 0}) = data;
+      // outputs.make<int>(OutputRef{"data", 0}) = data;
 
       fair::mq::Device& device = *(rds.device());
       auto transport = device.GetChannel(*channelName, 0).Transport();
@@ -327,14 +327,14 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
   Inputs checkerInputs;
   if (proxyMode != ProxyMode::All) {
     checkerInputs.emplace_back(InputSpec{"datain", ConcreteDataTypeMatcher{"TST", "DATA"}, Lifetime::Timeframe});
-    //for (unsigned int i = 0; i < pState->nChannels; i++) {
-    //  checkerInputs.emplace_back(InputSpec{{"datain"}, "TST", "DATA", i, Lifetime::Timeframe});
-    //}
+    // for (unsigned int i = 0; i < pState->nChannels; i++) {
+    //   checkerInputs.emplace_back(InputSpec{{"datain"}, "TST", "DATA", i, Lifetime::Timeframe});
+    // }
   } else {
     checkerInputs.emplace_back(InputSpec{"datain", ConcreteDataTypeMatcher{"PRX", "DATA"}, Lifetime::Timeframe});
-    //for (unsigned int i = 0; i < pState->nChannels; i++) {
-    //  checkerInputs.emplace_back(InputSpec{{"datain"}, "PRX", "DATA", i, Lifetime::Timeframe});
-    //}
+    // for (unsigned int i = 0; i < pState->nChannels; i++) {
+    //   checkerInputs.emplace_back(InputSpec{{"datain"}, "PRX", "DATA", i, Lifetime::Timeframe});
+    // }
   }
   if (proxyMode != ProxyMode::OnlyOutput) {
     // the checker is not added if the input proxy is skipped
@@ -349,22 +349,22 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
   // reads the messages from the output proxy via the out-of-band channel
 
   // converter callback for the external FairMQ device proxy ProcessorSpec generator
-  auto converter = [](TimingInfo&, ServiceRegistryRef const& services, fair::mq::Parts& inputs, ChannelRetriever channelRetriever, size_t newTimesliceId, bool&) {
+  InjectorFunction converter = [](TimingInfo&, ServiceRegistryRef const& services, fair::mq::Parts& inputs, ChannelRetriever channelRetriever, size_t newTimesliceId, bool&) -> bool {
     auto* device = services.get<RawDeviceService>().device();
     ASSERT_ERROR(inputs.Size() >= 2);
     if (inputs.Size() < 2) {
-      return;
+      return false;
     }
     int msgidx = 0;
     auto dh = o2::header::get<o2::header::DataHeader*>(inputs.At(msgidx)->GetData());
     if (!dh) {
       LOG(error) << "data on input " << msgidx << " does not follow the O2 data model, DataHeader missing";
-      return;
+      return false;
     }
     auto dph = o2::header::get<DataProcessingHeader*>(inputs.At(msgidx)->GetData());
     if (!dph) {
       LOG(error) << "data on input " << msgidx << " does not follow the O2 data model, DataProcessingHeader missing";
-      return;
+      return false;
     }
     // Note: we want to run both the output and input proxy in the same workflow and thus we need
     // different data identifiers and change the data origin in the forwarding
@@ -377,7 +377,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
     ASSERT_ERROR(!isData || !channelName.empty());
     LOG(debug) << "using channel '" << channelName << "' for " << DataSpecUtils::describe(OutputSpec{dh->dataOrigin, dh->dataDescription, dh->subSpecification});
     if (channelName.empty()) {
-      return;
+      return false;
     }
     fair::mq::Parts output;
     for (; msgidx < inputs.Size(); ++msgidx) {
@@ -402,7 +402,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
       }
     }
     o2::framework::sendOnChannel(*device, output, channelName, (size_t)-1);
-    return;
+    return output.Size() != 0;
   };
 
   // we use the same spec to build the configuration string, ideally we would have some helpers
diff --git a/Utilities/DataSampling/src/DataSamplingReadoutAdapter.cxx b/Utilities/DataSampling/src/DataSamplingReadoutAdapter.cxx
index 745c0cddbf835..66a872e265baf 100644
--- a/Utilities/DataSampling/src/DataSamplingReadoutAdapter.cxx
+++ b/Utilities/DataSampling/src/DataSamplingReadoutAdapter.cxx
@@ -41,6 +41,7 @@ InjectorFunction dataSamplingReadoutAdapter(OutputSpec const& spec)
       o2::header::Stack headerStack{dh, dph};
       sendOnChannel(*device, std::move(headerStack), std::move(parts.At(i)), spec, channelRetriever);
     }
+    return parts.Size() != 0;
   };
 }
 
diff --git a/run/o2sim_mctracks_proxy.cxx b/run/o2sim_mctracks_proxy.cxx
index cd032dd35c4fa..8b40c6a647cca 100644
--- a/run/o2sim_mctracks_proxy.cxx
+++ b/run/o2sim_mctracks_proxy.cxx
@@ -93,8 +93,9 @@ InjectorFunction o2simKinematicsConverter(std::vector<OutputSpec> const& specs,
   auto MCTracksMessageCache = std::make_shared<fair::mq::Parts>();
   auto Nparts = std::make_shared<int>(nPerTF);
 
-  return [timesliceId, specs, step, nevents, nPerTF, totalEventCounter, eventCounter, TFcounter, Nparts, MCHeadersMessageCache = MCHeadersMessageCache, MCTracksMessageCache = MCTracksMessageCache](TimingInfo& ti, ServiceRegistryRef const& services, fair::mq::Parts& parts, ChannelRetriever channelRetriever, size_t newTimesliceId, bool& stop) mutable {
+  return [timesliceId, specs, step, nevents, nPerTF, totalEventCounter, eventCounter, TFcounter, Nparts, MCHeadersMessageCache = MCHeadersMessageCache, MCTracksMessageCache = MCTracksMessageCache](TimingInfo& ti, ServiceRegistryRef const& services, fair::mq::Parts& parts, ChannelRetriever channelRetriever, size_t newTimesliceId, bool& stop) mutable -> bool {
     auto*device = services.get<RawDeviceService>().device();
+    bool didSendData = false;
     if (nPerTF < 0) {
       // if no aggregation requested, forward each message with the DPL header
       if (*timesliceId != newTimesliceId) {
@@ -108,6 +109,7 @@ InjectorFunction o2simKinematicsConverter(std::vector<OutputSpec> const& specs,
         // we have to move the incoming data
         o2::header::Stack headerStack{dh, dph};
         sendOnChannel(*device, std::move(headerStack), std::move(parts.At(i)), specs[i], channelRetriever);
+        didSendData |= parts.At(i)->GetSize() > 0;
       }
       *timesliceId += step;
     } else {
@@ -141,6 +143,8 @@ InjectorFunction o2simKinematicsConverter(std::vector<OutputSpec> const& specs,
       *eventCounter = 0;
       sendOnChannel(*device, *MCHeadersMessageCache.get(), channelRetriever(specs[0], *TFcounter), *TFcounter);
       sendOnChannel(*device, *MCTracksMessageCache.get(), channelRetriever(specs[1], *TFcounter), *TFcounter);
+      didSendData |= MCHeadersMessageCache->Size() > 0;
+      didSendData |= MCTracksMessageCache->Size() > 0;
       ++(*TFcounter);
       MCHeadersMessageCache->Clear();
       MCTracksMessageCache->Clear();
@@ -150,7 +154,7 @@ InjectorFunction o2simKinematicsConverter(std::vector<OutputSpec> const& specs,
       // I am done (I don't expect more events to convert); so tell the proxy device to shut-down
       stop = true;
     }
-    return;
+    return didSendData;
   };
 }
 

From e2f54d002112d3ec4520efd867fcdbbc15f60baa Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 19 Oct 2023 15:55:37 +0200
Subject: [PATCH 004/726] DPL: notify DPL about out of band data being sent by
 input proxy

This is needed to avoid skipping some of the callbacks, in particular
the CCDB one.
---
 .../Core/src/ExternalFairMQDeviceProxy.cxx    | 27 +++++++++++--------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
index 44a9b15d684b3..7e09bfb408ff1 100644
--- a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
+++ b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
@@ -561,13 +561,6 @@ InjectorFunction dplModelAdaptor(std::vector<OutputSpec> const& filterSpecs, DPL
       didSendParts = true;
       sendOnChannel(*device, channelParts, channelName, newTimesliceId);
     }
-    // In case we did not send any part at all, we need to rewind by one
-    // to avoid creating extra timeslices at the end of the run.
-    auto& decongestion = services.get<DecongestionService>();
-    decongestion.nextEnumerationTimesliceRewinded = !didSendParts;
-    if (didSendParts == false) {
-      decongestion.nextEnumerationTimeslice -= 1;
-    }
     if (not unmatchedDescriptions.empty()) {
       if (throwOnUnmatchedInputs) {
         std::string descriptions;
@@ -769,12 +762,13 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
       return count;
     };
 
-    auto dataHandler = [ref = ctx.services(), converter, doInjectMissingData, doPrintSizes,
+    // Data handler for incoming data. Must return true if it sent any data.
+    auto dataHandler = [converter, doInjectMissingData, doPrintSizes,
                         outputRoutes = std::move(outputRoutes),
                         control = &ctx.services().get<ControlService>(),
                         deviceState = &ctx.services().get<DeviceState>(),
                         timesliceIndex = &ctx.services().get<TimesliceIndex>(),
-                        outputChannels = std::move(outputChannels)](TimingInfo& timingInfo, fair::mq::Parts& inputs, int, size_t ci, bool newRun) {
+                        outputChannels = std::move(outputChannels)](ServiceRegistryRef ref, TimingInfo& timingInfo, fair::mq::Parts& inputs, int, size_t ci, bool newRun) -> bool {
       auto* device = ref.get<RawDeviceService>().device();
       // pass a copy of the outputRoutes
       auto channelRetriever = [&outputRoutes](OutputSpec const& query, DataProcessingHeader::StartTime timeslice) -> std::string {
@@ -803,7 +797,7 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
       if (doInjectMissingData) {
         injectMissingData(*device, inputs, outputRoutes, doInjectMissingData, doPrintSizes);
       }
-      converter(timingInfo, ref, inputs, channelRetriever, timesliceIndex->getOldestPossibleOutput().timeslice.value, shouldstop);
+      bool didSendParts = converter(timingInfo, ref, inputs, channelRetriever, timesliceIndex->getOldestPossibleOutput().timeslice.value, shouldstop);
 
       // If we have enough EoS messages, we can stop the device
       // Notice that this has a number of failure modes:
@@ -832,6 +826,7 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
         std::fill(eosPeersCount.begin(), eosPeersCount.end(), 0);
         control->endOfStream();
       }
+      return didSendParts;
     };
 
     auto runHandler = [dataHandler, minSHM, sendTFcounter](ProcessingContext& ctx) {
@@ -844,6 +839,7 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
         inStopTransition = true;
       }
 
+      bool didSendParts = false;
       for (size_t ci = 0; ci < channels.size(); ++ci) {
         std::string const& channel = channels[ci];
         int waitTime = channels.size() == 1 ? -1 : 1;
@@ -875,7 +871,7 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
               timingInfo.creation = dph->creation;
             }
             if (!inStopTransition) {
-              dataHandler(timingInfo, parts, 0, ci, newRun);
+              didSendParts |= dataHandler(ctx.services(), timingInfo, parts, 0, ci, newRun);
             }
             if (sendTFcounter) {
               ctx.services().get<o2::monitoring::Monitoring>().send(o2::monitoring::Metric{(uint64_t)timingInfo.tfCounter, "df-sent"}.addTag(o2::monitoring::tags::Key::Subsystem, o2::monitoring::tags::Value::DPL));
@@ -887,6 +883,15 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
           waitTime = 0;
         }
       }
+      // In case we did not send any part at all, we need to rewind by one
+      // to avoid creating extra timeslices.
+      auto& decongestion = ref.get<DecongestionService>();
+      decongestion.nextEnumerationTimesliceRewinded = !didSendParts;
+      if (didSendParts) {
+        ref.get<MessageContext>().fakeDispatch();
+      } else {
+        decongestion.nextEnumerationTimeslice -= 1;
+      }
     };
 
     return runHandler;

From e7ae95ae76c74e8ce0f455b5ce0f6b5be2d5c101 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 19 Oct 2023 16:15:34 +0200
Subject: [PATCH 005/726] DPL: notify DPL about out of band data being sent by
 input proxy

This is needed to avoid skipping some of the callbacks, in particular
the CCDB one.
---
 .../Core/src/ExternalFairMQDeviceProxy.cxx    | 27 +++++++++++--------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
index 44a9b15d684b3..6a10a7e8a0111 100644
--- a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
+++ b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
@@ -561,13 +561,6 @@ InjectorFunction dplModelAdaptor(std::vector<OutputSpec> const& filterSpecs, DPL
       didSendParts = true;
       sendOnChannel(*device, channelParts, channelName, newTimesliceId);
     }
-    // In case we did not send any part at all, we need to rewind by one
-    // to avoid creating extra timeslices at the end of the run.
-    auto& decongestion = services.get<DecongestionService>();
-    decongestion.nextEnumerationTimesliceRewinded = !didSendParts;
-    if (didSendParts == false) {
-      decongestion.nextEnumerationTimeslice -= 1;
-    }
     if (not unmatchedDescriptions.empty()) {
       if (throwOnUnmatchedInputs) {
         std::string descriptions;
@@ -769,12 +762,13 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
       return count;
     };
 
-    auto dataHandler = [ref = ctx.services(), converter, doInjectMissingData, doPrintSizes,
+    // Data handler for incoming data. Must return true if it sent any data.
+    auto dataHandler = [converter, doInjectMissingData, doPrintSizes,
                         outputRoutes = std::move(outputRoutes),
                         control = &ctx.services().get<ControlService>(),
                         deviceState = &ctx.services().get<DeviceState>(),
                         timesliceIndex = &ctx.services().get<TimesliceIndex>(),
-                        outputChannels = std::move(outputChannels)](TimingInfo& timingInfo, fair::mq::Parts& inputs, int, size_t ci, bool newRun) {
+                        outputChannels = std::move(outputChannels)](ServiceRegistryRef ref, TimingInfo& timingInfo, fair::mq::Parts& inputs, int, size_t ci, bool newRun) -> bool {
       auto* device = ref.get<RawDeviceService>().device();
       // pass a copy of the outputRoutes
       auto channelRetriever = [&outputRoutes](OutputSpec const& query, DataProcessingHeader::StartTime timeslice) -> std::string {
@@ -803,7 +797,7 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
       if (doInjectMissingData) {
         injectMissingData(*device, inputs, outputRoutes, doInjectMissingData, doPrintSizes);
       }
-      converter(timingInfo, ref, inputs, channelRetriever, timesliceIndex->getOldestPossibleOutput().timeslice.value, shouldstop);
+      bool didSendParts = converter(timingInfo, ref, inputs, channelRetriever, timesliceIndex->getOldestPossibleOutput().timeslice.value, shouldstop);
 
       // If we have enough EoS messages, we can stop the device
       // Notice that this has a number of failure modes:
@@ -832,6 +826,7 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
         std::fill(eosPeersCount.begin(), eosPeersCount.end(), 0);
         control->endOfStream();
       }
+      return didSendParts;
     };
 
     auto runHandler = [dataHandler, minSHM, sendTFcounter](ProcessingContext& ctx) {
@@ -844,6 +839,7 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
         inStopTransition = true;
       }
 
+      bool didSendParts = false;
       for (size_t ci = 0; ci < channels.size(); ++ci) {
         std::string const& channel = channels[ci];
         int waitTime = channels.size() == 1 ? -1 : 1;
@@ -875,7 +871,7 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
               timingInfo.creation = dph->creation;
             }
             if (!inStopTransition) {
-              dataHandler(timingInfo, parts, 0, ci, newRun);
+              didSendParts |= dataHandler(ctx.services(), timingInfo, parts, 0, ci, newRun);
             }
             if (sendTFcounter) {
               ctx.services().get<o2::monitoring::Monitoring>().send(o2::monitoring::Metric{(uint64_t)timingInfo.tfCounter, "df-sent"}.addTag(o2::monitoring::tags::Key::Subsystem, o2::monitoring::tags::Value::DPL));
@@ -887,6 +883,15 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
           waitTime = 0;
         }
       }
+      // In case we did not send any part at all, we need to rewind by one
+      // to avoid creating extra timeslices.
+      auto& decongestion = ctx.services().get<DecongestionService>();
+      decongestion.nextEnumerationTimesliceRewinded = !didSendParts;
+      if (didSendParts) {
+        ctx.services().get<MessageContext>().fakeDispatch();
+      } else {
+        decongestion.nextEnumerationTimeslice -= 1;
+      }
     };
 
     return runHandler;

From 7e04743709df64b14d9a510985a252088b61d266 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 20 Oct 2023 01:30:44 +0200
Subject: [PATCH 006/726] Avoid temporary objects

---
 Detectors/Raw/TFReaderDD/src/TFReaderSpec.cxx | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/Detectors/Raw/TFReaderDD/src/TFReaderSpec.cxx b/Detectors/Raw/TFReaderDD/src/TFReaderSpec.cxx
index a75254c490c6a..4c916fc2e70d4 100644
--- a/Detectors/Raw/TFReaderDD/src/TFReaderSpec.cxx
+++ b/Detectors/Raw/TFReaderDD/src/TFReaderSpec.cxx
@@ -465,28 +465,28 @@ o2f::DataProcessorSpec o2::rawdd::getTFReaderSpec(o2::rawdd::TFReaderInp& rinp)
           rinp.hdVec.emplace_back(o2h::DataHeader{"CELLS", DetID::getDataOrigin(id), 0, 0});       // in abcence of real data this will be sent
           rinp.hdVec.emplace_back(o2h::DataHeader{"CELLTRIGREC", DetID::getDataOrigin(id), 0, 0}); // in abcence of real data this will be sent
         } else if (id == DetID::CPV) {
-          spec.outputs.emplace_back(o2f::OutputSpec{DetID::getDataOrigin(id), "DIGITS", 0});
-          spec.outputs.emplace_back(o2f::OutputSpec{DetID::getDataOrigin(id), "DIGITTRIGREC", 0});
-          spec.outputs.emplace_back(o2f::OutputSpec{DetID::getDataOrigin(id), "RAWHWERRORS", 0});
-          rinp.hdVec.emplace_back(o2h::DataHeader{"DIGITS", DetID::getDataOrigin(id), 0, 0});       // in abcence of real data this will be sent
-          rinp.hdVec.emplace_back(o2h::DataHeader{"DIGITTRIGREC", DetID::getDataOrigin(id), 0, 0}); // in abcence of real data this will be sent
-          rinp.hdVec.emplace_back(o2h::DataHeader{"RAWHWERRORS", DetID::getDataOrigin(id), 0, 0});  // in abcence of real data this will be sent
+          spec.outputs.emplace_back(DetID::getDataOrigin(id), "DIGITS", 0);
+          spec.outputs.emplace_back(DetID::getDataOrigin(id), "DIGITTRIGREC", 0);
+          spec.outputs.emplace_back(DetID::getDataOrigin(id), "RAWHWERRORS", 0);
+          rinp.hdVec.emplace_back("DIGITS", DetID::getDataOrigin(id), 0, 0);       // in abcence of real data this will be sent
+          rinp.hdVec.emplace_back("DIGITTRIGREC", DetID::getDataOrigin(id), 0, 0); // in abcence of real data this will be sent
+          rinp.hdVec.emplace_back("RAWHWERRORS", DetID::getDataOrigin(id), 0, 0);  // in abcence of real data this will be sent
         } else if (id == DetID::EMC) {
           spec.outputs.emplace_back(o2f::OutputSpec{o2f::ConcreteDataTypeMatcher{DetID::getDataOrigin(id), "CELLS"}});
           spec.outputs.emplace_back(o2f::OutputSpec{o2f::ConcreteDataTypeMatcher{DetID::getDataOrigin(id), "CELLSTRGR"}});
           spec.outputs.emplace_back(o2f::OutputSpec{o2f::ConcreteDataTypeMatcher{DetID::getDataOrigin(id), "DECODERERR"}});
-          rinp.hdVec.emplace_back(o2h::DataHeader{"CELLS", DetID::getDataOrigin(id), 0, 0});      // in abcence of real data this will be sent
-          rinp.hdVec.emplace_back(o2h::DataHeader{"CELLSTRGR", DetID::getDataOrigin(id), 0, 0});  // in abcence of real data this will be sent
-          rinp.hdVec.emplace_back(o2h::DataHeader{"DECODERERR", DetID::getDataOrigin(id), 0, 0}); // in abcence of real data this will be sent
+          rinp.hdVec.emplace_back("CELLS", DetID::getDataOrigin(id), 0, 0);      // in abcence of real data this will be sent
+          rinp.hdVec.emplace_back("CELLSTRGR", DetID::getDataOrigin(id), 0, 0);  // in abcence of real data this will be sent
+          rinp.hdVec.emplace_back("DECODERERR", DetID::getDataOrigin(id), 0, 0); // in abcence of real data this will be sent
         } else if (id == DetID::FOC) {
           spec.outputs.emplace_back(o2f::OutputSpec{o2f::ConcreteDataTypeMatcher{DetID::getDataOrigin(id), "PADLAYERS"}});
           spec.outputs.emplace_back(o2f::OutputSpec{o2f::ConcreteDataTypeMatcher{DetID::getDataOrigin(id), "PIXELHITS"}});
           spec.outputs.emplace_back(o2f::OutputSpec{o2f::ConcreteDataTypeMatcher{DetID::getDataOrigin(id), "PIXELCHIPS"}});
           spec.outputs.emplace_back(o2f::OutputSpec{o2f::ConcreteDataTypeMatcher{DetID::getDataOrigin(id), "TRIGGERS"}});
-          rinp.hdVec.emplace_back(o2h::DataHeader{"PADLAYERS", DetID::getDataOrigin(id), 0, 0});  // in abcence of real data this will be sent
-          rinp.hdVec.emplace_back(o2h::DataHeader{"PIXELHITS", DetID::getDataOrigin(id), 0, 0});  // in abcence of real data this will be sent
-          rinp.hdVec.emplace_back(o2h::DataHeader{"PIXELCHIPS", DetID::getDataOrigin(id), 0, 0}); // in abcence of real data this will be sent
-          rinp.hdVec.emplace_back(o2h::DataHeader{"TRIGGERS", DetID::getDataOrigin(id), 0, 0});   // in abcence of real data this will be sent
+          rinp.hdVec.emplace_back("PADLAYERS", DetID::getDataOrigin(id), 0, 0);  // in abcence of real data this will be sent
+          rinp.hdVec.emplace_back("PIXELHITS", DetID::getDataOrigin(id), 0, 0);  // in abcence of real data this will be sent
+          rinp.hdVec.emplace_back("PIXELCHIPS", DetID::getDataOrigin(id), 0, 0); // in abcence of real data this will be sent
+          rinp.hdVec.emplace_back("TRIGGERS", DetID::getDataOrigin(id), 0, 0);   // in abcence of real data this will be sent
         }
       }
     }

From 375aab351c6acbab32fe566385bcf646297a8b56 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 23 Oct 2023 12:34:18 +0200
Subject: [PATCH 007/726] DPL: do not add extra DataHeader to EoS

Historically we had to have a DataHeader for EoS in any case.

This is not needed anymore and it might explain some weird behavior we
see online.
---
 .../Core/src/ExternalFairMQDeviceProxy.cxx    | 23 ++++---------------
 1 file changed, 4 insertions(+), 19 deletions(-)

diff --git a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
index 4d9d4c11df49f..d9f3a3b3e89ac 100644
--- a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
+++ b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
@@ -165,7 +165,8 @@ void sendOnChannel(fair::mq::Device& device, fair::mq::MessagePtr&& headerMessag
 {
   //  const auto* dph = o2::header::get<DataProcessingHeader*>( *reinterpret_cast<o2::header::Stack*>(headerMessage->GetData()) );
   const auto* dph = o2::header::get<DataProcessingHeader*>(headerMessage->GetData());
-  if (!dph) {
+  const auto* sih = o2::header::get<SourceInfoHeader*>(headerMessage->GetData());
+  if (!dph && !sih) {
     LOG(error) << "Header does not follow the O2 data model, DataProcessingHeader missing";
     return;
   }
@@ -963,19 +964,11 @@ DataProcessorSpec specifyFairMQDeviceOutputProxy(char const* name,
         if (channelName != outputChannelName) {
           continue;
         }
-        DataHeader dh;
-        dh.dataOrigin = "DPL";
-        dh.dataDescription = "EOS";
-        dh.subSpecification = 0;
-        dh.payloadSize = 0;
-        dh.payloadSerializationMethod = o2::header::gSerializationMethodNone;
-        dh.tfCounter = 0;
-        dh.firstTForbit = 0;
         SourceInfoHeader sih;
         sih.state = InputChannelState::Completed;
         // allocate the header message using the underlying transport of the channel
         auto channelAlloc = o2::pmr::getTransportAllocator(channelInfo.second[0].Transport());
-        auto headerMessage = o2::pmr::getMessage(o2::header::Stack{channelAlloc, dh, *lastDataProcessingHeader, sih});
+        auto headerMessage = o2::pmr::getMessage(o2::header::Stack{channelAlloc, sih});
         fair::mq::Parts out;
         out.AddPart(std::move(headerMessage));
         // add empty payload message
@@ -1071,19 +1064,11 @@ DataProcessorSpec specifyFairMQDeviceMultiOutputProxy(char const* name,
         if (!checkChannel(channelName)) {
           continue;
         }
-        DataHeader dh;
-        dh.dataOrigin = "DPL";
-        dh.dataDescription = "EOS";
-        dh.subSpecification = 0;
-        dh.payloadSize = 0;
-        dh.payloadSerializationMethod = o2::header::gSerializationMethodNone;
-        dh.tfCounter = 0;
-        dh.firstTForbit = 0;
         SourceInfoHeader sih;
         sih.state = InputChannelState::Completed;
         // allocate the header message using the underlying transport of the channel
         auto channelAlloc = o2::pmr::getTransportAllocator(channelInfo.second[0].Transport());
-        auto headerMessage = o2::pmr::getMessage(o2::header::Stack{channelAlloc, dh, *lastDataProcessingHeader, sih});
+        auto headerMessage = o2::pmr::getMessage(o2::header::Stack{channelAlloc, sih});
         fair::mq::Parts out;
         out.AddPart(std::move(headerMessage));
         // add empty payload message

From 091f9d18c8b66d4fcaf8a9971a7ea8c869f6b948 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 20 Oct 2023 15:37:59 +0200
Subject: [PATCH 008/726] Drop need to include ptree in ConfigurableParam.h

---
 Common/SimConfig/test/testSimCutParam.cxx     |  1 +
 .../include/CommonUtils/ConfigurableParam.h   | 59 +++++++------------
 Common/Utils/src/ConfigurableParam.cxx        | 35 +++++++++--
 .../MUON/MCH/Raw/test/testClosureCoDec.cxx    |  1 +
 .../MCH/Simulation/test/testDigitizer.cxx     |  1 +
 .../MCH/Triggering/test/testEventFinder.cxx   |  1 +
 Detectors/TPC/base/test/testTPCParameters.cxx |  7 +--
 .../include/TPCCalibration/IDCAverageGroup.h  |  1 +
 .../include/TPCCalibration/IDCFactorization.h |  1 +
 .../include/TPCCalibration/SACFactorization.h |  1 +
 .../calibration/src/IDCGroupingParameter.cxx  |  2 +
 .../TPC/reconstruction/macro/makeIonTail.C    |  1 +
 Detectors/TPC/simulation/macro/toyCluster.C   |  1 +
 .../TPC/workflow/src/ApplyCCDBCalibSpec.cxx   |  1 +
 macro/o2sim.C                                 |  1 +
 15 files changed, 66 insertions(+), 48 deletions(-)

diff --git a/Common/SimConfig/test/testSimCutParam.cxx b/Common/SimConfig/test/testSimCutParam.cxx
index 7a17ae9d1f2cf..468d76429cbba 100644
--- a/Common/SimConfig/test/testSimCutParam.cxx
+++ b/Common/SimConfig/test/testSimCutParam.cxx
@@ -15,6 +15,7 @@
 #include <boost/test/unit_test.hpp>
 #include "SimConfig/SimParams.h"
 #include "CommonUtils/ConfigurableParam.h"
+#include <boost/property_tree/ptree.hpp>
 
 using namespace o2::conf;
 
diff --git a/Common/Utils/include/CommonUtils/ConfigurableParam.h b/Common/Utils/include/CommonUtils/ConfigurableParam.h
index 08356dc462de8..4314308e9f93a 100644
--- a/Common/Utils/include/CommonUtils/ConfigurableParam.h
+++ b/Common/Utils/include/CommonUtils/ConfigurableParam.h
@@ -18,7 +18,7 @@
 #include <cassert>
 #include <map>
 #include <unordered_map>
-#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/ptree_fwd.hpp>
 #include <typeinfo>
 #include <iostream>
 
@@ -181,9 +181,6 @@ class ConfigurableParam
   static void setInputDir(const std::string& d) { sInputDir = d; }
   static void setOutputDir(const std::string& d) { sOutputDir = d; }
 
-  static boost::property_tree::ptree readINI(std::string const& filepath);
-  static boost::property_tree::ptree readJSON(std::string const& filepath);
-  static boost::property_tree::ptree readConfigFile(std::string const& filepath);
   static bool configFileExists(std::string const& filepath);
 
   // writes a human readable JSON file of all parameters
@@ -195,10 +192,12 @@ class ConfigurableParam
   template <typename T>
   static T getValueAs(std::string key)
   {
-    if (!sIsFullyInitialized) {
-      initialize();
-    }
-    return sPtree->get<T>(key);
+    return [](auto* tree, const std::string& key) -> T {
+      if (!sIsFullyInitialized) {
+        initialize();
+      }
+      return tree->template get<T>(key);
+    }(sPtree, key);
   }
 
   template <typename T>
@@ -207,19 +206,21 @@ class ConfigurableParam
     if (!sIsFullyInitialized) {
       initialize();
     }
-    assert(sPtree);
-    try {
-      auto key = mainkey + "." + subkey;
-      if (sPtree->get_optional<std::string>(key).is_initialized()) {
-        sPtree->put(key, x);
-        auto changed = updateThroughStorageMap(mainkey, subkey, typeid(T), (void*)&x);
-        if (changed != EParamUpdateStatus::Failed) {
-          sValueProvenanceMap->find(key)->second = kRT; // set to runtime
+    return [&subkey, &x, &mainkey](auto* tree) -> void {
+      assert(tree);
+      try {
+        auto key = mainkey + "." + subkey;
+        if (tree->template get_optional<std::string>(key).is_initialized()) {
+          tree->put(key, x);
+          auto changed = updateThroughStorageMap(mainkey, subkey, typeid(T), (void*)&x);
+          if (changed != EParamUpdateStatus::Failed) {
+            sValueProvenanceMap->find(key)->second = kRT; // set to runtime
+          }
         }
+      } catch (std::exception const& e) {
+        std::cerr << "Error in setValue (T) " << e.what() << "\n";
       }
-    } catch (std::exception const& e) {
-      std::cerr << "Error in setValue (T) " << e.what() << "\n";
-    }
+    }(sPtree);
   }
 
   static void setProvenance(std::string const& mainkey, std::string const& subkey, EParamProvenance p)
@@ -241,25 +242,7 @@ class ConfigurableParam
 
   // specialized for std::string
   // which means that the type will be converted internally
-  static void setValue(std::string const& key, std::string const& valuestring)
-  {
-    if (!sIsFullyInitialized) {
-      initialize();
-    }
-    assert(sPtree);
-    try {
-      if (sPtree->get_optional<std::string>(key).is_initialized()) {
-        sPtree->put(key, valuestring);
-        auto changed = updateThroughStorageMapWithConversion(key, valuestring);
-        if (changed != EParamUpdateStatus::Failed) {
-          sValueProvenanceMap->find(key)->second = kRT; // set to runtime
-        }
-      }
-    } catch (std::exception const& e) {
-      std::cerr << "Error in setValue (string) " << e.what() << "\n";
-    }
-  }
-
+  static void setValue(std::string const& key, std::string const& valuestring);
   static void setEnumValue(const std::string&, const std::string&);
   static void setArrayValue(const std::string&, const std::string&);
 
diff --git a/Common/Utils/src/ConfigurableParam.cxx b/Common/Utils/src/ConfigurableParam.cxx
index e581970fc81fb..29e1803096f33 100644
--- a/Common/Utils/src/ConfigurableParam.cxx
+++ b/Common/Utils/src/ConfigurableParam.cxx
@@ -43,6 +43,10 @@ namespace o2
 {
 namespace conf
 {
+boost::property_tree::ptree readINI(std::string const& filepath);
+boost::property_tree::ptree readJSON(std::string const& filepath);
+boost::property_tree::ptree readConfigFile(std::string const& filepath);
+
 std::vector<ConfigurableParam*>* ConfigurableParam::sRegisteredParamClasses = nullptr;
 boost::property_tree::ptree* ConfigurableParam::sPtree = nullptr;
 std::map<std::string, std::pair<std::type_info const&, void*>>* ConfigurableParam::sKeyToStorageMap = nullptr;
@@ -199,9 +203,9 @@ bool ConfigurableParam::configFileExists(std::string const& filepath)
 
 // ------------------------------------------------------------------
 
-boost::property_tree::ptree ConfigurableParam::readConfigFile(std::string const& filepath)
+boost::property_tree::ptree readConfigFile(std::string const& inputDir, std::string const& filepath)
 {
-  auto inpfilename = o2::utils::Str::concat_string(sInputDir, filepath);
+  auto inpfilename = o2::utils::Str::concat_string(inputDir, filepath);
   if (!std::filesystem::exists(inpfilename)) {
     LOG(fatal) << inpfilename << " : config file does not exist!";
   }
@@ -219,9 +223,28 @@ boost::property_tree::ptree ConfigurableParam::readConfigFile(std::string const&
   return pt;
 }
 
+void ConfigurableParam::setValue(std::string const& key, std::string const& valuestring)
+{
+  if (!sIsFullyInitialized) {
+    initialize();
+  }
+  assert(sPtree);
+  try {
+    if (sPtree->get_optional<std::string>(key).is_initialized()) {
+      sPtree->put(key, valuestring);
+      auto changed = updateThroughStorageMapWithConversion(key, valuestring);
+      if (changed != EParamUpdateStatus::Failed) {
+        sValueProvenanceMap->find(key)->second = kRT; // set to runtime
+      }
+    }
+  } catch (std::exception const& e) {
+    std::cerr << "Error in setValue (string) " << e.what() << "\n";
+  }
+}
+
 // ------------------------------------------------------------------
 
-boost::property_tree::ptree ConfigurableParam::readINI(std::string const& filepath)
+boost::property_tree::ptree readINI(std::string const& filepath)
 {
   boost::property_tree::ptree pt;
   try {
@@ -237,7 +260,7 @@ boost::property_tree::ptree ConfigurableParam::readINI(std::string const& filepa
 
 // ------------------------------------------------------------------
 
-boost::property_tree::ptree ConfigurableParam::readJSON(std::string const& filepath)
+boost::property_tree::ptree readJSON(std::string const& filepath)
 {
   boost::property_tree::ptree pt;
 
@@ -258,7 +281,7 @@ void ConfigurableParam::writeJSON(std::string const& filename, std::string const
     LOG(info) << "ignoring writing of json file " << filename;
     return;
   }
-  initPropertyTree();     // update the boost tree before writing
+  initPropertyTree(); // update the boost tree before writing
   auto outfilename = o2::utils::Str::concat_string(sOutputDir, filename);
   if (!keyOnly.empty()) { // write ini for selected key only
     try {
@@ -409,7 +432,7 @@ void ConfigurableParam::updateFromFile(std::string const& configFile, std::strin
     return;
   }
 
-  boost::property_tree::ptree pt = readConfigFile(cfgfile);
+  boost::property_tree::ptree pt = readConfigFile(ConfigurableParam::sInputDir, cfgfile);
 
   std::vector<std::pair<std::string, std::string>> keyValPairs;
   auto request = o2::utils::Str::tokenize(paramsList, ',', true);
diff --git a/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx b/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
index e545c15025d8e..060d3b816cc8e 100644
--- a/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
+++ b/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
@@ -16,6 +16,7 @@
 #define BOOST_TEST_DYN_LINK
 
 #include <boost/test/unit_test.hpp>
+#include <boost/property_tree/ptree.hpp>
 #include "DetectorsRaw/HBFUtils.h"
 #include "DetectorsRaw/RawFileWriter.h"
 #include "Framework/Logger.h"
diff --git a/Detectors/MUON/MCH/Simulation/test/testDigitizer.cxx b/Detectors/MUON/MCH/Simulation/test/testDigitizer.cxx
index 791ab8e7a3909..2a550220bb137 100644
--- a/Detectors/MUON/MCH/Simulation/test/testDigitizer.cxx
+++ b/Detectors/MUON/MCH/Simulation/test/testDigitizer.cxx
@@ -31,6 +31,7 @@
 #include "TGeoManager.h"
 #include "boost/format.hpp"
 #include <boost/test/data/test_case.hpp>
+#include <boost/property_tree/ptree.hpp>
 #include <algorithm>
 #include <unordered_map>
 
diff --git a/Detectors/MUON/MCH/Triggering/test/testEventFinder.cxx b/Detectors/MUON/MCH/Triggering/test/testEventFinder.cxx
index eb786fc1bb068..60275f2cad2f4 100644
--- a/Detectors/MUON/MCH/Triggering/test/testEventFinder.cxx
+++ b/Detectors/MUON/MCH/Triggering/test/testEventFinder.cxx
@@ -27,6 +27,7 @@
 #include <vector>
 
 #include <boost/test/unit_test.hpp>
+#include <boost/property_tree/ptree.hpp>
 
 #include "CommonUtils/ConfigurableParam.h"
 #include "SimulationDataFormat/MCCompLabel.h"
diff --git a/Detectors/TPC/base/test/testTPCParameters.cxx b/Detectors/TPC/base/test/testTPCParameters.cxx
index 14b84da2fbf62..98713e6f5ba57 100644
--- a/Detectors/TPC/base/test/testTPCParameters.cxx
+++ b/Detectors/TPC/base/test/testTPCParameters.cxx
@@ -23,9 +23,9 @@
 #include "TPCBase/ParameterGas.h"
 #include <CommonUtils/ConfigurableParam.h>
 #include <CommonUtils/ConfigurableParamHelper.h>
-namespace o2
-{
-namespace tpc
+#include <boost/property_tree/ptree.hpp>
+
+namespace o2::tpc
 {
 
 constexpr float NominalTimeBin = 8 * o2::constants::lhc::LHCBunchSpacingNS * 1e-3;
@@ -367,5 +367,4 @@ BOOST_AUTO_TEST_CASE(ParameterGEM_test2)
   BOOST_CHECK_CLOSE(o2::conf::ConfigurableParam::getValueAs<float>("TPCGEMParam.KappaStack"), 32, 1e-12);
   BOOST_CHECK_CLOSE(o2::conf::ConfigurableParam::getValueAs<float>("TPCGEMParam.EfficiencyStack"), 33, 1e-3);
 }
-} // namespace tpc
 } // namespace o2
diff --git a/Detectors/TPC/calibration/include/TPCCalibration/IDCAverageGroup.h b/Detectors/TPC/calibration/include/TPCCalibration/IDCAverageGroup.h
index fecbe9f14f18b..dd9a5fc257935 100644
--- a/Detectors/TPC/calibration/include/TPCCalibration/IDCAverageGroup.h
+++ b/Detectors/TPC/calibration/include/TPCCalibration/IDCAverageGroup.h
@@ -20,6 +20,7 @@
 #include "TPCCalibration/IDCAverageGroupBase.h"
 #include "TPCBase/Sector.h"
 #include "TPCBase/CalDet.h"
+#include <boost/property_tree/ptree.hpp>
 #include <vector>
 #include "Rtypes.h"
 
diff --git a/Detectors/TPC/calibration/include/TPCCalibration/IDCFactorization.h b/Detectors/TPC/calibration/include/TPCCalibration/IDCFactorization.h
index 4e9d3ab542ff3..37f4e7c17aa8a 100644
--- a/Detectors/TPC/calibration/include/TPCCalibration/IDCFactorization.h
+++ b/Detectors/TPC/calibration/include/TPCCalibration/IDCFactorization.h
@@ -24,6 +24,7 @@
 #include "TPCCalibration/IDCContainer.h"
 #include "TPCCalibration/IDCGroupHelperSector.h"
 #include "DataFormatsTPC/Defs.h"
+#include <boost/property_tree/ptree.hpp>
 
 namespace o2::tpc
 {
diff --git a/Detectors/TPC/calibration/include/TPCCalibration/SACFactorization.h b/Detectors/TPC/calibration/include/TPCCalibration/SACFactorization.h
index b1b4430c11a00..7141750b8ce37 100644
--- a/Detectors/TPC/calibration/include/TPCCalibration/SACFactorization.h
+++ b/Detectors/TPC/calibration/include/TPCCalibration/SACFactorization.h
@@ -21,6 +21,7 @@
 #include "Rtypes.h"
 #include "TPCCalibration/IDCContainer.h"
 #include "DataFormatsTPC/Defs.h"
+#include <boost/property_tree/ptree.hpp>
 
 namespace o2::tpc
 {
diff --git a/Detectors/TPC/calibration/src/IDCGroupingParameter.cxx b/Detectors/TPC/calibration/src/IDCGroupingParameter.cxx
index bbecb4df8f8fd..895b6957d8a01 100644
--- a/Detectors/TPC/calibration/src/IDCGroupingParameter.cxx
+++ b/Detectors/TPC/calibration/src/IDCGroupingParameter.cxx
@@ -12,6 +12,8 @@
 #include "TPCCalibration/IDCGroupingParameter.h"
 #include "Framework/Logger.h"
 #include "Algorithm/RangeTokenizer.h"
+#include <boost/property_tree/ptree.hpp>
+#include <cassert>
 #include <stdexcept>
 
 using namespace o2::tpc;
diff --git a/Detectors/TPC/reconstruction/macro/makeIonTail.C b/Detectors/TPC/reconstruction/macro/makeIonTail.C
index 98db6540d20f7..d8d1be6830626 100644
--- a/Detectors/TPC/reconstruction/macro/makeIonTail.C
+++ b/Detectors/TPC/reconstruction/macro/makeIonTail.C
@@ -32,6 +32,7 @@
 #include "TPCBase/CRUCalibHelpers.h"
 #include "TPCBase/CRU.h"
 #include "CommonUtils/TreeStreamRedirector.h"
+#include <boost/property_tree/ptree.hpp>
 
 using namespace o2::tpc;
 size_t digitsInSaturateion(std::vector<Digit>& digits, bool correctCharge = false, CalPad* pedestals = nullptr, o2::utils::TreeStreamRedirector* stream = nullptr);
diff --git a/Detectors/TPC/simulation/macro/toyCluster.C b/Detectors/TPC/simulation/macro/toyCluster.C
index 257adaf5f79fc..2087a7773fe22 100644
--- a/Detectors/TPC/simulation/macro/toyCluster.C
+++ b/Detectors/TPC/simulation/macro/toyCluster.C
@@ -58,6 +58,7 @@
 #include "TPCReconstruction/HwClusterer.h"
 #include "TPCSimulation/GEMAmplification.h"
 #endif
+#include <boost/property_tree/ptree.hpp>
 
 using namespace o2::tpc;
 void fillTPCHits(const float theta, const float phi, const float dedx, std::vector<HitGroup>& hitGroupSector, std::pair<GlobalPosition3D, GlobalPosition3D>& trackInfo);
diff --git a/Detectors/TPC/workflow/src/ApplyCCDBCalibSpec.cxx b/Detectors/TPC/workflow/src/ApplyCCDBCalibSpec.cxx
index 37429a04c4b34..9c588ff50a45c 100644
--- a/Detectors/TPC/workflow/src/ApplyCCDBCalibSpec.cxx
+++ b/Detectors/TPC/workflow/src/ApplyCCDBCalibSpec.cxx
@@ -19,6 +19,7 @@
 #include "TPCBase/ParameterGas.h"
 #include "DataFormatsTPC/LtrCalibData.h"
 #include "TPCWorkflow/ApplyCCDBCalibSpec.h"
+#include <boost/property_tree/ptree.hpp>
 
 using namespace o2::framework;
 
diff --git a/macro/o2sim.C b/macro/o2sim.C
index 5257a225d3ed9..f610be730eeb7 100644
--- a/macro/o2sim.C
+++ b/macro/o2sim.C
@@ -39,6 +39,7 @@
 #include <sstream>
 #endif
 #include "migrateSimFiles.C"
+#include <boost/property_tree/ptree.hpp>
 
 void check_notransport()
 {

From 63a61b8a97fde7d52856b07bdff6de05940748ab Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 27 Oct 2023 10:26:44 +0200
Subject: [PATCH 009/726] Modernize Headers/TimeStamp.h

---
 DataFormats/Headers/include/Headers/TimeStamp.h | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/DataFormats/Headers/include/Headers/TimeStamp.h b/DataFormats/Headers/include/Headers/TimeStamp.h
index bd4bd8c8b94e9..5867cf6bb4949 100644
--- a/DataFormats/Headers/include/Headers/TimeStamp.h
+++ b/DataFormats/Headers/include/Headers/TimeStamp.h
@@ -23,9 +23,7 @@
 #include <cassert>
 #include <type_traits> // for std::integral_constant
 
-namespace o2
-{
-namespace header
+namespace o2::header
 {
 
 // https://lhc-machine-outreach.web.cern.ch/lhc-machine-outreach/collisions.htm
@@ -159,7 +157,7 @@ class TimeStamp
   }
 
   // TODO: implement transformation from one unit to the other
-  //void transform(const TimeUnitID& unit) {
+  // void transform(const TimeUnitID& unit) {
   //  if (mUnit == unit) return;
   //  ...
   //}
@@ -182,7 +180,6 @@ class TimeStamp
     };
   };
 };
-} //namespace header
-} //namespace o2
+} // namespace o2::header
 
 #endif

From cfe60f1bf2604d872a706a3d600ef8a93038c54c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 27 Oct 2023 10:26:45 +0200
Subject: [PATCH 010/726] Fix type comparison for C++20

The two are effectively different types and there is nothing which allows
implicitly casting between them. Also add some protection making sure we notice
if we change one of the two sides inadvertedly.
---
 DataFormats/Headers/include/Headers/TimeStamp.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/DataFormats/Headers/include/Headers/TimeStamp.h b/DataFormats/Headers/include/Headers/TimeStamp.h
index 5867cf6bb4949..b6d2b26ea4822 100644
--- a/DataFormats/Headers/include/Headers/TimeStamp.h
+++ b/DataFormats/Headers/include/Headers/TimeStamp.h
@@ -138,12 +138,14 @@ class TimeStamp
     static_assert(std::is_same<typename T::rep, Rep>::value && std::is_same<typename T::period, Period>::value,
                   "only clock and duration types defining the rep and period member types are allowed");
     using duration = std::chrono::duration<Rep, Period>;
-    if (mUnit == sClockLHC) {
+    static_assert(sizeof(mUnit) == sizeof(sClockLHC), "size mismatch of mUnit and sClockLHC");
+    if (memcmp(&mUnit, &sClockLHC, sizeof(sClockLHC)) == 0) {
       // cast each part individually, if the precision of the return type
       // is smaller the values are simply truncated
       return std::chrono::duration_cast<duration>(LHCOrbitClock::duration(mPeriod) + LHCBunchClock::duration(mBCNumber));
     }
-    if (mUnit == sMicroSeconds) {
+    static_assert(sizeof(mUnit) == sizeof(sMicroSeconds), "size mismatch of mUnit and sMicroSeconds");
+    if (memcmp(&mUnit, &sMicroSeconds, sizeof(sMicroSeconds)) == 0) {
       // TODO: is there a better way to mark the subticks invalid for the
       // micro seconds representation? First step is probably to remove/rename the
       // variable

From 60e7d3b6641f1044eb208cba10abff3318770631 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 27 Oct 2023 11:27:52 +0200
Subject: [PATCH 011/726] Adhere to coding conventions

---
 Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx b/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx
index 2372ba07a28f7..a1f9b385fd81a 100644
--- a/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx
+++ b/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx
@@ -47,7 +47,7 @@ InjectorFunction dcs2dpl()
 // InjectorFunction dcs2dpl()
 {
   return [](TimingInfo&, ServiceRegistryRef const& services, fair::mq::Parts& parts, ChannelRetriever channelRetriever, size_t newTimesliceId, bool&) -> bool {
-    auto *device = services.get<RawDeviceService>().device();
+    auto* device = services.get<RawDeviceService>().device();
     std::string messageHeader{static_cast<const char*>(parts.At(0)->GetData()), parts.At(0)->GetSize()};
     size_t dataSize = parts.At(1)->GetSize();
     std::string messageData{static_cast<const char*>(parts.At(1)->GetData()), parts.At(1)->GetSize()};

From b1074f9135dd69ce6f088e6e323880567222a84b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 27 Oct 2023 11:27:52 +0200
Subject: [PATCH 012/726] Notify DPL about dispatched out-of-band data

---
 Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx b/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx
index a1f9b385fd81a..73615bc5c47b4 100644
--- a/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx
+++ b/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx
@@ -91,6 +91,7 @@ InjectorFunction dcs2dpl()
     outParts.AddPart(std::move(plMessageF));
     sendOnChannel(*device, outParts, channel, newTimesliceId);
     LOG(info) << "Sent CTP counters DPL message" << std::flush;
+    services.get<MessageContext>().fakeDispatch();
     return true;
   };
 }

From 12d0af3ffb6bf3b1f49f61d39126f93a459ab95d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 27 Oct 2023 11:35:23 +0200
Subject: [PATCH 013/726] Ignore checking for FairMQ version

* Version is way too obsolete in any case.
* Anything which is not the version in alidist is unsupported in any case.
* It breaks whenever git is not able to determine a valid version.
---
 dependencies/O2Dependencies.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dependencies/O2Dependencies.cmake b/dependencies/O2Dependencies.cmake
index 4d90d59c9f2be..f4d591e07ab65 100644
--- a/dependencies/O2Dependencies.cmake
+++ b/dependencies/O2Dependencies.cmake
@@ -88,7 +88,7 @@ set_package_properties(Microsoft.GSL
                        TYPE REQUIRED
                        PURPOSE "Mainly for its span")
 
-find_package(FairMQ 1.4.41 CONFIG)
+find_package(FairMQ CONFIG)
 set_package_properties(FairMQ PROPERTIES TYPE REQUIRED)
 
 # find_package(protobuf CONFIG)

From d9e40a2194fc468bdaa5d0902e642688423fe8e4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 27 Oct 2023 12:28:27 +0200
Subject: [PATCH 014/726] DPL: propaedeutic for better GUI shutdown

This will allow in the GUI to know wether the children have
all shutdown and will keep displaying the debug gui, even
when clicking on the close button, until that is the case.
---
 Framework/Core/include/Framework/GuiCallbackContext.h | 1 +
 Framework/Core/src/runDataProcessing.cxx              | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/include/Framework/GuiCallbackContext.h b/Framework/Core/include/Framework/GuiCallbackContext.h
index d1efd5717a4de..9c0f21f3aec9c 100644
--- a/Framework/Core/include/Framework/GuiCallbackContext.h
+++ b/Framework/Core/include/Framework/GuiCallbackContext.h
@@ -43,6 +43,7 @@ struct GuiCallbackContext {
   DebugGUI* plugin = nullptr;
   void* window = nullptr;
   bool* guiQuitRequested = nullptr;
+  bool* allChildrenGone = nullptr;
   std::function<void(void)> callback;
   std::set<GuiRenderer*> renderers;
 };
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index c3d6dc1a7d4b7..0082a79906c61 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1424,6 +1424,8 @@ int runStateMachine(DataProcessorSpecs const& workflow,
 
   uv_timer_t metricDumpTimer;
   metricDumpTimer.data = &serverContext;
+  bool allChildrenGone = false;
+  guiContext.allChildrenGone = &allChildrenGone;
 
   while (true) {
     // If control forced some transition on us, we push it to the queue.
@@ -2093,7 +2095,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
         driverInfo.sigchldRequested = false;
         processChildrenOutput(driverInfo, infos, runningWorkflow.devices, controls);
         hasError = processSigChild(infos, runningWorkflow.devices);
-        bool allChildrenGone = areAllChildrenGone(infos);
+        allChildrenGone = areAllChildrenGone(infos);
         bool canExit = checkIfCanExit(infos);
         bool supposedToQuit = (guiQuitRequested || canExit || graceful_exit);
 

From 25910215071a38917b1f394d259517510d961434 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 27 Oct 2023 13:00:41 +0200
Subject: [PATCH 015/726] DPL: dump walltime for last iteration

This will dump the elapsed_time_ms metric in the performance metric
providing every 5s the value of the last completed iteration.
---
 Framework/Core/src/runDataProcessing.cxx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index c3d6dc1a7d4b7..ffdb0173abbff 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1152,6 +1152,7 @@ std::vector<std::regex> getDumpableMetrics()
   dumpableMetrics.emplace_back("^table-bytes-.*");
   dumpableMetrics.emplace_back("^total-timeframes.*");
   dumpableMetrics.emplace_back("^device_state.*");
+  dumpableMetrics.emplace_back("^elapsed_time_ms$");
   return dumpableMetrics;
 }
 

From 188e115115681e0d2266baa747acf1adbe643b0d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 27 Oct 2023 13:46:32 +0200
Subject: [PATCH 016/726] DPL: dump walltime for last iteration

This will dump the elapsed_time_ms metric in the performance metric
providing every 5s the value of the last completed iteration.
---
 Framework/Core/include/Framework/DataProcessingStats.h | 1 +
 Framework/Core/src/CommonServices.cxx                  | 4 ++++
 Framework/Core/src/DataProcessingDevice.cxx            | 8 ++++++--
 Framework/Core/src/runDataProcessing.cxx               | 1 +
 4 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/include/Framework/DataProcessingStats.h b/Framework/Core/include/Framework/DataProcessingStats.h
index 6d9bceeeafa6f..ae8ac29d89e15 100644
--- a/Framework/Core/include/Framework/DataProcessingStats.h
+++ b/Framework/Core/include/Framework/DataProcessingStats.h
@@ -30,6 +30,7 @@ enum struct ProcessingStatsId : short {
   INCOMPLETE_INPUTS,
   TOTAL_INPUTS,
   LAST_ELAPSED_TIME_MS,
+  TOTAL_WALL_TIME_MS,
   LAST_PROCESSED_SIZE,
   TOTAL_PROCESSED_SIZE,
   TOTAL_SIGUSR1,
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 460702d654213..44d63cd2ab1f5 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -748,6 +748,10 @@ o2::framework::ServiceSpec CommonServices::dataProcessingStats()
                    .metricId = (int)ProcessingStatsId::LAST_ELAPSED_TIME_MS,
                    .kind = Kind::UInt64,
                    .minPublishInterval = quickUpdateInterval},
+        MetricSpec{.name = "total_wall_time_ms",
+                   .metricId = (int)ProcessingStatsId::TOTAL_WALL_TIME_MS,
+                   .kind = Kind::UInt64,
+                   .minPublishInterval = quickUpdateInterval},
         MetricSpec{.name = "last_processed_input_size_byte",
                    .metricId = (int)ProcessingStatsId::LAST_PROCESSED_SIZE,
                    .kind = Kind::UInt64,
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 23bdc9e75806b..b39607366adec 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -2179,9 +2179,13 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
     buffer[record.size()] = 0;
     states.updateState({.id = short((int)ProcessingStateId::DATA_RELAYER_BASE + action.slot.index), (int)(record.size() + buffer - relayerSlotState), relayerSlotState});
     uint64_t tEnd = uv_hrtime();
-    stats.updateStats({(int)ProcessingStatsId::LAST_ELAPSED_TIME_MS, DataProcessingStats::Op::Set, (int64_t)(tEnd - tStart)});
+    // tEnd and tStart are in nanoseconds according to https://docs.libuv.org/en/v1.x/misc.html#c.uv_hrtime
+    int64_t wallTimeMs = (tEnd - tStart) / 1000000;
+    stats.updateStats({(int)ProcessingStatsId::LAST_ELAPSED_TIME_MS, DataProcessingStats::Op::Set, wallTimeMs});
+    // Sum up the total wall time, in milliseconds.
+    stats.updateStats({(int)ProcessingStatsId::TOTAL_WALL_TIME_MS, DataProcessingStats::Op::Add, wallTimeMs});
     // The time interval is in seconds while tEnd - tStart is in nanoseconds, so we divide by 1000000 to get the fraction in ms/s.
-    stats.updateStats({(short)ProcessingStatsId::CPU_USAGE_FRACTION, DataProcessingStats::Op::CumulativeRate, (int64_t)(tEnd - tStart) / 1000000});
+    stats.updateStats({(short)ProcessingStatsId::CPU_USAGE_FRACTION, DataProcessingStats::Op::CumulativeRate, wallTimeMs});
     stats.updateStats({(int)ProcessingStatsId::LAST_PROCESSED_SIZE, DataProcessingStats::Op::Set, calculateTotalInputRecordSize(record)});
     stats.updateStats({(int)ProcessingStatsId::TOTAL_PROCESSED_SIZE, DataProcessingStats::Op::Add, calculateTotalInputRecordSize(record)});
     auto latency = calculateInputRecordLatency(record, tStartMilli);
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index c3d6dc1a7d4b7..9216ea0463868 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1152,6 +1152,7 @@ std::vector<std::regex> getDumpableMetrics()
   dumpableMetrics.emplace_back("^table-bytes-.*");
   dumpableMetrics.emplace_back("^total-timeframes.*");
   dumpableMetrics.emplace_back("^device_state.*");
+  dumpableMetrics.emplace_back("^total_wall_time_ms$");
   return dumpableMetrics;
 }
 

From 8cd4b7af7e6901b5a16caf171e1c6fc647bcf564 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 27 Oct 2023 13:53:41 +0200
Subject: [PATCH 017/726] DPL: dump total wall time per device

---
 Framework/Core/include/Framework/DataProcessingStats.h | 1 +
 Framework/Core/src/CommonServices.cxx                  | 4 ++++
 Framework/Core/src/DataProcessingDevice.cxx            | 8 ++++++--
 Framework/Core/src/runDataProcessing.cxx               | 1 +
 4 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/include/Framework/DataProcessingStats.h b/Framework/Core/include/Framework/DataProcessingStats.h
index 6d9bceeeafa6f..ae8ac29d89e15 100644
--- a/Framework/Core/include/Framework/DataProcessingStats.h
+++ b/Framework/Core/include/Framework/DataProcessingStats.h
@@ -30,6 +30,7 @@ enum struct ProcessingStatsId : short {
   INCOMPLETE_INPUTS,
   TOTAL_INPUTS,
   LAST_ELAPSED_TIME_MS,
+  TOTAL_WALL_TIME_MS,
   LAST_PROCESSED_SIZE,
   TOTAL_PROCESSED_SIZE,
   TOTAL_SIGUSR1,
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 460702d654213..44d63cd2ab1f5 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -748,6 +748,10 @@ o2::framework::ServiceSpec CommonServices::dataProcessingStats()
                    .metricId = (int)ProcessingStatsId::LAST_ELAPSED_TIME_MS,
                    .kind = Kind::UInt64,
                    .minPublishInterval = quickUpdateInterval},
+        MetricSpec{.name = "total_wall_time_ms",
+                   .metricId = (int)ProcessingStatsId::TOTAL_WALL_TIME_MS,
+                   .kind = Kind::UInt64,
+                   .minPublishInterval = quickUpdateInterval},
         MetricSpec{.name = "last_processed_input_size_byte",
                    .metricId = (int)ProcessingStatsId::LAST_PROCESSED_SIZE,
                    .kind = Kind::UInt64,
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 23bdc9e75806b..b39607366adec 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -2179,9 +2179,13 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
     buffer[record.size()] = 0;
     states.updateState({.id = short((int)ProcessingStateId::DATA_RELAYER_BASE + action.slot.index), (int)(record.size() + buffer - relayerSlotState), relayerSlotState});
     uint64_t tEnd = uv_hrtime();
-    stats.updateStats({(int)ProcessingStatsId::LAST_ELAPSED_TIME_MS, DataProcessingStats::Op::Set, (int64_t)(tEnd - tStart)});
+    // tEnd and tStart are in nanoseconds according to https://docs.libuv.org/en/v1.x/misc.html#c.uv_hrtime
+    int64_t wallTimeMs = (tEnd - tStart) / 1000000;
+    stats.updateStats({(int)ProcessingStatsId::LAST_ELAPSED_TIME_MS, DataProcessingStats::Op::Set, wallTimeMs});
+    // Sum up the total wall time, in milliseconds.
+    stats.updateStats({(int)ProcessingStatsId::TOTAL_WALL_TIME_MS, DataProcessingStats::Op::Add, wallTimeMs});
     // The time interval is in seconds while tEnd - tStart is in nanoseconds, so we divide by 1000000 to get the fraction in ms/s.
-    stats.updateStats({(short)ProcessingStatsId::CPU_USAGE_FRACTION, DataProcessingStats::Op::CumulativeRate, (int64_t)(tEnd - tStart) / 1000000});
+    stats.updateStats({(short)ProcessingStatsId::CPU_USAGE_FRACTION, DataProcessingStats::Op::CumulativeRate, wallTimeMs});
     stats.updateStats({(int)ProcessingStatsId::LAST_PROCESSED_SIZE, DataProcessingStats::Op::Set, calculateTotalInputRecordSize(record)});
     stats.updateStats({(int)ProcessingStatsId::TOTAL_PROCESSED_SIZE, DataProcessingStats::Op::Add, calculateTotalInputRecordSize(record)});
     auto latency = calculateInputRecordLatency(record, tStartMilli);
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index c3d6dc1a7d4b7..9216ea0463868 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1152,6 +1152,7 @@ std::vector<std::regex> getDumpableMetrics()
   dumpableMetrics.emplace_back("^table-bytes-.*");
   dumpableMetrics.emplace_back("^total-timeframes.*");
   dumpableMetrics.emplace_back("^device_state.*");
+  dumpableMetrics.emplace_back("^total_wall_time_ms$");
   return dumpableMetrics;
 }
 

From 9125ce1bef7599253ab51db3b11acd7bad72c567 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 27 Oct 2023 15:05:10 +0200
Subject: [PATCH 018/726] Fix deprecation error / warning

---
 Utilities/Tools/cpulimit/cpulimit.c |  7 +-----
 Utilities/Tools/cpulimit/memrchr.c  | 38 -----------------------------
 2 files changed, 1 insertion(+), 44 deletions(-)
 delete mode 100644 Utilities/Tools/cpulimit/memrchr.c

diff --git a/Utilities/Tools/cpulimit/cpulimit.c b/Utilities/Tools/cpulimit/cpulimit.c
index e7c801496ba3d..e35a1565253e7 100644
--- a/Utilities/Tools/cpulimit/cpulimit.c
+++ b/Utilities/Tools/cpulimit/cpulimit.c
@@ -33,7 +33,6 @@
 #include <getopt.h>
 #include <time.h>
 #include <signal.h>
-#include <string.h>
 #include <errno.h>
 #include <string.h>
 #include <sys/stat.h>
@@ -56,10 +55,6 @@
 #include <sys/sysinfo.h>
 #endif
 
-#ifdef __APPLE__
-#include "memrchr.c"
-#endif
-
 //some useful macro
 #ifndef MIN
 #define MIN(a,b) (((a)<(b))?(a):(b))
@@ -329,7 +324,7 @@ int main(int argc, char **argv) {
 	int include_children = 0;
 
 	//get program name
-	char *p = (char*)memrchr(argv[0], (unsigned int)'/', strlen(argv[0]));
+	char *p = (char*)strrchr(argv[0], '/');
 	program_name = p==NULL ? argv[0] : (p+1);
 	//get current pid
 	cpulimit_pid = getpid();
diff --git a/Utilities/Tools/cpulimit/memrchr.c b/Utilities/Tools/cpulimit/memrchr.c
deleted file mode 100644
index 1f3787020afad..0000000000000
--- a/Utilities/Tools/cpulimit/memrchr.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright (c) 2007 Todd C. Miller <Todd.Miller@courtesan.com>
- *
- * Permission to use, copy, modify, and distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
- * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
- * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
- * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
- * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
- * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
- */
-
-#include <sys/types.h>
-
-/*
- * Reverse memchr()
- * Find the last occurrence of 'c' in the buffer 's' of size 'n'.
- */
-void *
-memrchr(s, c, n)
-    const void *s;
-    int c;
-    size_t n;
-{
-    if (n != 0) {
-    	const unsigned char *cp;
-        cp = (unsigned char *)s + n;
-	do {
-	    if (*(--cp) == (unsigned char)c)
-		return((void *)cp);
-	} while (--n != 0);
-    }
-    return((void *)0);
-}

From 0dc062532c403ca98d0d5f7d9b17fb488bdcacc6 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 27 Oct 2023 22:17:54 +0200
Subject: [PATCH 019/726] DPL Analysis: drop unused API

Since we use bulk reading, this is not really needed anymore.
---
 .../AnalysisSupport/src/DataInputDirector.cxx | 21 -------------------
 .../AnalysisSupport/src/DataInputDirector.h   |  1 -
 2 files changed, 22 deletions(-)

diff --git a/Framework/AnalysisSupport/src/DataInputDirector.cxx b/Framework/AnalysisSupport/src/DataInputDirector.cxx
index c7f1a0c7d5de1..d8df1dab09feb 100644
--- a/Framework/AnalysisSupport/src/DataInputDirector.cxx
+++ b/Framework/AnalysisSupport/src/DataInputDirector.cxx
@@ -710,27 +710,6 @@ DataInputDescriptor* DataInputDirector::getDataInputDescriptor(header::DataHeade
   return result;
 }
 
-std::unique_ptr<TTreeReader> DataInputDirector::getTreeReader(header::DataHeader dh, int counter, int numTF, std::string treename)
-{
-  std::unique_ptr<TTreeReader> reader = nullptr;
-  auto didesc = getDataInputDescriptor(dh);
-  // if NOT match then use defaultDataInputDescriptor
-  if (!didesc) {
-    didesc = mdefaultDataInputDescriptor;
-  }
-
-  auto fileAndFolder = didesc->getFileFolder(counter, numTF);
-  if (fileAndFolder.file) {
-    treename = fileAndFolder.folderName + "/" + treename;
-    reader = std::make_unique<TTreeReader>(treename.c_str(), fileAndFolder.file);
-    if (!reader) {
-      throw std::runtime_error(fmt::format(R"(Couldn't create TTreeReader for tree "{}" in file "{}")", treename, fileAndFolder.file->GetName()));
-    }
-  }
-
-  return reader;
-}
-
 FileAndFolder DataInputDirector::getFileFolder(header::DataHeader dh, int counter, int numTF)
 {
   auto didesc = getDataInputDescriptor(dh);
diff --git a/Framework/AnalysisSupport/src/DataInputDirector.h b/Framework/AnalysisSupport/src/DataInputDirector.h
index e9b483bbc729a..ec3f99779a5b1 100644
--- a/Framework/AnalysisSupport/src/DataInputDirector.h
+++ b/Framework/AnalysisSupport/src/DataInputDirector.h
@@ -138,7 +138,6 @@ class DataInputDirector
   DataInputDescriptor* getDataInputDescriptor(header::DataHeader dh);
   int getNumberInputDescriptors() { return mdataInputDescriptors.size(); }
 
-  std::unique_ptr<TTreeReader> getTreeReader(header::DataHeader dh, int counter, int numTF, std::string treeName);
   bool readTree(DataAllocator& outputs, header::DataHeader dh, int counter, int numTF, size_t& totalSizeCompressed, size_t& totalSizeUncompressed);
   uint64_t getTimeFrameNumber(header::DataHeader dh, int counter, int numTF);
   FileAndFolder getFileFolder(header::DataHeader dh, int counter, int numTF);

From 25adbd8a3d5d72424fa1d83b84dc934a78059bc2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 27 Oct 2023 22:29:20 +0200
Subject: [PATCH 020/726] DPL Analysis: drop unused API

Since we use bulk reading, TTreeReader usage is not really needed anymore.
---
 .../AnalysisSupport/src/DataInputDirector.cxx | 21 -------------------
 .../AnalysisSupport/src/DataInputDirector.h   |  2 --
 2 files changed, 23 deletions(-)

diff --git a/Framework/AnalysisSupport/src/DataInputDirector.cxx b/Framework/AnalysisSupport/src/DataInputDirector.cxx
index c7f1a0c7d5de1..d8df1dab09feb 100644
--- a/Framework/AnalysisSupport/src/DataInputDirector.cxx
+++ b/Framework/AnalysisSupport/src/DataInputDirector.cxx
@@ -710,27 +710,6 @@ DataInputDescriptor* DataInputDirector::getDataInputDescriptor(header::DataHeade
   return result;
 }
 
-std::unique_ptr<TTreeReader> DataInputDirector::getTreeReader(header::DataHeader dh, int counter, int numTF, std::string treename)
-{
-  std::unique_ptr<TTreeReader> reader = nullptr;
-  auto didesc = getDataInputDescriptor(dh);
-  // if NOT match then use defaultDataInputDescriptor
-  if (!didesc) {
-    didesc = mdefaultDataInputDescriptor;
-  }
-
-  auto fileAndFolder = didesc->getFileFolder(counter, numTF);
-  if (fileAndFolder.file) {
-    treename = fileAndFolder.folderName + "/" + treename;
-    reader = std::make_unique<TTreeReader>(treename.c_str(), fileAndFolder.file);
-    if (!reader) {
-      throw std::runtime_error(fmt::format(R"(Couldn't create TTreeReader for tree "{}" in file "{}")", treename, fileAndFolder.file->GetName()));
-    }
-  }
-
-  return reader;
-}
-
 FileAndFolder DataInputDirector::getFileFolder(header::DataHeader dh, int counter, int numTF)
 {
   auto didesc = getDataInputDescriptor(dh);
diff --git a/Framework/AnalysisSupport/src/DataInputDirector.h b/Framework/AnalysisSupport/src/DataInputDirector.h
index e9b483bbc729a..ea54d6b3c7958 100644
--- a/Framework/AnalysisSupport/src/DataInputDirector.h
+++ b/Framework/AnalysisSupport/src/DataInputDirector.h
@@ -12,7 +12,6 @@
 #define o2_framework_DataInputDirector_H_INCLUDED
 
 #include "TFile.h"
-#include "TTreeReader.h"
 
 #include "Framework/DataDescriptorMatcher.h"
 #include "Framework/DataAllocator.h"
@@ -138,7 +137,6 @@ class DataInputDirector
   DataInputDescriptor* getDataInputDescriptor(header::DataHeader dh);
   int getNumberInputDescriptors() { return mdataInputDescriptors.size(); }
 
-  std::unique_ptr<TTreeReader> getTreeReader(header::DataHeader dh, int counter, int numTF, std::string treeName);
   bool readTree(DataAllocator& outputs, header::DataHeader dh, int counter, int numTF, size_t& totalSizeCompressed, size_t& totalSizeUncompressed);
   uint64_t getTimeFrameNumber(header::DataHeader dh, int counter, int numTF);
   FileAndFolder getFileFolder(header::DataHeader dh, int counter, int numTF);

From 89058ee6be5412e600e8d4bbf7ec69c7d6829a37 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 27 Oct 2023 22:27:44 +0200
Subject: [PATCH 021/726] DPL: forward declare Monitoring

---
 Framework/AnalysisSupport/src/DataInputDirector.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Framework/AnalysisSupport/src/DataInputDirector.h b/Framework/AnalysisSupport/src/DataInputDirector.h
index ceb43c2fec6f9..eca0ef195d111 100644
--- a/Framework/AnalysisSupport/src/DataInputDirector.h
+++ b/Framework/AnalysisSupport/src/DataInputDirector.h
@@ -15,11 +15,15 @@
 
 #include "Framework/DataDescriptorMatcher.h"
 #include "Framework/DataAllocator.h"
-#include "Monitoring/Monitoring.h"
 
 #include <regex>
 #include "rapidjson/fwd.h"
 
+namespace o2::monitoring
+{
+class Monitoring;
+}
+
 namespace o2::framework
 {
 

From a947355c437ea57dae0f4dfe6511e0fbcc741b55 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 27 Oct 2023 22:27:38 +0200
Subject: [PATCH 022/726] DPL: follow coding conventions

---
 Framework/AnalysisSupport/src/DataInputDirector.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Framework/AnalysisSupport/src/DataInputDirector.h b/Framework/AnalysisSupport/src/DataInputDirector.h
index ea54d6b3c7958..ceb43c2fec6f9 100644
--- a/Framework/AnalysisSupport/src/DataInputDirector.h
+++ b/Framework/AnalysisSupport/src/DataInputDirector.h
@@ -8,8 +8,8 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
-#ifndef o2_framework_DataInputDirector_H_INCLUDED
-#define o2_framework_DataInputDirector_H_INCLUDED
+#ifndef O2_FRAMEWORK_DATAINPUTDIRECTOR_H_
+#define O2_FRAMEWORK_DATAINPUTDIRECTOR_H_
 
 #include "TFile.h"
 
@@ -167,4 +167,4 @@ class DataInputDirector
 
 } // namespace o2::framework
 
-#endif // o2_framework_DataInputDirector_H_INCLUDED
+#endif // O2_FRAMEWORK_DATAINPUTDIRECTOR_H_

From 20e7dea9b76df3d1763ed322ef58832d53db6fca Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 28 Oct 2023 11:04:09 +0200
Subject: [PATCH 023/726] DPL: forward declare Monitoring

---
 Framework/AnalysisSupport/src/DataInputDirector.cxx | 2 ++
 Framework/AnalysisSupport/src/DataInputDirector.h   | 6 +++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/Framework/AnalysisSupport/src/DataInputDirector.cxx b/Framework/AnalysisSupport/src/DataInputDirector.cxx
index d8df1dab09feb..6e072d47e523f 100644
--- a/Framework/AnalysisSupport/src/DataInputDirector.cxx
+++ b/Framework/AnalysisSupport/src/DataInputDirector.cxx
@@ -16,6 +16,8 @@
 #include "Headers/DataHeader.h"
 #include "Framework/TableTreeHelpers.h"
 #include "Monitoring/Tags.h"
+#include "Monitoring/Metric.h"
+#include "Monitoring/Monitoring.h"
 
 #include "rapidjson/document.h"
 #include "rapidjson/prettywriter.h"
diff --git a/Framework/AnalysisSupport/src/DataInputDirector.h b/Framework/AnalysisSupport/src/DataInputDirector.h
index ceb43c2fec6f9..eca0ef195d111 100644
--- a/Framework/AnalysisSupport/src/DataInputDirector.h
+++ b/Framework/AnalysisSupport/src/DataInputDirector.h
@@ -15,11 +15,15 @@
 
 #include "Framework/DataDescriptorMatcher.h"
 #include "Framework/DataAllocator.h"
-#include "Monitoring/Monitoring.h"
 
 #include <regex>
 #include "rapidjson/fwd.h"
 
+namespace o2::monitoring
+{
+class Monitoring;
+}
+
 namespace o2::framework
 {
 

From 1afdb318ae37de7cff7ac68992b0b3c34d8bf91f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 28 Oct 2023 16:03:15 +0200
Subject: [PATCH 024/726] DPL: use Catch2 for ROOT tests

---
 Framework/Core/CMakeLists.txt                | 14 ++--
 Framework/Core/test/test_Root2ArrowTable.cxx | 68 +++++++++-----------
 2 files changed, 41 insertions(+), 41 deletions(-)

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index 898fb3f6441fd..e58a6414e06ac 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -275,11 +275,15 @@ o2_add_test(O2DatabasePDG NAME test_Framework_test_O2DatabasePDG
             LABELS framework
             PUBLIC_LINK_LIBRARIES O2::Framework O2::FrameworkPhysicsSupport)
 
-o2_add_test(Root2ArrowTable NAME test_Framework_test_Root2ArrowTable
-            SOURCES test/test_Root2ArrowTable.cxx
-            COMPONENT_NAME Framework
-            LABELS framework
-            PUBLIC_LINK_LIBRARIES O2::Framework ROOT::ROOTDataFrame)
+# All the tests which require ROOT to work
+add_executable(o2-test-framework-root
+               test/test_Root2ArrowTable.cxx
+               )
+target_link_libraries(o2-test-framework-root PRIVATE O2::Framework)
+target_link_libraries(o2-test-framework-root PRIVATE O2::Catch2)
+target_link_libraries(o2-test-framework-root PRIVATE ROOT::ROOTDataFrame)
+set_property(TARGET o2-test-framework-root PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
+add_test(NAME framework:root COMMAND o2-test-framework-root --skip-benchmarks)
 
 o2_add_test(InfoLogger NAME test_Framework_test_InfoLogger
             SOURCES test/test_InfoLogger.cxx
diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx
index 43876b3230bae..b22432d8565a5 100644
--- a/Framework/Core/test/test_Root2ArrowTable.cxx
+++ b/Framework/Core/test/test_Root2ArrowTable.cxx
@@ -9,11 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
-#define BOOST_TEST_MODULE Test Framework TableBuilder
-#define BOOST_TEST_MAIN
-#define BOOST_TEST_DYN_LINK
-
-#include <boost/test/unit_test.hpp>
+#include <catch_amalgamated.hpp>
 
 #include "Framework/TableBuilder.h"
 #include "Framework/RootTableBuilderHelpers.h"
@@ -32,7 +28,7 @@
 
 using namespace o2::framework;
 
-BOOST_AUTO_TEST_CASE(RootTree2Table)
+TEST_CASE("RootTree2Table")
 {
   using namespace o2::framework;
   /// Create a simple TTree
@@ -78,43 +74,43 @@ BOOST_AUTO_TEST_CASE(RootTree2Table)
 
   RootTableBuilderHelpers::convertTTree(builder, reader, std::move(xyzReader), std::move(ijkReader), std::move(pxReader), std::move(pyReader), std::move(pzReader), std::move(randomReader), std::move(evReader));
   auto table = builder.finalize();
-  BOOST_REQUIRE_EQUAL(table->num_rows(), 1000);
-  BOOST_REQUIRE_EQUAL(table->num_columns(), 7);
-  BOOST_REQUIRE_EQUAL(table->schema()->field(0)->type()->id(), arrow::fixed_size_list(arrow::float32(), 3)->id());
-  BOOST_REQUIRE_EQUAL(table->schema()->field(1)->type()->id(), arrow::fixed_size_list(arrow::int32(), 2)->id());
-  BOOST_REQUIRE_EQUAL(table->schema()->field(2)->type()->id(), arrow::float32()->id());
-  BOOST_REQUIRE_EQUAL(table->schema()->field(3)->type()->id(), arrow::float32()->id());
-  BOOST_REQUIRE_EQUAL(table->schema()->field(4)->type()->id(), arrow::float32()->id());
-  BOOST_REQUIRE_EQUAL(table->schema()->field(5)->type()->id(), arrow::float64()->id());
-  BOOST_REQUIRE_EQUAL(table->schema()->field(6)->type()->id(), arrow::int32()->id());
+  REQUIRE(table->num_rows() == 1000);
+  REQUIRE(table->num_columns() == 7);
+  REQUIRE(table->schema()->field(0)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id());
+  REQUIRE(table->schema()->field(1)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id());
+  REQUIRE(table->schema()->field(2)->type()->id() == arrow::float32()->id());
+  REQUIRE(table->schema()->field(3)->type()->id() == arrow::float32()->id());
+  REQUIRE(table->schema()->field(4)->type()->id() == arrow::float32()->id());
+  REQUIRE(table->schema()->field(5)->type()->id() == arrow::float64()->id());
+  REQUIRE(table->schema()->field(6)->type()->id() == arrow::int32()->id());
 
   {
     auto chunkToUse = table->column(0)->chunk(0);
     chunkToUse = std::dynamic_pointer_cast<arrow::FixedSizeListArray>(chunkToUse)->values();
     auto array = std::static_pointer_cast<arrow::FloatArray>(chunkToUse);
     // array of 3 floats, time 1000.
-    BOOST_REQUIRE_EQUAL(array->length(), 3000);
+    REQUIRE(array->length() == 3000);
     const float* c = reinterpret_cast<float const*>(array->values()->data());
 
     //auto array = std::static_pointer_cast<arrow::FixedSizeBinaryArray>(table->column(0)->chunk(0));
-    //BOOST_CHECK_EQUAL(array->byte_width(), sizeof(float[3]));
+    //CHECK_EQUAL(array->byte_width(), sizeof(float[3]));
     //const float* c = reinterpret_cast<float const*>(array->Value(0));
 
-    BOOST_CHECK_EQUAL(c[0], 1);
-    BOOST_CHECK_EQUAL(c[1], 2);
-    BOOST_CHECK_EQUAL(c[2], 1);
+    CHECK(c[0] == 1);
+    CHECK(c[1] == 2);
+    CHECK(c[2] == 1);
   }
   {
     //auto values = std::static_pointer_cast<arrow::FixedSizeBinaryArray>(table->column(1)->chunk(0));
     auto chunkToUse = table->column(1)->chunk(0);
     chunkToUse = std::dynamic_pointer_cast<arrow::FixedSizeListArray>(chunkToUse)->values();
     auto array = std::static_pointer_cast<arrow::Int32Array>(chunkToUse);
-    BOOST_REQUIRE_EQUAL(array->length(), 2000);
+    REQUIRE(array->length() == 2000);
 
     const int* ptr = reinterpret_cast<int const*>(array->values()->data());
     for (size_t i = 0; i < 1000; i++) {
-      BOOST_CHECK_EQUAL(ptr[2 * i + 0], i);
-      BOOST_CHECK_EQUAL(ptr[2 * i + 1], i + 1);
+      CHECK(ptr[2 * i + 0] == i);
+      CHECK(ptr[2 * i + 1] == i + 1);
     }
   }
 }
@@ -138,7 +134,7 @@ DECLARE_SOA_TABLE(Test, "AOD", "ETAPHI",
                   test::Random, test::Ev);
 } // namespace o2::aod
 
-BOOST_AUTO_TEST_CASE(RootTree2TableViaASoA)
+TEST_CASE("RootTree2TableViaASoA")
 {
   using namespace o2::framework;
   /// Create a simple TTree
@@ -171,23 +167,23 @@ BOOST_AUTO_TEST_CASE(RootTree2TableViaASoA)
   // Create an arrow table from this.
   TableBuilder builder;
   TTreeReader reader(&t2);
-  BOOST_REQUIRE_EQUAL(t2.GetEntries(), 1000);
+  REQUIRE(t2.GetEntries() == 1000);
 
   RootTableBuilderHelpers::convertASoA<o2::aod::Test>(builder, reader);
   auto table = builder.finalize();
-  BOOST_REQUIRE_EQUAL(table->num_rows(), 1000);
-  BOOST_REQUIRE_EQUAL(table->num_columns(), 7);
-  BOOST_REQUIRE_EQUAL(table->column(0)->type()->id(), arrow::float32()->id());
-  BOOST_REQUIRE_EQUAL(table->column(1)->type()->id(), arrow::float32()->id());
-  BOOST_REQUIRE_EQUAL(table->column(2)->type()->id(), arrow::float32()->id());
-  BOOST_REQUIRE_EQUAL(table->column(3)->type()->id(), arrow::fixed_size_list(arrow::float32(), 3)->id());
-  BOOST_REQUIRE_EQUAL(table->column(4)->type()->id(), arrow::fixed_size_list(arrow::int32(), 2)->id());
-  BOOST_REQUIRE_EQUAL(table->column(5)->type()->id(), arrow::float64()->id());
-  BOOST_REQUIRE_EQUAL(table->column(6)->type()->id(), arrow::int32()->id());
+  REQUIRE(table->num_rows() == 1000);
+  REQUIRE(table->num_columns() == 7);
+  REQUIRE(table->column(0)->type()->id() == arrow::float32()->id());
+  REQUIRE(table->column(1)->type()->id() == arrow::float32()->id());
+  REQUIRE(table->column(2)->type()->id() == arrow::float32()->id());
+  REQUIRE(table->column(3)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id());
+  REQUIRE(table->column(4)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id());
+  REQUIRE(table->column(5)->type()->id() == arrow::float64()->id());
+  REQUIRE(table->column(6)->type()->id() == arrow::int32()->id());
 
   o2::aod::Test testTable{table};
   for (auto& row : testTable) {
-    BOOST_REQUIRE_EQUAL(row.ij()[0], row.ij()[1] - 1);
-    BOOST_REQUIRE_EQUAL(row.ij()[1], row.ev());
+    REQUIRE(row.ij()[0] == row.ij()[1] - 1);
+    REQUIRE(row.ij()[1] == row.ev());
   }
 }

From 5f96d885dd967acb49f4d762841bad9369e87f9a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 30 Oct 2023 11:01:07 +0100
Subject: [PATCH 025/726] Drop old TOF compressor

---
 Detectors/TOF/compression/CMakeLists.txt      |   1 -
 .../TOFCompression/CompressorTaskOld.h        |  50 ----
 .../TOF/compression/src/CompressorTaskOld.cxx | 216 ------------------
 .../TOF/compression/src/tof-compressor.cxx    |  27 +--
 4 files changed, 4 insertions(+), 290 deletions(-)
 delete mode 100644 Detectors/TOF/compression/include/TOFCompression/CompressorTaskOld.h
 delete mode 100644 Detectors/TOF/compression/src/CompressorTaskOld.cxx

diff --git a/Detectors/TOF/compression/CMakeLists.txt b/Detectors/TOF/compression/CMakeLists.txt
index 00b29e267eb92..8ea77eb8028ba 100644
--- a/Detectors/TOF/compression/CMakeLists.txt
+++ b/Detectors/TOF/compression/CMakeLists.txt
@@ -12,7 +12,6 @@
 o2_add_library(TOFCompression
                SOURCES src/Compressor.cxx
                	       src/CompressorTask.cxx
-                       src/CompressorTaskOld.cxx
                PUBLIC_LINK_LIBRARIES O2::TOFBase O2::Framework O2::Headers O2::DataFormatsTOF
 	                             O2::DetectorsRaw
 	       )
diff --git a/Detectors/TOF/compression/include/TOFCompression/CompressorTaskOld.h b/Detectors/TOF/compression/include/TOFCompression/CompressorTaskOld.h
deleted file mode 100644
index ed78dd88caa64..0000000000000
--- a/Detectors/TOF/compression/include/TOFCompression/CompressorTaskOld.h
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// @file   CompressorTask.h
-/// @author Roberto Preghenella
-/// @since  2019-12-18
-/// @brief  TOF raw data compressor task
-
-#ifndef O2_TOF_COMPRESSORTASK_OLD
-#define O2_TOF_COMPRESSORTASK_OLD
-
-#include "Framework/Task.h"
-#include "Framework/DataProcessorSpec.h"
-#include "TOFCompression/Compressor.h"
-#include <fstream>
-
-using namespace o2::framework;
-
-namespace o2
-{
-namespace tof
-{
-
-template <typename RDH, bool verbose, bool paranoid>
-class CompressorTaskOld : public Task
-{
- public:
-  CompressorTaskOld(long payloadLim = -1) : mPayloadLimit(payloadLim) {}
-  ~CompressorTaskOld() override = default;
-  void init(InitContext& ic) final;
-  void run(ProcessingContext& pc) final;
-
- private:
-  Compressor<RDH, verbose, paranoid> mCompressor;
-  int mOutputBufferSize;
-  long mPayloadLimit = -1;
-};
-
-} // namespace tof
-} // namespace o2
-
-#endif /** O2_TOF_COMPRESSORTASK_OLD **/
diff --git a/Detectors/TOF/compression/src/CompressorTaskOld.cxx b/Detectors/TOF/compression/src/CompressorTaskOld.cxx
deleted file mode 100644
index 1a4e28393a8c3..0000000000000
--- a/Detectors/TOF/compression/src/CompressorTaskOld.cxx
+++ /dev/null
@@ -1,216 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// @file   CompressorTaskOld.cxx
-/// @author Roberto Preghenella
-/// @since  2019-12-18
-/// @brief  TOF raw data compressor task
-
-#include "TOFCompression/CompressorTaskOld.h"
-#include "Framework/ControlService.h"
-#include "Framework/ConfigParamRegistry.h"
-#include "Framework/RawDeviceService.h"
-#include "Framework/DeviceSpec.h"
-#include "Framework/DataSpecUtils.h"
-#include "Framework/InputRecordWalker.h"
-#include "CommonUtils/VerbosityConfig.h"
-
-#include <fairmq/Device.h>
-#include <fairmq/Parts.h>
-
-using namespace o2::framework;
-
-namespace o2
-{
-namespace tof
-{
-
-template <typename RDH, bool verbose, bool paranoid>
-void CompressorTaskOld<RDH, verbose, paranoid>::init(InitContext& ic)
-{
-  if (mPayloadLimit < 0) {
-    LOG(info) << "Compressor init";
-  } else {
-    LOG(info) << "Compressor init with Payload limit at " << mPayloadLimit;
-  }
-
-  auto decoderCONET = ic.options().get<bool>("tof-compressor-conet-mode");
-  auto decoderVerbose = ic.options().get<bool>("tof-compressor-decoder-verbose");
-  auto encoderVerbose = ic.options().get<bool>("tof-compressor-encoder-verbose");
-  auto checkerVerbose = ic.options().get<bool>("tof-compressor-checker-verbose");
-  mOutputBufferSize = ic.options().get<int>("tof-compressor-output-buffer-size");
-
-  mCompressor.setDecoderCONET(decoderCONET);
-  mCompressor.setDecoderVerbose(decoderVerbose);
-  mCompressor.setEncoderVerbose(encoderVerbose);
-  mCompressor.setCheckerVerbose(checkerVerbose);
-
-  auto finishFunction = [this]() {
-    mCompressor.checkSummary();
-  };
-
-  ic.services().get<CallbackService>().set<CallbackService::Id::Stop>(finishFunction);
-}
-
-template <typename RDH, bool verbose, bool paranoid>
-void CompressorTaskOld<RDH, verbose, paranoid>::run(ProcessingContext& pc)
-{
-  LOG(debug) << "Compressor run";
-
-  auto device = pc.services().get<o2::framework::RawDeviceService>().device();
-  auto outputRoutes = pc.services().get<o2::framework::RawDeviceService>().spec().outputs;
-  if (outputRoutes.size() != 1) {
-    LOG(error) << "Compressor output routes size != 1";
-    return;
-  }
-  auto fairMQChannel = outputRoutes.at(0).channel;
-  fair::mq::Parts partsOut;
-
-  /** to store data sorted by subspec id **/
-  std::map<int, std::vector<o2::framework::DataRef>> subspecPartMap;
-  std::map<int, int> subspecBufferSize;
-
-  // if we see requested data type input with 0xDEADBEEF subspec and 0 payload this means that the "delayed message"
-  // mechanism created it in absence of real data from upstream. Processor should send empty output to not block the workflow
-  {
-    auto& inputs = pc.inputs();
-    static size_t contDeadBeef = 0; // number of times 0xDEADBEEF was seen continuously
-    std::vector<InputSpec> dummy{InputSpec{"dummy", ConcreteDataMatcher{"TOF", "RAWDATA", 0xDEADBEEF}}};
-    for (const auto& ref : InputRecordWalker(inputs, dummy)) {
-      const auto* dh = o2::framework::DataRefUtils::getHeader<o2::header::DataHeader*>(ref);
-      auto payloadSize = DataRefUtils::getPayloadSize(ref);
-      if (payloadSize == 0) {
-        auto maxWarn = o2::conf::VerbosityConfig::Instance().maxWarnDeadBeef;
-        if (++contDeadBeef <= maxWarn) {
-          LOGP(alarm, "Found input [{}/{}/{:#x}] TF#{} 1st_orbit:{} Payload {} : assuming no payload for all links in this TF{}",
-               dh->dataOrigin.str, dh->dataDescription.str, dh->subSpecification, dh->tfCounter, dh->firstTForbit, payloadSize,
-               contDeadBeef == maxWarn ? fmt::format(". {} such inputs in row received, stopping reporting", contDeadBeef) : "");
-        }
-        // send empty message with DEADBEEF subspec
-        const auto* dph = o2::framework::DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(ref);
-        o2::header::DataHeader emptyDH("CRAWDATA", "TOF", 0xdeadbeef, 0, 0, 1);
-        emptyDH.runNumber = dh->runNumber;
-        emptyDH.payloadSerializationMethod = o2::header::gSerializationMethodNone;
-        emptyDH.firstTForbit = dh->firstTForbit;
-        emptyDH.tfCounter = dh->tfCounter;
-
-        o2::header::Stack emptyStack{emptyDH, o2::framework::DataProcessingHeader{dph->startTime, dph->duration, dph->creation}};
-
-        auto headerMessage = device->NewMessage(emptyStack.size());
-        auto payloadMessage = device->NewMessage(0);
-        std::memcpy(headerMessage->GetData(), emptyStack.data(), emptyStack.size());
-        partsOut.AddPart(std::move(headerMessage));
-        partsOut.AddPart(std::move(payloadMessage));
-        device->Send(partsOut, fairMQChannel);
-        return;
-      }
-    }
-    contDeadBeef = 0; // if good data, reset the counter
-  }
-
-  /** loop over inputs routes **/
-  std::vector<InputSpec> sel{InputSpec{"filter", ConcreteDataTypeMatcher{"TOF", "RAWDATA"}}};
-  for (const auto& ref : InputRecordWalker(pc.inputs(), sel)) {
-    //  for (auto iit = pc.inputs().begin(), iend = pc.inputs().end(); iit != iend; ++iit) {
-    //    if (!iit.isValid()) {
-    //      continue;
-    //    }
-
-    /** loop over input parts **/
-    //    for (auto const& ref : iit) {
-
-    /** store parts in map **/
-    auto headerIn = DataRefUtils::getHeader<o2::header::DataHeader*>(ref);
-    auto payloadInSize = DataRefUtils::getPayloadSize(ref);
-    auto subspec = headerIn->subSpecification;
-    subspecPartMap[subspec].push_back(ref);
-
-    /** increase subspec buffer size **/
-    if (!subspecBufferSize.count(subspec)) {
-      subspecBufferSize[subspec] = 0;
-    }
-    subspecBufferSize[subspec] += payloadInSize;
-    //  }
-  }
-
-  /** loop over subspecs **/
-  for (auto& subspecPartEntry : subspecPartMap) {
-
-    auto subspec = subspecPartEntry.first;
-    auto parts = subspecPartEntry.second;
-    auto& firstPart = parts.at(0);
-
-    /** use the first part to define output headers **/
-    auto headerOut = *DataRefUtils::getHeader<o2::header::DataHeader*>(firstPart);
-    auto dataProcessingHeaderOut = *DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(firstPart);
-    headerOut.dataDescription = "CRAWDATA";
-    headerOut.payloadSize = 0;
-    headerOut.splitPayloadParts = 1;
-
-    /** initialise output message **/
-    auto bufferSize = mOutputBufferSize >= 0 ? mOutputBufferSize + subspecBufferSize[subspec] : std::abs(mOutputBufferSize);
-    auto bufferSizeDouble = bufferSize * 2;
-    auto payloadMessage = device->NewMessage(bufferSizeDouble); // start with double size since later resized (but resize cannot resize more than original one)
-    auto bufferPointer = (char*)payloadMessage->GetData();
-
-    /** loop over subspec parts **/
-    for (const auto& ref : parts) {
-
-      /** input **/
-      auto headerIn = DataRefUtils::getHeader<o2::header::DataHeader*>(ref);
-      auto dataProcessingHeaderIn = DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(ref);
-      auto payloadIn = ref.payload;
-      auto payloadInSize = DataRefUtils::getPayloadSize(ref);
-
-      if (mPayloadLimit > -1 && payloadInSize > mPayloadLimit) {
-        LOG(error) << "Payload larger than limit (" << mPayloadLimit << "), payload = " << payloadInSize;
-        continue;
-      }
-
-      /** prepare compressor **/
-      mCompressor.setDecoderBuffer(payloadIn);
-      mCompressor.setDecoderBufferSize(payloadInSize);
-      mCompressor.setEncoderBuffer(bufferPointer);
-      mCompressor.setEncoderBufferSize(bufferSize);
-
-      /** run **/
-      mCompressor.run();
-      auto payloadOutSize = mCompressor.getEncoderByteCounter();
-      bufferPointer += payloadOutSize;
-      bufferSize -= payloadOutSize;
-      headerOut.payloadSize += payloadOutSize;
-    }
-
-    /** finalise output message **/
-    if (headerOut.payloadSize > bufferSizeDouble) {
-      headerOut.payloadSize = 0; // put payload to zero, otherwise it will trigger a crash
-    }
-    payloadMessage->SetUsedSize(headerOut.payloadSize);
-    o2::header::Stack headerStack{headerOut, dataProcessingHeaderOut};
-    auto headerMessage = device->NewMessage(headerStack.size());
-    std::memcpy(headerMessage->GetData(), headerStack.data(), headerStack.size());
-
-    /** add parts **/
-    partsOut.AddPart(std::move(headerMessage));
-    partsOut.AddPart(std::move(payloadMessage));
-  }
-
-  /** send message **/
-  device->Send(partsOut, fairMQChannel);
-}
-
-template class CompressorTaskOld<o2::header::RAWDataHeader, false, false>;
-template class CompressorTaskOld<o2::header::RAWDataHeader, false, true>;
-template class CompressorTaskOld<o2::header::RAWDataHeader, true, false>;
-template class CompressorTaskOld<o2::header::RAWDataHeader, true, true>;
-
-} // namespace tof
-} // namespace o2
diff --git a/Detectors/TOF/compression/src/tof-compressor.cxx b/Detectors/TOF/compression/src/tof-compressor.cxx
index 4358ceae84bdb..29286911957d3 100644
--- a/Detectors/TOF/compression/src/tof-compressor.cxx
+++ b/Detectors/TOF/compression/src/tof-compressor.cxx
@@ -15,7 +15,6 @@
 /// @brief  Basic DPL workflow for TOF raw data compression
 
 #include "TOFCompression/CompressorTask.h"
-#include "TOFCompression/CompressorTaskOld.h"
 #include "Framework/WorkflowSpec.h"
 #include "Framework/ConfigParamSpec.h"
 #include "Framework/ConcreteDataMatcher.h"
@@ -45,7 +44,6 @@ void customize(std::vector<ConfigParamSpec>& workflowOptions)
   workflowOptions.push_back(paranoid);
   workflowOptions.push_back(ignoreStf);
   workflowOptions.push_back(payloadlim);
-  workflowOptions.emplace_back(ConfigParamSpec{"old", VariantType::Bool, false, {"use the non-DPL version of the compressor"}});
   workflowOptions.push_back(ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}});
 }
 
@@ -61,7 +59,6 @@ WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
   auto verbose = cfgc.options().get<bool>("tof-compressor-verbose");
   auto paranoid = cfgc.options().get<bool>("tof-compressor-paranoid");
   auto ignoreStf = cfgc.options().get<bool>("ignore-dist-stf");
-  auto old = cfgc.options().get<bool>("old");
   auto payloadLim = cfgc.options().get<long>("payload-limit");
 
   std::vector<OutputSpec> outputs;
@@ -70,32 +67,16 @@ WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
   AlgorithmSpec algoSpec;
   if (rdhVersion == o2::raw::RDHUtils::getVersion<o2::header::RAWDataHeader>()) {
     if (!verbose && !paranoid) {
-      if (old) {
-        algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTaskOld<o2::header::RAWDataHeader, false, false>>(payloadLim)};
-      } else {
-        algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTask<o2::header::RAWDataHeader, false, false>>(payloadLim)};
-      }
+      algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTask<o2::header::RAWDataHeader, false, false>>(payloadLim)};
     }
     if (!verbose && paranoid) {
-      if (old) {
-        algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTaskOld<o2::header::RAWDataHeader, false, true>>(payloadLim)};
-      } else {
-        algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTask<o2::header::RAWDataHeader, false, true>>(payloadLim)};
-      }
+      algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTask<o2::header::RAWDataHeader, false, true>>(payloadLim)};
     }
     if (verbose && !paranoid) {
-      if (old) {
-        algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTaskOld<o2::header::RAWDataHeader, true, false>>(payloadLim)};
-      } else {
-        algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTask<o2::header::RAWDataHeader, true, false>>(payloadLim)};
-      }
+      algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTask<o2::header::RAWDataHeader, true, false>>(payloadLim)};
     }
     if (verbose && paranoid) {
-      if (old) {
-        algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTaskOld<o2::header::RAWDataHeader, true, true>>(payloadLim)};
-      } else {
-        algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTask<o2::header::RAWDataHeader, true, true>>(payloadLim)};
-      }
+      algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTask<o2::header::RAWDataHeader, true, true>>(payloadLim)};
     }
   }
 

From 357a86db4004fc07427d1504a003d0255767ea75 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 30 Oct 2023 11:53:46 +0100
Subject: [PATCH 026/726] Drop old TOF compressor

---
 Detectors/TOF/compression/CMakeLists.txt      |   1 -
 .../TOFCompression/CompressorTaskOld.h        |  50 ----
 .../TOF/compression/src/CompressorTaskOld.cxx | 216 ------------------
 .../TOF/compression/src/tof-compressor.cxx    |  28 +--
 4 files changed, 7 insertions(+), 288 deletions(-)
 delete mode 100644 Detectors/TOF/compression/include/TOFCompression/CompressorTaskOld.h
 delete mode 100644 Detectors/TOF/compression/src/CompressorTaskOld.cxx

diff --git a/Detectors/TOF/compression/CMakeLists.txt b/Detectors/TOF/compression/CMakeLists.txt
index 00b29e267eb92..8ea77eb8028ba 100644
--- a/Detectors/TOF/compression/CMakeLists.txt
+++ b/Detectors/TOF/compression/CMakeLists.txt
@@ -12,7 +12,6 @@
 o2_add_library(TOFCompression
                SOURCES src/Compressor.cxx
                	       src/CompressorTask.cxx
-                       src/CompressorTaskOld.cxx
                PUBLIC_LINK_LIBRARIES O2::TOFBase O2::Framework O2::Headers O2::DataFormatsTOF
 	                             O2::DetectorsRaw
 	       )
diff --git a/Detectors/TOF/compression/include/TOFCompression/CompressorTaskOld.h b/Detectors/TOF/compression/include/TOFCompression/CompressorTaskOld.h
deleted file mode 100644
index ed78dd88caa64..0000000000000
--- a/Detectors/TOF/compression/include/TOFCompression/CompressorTaskOld.h
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// @file   CompressorTask.h
-/// @author Roberto Preghenella
-/// @since  2019-12-18
-/// @brief  TOF raw data compressor task
-
-#ifndef O2_TOF_COMPRESSORTASK_OLD
-#define O2_TOF_COMPRESSORTASK_OLD
-
-#include "Framework/Task.h"
-#include "Framework/DataProcessorSpec.h"
-#include "TOFCompression/Compressor.h"
-#include <fstream>
-
-using namespace o2::framework;
-
-namespace o2
-{
-namespace tof
-{
-
-template <typename RDH, bool verbose, bool paranoid>
-class CompressorTaskOld : public Task
-{
- public:
-  CompressorTaskOld(long payloadLim = -1) : mPayloadLimit(payloadLim) {}
-  ~CompressorTaskOld() override = default;
-  void init(InitContext& ic) final;
-  void run(ProcessingContext& pc) final;
-
- private:
-  Compressor<RDH, verbose, paranoid> mCompressor;
-  int mOutputBufferSize;
-  long mPayloadLimit = -1;
-};
-
-} // namespace tof
-} // namespace o2
-
-#endif /** O2_TOF_COMPRESSORTASK_OLD **/
diff --git a/Detectors/TOF/compression/src/CompressorTaskOld.cxx b/Detectors/TOF/compression/src/CompressorTaskOld.cxx
deleted file mode 100644
index 1a4e28393a8c3..0000000000000
--- a/Detectors/TOF/compression/src/CompressorTaskOld.cxx
+++ /dev/null
@@ -1,216 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// @file   CompressorTaskOld.cxx
-/// @author Roberto Preghenella
-/// @since  2019-12-18
-/// @brief  TOF raw data compressor task
-
-#include "TOFCompression/CompressorTaskOld.h"
-#include "Framework/ControlService.h"
-#include "Framework/ConfigParamRegistry.h"
-#include "Framework/RawDeviceService.h"
-#include "Framework/DeviceSpec.h"
-#include "Framework/DataSpecUtils.h"
-#include "Framework/InputRecordWalker.h"
-#include "CommonUtils/VerbosityConfig.h"
-
-#include <fairmq/Device.h>
-#include <fairmq/Parts.h>
-
-using namespace o2::framework;
-
-namespace o2
-{
-namespace tof
-{
-
-template <typename RDH, bool verbose, bool paranoid>
-void CompressorTaskOld<RDH, verbose, paranoid>::init(InitContext& ic)
-{
-  if (mPayloadLimit < 0) {
-    LOG(info) << "Compressor init";
-  } else {
-    LOG(info) << "Compressor init with Payload limit at " << mPayloadLimit;
-  }
-
-  auto decoderCONET = ic.options().get<bool>("tof-compressor-conet-mode");
-  auto decoderVerbose = ic.options().get<bool>("tof-compressor-decoder-verbose");
-  auto encoderVerbose = ic.options().get<bool>("tof-compressor-encoder-verbose");
-  auto checkerVerbose = ic.options().get<bool>("tof-compressor-checker-verbose");
-  mOutputBufferSize = ic.options().get<int>("tof-compressor-output-buffer-size");
-
-  mCompressor.setDecoderCONET(decoderCONET);
-  mCompressor.setDecoderVerbose(decoderVerbose);
-  mCompressor.setEncoderVerbose(encoderVerbose);
-  mCompressor.setCheckerVerbose(checkerVerbose);
-
-  auto finishFunction = [this]() {
-    mCompressor.checkSummary();
-  };
-
-  ic.services().get<CallbackService>().set<CallbackService::Id::Stop>(finishFunction);
-}
-
-template <typename RDH, bool verbose, bool paranoid>
-void CompressorTaskOld<RDH, verbose, paranoid>::run(ProcessingContext& pc)
-{
-  LOG(debug) << "Compressor run";
-
-  auto device = pc.services().get<o2::framework::RawDeviceService>().device();
-  auto outputRoutes = pc.services().get<o2::framework::RawDeviceService>().spec().outputs;
-  if (outputRoutes.size() != 1) {
-    LOG(error) << "Compressor output routes size != 1";
-    return;
-  }
-  auto fairMQChannel = outputRoutes.at(0).channel;
-  fair::mq::Parts partsOut;
-
-  /** to store data sorted by subspec id **/
-  std::map<int, std::vector<o2::framework::DataRef>> subspecPartMap;
-  std::map<int, int> subspecBufferSize;
-
-  // if we see requested data type input with 0xDEADBEEF subspec and 0 payload this means that the "delayed message"
-  // mechanism created it in absence of real data from upstream. Processor should send empty output to not block the workflow
-  {
-    auto& inputs = pc.inputs();
-    static size_t contDeadBeef = 0; // number of times 0xDEADBEEF was seen continuously
-    std::vector<InputSpec> dummy{InputSpec{"dummy", ConcreteDataMatcher{"TOF", "RAWDATA", 0xDEADBEEF}}};
-    for (const auto& ref : InputRecordWalker(inputs, dummy)) {
-      const auto* dh = o2::framework::DataRefUtils::getHeader<o2::header::DataHeader*>(ref);
-      auto payloadSize = DataRefUtils::getPayloadSize(ref);
-      if (payloadSize == 0) {
-        auto maxWarn = o2::conf::VerbosityConfig::Instance().maxWarnDeadBeef;
-        if (++contDeadBeef <= maxWarn) {
-          LOGP(alarm, "Found input [{}/{}/{:#x}] TF#{} 1st_orbit:{} Payload {} : assuming no payload for all links in this TF{}",
-               dh->dataOrigin.str, dh->dataDescription.str, dh->subSpecification, dh->tfCounter, dh->firstTForbit, payloadSize,
-               contDeadBeef == maxWarn ? fmt::format(". {} such inputs in row received, stopping reporting", contDeadBeef) : "");
-        }
-        // send empty message with DEADBEEF subspec
-        const auto* dph = o2::framework::DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(ref);
-        o2::header::DataHeader emptyDH("CRAWDATA", "TOF", 0xdeadbeef, 0, 0, 1);
-        emptyDH.runNumber = dh->runNumber;
-        emptyDH.payloadSerializationMethod = o2::header::gSerializationMethodNone;
-        emptyDH.firstTForbit = dh->firstTForbit;
-        emptyDH.tfCounter = dh->tfCounter;
-
-        o2::header::Stack emptyStack{emptyDH, o2::framework::DataProcessingHeader{dph->startTime, dph->duration, dph->creation}};
-
-        auto headerMessage = device->NewMessage(emptyStack.size());
-        auto payloadMessage = device->NewMessage(0);
-        std::memcpy(headerMessage->GetData(), emptyStack.data(), emptyStack.size());
-        partsOut.AddPart(std::move(headerMessage));
-        partsOut.AddPart(std::move(payloadMessage));
-        device->Send(partsOut, fairMQChannel);
-        return;
-      }
-    }
-    contDeadBeef = 0; // if good data, reset the counter
-  }
-
-  /** loop over inputs routes **/
-  std::vector<InputSpec> sel{InputSpec{"filter", ConcreteDataTypeMatcher{"TOF", "RAWDATA"}}};
-  for (const auto& ref : InputRecordWalker(pc.inputs(), sel)) {
-    //  for (auto iit = pc.inputs().begin(), iend = pc.inputs().end(); iit != iend; ++iit) {
-    //    if (!iit.isValid()) {
-    //      continue;
-    //    }
-
-    /** loop over input parts **/
-    //    for (auto const& ref : iit) {
-
-    /** store parts in map **/
-    auto headerIn = DataRefUtils::getHeader<o2::header::DataHeader*>(ref);
-    auto payloadInSize = DataRefUtils::getPayloadSize(ref);
-    auto subspec = headerIn->subSpecification;
-    subspecPartMap[subspec].push_back(ref);
-
-    /** increase subspec buffer size **/
-    if (!subspecBufferSize.count(subspec)) {
-      subspecBufferSize[subspec] = 0;
-    }
-    subspecBufferSize[subspec] += payloadInSize;
-    //  }
-  }
-
-  /** loop over subspecs **/
-  for (auto& subspecPartEntry : subspecPartMap) {
-
-    auto subspec = subspecPartEntry.first;
-    auto parts = subspecPartEntry.second;
-    auto& firstPart = parts.at(0);
-
-    /** use the first part to define output headers **/
-    auto headerOut = *DataRefUtils::getHeader<o2::header::DataHeader*>(firstPart);
-    auto dataProcessingHeaderOut = *DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(firstPart);
-    headerOut.dataDescription = "CRAWDATA";
-    headerOut.payloadSize = 0;
-    headerOut.splitPayloadParts = 1;
-
-    /** initialise output message **/
-    auto bufferSize = mOutputBufferSize >= 0 ? mOutputBufferSize + subspecBufferSize[subspec] : std::abs(mOutputBufferSize);
-    auto bufferSizeDouble = bufferSize * 2;
-    auto payloadMessage = device->NewMessage(bufferSizeDouble); // start with double size since later resized (but resize cannot resize more than original one)
-    auto bufferPointer = (char*)payloadMessage->GetData();
-
-    /** loop over subspec parts **/
-    for (const auto& ref : parts) {
-
-      /** input **/
-      auto headerIn = DataRefUtils::getHeader<o2::header::DataHeader*>(ref);
-      auto dataProcessingHeaderIn = DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(ref);
-      auto payloadIn = ref.payload;
-      auto payloadInSize = DataRefUtils::getPayloadSize(ref);
-
-      if (mPayloadLimit > -1 && payloadInSize > mPayloadLimit) {
-        LOG(error) << "Payload larger than limit (" << mPayloadLimit << "), payload = " << payloadInSize;
-        continue;
-      }
-
-      /** prepare compressor **/
-      mCompressor.setDecoderBuffer(payloadIn);
-      mCompressor.setDecoderBufferSize(payloadInSize);
-      mCompressor.setEncoderBuffer(bufferPointer);
-      mCompressor.setEncoderBufferSize(bufferSize);
-
-      /** run **/
-      mCompressor.run();
-      auto payloadOutSize = mCompressor.getEncoderByteCounter();
-      bufferPointer += payloadOutSize;
-      bufferSize -= payloadOutSize;
-      headerOut.payloadSize += payloadOutSize;
-    }
-
-    /** finalise output message **/
-    if (headerOut.payloadSize > bufferSizeDouble) {
-      headerOut.payloadSize = 0; // put payload to zero, otherwise it will trigger a crash
-    }
-    payloadMessage->SetUsedSize(headerOut.payloadSize);
-    o2::header::Stack headerStack{headerOut, dataProcessingHeaderOut};
-    auto headerMessage = device->NewMessage(headerStack.size());
-    std::memcpy(headerMessage->GetData(), headerStack.data(), headerStack.size());
-
-    /** add parts **/
-    partsOut.AddPart(std::move(headerMessage));
-    partsOut.AddPart(std::move(payloadMessage));
-  }
-
-  /** send message **/
-  device->Send(partsOut, fairMQChannel);
-}
-
-template class CompressorTaskOld<o2::header::RAWDataHeader, false, false>;
-template class CompressorTaskOld<o2::header::RAWDataHeader, false, true>;
-template class CompressorTaskOld<o2::header::RAWDataHeader, true, false>;
-template class CompressorTaskOld<o2::header::RAWDataHeader, true, true>;
-
-} // namespace tof
-} // namespace o2
diff --git a/Detectors/TOF/compression/src/tof-compressor.cxx b/Detectors/TOF/compression/src/tof-compressor.cxx
index 4358ceae84bdb..e305911ee152f 100644
--- a/Detectors/TOF/compression/src/tof-compressor.cxx
+++ b/Detectors/TOF/compression/src/tof-compressor.cxx
@@ -15,7 +15,6 @@
 /// @brief  Basic DPL workflow for TOF raw data compression
 
 #include "TOFCompression/CompressorTask.h"
-#include "TOFCompression/CompressorTaskOld.h"
 #include "Framework/WorkflowSpec.h"
 #include "Framework/ConfigParamSpec.h"
 #include "Framework/ConcreteDataMatcher.h"
@@ -62,6 +61,9 @@ WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
   auto paranoid = cfgc.options().get<bool>("tof-compressor-paranoid");
   auto ignoreStf = cfgc.options().get<bool>("ignore-dist-stf");
   auto old = cfgc.options().get<bool>("old");
+  if (old) {
+    LOGP(error, "--old option is not actually supported, using the new version");
+  }
   auto payloadLim = cfgc.options().get<long>("payload-limit");
 
   std::vector<OutputSpec> outputs;
@@ -70,32 +72,16 @@ WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
   AlgorithmSpec algoSpec;
   if (rdhVersion == o2::raw::RDHUtils::getVersion<o2::header::RAWDataHeader>()) {
     if (!verbose && !paranoid) {
-      if (old) {
-        algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTaskOld<o2::header::RAWDataHeader, false, false>>(payloadLim)};
-      } else {
-        algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTask<o2::header::RAWDataHeader, false, false>>(payloadLim)};
-      }
+      algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTask<o2::header::RAWDataHeader, false, false>>(payloadLim)};
     }
     if (!verbose && paranoid) {
-      if (old) {
-        algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTaskOld<o2::header::RAWDataHeader, false, true>>(payloadLim)};
-      } else {
-        algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTask<o2::header::RAWDataHeader, false, true>>(payloadLim)};
-      }
+      algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTask<o2::header::RAWDataHeader, false, true>>(payloadLim)};
     }
     if (verbose && !paranoid) {
-      if (old) {
-        algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTaskOld<o2::header::RAWDataHeader, true, false>>(payloadLim)};
-      } else {
-        algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTask<o2::header::RAWDataHeader, true, false>>(payloadLim)};
-      }
+      algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTask<o2::header::RAWDataHeader, true, false>>(payloadLim)};
     }
     if (verbose && paranoid) {
-      if (old) {
-        algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTaskOld<o2::header::RAWDataHeader, true, true>>(payloadLim)};
-      } else {
-        algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTask<o2::header::RAWDataHeader, true, true>>(payloadLim)};
-      }
+      algoSpec = AlgorithmSpec{adaptFromTask<o2::tof::CompressorTask<o2::header::RAWDataHeader, true, true>>(payloadLim)};
     }
   }
 

From 30e0c6b21269450915194a7a66139ae5af40d29f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 30 Oct 2023 12:49:15 +0100
Subject: [PATCH 027/726] Avoid duplicate dependency

---
 Framework/Core/CMakeLists.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index e58a6414e06ac..25f2e60f402c4 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -158,7 +158,6 @@ o2_add_library(Framework
                                      O2::Headers
                                      O2::MemoryResources
                                      O2::PCG
-                                     ROOT::Tree
                                      RapidJSON::RapidJSON
                                      Arrow::arrow_shared
                                      Microsoft.GSL::GSL

From 221bf8caf6ef5fea05734bbc0e91402a4f395a93 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 30 Oct 2023 16:27:01 +0100
Subject: [PATCH 028/726] DPL: improve homegeneous_apply_ref

* Return something indexable for the case N == 0.
* Improve tests.
---
 Framework/Foundation/include/Framework/StructToTuple.h | 2 +-
 Framework/Foundation/test/test_StructToTuple.cxx       | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Framework/Foundation/include/Framework/StructToTuple.h b/Framework/Foundation/include/Framework/StructToTuple.h
index 94655f605c9a3..4ea93e113249d 100644
--- a/Framework/Foundation/include/Framework/StructToTuple.h
+++ b/Framework/Foundation/include/Framework/StructToTuple.h
@@ -382,7 +382,7 @@ auto homogeneous_apply_refs(L l, T&& object)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY_LOW (3)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY_LOW (2)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY_LOW (1)
-  else { return false; }
+  else { return std::array<bool,0>(); }
   // clang-format on
 }
 
diff --git a/Framework/Foundation/test/test_StructToTuple.cxx b/Framework/Foundation/test/test_StructToTuple.cxx
index 342d51a23bf80..06cf4b47a9f7c 100644
--- a/Framework/Foundation/test/test_StructToTuple.cxx
+++ b/Framework/Foundation/test/test_StructToTuple.cxx
@@ -135,4 +135,6 @@ TEST_CASE("TestStructToTuple")
     }
   },
                                                   nestedFoo);
+  REQUIRE(t6.size() == 3);
+  REQUIRE(t6[0] == true);
 }

From bdb112db4e8ba43fc9e3ef5b1c6ba616a3b24126 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 30 Oct 2023 16:43:05 +0100
Subject: [PATCH 029/726] DPL: flush messages immediately when notifying state
 change

---
 Framework/Core/src/ControlService.cxx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Framework/Core/src/ControlService.cxx b/Framework/Core/src/ControlService.cxx
index 65619baba1037..13940867d61e3 100644
--- a/Framework/Core/src/ControlService.cxx
+++ b/Framework/Core/src/ControlService.cxx
@@ -86,6 +86,7 @@ void ControlService::notifyDeviceState(std::string currentState)
 {
   std::scoped_lock lock(mMutex);
   mDriverClient.tell(fmt::format("CONTROL_ACTION: NOTIFY_DEVICE_STATE {}", currentState));
+  mDriverClient.flushPending(mRegistry);
 }
 
 } // namespace o2::framework

From d1371dc2c4e69d22b8962cb7ff0629b94ad9e5e8 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 30 Oct 2023 22:13:17 +0100
Subject: [PATCH 030/726] DPL: have common baseclass to abstract tree writing

This is the the first step towards writing something else,
e.g. a RNTuple.
---
 .../Core/include/Framework/TableTreeHelpers.h | 28 +++++++++---
 Framework/Core/src/CommonDataProcessors.cxx   | 10 ++---
 Framework/Core/src/TableTreeHelpers.cxx       | 44 +++++++++++--------
 Framework/Core/test/benchmark_TableToTree.cxx |  2 +-
 Framework/Core/test/benchmark_TreeToTable.cxx |  2 +-
 Framework/Core/test/test_TreeToTable.cxx      |  9 ++--
 6 files changed, 59 insertions(+), 36 deletions(-)

diff --git a/Framework/Core/include/Framework/TableTreeHelpers.h b/Framework/Core/include/Framework/TableTreeHelpers.h
index e12f4a38a973b..78bf0bba11389 100644
--- a/Framework/Core/include/Framework/TableTreeHelpers.h
+++ b/Framework/Core/include/Framework/TableTreeHelpers.h
@@ -105,18 +105,34 @@ class ColumnToBranch
   int mFieldSize = 0;
 };
 
-class TableToTree
+class TableToRoot
 {
  public:
-  TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* treename);
+  TableToRoot(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* name);
+  ~TableToRoot() = default;
 
-  std::shared_ptr<TTree> process();
-  void addBranch(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field);
-  void addAllBranches();
+  virtual void addColumn(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field) = 0;
+  virtual void process() = 0;
+  // Helper function which marks all columns in the table for writing
+  void addAllColumns();
 
- private:
+ protected:
   arrow::Table* mTable;
+  TFile* mFile;
+  std::string mName;
   int64_t mRows = 0;
+};
+
+class TableToTree : public TableToRoot
+{
+ public:
+  TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* treename);
+
+  void process() final;
+  void addColumn(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field) final;
+  std::shared_ptr<TTree> tree() { return mTree; }
+
+ private:
   std::shared_ptr<TTree> mTree;
   std::vector<std::unique_ptr<ColumnToBranch>> mColumnReaders;
 };
diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 08283b5e49bb5..c9d8aa835d734 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -417,9 +417,7 @@ DataProcessorSpec
         for (auto d : ds) {
           auto fileAndFolder = dod->getFileFolder(d, tfNumber, aodInputFile);
           auto treename = fileAndFolder.folderName + "/" + d->treename;
-          TableToTree ta2tr(table,
-                            fileAndFolder.file,
-                            treename.c_str());
+          std::unique_ptr<TableToRoot> ta2r = std::make_unique<TableToTree>(table, fileAndFolder.file, treename.c_str());
 
           // update metadata
           if (fileAndFolder.file->FindObjectAny("metaData")) {
@@ -438,13 +436,13 @@ DataProcessorSpec
               auto col = table->column(idx);
               auto field = table->schema()->field(idx);
               if (idx != -1) {
-                ta2tr.addBranch(col, field);
+                ta2r->addColumn(col, field);
               }
             }
           } else {
-            ta2tr.addAllBranches();
+            ta2r->addAllColumns();
           }
-          ta2tr.process();
+          ta2r->process();
         }
       }
     };
diff --git a/Framework/Core/src/TableTreeHelpers.cxx b/Framework/Core/src/TableTreeHelpers.cxx
index e6e6a1349e619..4db7a8630ef3a 100644
--- a/Framework/Core/src/TableTreeHelpers.cxx
+++ b/Framework/Core/src/TableTreeHelpers.cxx
@@ -407,14 +407,33 @@ void ColumnToBranch::nextChunk()
   accessChunk();
 }
 
-TableToTree::TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* treename)
+TableToRoot::TableToRoot(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* name)
+  : mTable{table.get()},
+    mFile{file},
+    mName{name}
 {
-  mTable = table.get();
-  mTree.reset(static_cast<TTree*>(file->Get(treename)));
+}
+
+void TableToRoot::addAllColumns()
+{
+  mRows = mTable->num_rows();
+  auto columns = mTable->columns();
+  auto fields = mTable->schema()->fields();
+  assert(columns.size() == fields.size());
+  for (auto i = 0u; i < columns.size(); ++i) {
+    addColumn(columns[i], fields[i]);
+  }
+}
+
+
+TableToTree::TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* name)
+  : TableToRoot{table, file, name}
+{
+  mTree.reset(static_cast<TTree*>(mFile->Get(mName.c_str())));
   if (mTree) {
     return;
   }
-  std::string treeName(treename);
+  std::string treeName(mName);
   auto pos = treeName.find_first_of('/');
   if (pos != std::string::npos) {
     file->cd(treeName.substr(0, pos).c_str());
@@ -423,18 +442,8 @@ TableToTree::TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file
   mTree = std::make_shared<TTree>(treeName.c_str(), treeName.c_str());
 }
 
-void TableToTree::addAllBranches()
-{
-  mRows = mTable->num_rows();
-  auto columns = mTable->columns();
-  auto fields = mTable->schema()->fields();
-  assert(columns.size() == fields.size());
-  for (auto i = 0u; i < columns.size(); ++i) {
-    addBranch(columns[i], fields[i]);
-  }
-}
 
-void TableToTree::addBranch(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field)
+void TableToTree::addColumn(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field)
 {
   if (mRows == 0) {
     mRows = column->length();
@@ -444,13 +453,13 @@ void TableToTree::addBranch(std::shared_ptr<arrow::ChunkedArray> const& column,
   mColumnReaders.emplace_back(new ColumnToBranch{mTree.get(), column, field});
 }
 
-std::shared_ptr<TTree> TableToTree::process()
+void TableToTree::process()
 {
   int64_t row = 0;
   if (mTree->GetNbranches() == 0 || mRows == 0) {
     mTree->Write("", TObject::kOverwrite);
     mTree->SetDirectory(nullptr);
-    return mTree;
+    return;
   }
 
   for (auto& reader : mColumnReaders) {
@@ -468,7 +477,6 @@ std::shared_ptr<TTree> TableToTree::process()
   }
   mTree->Write("", TObject::kOverwrite);
   mTree->SetDirectory(nullptr);
-  return mTree;
 }
 
 TreeToTable::TreeToTable(arrow::MemoryPool* pool)
diff --git a/Framework/Core/test/benchmark_TableToTree.cxx b/Framework/Core/test/benchmark_TableToTree.cxx
index 0a7963518f00b..0ca4eb81112b8 100644
--- a/Framework/Core/test/benchmark_TableToTree.cxx
+++ b/Framework/Core/test/benchmark_TableToTree.cxx
@@ -62,7 +62,7 @@ static void BM_TableToTree(benchmark::State& state)
 
     // benchmark TableToTree
     TableToTree ta2tr(table, &fout, "table2tree");
-    ta2tr.addAllBranches();
+    ta2tr.addAllColumns();
     ta2tr.process();
 
     // clean up
diff --git a/Framework/Core/test/benchmark_TreeToTable.cxx b/Framework/Core/test/benchmark_TreeToTable.cxx
index 6eca853a90ce4..476e653acaf83 100644
--- a/Framework/Core/test/benchmark_TreeToTable.cxx
+++ b/Framework/Core/test/benchmark_TreeToTable.cxx
@@ -58,7 +58,7 @@ static void BM_TreeToTable(benchmark::State& state)
   // now convert the table to a tree
   TFile fout("tree2table.root", "RECREATE");
   TableToTree ta2tr(table, &fout, "tree2table");
-  ta2tr.addAllBranches();
+  ta2tr.addAllColumns();
   ta2tr.process();
   fout.Close();
 
diff --git a/Framework/Core/test/test_TreeToTable.cxx b/Framework/Core/test/test_TreeToTable.cxx
index 65569b72b9f35..32575a8caf3e0 100644
--- a/Framework/Core/test/test_TreeToTable.cxx
+++ b/Framework/Core/test/test_TreeToTable.cxx
@@ -143,9 +143,10 @@ TEST_CASE("TreeToTableConversion")
   // save table as tree
   TFile* f2 = TFile::Open("table2tree.root", "RECREATE");
   TableToTree ta2tr(table, f2, "mytree");
-  ta2tr.addAllBranches();
+  ta2tr.addAllColumns();
 
-  auto t2 = ta2tr.process();
+  ta2tr.process();
+  auto t2 = ta2tr.tree();
   auto br = (TBranch*)t2->GetBranch("ok");
   REQUIRE(t2->GetEntries() == ndp);
   REQUIRE(br->GetEntries() == ndp);
@@ -201,8 +202,8 @@ TEST_CASE("VariableLists")
 
   auto* f = TFile::Open("variable_lists.root", "RECREATE");
   TableToTree ta2tr(table, f, "lists");
-  ta2tr.addAllBranches();
-  auto tree = ta2tr.process();
+  ta2tr.addAllColumns();
+  ta2tr.process();
   f->Close();
 
   auto* f2 = TFile::Open("variable_lists.root", "READ");

From 83003d35172e4abe0ecf8e2a3fd4efe42000439b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 31 Oct 2023 15:37:09 +0100
Subject: [PATCH 031/726] DPL: have common baseclass to abstract tree writing

This is the the first step towards writing something else,
e.g. a RNTuple.
---
 .../Core/include/Framework/TableTreeHelpers.h | 28 +++++++++---
 Framework/Core/src/CommonDataProcessors.cxx   | 10 ++---
 Framework/Core/src/TableTreeHelpers.cxx       | 43 +++++++++++--------
 Framework/Core/test/benchmark_TableToTree.cxx |  2 +-
 Framework/Core/test/benchmark_TreeToTable.cxx |  2 +-
 Framework/Core/test/test_TreeToTable.cxx      |  9 ++--
 6 files changed, 58 insertions(+), 36 deletions(-)

diff --git a/Framework/Core/include/Framework/TableTreeHelpers.h b/Framework/Core/include/Framework/TableTreeHelpers.h
index e12f4a38a973b..78bf0bba11389 100644
--- a/Framework/Core/include/Framework/TableTreeHelpers.h
+++ b/Framework/Core/include/Framework/TableTreeHelpers.h
@@ -105,18 +105,34 @@ class ColumnToBranch
   int mFieldSize = 0;
 };
 
-class TableToTree
+class TableToRoot
 {
  public:
-  TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* treename);
+  TableToRoot(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* name);
+  ~TableToRoot() = default;
 
-  std::shared_ptr<TTree> process();
-  void addBranch(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field);
-  void addAllBranches();
+  virtual void addColumn(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field) = 0;
+  virtual void process() = 0;
+  // Helper function which marks all columns in the table for writing
+  void addAllColumns();
 
- private:
+ protected:
   arrow::Table* mTable;
+  TFile* mFile;
+  std::string mName;
   int64_t mRows = 0;
+};
+
+class TableToTree : public TableToRoot
+{
+ public:
+  TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* treename);
+
+  void process() final;
+  void addColumn(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field) final;
+  std::shared_ptr<TTree> tree() { return mTree; }
+
+ private:
   std::shared_ptr<TTree> mTree;
   std::vector<std::unique_ptr<ColumnToBranch>> mColumnReaders;
 };
diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 08283b5e49bb5..c9d8aa835d734 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -417,9 +417,7 @@ DataProcessorSpec
         for (auto d : ds) {
           auto fileAndFolder = dod->getFileFolder(d, tfNumber, aodInputFile);
           auto treename = fileAndFolder.folderName + "/" + d->treename;
-          TableToTree ta2tr(table,
-                            fileAndFolder.file,
-                            treename.c_str());
+          std::unique_ptr<TableToRoot> ta2r = std::make_unique<TableToTree>(table, fileAndFolder.file, treename.c_str());
 
           // update metadata
           if (fileAndFolder.file->FindObjectAny("metaData")) {
@@ -438,13 +436,13 @@ DataProcessorSpec
               auto col = table->column(idx);
               auto field = table->schema()->field(idx);
               if (idx != -1) {
-                ta2tr.addBranch(col, field);
+                ta2r->addColumn(col, field);
               }
             }
           } else {
-            ta2tr.addAllBranches();
+            ta2r->addAllColumns();
           }
-          ta2tr.process();
+          ta2r->process();
         }
       }
     };
diff --git a/Framework/Core/src/TableTreeHelpers.cxx b/Framework/Core/src/TableTreeHelpers.cxx
index e6e6a1349e619..015bff26a0606 100644
--- a/Framework/Core/src/TableTreeHelpers.cxx
+++ b/Framework/Core/src/TableTreeHelpers.cxx
@@ -407,14 +407,32 @@ void ColumnToBranch::nextChunk()
   accessChunk();
 }
 
-TableToTree::TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* treename)
+TableToRoot::TableToRoot(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* name)
+  : mTable{table.get()},
+    mFile{file},
+    mName{name}
 {
-  mTable = table.get();
-  mTree.reset(static_cast<TTree*>(file->Get(treename)));
+}
+
+void TableToRoot::addAllColumns()
+{
+  mRows = mTable->num_rows();
+  auto columns = mTable->columns();
+  auto fields = mTable->schema()->fields();
+  assert(columns.size() == fields.size());
+  for (auto i = 0u; i < columns.size(); ++i) {
+    addColumn(columns[i], fields[i]);
+  }
+}
+
+TableToTree::TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* name)
+  : TableToRoot{table, file, name}
+{
+  mTree.reset(static_cast<TTree*>(mFile->Get(mName.c_str())));
   if (mTree) {
     return;
   }
-  std::string treeName(treename);
+  std::string treeName(mName);
   auto pos = treeName.find_first_of('/');
   if (pos != std::string::npos) {
     file->cd(treeName.substr(0, pos).c_str());
@@ -423,18 +441,8 @@ TableToTree::TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file
   mTree = std::make_shared<TTree>(treeName.c_str(), treeName.c_str());
 }
 
-void TableToTree::addAllBranches()
-{
-  mRows = mTable->num_rows();
-  auto columns = mTable->columns();
-  auto fields = mTable->schema()->fields();
-  assert(columns.size() == fields.size());
-  for (auto i = 0u; i < columns.size(); ++i) {
-    addBranch(columns[i], fields[i]);
-  }
-}
 
-void TableToTree::addBranch(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field)
+void TableToTree::addColumn(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field)
 {
   if (mRows == 0) {
     mRows = column->length();
@@ -444,13 +452,13 @@ void TableToTree::addBranch(std::shared_ptr<arrow::ChunkedArray> const& column,
   mColumnReaders.emplace_back(new ColumnToBranch{mTree.get(), column, field});
 }
 
-std::shared_ptr<TTree> TableToTree::process()
+void TableToTree::process()
 {
   int64_t row = 0;
   if (mTree->GetNbranches() == 0 || mRows == 0) {
     mTree->Write("", TObject::kOverwrite);
     mTree->SetDirectory(nullptr);
-    return mTree;
+    return;
   }
 
   for (auto& reader : mColumnReaders) {
@@ -468,7 +476,6 @@ std::shared_ptr<TTree> TableToTree::process()
   }
   mTree->Write("", TObject::kOverwrite);
   mTree->SetDirectory(nullptr);
-  return mTree;
 }
 
 TreeToTable::TreeToTable(arrow::MemoryPool* pool)
diff --git a/Framework/Core/test/benchmark_TableToTree.cxx b/Framework/Core/test/benchmark_TableToTree.cxx
index 0a7963518f00b..0ca4eb81112b8 100644
--- a/Framework/Core/test/benchmark_TableToTree.cxx
+++ b/Framework/Core/test/benchmark_TableToTree.cxx
@@ -62,7 +62,7 @@ static void BM_TableToTree(benchmark::State& state)
 
     // benchmark TableToTree
     TableToTree ta2tr(table, &fout, "table2tree");
-    ta2tr.addAllBranches();
+    ta2tr.addAllColumns();
     ta2tr.process();
 
     // clean up
diff --git a/Framework/Core/test/benchmark_TreeToTable.cxx b/Framework/Core/test/benchmark_TreeToTable.cxx
index 6eca853a90ce4..476e653acaf83 100644
--- a/Framework/Core/test/benchmark_TreeToTable.cxx
+++ b/Framework/Core/test/benchmark_TreeToTable.cxx
@@ -58,7 +58,7 @@ static void BM_TreeToTable(benchmark::State& state)
   // now convert the table to a tree
   TFile fout("tree2table.root", "RECREATE");
   TableToTree ta2tr(table, &fout, "tree2table");
-  ta2tr.addAllBranches();
+  ta2tr.addAllColumns();
   ta2tr.process();
   fout.Close();
 
diff --git a/Framework/Core/test/test_TreeToTable.cxx b/Framework/Core/test/test_TreeToTable.cxx
index 65569b72b9f35..32575a8caf3e0 100644
--- a/Framework/Core/test/test_TreeToTable.cxx
+++ b/Framework/Core/test/test_TreeToTable.cxx
@@ -143,9 +143,10 @@ TEST_CASE("TreeToTableConversion")
   // save table as tree
   TFile* f2 = TFile::Open("table2tree.root", "RECREATE");
   TableToTree ta2tr(table, f2, "mytree");
-  ta2tr.addAllBranches();
+  ta2tr.addAllColumns();
 
-  auto t2 = ta2tr.process();
+  ta2tr.process();
+  auto t2 = ta2tr.tree();
   auto br = (TBranch*)t2->GetBranch("ok");
   REQUIRE(t2->GetEntries() == ndp);
   REQUIRE(br->GetEntries() == ndp);
@@ -201,8 +202,8 @@ TEST_CASE("VariableLists")
 
   auto* f = TFile::Open("variable_lists.root", "RECREATE");
   TableToTree ta2tr(table, f, "lists");
-  ta2tr.addAllBranches();
-  auto tree = ta2tr.process();
+  ta2tr.addAllColumns();
+  ta2tr.process();
   f->Close();
 
   auto* f2 = TFile::Open("variable_lists.root", "READ");

From 695d9f1810e8f6f294915cf187a16126c94823c3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 31 Oct 2023 15:40:29 +0100
Subject: [PATCH 032/726] DPL: have common baseclass to abstract tree writing

This is the the first step towards writing something else,
e.g. a RNTuple.
---
 .../Core/include/Framework/TableTreeHelpers.h | 28 +++++++++---
 Framework/Core/src/CommonDataProcessors.cxx   | 10 ++---
 Framework/Core/src/TableTreeHelpers.cxx       | 44 +++++++++++--------
 Framework/Core/test/benchmark_TableToTree.cxx |  2 +-
 Framework/Core/test/benchmark_TreeToTable.cxx |  2 +-
 Framework/Core/test/test_TreeToTable.cxx      |  9 ++--
 6 files changed, 58 insertions(+), 37 deletions(-)

diff --git a/Framework/Core/include/Framework/TableTreeHelpers.h b/Framework/Core/include/Framework/TableTreeHelpers.h
index e12f4a38a973b..78bf0bba11389 100644
--- a/Framework/Core/include/Framework/TableTreeHelpers.h
+++ b/Framework/Core/include/Framework/TableTreeHelpers.h
@@ -105,18 +105,34 @@ class ColumnToBranch
   int mFieldSize = 0;
 };
 
-class TableToTree
+class TableToRoot
 {
  public:
-  TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* treename);
+  TableToRoot(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* name);
+  ~TableToRoot() = default;
 
-  std::shared_ptr<TTree> process();
-  void addBranch(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field);
-  void addAllBranches();
+  virtual void addColumn(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field) = 0;
+  virtual void process() = 0;
+  // Helper function which marks all columns in the table for writing
+  void addAllColumns();
 
- private:
+ protected:
   arrow::Table* mTable;
+  TFile* mFile;
+  std::string mName;
   int64_t mRows = 0;
+};
+
+class TableToTree : public TableToRoot
+{
+ public:
+  TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* treename);
+
+  void process() final;
+  void addColumn(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field) final;
+  std::shared_ptr<TTree> tree() { return mTree; }
+
+ private:
   std::shared_ptr<TTree> mTree;
   std::vector<std::unique_ptr<ColumnToBranch>> mColumnReaders;
 };
diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 08283b5e49bb5..c9d8aa835d734 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -417,9 +417,7 @@ DataProcessorSpec
         for (auto d : ds) {
           auto fileAndFolder = dod->getFileFolder(d, tfNumber, aodInputFile);
           auto treename = fileAndFolder.folderName + "/" + d->treename;
-          TableToTree ta2tr(table,
-                            fileAndFolder.file,
-                            treename.c_str());
+          std::unique_ptr<TableToRoot> ta2r = std::make_unique<TableToTree>(table, fileAndFolder.file, treename.c_str());
 
           // update metadata
           if (fileAndFolder.file->FindObjectAny("metaData")) {
@@ -438,13 +436,13 @@ DataProcessorSpec
               auto col = table->column(idx);
               auto field = table->schema()->field(idx);
               if (idx != -1) {
-                ta2tr.addBranch(col, field);
+                ta2r->addColumn(col, field);
               }
             }
           } else {
-            ta2tr.addAllBranches();
+            ta2r->addAllColumns();
           }
-          ta2tr.process();
+          ta2r->process();
         }
       }
     };
diff --git a/Framework/Core/src/TableTreeHelpers.cxx b/Framework/Core/src/TableTreeHelpers.cxx
index e6e6a1349e619..1c96157637433 100644
--- a/Framework/Core/src/TableTreeHelpers.cxx
+++ b/Framework/Core/src/TableTreeHelpers.cxx
@@ -407,34 +407,41 @@ void ColumnToBranch::nextChunk()
   accessChunk();
 }
 
-TableToTree::TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* treename)
+TableToRoot::TableToRoot(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* name)
+  : mTable{table.get()},
+    mFile{file},
+    mName{name}
 {
-  mTable = table.get();
-  mTree.reset(static_cast<TTree*>(file->Get(treename)));
-  if (mTree) {
-    return;
-  }
-  std::string treeName(treename);
-  auto pos = treeName.find_first_of('/');
-  if (pos != std::string::npos) {
-    file->cd(treeName.substr(0, pos).c_str());
-    treeName = treeName.substr(pos + 1, std::string::npos);
-  }
-  mTree = std::make_shared<TTree>(treeName.c_str(), treeName.c_str());
 }
 
-void TableToTree::addAllBranches()
+void TableToRoot::addAllColumns()
 {
   mRows = mTable->num_rows();
   auto columns = mTable->columns();
   auto fields = mTable->schema()->fields();
   assert(columns.size() == fields.size());
   for (auto i = 0u; i < columns.size(); ++i) {
-    addBranch(columns[i], fields[i]);
+    addColumn(columns[i], fields[i]);
+  }
+}
+
+TableToTree::TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* name)
+  : TableToRoot{table, file, name}
+{
+  mTree.reset(static_cast<TTree*>(mFile->Get(mName.c_str())));
+  if (mTree) {
+    return;
   }
+  std::string treeName(mName);
+  auto pos = treeName.find_first_of('/');
+  if (pos != std::string::npos) {
+    file->cd(treeName.substr(0, pos).c_str());
+    treeName = treeName.substr(pos + 1, std::string::npos);
+  }
+  mTree = std::make_shared<TTree>(treeName.c_str(), treeName.c_str());
 }
 
-void TableToTree::addBranch(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field)
+void TableToTree::addColumn(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field)
 {
   if (mRows == 0) {
     mRows = column->length();
@@ -444,13 +451,13 @@ void TableToTree::addBranch(std::shared_ptr<arrow::ChunkedArray> const& column,
   mColumnReaders.emplace_back(new ColumnToBranch{mTree.get(), column, field});
 }
 
-std::shared_ptr<TTree> TableToTree::process()
+void TableToTree::process()
 {
   int64_t row = 0;
   if (mTree->GetNbranches() == 0 || mRows == 0) {
     mTree->Write("", TObject::kOverwrite);
     mTree->SetDirectory(nullptr);
-    return mTree;
+    return;
   }
 
   for (auto& reader : mColumnReaders) {
@@ -468,7 +475,6 @@ std::shared_ptr<TTree> TableToTree::process()
   }
   mTree->Write("", TObject::kOverwrite);
   mTree->SetDirectory(nullptr);
-  return mTree;
 }
 
 TreeToTable::TreeToTable(arrow::MemoryPool* pool)
diff --git a/Framework/Core/test/benchmark_TableToTree.cxx b/Framework/Core/test/benchmark_TableToTree.cxx
index 0a7963518f00b..0ca4eb81112b8 100644
--- a/Framework/Core/test/benchmark_TableToTree.cxx
+++ b/Framework/Core/test/benchmark_TableToTree.cxx
@@ -62,7 +62,7 @@ static void BM_TableToTree(benchmark::State& state)
 
     // benchmark TableToTree
     TableToTree ta2tr(table, &fout, "table2tree");
-    ta2tr.addAllBranches();
+    ta2tr.addAllColumns();
     ta2tr.process();
 
     // clean up
diff --git a/Framework/Core/test/benchmark_TreeToTable.cxx b/Framework/Core/test/benchmark_TreeToTable.cxx
index 6eca853a90ce4..476e653acaf83 100644
--- a/Framework/Core/test/benchmark_TreeToTable.cxx
+++ b/Framework/Core/test/benchmark_TreeToTable.cxx
@@ -58,7 +58,7 @@ static void BM_TreeToTable(benchmark::State& state)
   // now convert the table to a tree
   TFile fout("tree2table.root", "RECREATE");
   TableToTree ta2tr(table, &fout, "tree2table");
-  ta2tr.addAllBranches();
+  ta2tr.addAllColumns();
   ta2tr.process();
   fout.Close();
 
diff --git a/Framework/Core/test/test_TreeToTable.cxx b/Framework/Core/test/test_TreeToTable.cxx
index 65569b72b9f35..32575a8caf3e0 100644
--- a/Framework/Core/test/test_TreeToTable.cxx
+++ b/Framework/Core/test/test_TreeToTable.cxx
@@ -143,9 +143,10 @@ TEST_CASE("TreeToTableConversion")
   // save table as tree
   TFile* f2 = TFile::Open("table2tree.root", "RECREATE");
   TableToTree ta2tr(table, f2, "mytree");
-  ta2tr.addAllBranches();
+  ta2tr.addAllColumns();
 
-  auto t2 = ta2tr.process();
+  ta2tr.process();
+  auto t2 = ta2tr.tree();
   auto br = (TBranch*)t2->GetBranch("ok");
   REQUIRE(t2->GetEntries() == ndp);
   REQUIRE(br->GetEntries() == ndp);
@@ -201,8 +202,8 @@ TEST_CASE("VariableLists")
 
   auto* f = TFile::Open("variable_lists.root", "RECREATE");
   TableToTree ta2tr(table, f, "lists");
-  ta2tr.addAllBranches();
-  auto tree = ta2tr.process();
+  ta2tr.addAllColumns();
+  ta2tr.process();
   f->Close();
 
   auto* f2 = TFile::Open("variable_lists.root", "READ");

From bdc4e1f2b9c45cb3a9e319596683ec96e2aafd2a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 31 Oct 2023 15:52:46 +0100
Subject: [PATCH 033/726] Fix warning on unused result

---
 Detectors/TPC/base/src/Painter.cxx | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Detectors/TPC/base/src/Painter.cxx b/Detectors/TPC/base/src/Painter.cxx
index 7339a11e4c8e0..0f7b89b71e07a 100644
--- a/Detectors/TPC/base/src/Painter.cxx
+++ b/Detectors/TPC/base/src/Painter.cxx
@@ -927,9 +927,7 @@ TH3F painter::convertCalDetToTH3(const std::vector<CalDet<DataT>>& calDet, const
           const int binRBottomStart = std::clamp(histConvSum.GetYaxis()->FindBin(radiusPadCoord[0]) - 1, 1, nRBins);
           const int binRTopEnd = std::clamp(histConvSum.GetYaxis()->FindBin(radiusPadCoord[1]) + 1, 1, nRBins);
           int binPhiStart = std::min(histConvSum.GetXaxis()->FindBin(phiPadCoord[0]), histConvSum.GetXaxis()->FindBin(phiPadCoord[1]));
-          std::clamp(binPhiStart - 1, 1, nPhiBins);
           int binPhiEnd = std::max(histConvSum.GetXaxis()->FindBin(phiPadCoord[2]), histConvSum.GetXaxis()->FindBin(phiPadCoord[3]));
-          std::clamp(binPhiEnd + 1, 1, nPhiBins);
 
           // define boost geoemtry object
           polygon geoPad;

From 5ff18e2b36b14c28d2a983b52f1c846420ad9602 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 31 Oct 2023 22:08:44 +0100
Subject: [PATCH 034/726] Make sure all initializer clauses are designated

Not doing so is actually an error and breaks C++20.
---
 Framework/Core/src/DataProcessingDevice.cxx |  4 +++-
 Framework/Core/src/DataRelayer.cxx          | 16 +++++++++-------
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index b39607366adec..0bb0d6bf543ac 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -2177,7 +2177,9 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       buffer[ai] = record.isValid(ai) ? '3' : '0';
     }
     buffer[record.size()] = 0;
-    states.updateState({.id = short((int)ProcessingStateId::DATA_RELAYER_BASE + action.slot.index), (int)(record.size() + buffer - relayerSlotState), relayerSlotState});
+    states.updateState({.id = short((int)ProcessingStateId::DATA_RELAYER_BASE + action.slot.index), 
+                        .size = (int)(record.size() + buffer - relayerSlotState),
+                        .data = relayerSlotState});
     uint64_t tEnd = uv_hrtime();
     // tEnd and tStart are in nanoseconds according to https://docs.libuv.org/en/v1.x/misc.html#c.uv_hrtime
     int64_t wallTimeMs = (tEnd - tStart) / 1000000;
diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index 7dc6d28a1a11f..94711ac5d954b 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -271,7 +271,9 @@ void sendVariableContextMetrics(VariableContext& context, TimesliceSlot slot, Da
       }
       state += ";";
     }
-    states.updateState({.id = short((int)ProcessingStateId::CONTEXT_VARIABLES_BASE + slot.index), (int)state.size(), state.data()});
+    states.updateState({.id = short((int)ProcessingStateId::CONTEXT_VARIABLES_BASE + slot.index),
+                        .size = (int)state.size(), 
+                        .data = state.data()});
   },
                   &states, slot);
 }
@@ -516,7 +518,7 @@ DataRelayer::RelayChoice
     index.publishSlot(slot);
     index.markAsDirty(slot, true);
     stats.updateStats({static_cast<short>(ProcessingStatsId::RELAYED_MESSAGES), DataProcessingStats::Op::Add, (int)1});
-    return RelayChoice{RelayChoice::Type::WillRelay, timeslice};
+    return RelayChoice{.type = RelayChoice::Type::WillRelay, .timeslice = timeslice};
   }
 
   /// If not, we find which timeslice we really were looking at
@@ -543,7 +545,7 @@ DataRelayer::RelayChoice
     for (size_t pi = 0; pi < nMessages; ++pi) {
       messages[pi].reset(nullptr);
     }
-    return RelayChoice{.type = RelayChoice::Type::Invalid, timeslice};
+    return RelayChoice{.type = RelayChoice::Type::Invalid, .timeslice = timeslice};
   }
 
   if (TimesliceId::isValid(timeslice) == false) {
@@ -553,7 +555,7 @@ DataRelayer::RelayChoice
     for (size_t pi = 0; pi < nMessages; ++pi) {
       messages[pi].reset(nullptr);
     }
-    return RelayChoice{.type = RelayChoice::Type::Invalid, timeslice};
+    return RelayChoice{.type = RelayChoice::Type::Invalid, .timeslice = timeslice};
   }
 
   TimesliceIndex::ActionTaken action;
@@ -563,7 +565,7 @@ DataRelayer::RelayChoice
 
   switch (action) {
     case TimesliceIndex::ActionTaken::Wait:
-      return RelayChoice{.type = RelayChoice::Type::Backpressured, timeslice};
+      return RelayChoice{.type = RelayChoice::Type::Backpressured, .timeslice = timeslice};
     case TimesliceIndex::ActionTaken::DropObsolete:
       static std::atomic<size_t> obsoleteCount = 0;
       static std::atomic<size_t> mult = 1;
@@ -573,7 +575,7 @@ DataRelayer::RelayChoice
           mult = mult * 10;
         }
       }
-      return RelayChoice{.type = RelayChoice::Type::Dropped, timeslice};
+      return RelayChoice{.type = RelayChoice::Type::Dropped, .timeslice = timeslice};
     case TimesliceIndex::ActionTaken::DropInvalid:
       LOG(warning) << "Incoming data is invalid, not relaying.";
       stats.updateStats({static_cast<short>(ProcessingStatsId::MALFORMED_INPUTS), DataProcessingStats::Op::Add, (int)1});
@@ -581,7 +583,7 @@ DataRelayer::RelayChoice
       for (size_t pi = 0; pi < nMessages; ++pi) {
         messages[pi].reset(nullptr);
       }
-      return RelayChoice{.type = RelayChoice::Type::Invalid, timeslice};
+      return RelayChoice{.type = RelayChoice::Type::Invalid, .timeslice = timeslice};
     case TimesliceIndex::ActionTaken::ReplaceUnused:
     case TimesliceIndex::ActionTaken::ReplaceObsolete:
       // At this point the variables match the new input but the

From b4f312d7191d3e997e64a3e95413cdbee3fe90d4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 31 Oct 2023 22:08:44 +0100
Subject: [PATCH 035/726] DPL: fix bad API usage

---
 Framework/Core/src/DataSender.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/src/DataSender.cxx b/Framework/Core/src/DataSender.cxx
index 8b63a2c45ec32..f0753102e2d40 100644
--- a/Framework/Core/src/DataSender.cxx
+++ b/Framework/Core/src/DataSender.cxx
@@ -119,7 +119,7 @@ void DataSender::verifyMissingSporadic() const
 {
   for (auto present : mPresent) {
     if (!present) {
-      LOGP(debug, O2DataModelHelpers::describeMissingOutputs(mOutputs, mPresent).c_str());
+      LOGP(debug, "{}", O2DataModelHelpers::describeMissingOutputs(mOutputs, mPresent).c_str());
       return;
     }
   }

From b9ddf10d376877b581ae1c2d3df514bba7477b11 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 31 Oct 2023 22:18:03 +0100
Subject: [PATCH 036/726] Make sure all initializer clauses are designated

Not doing so is actually an error and breaks C++20.
---
 Framework/Core/src/DataProcessingDevice.cxx |  4 +++-
 Framework/Core/src/DataRelayer.cxx          | 16 +++++++++-------
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index b39607366adec..9a575a95b078a 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -2177,7 +2177,9 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       buffer[ai] = record.isValid(ai) ? '3' : '0';
     }
     buffer[record.size()] = 0;
-    states.updateState({.id = short((int)ProcessingStateId::DATA_RELAYER_BASE + action.slot.index), (int)(record.size() + buffer - relayerSlotState), relayerSlotState});
+    states.updateState({.id = short((int)ProcessingStateId::DATA_RELAYER_BASE + action.slot.index),
+                        .size = (int)(record.size() + buffer - relayerSlotState),
+                        .data = relayerSlotState});
     uint64_t tEnd = uv_hrtime();
     // tEnd and tStart are in nanoseconds according to https://docs.libuv.org/en/v1.x/misc.html#c.uv_hrtime
     int64_t wallTimeMs = (tEnd - tStart) / 1000000;
diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index 7dc6d28a1a11f..c507d6eb308e3 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -271,7 +271,9 @@ void sendVariableContextMetrics(VariableContext& context, TimesliceSlot slot, Da
       }
       state += ";";
     }
-    states.updateState({.id = short((int)ProcessingStateId::CONTEXT_VARIABLES_BASE + slot.index), (int)state.size(), state.data()});
+    states.updateState({.id = short((int)ProcessingStateId::CONTEXT_VARIABLES_BASE + slot.index),
+                        .size = (int)state.size(),
+                        .data = state.data()});
   },
                   &states, slot);
 }
@@ -516,7 +518,7 @@ DataRelayer::RelayChoice
     index.publishSlot(slot);
     index.markAsDirty(slot, true);
     stats.updateStats({static_cast<short>(ProcessingStatsId::RELAYED_MESSAGES), DataProcessingStats::Op::Add, (int)1});
-    return RelayChoice{RelayChoice::Type::WillRelay, timeslice};
+    return RelayChoice{.type = RelayChoice::Type::WillRelay, .timeslice = timeslice};
   }
 
   /// If not, we find which timeslice we really were looking at
@@ -543,7 +545,7 @@ DataRelayer::RelayChoice
     for (size_t pi = 0; pi < nMessages; ++pi) {
       messages[pi].reset(nullptr);
     }
-    return RelayChoice{.type = RelayChoice::Type::Invalid, timeslice};
+    return RelayChoice{.type = RelayChoice::Type::Invalid, .timeslice = timeslice};
   }
 
   if (TimesliceId::isValid(timeslice) == false) {
@@ -553,7 +555,7 @@ DataRelayer::RelayChoice
     for (size_t pi = 0; pi < nMessages; ++pi) {
       messages[pi].reset(nullptr);
     }
-    return RelayChoice{.type = RelayChoice::Type::Invalid, timeslice};
+    return RelayChoice{.type = RelayChoice::Type::Invalid, .timeslice = timeslice};
   }
 
   TimesliceIndex::ActionTaken action;
@@ -563,7 +565,7 @@ DataRelayer::RelayChoice
 
   switch (action) {
     case TimesliceIndex::ActionTaken::Wait:
-      return RelayChoice{.type = RelayChoice::Type::Backpressured, timeslice};
+      return RelayChoice{.type = RelayChoice::Type::Backpressured, .timeslice = timeslice};
     case TimesliceIndex::ActionTaken::DropObsolete:
       static std::atomic<size_t> obsoleteCount = 0;
       static std::atomic<size_t> mult = 1;
@@ -573,7 +575,7 @@ DataRelayer::RelayChoice
           mult = mult * 10;
         }
       }
-      return RelayChoice{.type = RelayChoice::Type::Dropped, timeslice};
+      return RelayChoice{.type = RelayChoice::Type::Dropped, .timeslice = timeslice};
     case TimesliceIndex::ActionTaken::DropInvalid:
       LOG(warning) << "Incoming data is invalid, not relaying.";
       stats.updateStats({static_cast<short>(ProcessingStatsId::MALFORMED_INPUTS), DataProcessingStats::Op::Add, (int)1});
@@ -581,7 +583,7 @@ DataRelayer::RelayChoice
       for (size_t pi = 0; pi < nMessages; ++pi) {
         messages[pi].reset(nullptr);
       }
-      return RelayChoice{.type = RelayChoice::Type::Invalid, timeslice};
+      return RelayChoice{.type = RelayChoice::Type::Invalid, .timeslice = timeslice};
     case TimesliceIndex::ActionTaken::ReplaceUnused:
     case TimesliceIndex::ActionTaken::ReplaceObsolete:
       // At this point the variables match the new input but the

From 63f4e3579ea1dad836f5b5614037bda350892d68 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 31 Oct 2023 22:18:03 +0100
Subject: [PATCH 037/726] DPL: fix bad API usage

---
 Framework/Core/src/DataSender.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/src/DataSender.cxx b/Framework/Core/src/DataSender.cxx
index 8b63a2c45ec32..f0753102e2d40 100644
--- a/Framework/Core/src/DataSender.cxx
+++ b/Framework/Core/src/DataSender.cxx
@@ -119,7 +119,7 @@ void DataSender::verifyMissingSporadic() const
 {
   for (auto present : mPresent) {
     if (!present) {
-      LOGP(debug, O2DataModelHelpers::describeMissingOutputs(mOutputs, mPresent).c_str());
+      LOGP(debug, "{}", O2DataModelHelpers::describeMissingOutputs(mOutputs, mPresent).c_str());
       return;
     }
   }

From 3214f289f6402cedd443d5d6b1691928048ad879 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 2 Nov 2023 14:26:05 +0100
Subject: [PATCH 038/726] DPL Analysis: add version to the table size

---
 Framework/Core/src/DataProcessor.cxx | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/src/DataProcessor.cxx b/Framework/Core/src/DataProcessor.cxx
index bdfd740a998b3..e9938a304b8a6 100644
--- a/Framework/Core/src/DataProcessor.cxx
+++ b/Framework/Core/src/DataProcessor.cxx
@@ -107,12 +107,14 @@ void DataProcessor::doSend(DataSender& sender, ArrowContext& context, ServiceReg
 
     auto origin = std::regex_replace(dh->dataOrigin.as<std::string>(), invalid_metric, "_");
     auto description = std::regex_replace(dh->dataDescription.as<std::string>(), invalid_metric, "_");
+    uint64_t version = dh->subSpecification;
     monitoring.send(Metric{(uint64_t)payload->GetSize(),
-                           fmt::format("table-bytes-{}-{}-created",
+                           fmt::format("table-bytes-{}-{}-{}-created",
                                        origin,
-                                       description)}
+                                       description,
+                                       version)}
                       .addTag(Key::Subsystem, Value::DPL));
-    LOGP(detail, "Creating {}MB for table {}/{}.", payload->GetSize() / 1000000., dh->dataOrigin, dh->dataDescription);
+    LOGP(detail, "Creating {}MB for table {}/{}/{}.", payload->GetSize() / 1000000., dh->dataOrigin, dh->dataDescription, version);
     context.updateBytesSent(payload->GetSize());
     context.updateMessagesSent(1);
     parts.AddPart(std::move(messageRef.header));

From e6be56b71f47ea24b8248dfa66b1853031120916 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 3 Nov 2023 11:02:04 +0100
Subject: [PATCH 039/726] DPL: only listen to SIGUSR1 when the task is alive

---
 Framework/Core/include/Framework/DeviceContext.h |  2 ++
 Framework/Core/src/DataProcessingDevice.cxx      | 15 +++++++++++----
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/Framework/Core/include/Framework/DeviceContext.h b/Framework/Core/include/Framework/DeviceContext.h
index d34192ec6ad3d..3777e7f608b75 100644
--- a/Framework/Core/include/Framework/DeviceContext.h
+++ b/Framework/Core/include/Framework/DeviceContext.h
@@ -16,6 +16,7 @@
 #define O2_FRAMEWORK_DEVICECONTEXT_H_
 
 typedef struct uv_timer_s uv_timer_t;
+typedef struct uv_signal_s uv_signal_t;
 
 namespace o2::framework
 {
@@ -27,6 +28,7 @@ struct ComputingQuotaStats;
 struct DeviceContext {
   ComputingQuotaStats* quotaStats = nullptr;
   uv_timer_t* gracePeriodTimer = nullptr;
+  uv_signal_t* sigusr1Handle = nullptr;
   int expectedRegionCallbacks = 0;
   int exitTransitionTimeout = 0;
 };
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 9a575a95b078a..0bfeb7df2fdc5 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -972,10 +972,12 @@ void DataProcessingDevice::InitTask()
   // an event from the outside, making sure that the event loop can
   // be unblocked (e.g. by a quitting DPL driver) even when there
   // is no data pending to be processed.
-  auto* sigusr1Handle = (uv_signal_t*)malloc(sizeof(uv_signal_t));
-  uv_signal_init(state.loop, sigusr1Handle);
-  sigusr1Handle->data = &mServiceRegistry;
-  uv_signal_start(sigusr1Handle, on_signal_callback, SIGUSR1);
+  if (deviceContext.sigusr1Handle == nullptr) {
+    deviceContext.sigusr1Handle = (uv_signal_t*)malloc(sizeof(uv_signal_t));
+    deviceContext.sigusr1Handle->data = &mServiceRegistry;
+    uv_signal_init(state.loop, deviceContext.sigusr1Handle);
+  }
+  uv_signal_start(deviceContext.sigusr1Handle, on_signal_callback, SIGUSR1);
 
   /// Initialise the pollers
   DataProcessingDevice::initPollers();
@@ -1675,6 +1677,11 @@ void DataProcessingDevice::ResetTask()
 {
   ServiceRegistryRef ref{mServiceRegistry};
   ref.get<DataRelayer>().clear();
+  auto &deviceContext = ref.get<DeviceContext>();
+  // Stop the signal handler
+  if (deviceContext.sigusr1Handle) {
+    uv_signal_stop(deviceContext.sigusr1Handle);
+  }
 }
 
 struct WaitBackpressurePolicy {

From 484aff8091d90b5e9b1d6da59b9feb3f18aa30c1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 3 Nov 2023 11:37:46 +0100
Subject: [PATCH 040/726] DPL: only listen to SIGUSR1 when the task is alive

---
 Framework/Core/include/Framework/DeviceContext.h |  2 ++
 Framework/Core/src/DataProcessingDevice.cxx      | 15 +++++++++++----
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/Framework/Core/include/Framework/DeviceContext.h b/Framework/Core/include/Framework/DeviceContext.h
index d34192ec6ad3d..3777e7f608b75 100644
--- a/Framework/Core/include/Framework/DeviceContext.h
+++ b/Framework/Core/include/Framework/DeviceContext.h
@@ -16,6 +16,7 @@
 #define O2_FRAMEWORK_DEVICECONTEXT_H_
 
 typedef struct uv_timer_s uv_timer_t;
+typedef struct uv_signal_s uv_signal_t;
 
 namespace o2::framework
 {
@@ -27,6 +28,7 @@ struct ComputingQuotaStats;
 struct DeviceContext {
   ComputingQuotaStats* quotaStats = nullptr;
   uv_timer_t* gracePeriodTimer = nullptr;
+  uv_signal_t* sigusr1Handle = nullptr;
   int expectedRegionCallbacks = 0;
   int exitTransitionTimeout = 0;
 };
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 9a575a95b078a..1a4ec7ae1b04f 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -972,10 +972,12 @@ void DataProcessingDevice::InitTask()
   // an event from the outside, making sure that the event loop can
   // be unblocked (e.g. by a quitting DPL driver) even when there
   // is no data pending to be processed.
-  auto* sigusr1Handle = (uv_signal_t*)malloc(sizeof(uv_signal_t));
-  uv_signal_init(state.loop, sigusr1Handle);
-  sigusr1Handle->data = &mServiceRegistry;
-  uv_signal_start(sigusr1Handle, on_signal_callback, SIGUSR1);
+  if (deviceContext.sigusr1Handle == nullptr) {
+    deviceContext.sigusr1Handle = (uv_signal_t*)malloc(sizeof(uv_signal_t));
+    deviceContext.sigusr1Handle->data = &mServiceRegistry;
+    uv_signal_init(state.loop, deviceContext.sigusr1Handle);
+  }
+  uv_signal_start(deviceContext.sigusr1Handle, on_signal_callback, SIGUSR1);
 
   /// Initialise the pollers
   DataProcessingDevice::initPollers();
@@ -1675,6 +1677,11 @@ void DataProcessingDevice::ResetTask()
 {
   ServiceRegistryRef ref{mServiceRegistry};
   ref.get<DataRelayer>().clear();
+  auto& deviceContext = ref.get<DeviceContext>();
+  // Stop the signal handler
+  if (deviceContext.sigusr1Handle) {
+    uv_signal_stop(deviceContext.sigusr1Handle);
+  }
 }
 
 struct WaitBackpressurePolicy {

From 96f12c21eab217649188f26bde5bd93355252ccb Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 3 Nov 2023 13:29:36 +0100
Subject: [PATCH 041/726] DPL: only listen to SIGUSR1 when the task is alive

---
 .../Core/include/Framework/DeviceContext.h    |  2 ++
 Framework/Core/src/DataProcessingDevice.cxx   | 24 +++++++++++++++----
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/Framework/Core/include/Framework/DeviceContext.h b/Framework/Core/include/Framework/DeviceContext.h
index d34192ec6ad3d..3777e7f608b75 100644
--- a/Framework/Core/include/Framework/DeviceContext.h
+++ b/Framework/Core/include/Framework/DeviceContext.h
@@ -16,6 +16,7 @@
 #define O2_FRAMEWORK_DEVICECONTEXT_H_
 
 typedef struct uv_timer_s uv_timer_t;
+typedef struct uv_signal_s uv_signal_t;
 
 namespace o2::framework
 {
@@ -27,6 +28,7 @@ struct ComputingQuotaStats;
 struct DeviceContext {
   ComputingQuotaStats* quotaStats = nullptr;
   uv_timer_t* gracePeriodTimer = nullptr;
+  uv_signal_t* sigusr1Handle = nullptr;
   int expectedRegionCallbacks = 0;
   int exitTransitionTimeout = 0;
 };
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 9a575a95b078a..5542be850fb70 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -493,6 +493,10 @@ void on_signal_callback(uv_signal_t* handle, int signum)
   ZoneScopedN("Signal callaback");
   LOG(debug) << "Signal " << signum << " received.";
   auto* registry = (ServiceRegistry*)handle->data;
+  if (!registry) {
+    LOG(debug) << "No registry active. Ignoring signal";
+    return;
+  }
   ServiceRegistryRef ref{*registry};
   auto& state = ref.get<DeviceState>();
   auto& quotaEvaluator = ref.get<ComputingQuotaEvaluator>();
@@ -972,10 +976,14 @@ void DataProcessingDevice::InitTask()
   // an event from the outside, making sure that the event loop can
   // be unblocked (e.g. by a quitting DPL driver) even when there
   // is no data pending to be processed.
-  auto* sigusr1Handle = (uv_signal_t*)malloc(sizeof(uv_signal_t));
-  uv_signal_init(state.loop, sigusr1Handle);
-  sigusr1Handle->data = &mServiceRegistry;
-  uv_signal_start(sigusr1Handle, on_signal_callback, SIGUSR1);
+  if (deviceContext.sigusr1Handle == nullptr) {
+    deviceContext.sigusr1Handle = (uv_signal_t*)malloc(sizeof(uv_signal_t));
+    deviceContext.sigusr1Handle->data = &mServiceRegistry;
+    uv_signal_init(state.loop, deviceContext.sigusr1Handle);
+    uv_signal_start(deviceContext.sigusr1Handle, on_signal_callback, SIGUSR1);
+  }
+  // When we start, we must make sure that we do listen to the signal
+  deviceContext.sigusr1Handle->data = &mServiceRegistry;
 
   /// Initialise the pollers
   DataProcessingDevice::initPollers();
@@ -1675,6 +1683,14 @@ void DataProcessingDevice::ResetTask()
 {
   ServiceRegistryRef ref{mServiceRegistry};
   ref.get<DataRelayer>().clear();
+  auto& deviceContext = ref.get<DeviceContext>();
+  // If the signal handler is there, we should
+  // hide the registry from it, so that we do not
+  // end up calling the signal handler on something
+  // which might not be there anymore.
+  if (deviceContext.sigusr1Handle) {
+    deviceContext.sigusr1Handle->data = nullptr;
+  }
 }
 
 struct WaitBackpressurePolicy {

From 0d7959dc6e6921d18fe48832db0a1c77d01d472c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 4 Nov 2023 10:02:21 +0100
Subject: [PATCH 042/726] DPL: have common baseclass to abstract tree writing

This is the the first step towards writing something else,
e.g. a RNTuple.
---
 Detectors/AOD/src/StandaloneAODProducer.cxx   |  6 +--
 .../Core/include/Framework/TableTreeHelpers.h | 28 +++++++++---
 Framework/Core/src/CommonDataProcessors.cxx   | 10 ++---
 Framework/Core/src/TableTreeHelpers.cxx       | 44 +++++++++++--------
 Framework/Core/test/benchmark_TableToTree.cxx |  2 +-
 Framework/Core/test/benchmark_TreeToTable.cxx |  2 +-
 Framework/Core/test/test_TreeToTable.cxx      |  9 ++--
 7 files changed, 61 insertions(+), 40 deletions(-)

diff --git a/Detectors/AOD/src/StandaloneAODProducer.cxx b/Detectors/AOD/src/StandaloneAODProducer.cxx
index 151038465c1d1..e37a877cf05b1 100644
--- a/Detectors/AOD/src/StandaloneAODProducer.cxx
+++ b/Detectors/AOD/src/StandaloneAODProducer.cxx
@@ -95,7 +95,7 @@ void fillMCollisionTable(o2::steer::MCKinematicsReader const& mcreader)
   TFile outfile("aod.root", "UPDATE");
   {
     TableToTree t2t(mccoltable, &outfile, aod::MetadataTrait<o2::aod::McCollisions>::metadata::tableLabel());
-    t2t.addAllBranches();
+    t2t.addAllColumns();
     t2t.process();
   }
 }
@@ -201,12 +201,12 @@ void fillCollisionAndTrackTable()
       TFile outfile("aod.root", "RECREATE");
       {
         TableToTree t2t(colltable, &outfile, aod::MetadataTrait<o2::aod::Collisions>::metadata::tableLabel());
-        t2t.addAllBranches();
+        t2t.addAllColumns();
         t2t.process();
       }
       {
         TableToTree t2t(tracktable, &outfile, "Tracks" /* aod::MetadataTrait<o2::aod::Tracks>::metadata::tableLabel() */);
-        t2t.addAllBranches();
+        t2t.addAllColumns();
         t2t.process();
       }
     }
diff --git a/Framework/Core/include/Framework/TableTreeHelpers.h b/Framework/Core/include/Framework/TableTreeHelpers.h
index e12f4a38a973b..78bf0bba11389 100644
--- a/Framework/Core/include/Framework/TableTreeHelpers.h
+++ b/Framework/Core/include/Framework/TableTreeHelpers.h
@@ -105,18 +105,34 @@ class ColumnToBranch
   int mFieldSize = 0;
 };
 
-class TableToTree
+class TableToRoot
 {
  public:
-  TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* treename);
+  TableToRoot(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* name);
+  ~TableToRoot() = default;
 
-  std::shared_ptr<TTree> process();
-  void addBranch(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field);
-  void addAllBranches();
+  virtual void addColumn(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field) = 0;
+  virtual void process() = 0;
+  // Helper function which marks all columns in the table for writing
+  void addAllColumns();
 
- private:
+ protected:
   arrow::Table* mTable;
+  TFile* mFile;
+  std::string mName;
   int64_t mRows = 0;
+};
+
+class TableToTree : public TableToRoot
+{
+ public:
+  TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* treename);
+
+  void process() final;
+  void addColumn(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field) final;
+  std::shared_ptr<TTree> tree() { return mTree; }
+
+ private:
   std::shared_ptr<TTree> mTree;
   std::vector<std::unique_ptr<ColumnToBranch>> mColumnReaders;
 };
diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 08283b5e49bb5..c9d8aa835d734 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -417,9 +417,7 @@ DataProcessorSpec
         for (auto d : ds) {
           auto fileAndFolder = dod->getFileFolder(d, tfNumber, aodInputFile);
           auto treename = fileAndFolder.folderName + "/" + d->treename;
-          TableToTree ta2tr(table,
-                            fileAndFolder.file,
-                            treename.c_str());
+          std::unique_ptr<TableToRoot> ta2r = std::make_unique<TableToTree>(table, fileAndFolder.file, treename.c_str());
 
           // update metadata
           if (fileAndFolder.file->FindObjectAny("metaData")) {
@@ -438,13 +436,13 @@ DataProcessorSpec
               auto col = table->column(idx);
               auto field = table->schema()->field(idx);
               if (idx != -1) {
-                ta2tr.addBranch(col, field);
+                ta2r->addColumn(col, field);
               }
             }
           } else {
-            ta2tr.addAllBranches();
+            ta2r->addAllColumns();
           }
-          ta2tr.process();
+          ta2r->process();
         }
       }
     };
diff --git a/Framework/Core/src/TableTreeHelpers.cxx b/Framework/Core/src/TableTreeHelpers.cxx
index e6e6a1349e619..1c96157637433 100644
--- a/Framework/Core/src/TableTreeHelpers.cxx
+++ b/Framework/Core/src/TableTreeHelpers.cxx
@@ -407,34 +407,41 @@ void ColumnToBranch::nextChunk()
   accessChunk();
 }
 
-TableToTree::TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* treename)
+TableToRoot::TableToRoot(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* name)
+  : mTable{table.get()},
+    mFile{file},
+    mName{name}
 {
-  mTable = table.get();
-  mTree.reset(static_cast<TTree*>(file->Get(treename)));
-  if (mTree) {
-    return;
-  }
-  std::string treeName(treename);
-  auto pos = treeName.find_first_of('/');
-  if (pos != std::string::npos) {
-    file->cd(treeName.substr(0, pos).c_str());
-    treeName = treeName.substr(pos + 1, std::string::npos);
-  }
-  mTree = std::make_shared<TTree>(treeName.c_str(), treeName.c_str());
 }
 
-void TableToTree::addAllBranches()
+void TableToRoot::addAllColumns()
 {
   mRows = mTable->num_rows();
   auto columns = mTable->columns();
   auto fields = mTable->schema()->fields();
   assert(columns.size() == fields.size());
   for (auto i = 0u; i < columns.size(); ++i) {
-    addBranch(columns[i], fields[i]);
+    addColumn(columns[i], fields[i]);
+  }
+}
+
+TableToTree::TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* name)
+  : TableToRoot{table, file, name}
+{
+  mTree.reset(static_cast<TTree*>(mFile->Get(mName.c_str())));
+  if (mTree) {
+    return;
   }
+  std::string treeName(mName);
+  auto pos = treeName.find_first_of('/');
+  if (pos != std::string::npos) {
+    file->cd(treeName.substr(0, pos).c_str());
+    treeName = treeName.substr(pos + 1, std::string::npos);
+  }
+  mTree = std::make_shared<TTree>(treeName.c_str(), treeName.c_str());
 }
 
-void TableToTree::addBranch(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field)
+void TableToTree::addColumn(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field)
 {
   if (mRows == 0) {
     mRows = column->length();
@@ -444,13 +451,13 @@ void TableToTree::addBranch(std::shared_ptr<arrow::ChunkedArray> const& column,
   mColumnReaders.emplace_back(new ColumnToBranch{mTree.get(), column, field});
 }
 
-std::shared_ptr<TTree> TableToTree::process()
+void TableToTree::process()
 {
   int64_t row = 0;
   if (mTree->GetNbranches() == 0 || mRows == 0) {
     mTree->Write("", TObject::kOverwrite);
     mTree->SetDirectory(nullptr);
-    return mTree;
+    return;
   }
 
   for (auto& reader : mColumnReaders) {
@@ -468,7 +475,6 @@ std::shared_ptr<TTree> TableToTree::process()
   }
   mTree->Write("", TObject::kOverwrite);
   mTree->SetDirectory(nullptr);
-  return mTree;
 }
 
 TreeToTable::TreeToTable(arrow::MemoryPool* pool)
diff --git a/Framework/Core/test/benchmark_TableToTree.cxx b/Framework/Core/test/benchmark_TableToTree.cxx
index 0a7963518f00b..0ca4eb81112b8 100644
--- a/Framework/Core/test/benchmark_TableToTree.cxx
+++ b/Framework/Core/test/benchmark_TableToTree.cxx
@@ -62,7 +62,7 @@ static void BM_TableToTree(benchmark::State& state)
 
     // benchmark TableToTree
     TableToTree ta2tr(table, &fout, "table2tree");
-    ta2tr.addAllBranches();
+    ta2tr.addAllColumns();
     ta2tr.process();
 
     // clean up
diff --git a/Framework/Core/test/benchmark_TreeToTable.cxx b/Framework/Core/test/benchmark_TreeToTable.cxx
index 6eca853a90ce4..476e653acaf83 100644
--- a/Framework/Core/test/benchmark_TreeToTable.cxx
+++ b/Framework/Core/test/benchmark_TreeToTable.cxx
@@ -58,7 +58,7 @@ static void BM_TreeToTable(benchmark::State& state)
   // now convert the table to a tree
   TFile fout("tree2table.root", "RECREATE");
   TableToTree ta2tr(table, &fout, "tree2table");
-  ta2tr.addAllBranches();
+  ta2tr.addAllColumns();
   ta2tr.process();
   fout.Close();
 
diff --git a/Framework/Core/test/test_TreeToTable.cxx b/Framework/Core/test/test_TreeToTable.cxx
index 65569b72b9f35..32575a8caf3e0 100644
--- a/Framework/Core/test/test_TreeToTable.cxx
+++ b/Framework/Core/test/test_TreeToTable.cxx
@@ -143,9 +143,10 @@ TEST_CASE("TreeToTableConversion")
   // save table as tree
   TFile* f2 = TFile::Open("table2tree.root", "RECREATE");
   TableToTree ta2tr(table, f2, "mytree");
-  ta2tr.addAllBranches();
+  ta2tr.addAllColumns();
 
-  auto t2 = ta2tr.process();
+  ta2tr.process();
+  auto t2 = ta2tr.tree();
   auto br = (TBranch*)t2->GetBranch("ok");
   REQUIRE(t2->GetEntries() == ndp);
   REQUIRE(br->GetEntries() == ndp);
@@ -201,8 +202,8 @@ TEST_CASE("VariableLists")
 
   auto* f = TFile::Open("variable_lists.root", "RECREATE");
   TableToTree ta2tr(table, f, "lists");
-  ta2tr.addAllBranches();
-  auto tree = ta2tr.process();
+  ta2tr.addAllColumns();
+  ta2tr.process();
   f->Close();
 
   auto* f2 = TFile::Open("variable_lists.root", "READ");

From 7ca1f03bbd7f825e45511460a3dfa24acb31457d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 6 Nov 2023 14:11:40 +0100
Subject: [PATCH 043/726] DPL: have common baseclass to abstract tree writing

This is the the first step towards writing something else,
e.g. a RNTuple.
---
 Detectors/AOD/src/StandaloneAODProducer.cxx   |  6 +--
 .../Core/include/Framework/TableTreeHelpers.h | 28 +++++++++---
 Framework/Core/src/CommonDataProcessors.cxx   | 10 ++---
 Framework/Core/src/TableTreeHelpers.cxx       | 44 +++++++++++--------
 Framework/Core/test/benchmark_TableToTree.cxx |  2 +-
 Framework/Core/test/benchmark_TreeToTable.cxx |  2 +-
 Framework/Core/test/test_TreeToTable.cxx      |  9 ++--
 7 files changed, 61 insertions(+), 40 deletions(-)

diff --git a/Detectors/AOD/src/StandaloneAODProducer.cxx b/Detectors/AOD/src/StandaloneAODProducer.cxx
index 151038465c1d1..e37a877cf05b1 100644
--- a/Detectors/AOD/src/StandaloneAODProducer.cxx
+++ b/Detectors/AOD/src/StandaloneAODProducer.cxx
@@ -95,7 +95,7 @@ void fillMCollisionTable(o2::steer::MCKinematicsReader const& mcreader)
   TFile outfile("aod.root", "UPDATE");
   {
     TableToTree t2t(mccoltable, &outfile, aod::MetadataTrait<o2::aod::McCollisions>::metadata::tableLabel());
-    t2t.addAllBranches();
+    t2t.addAllColumns();
     t2t.process();
   }
 }
@@ -201,12 +201,12 @@ void fillCollisionAndTrackTable()
       TFile outfile("aod.root", "RECREATE");
       {
         TableToTree t2t(colltable, &outfile, aod::MetadataTrait<o2::aod::Collisions>::metadata::tableLabel());
-        t2t.addAllBranches();
+        t2t.addAllColumns();
         t2t.process();
       }
       {
         TableToTree t2t(tracktable, &outfile, "Tracks" /* aod::MetadataTrait<o2::aod::Tracks>::metadata::tableLabel() */);
-        t2t.addAllBranches();
+        t2t.addAllColumns();
         t2t.process();
       }
     }
diff --git a/Framework/Core/include/Framework/TableTreeHelpers.h b/Framework/Core/include/Framework/TableTreeHelpers.h
index e12f4a38a973b..bcf68e57844ec 100644
--- a/Framework/Core/include/Framework/TableTreeHelpers.h
+++ b/Framework/Core/include/Framework/TableTreeHelpers.h
@@ -105,18 +105,34 @@ class ColumnToBranch
   int mFieldSize = 0;
 };
 
-class TableToTree
+class TableToRoot
 {
  public:
-  TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* treename);
+  TableToRoot(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* name);
+  virtual ~TableToRoot() = default;
 
-  std::shared_ptr<TTree> process();
-  void addBranch(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field);
-  void addAllBranches();
+  virtual void addColumn(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field) = 0;
+  virtual void process() = 0;
+  // Helper function which marks all columns in the table for writing
+  void addAllColumns();
 
- private:
+ protected:
   arrow::Table* mTable;
+  TFile* mFile;
+  std::string mName;
   int64_t mRows = 0;
+};
+
+class TableToTree : public TableToRoot
+{
+ public:
+  TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* treename);
+
+  void process() final;
+  void addColumn(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field) final;
+  std::shared_ptr<TTree> tree() { return mTree; }
+
+ private:
   std::shared_ptr<TTree> mTree;
   std::vector<std::unique_ptr<ColumnToBranch>> mColumnReaders;
 };
diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 08283b5e49bb5..c9d8aa835d734 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -417,9 +417,7 @@ DataProcessorSpec
         for (auto d : ds) {
           auto fileAndFolder = dod->getFileFolder(d, tfNumber, aodInputFile);
           auto treename = fileAndFolder.folderName + "/" + d->treename;
-          TableToTree ta2tr(table,
-                            fileAndFolder.file,
-                            treename.c_str());
+          std::unique_ptr<TableToRoot> ta2r = std::make_unique<TableToTree>(table, fileAndFolder.file, treename.c_str());
 
           // update metadata
           if (fileAndFolder.file->FindObjectAny("metaData")) {
@@ -438,13 +436,13 @@ DataProcessorSpec
               auto col = table->column(idx);
               auto field = table->schema()->field(idx);
               if (idx != -1) {
-                ta2tr.addBranch(col, field);
+                ta2r->addColumn(col, field);
               }
             }
           } else {
-            ta2tr.addAllBranches();
+            ta2r->addAllColumns();
           }
-          ta2tr.process();
+          ta2r->process();
         }
       }
     };
diff --git a/Framework/Core/src/TableTreeHelpers.cxx b/Framework/Core/src/TableTreeHelpers.cxx
index e6e6a1349e619..1c96157637433 100644
--- a/Framework/Core/src/TableTreeHelpers.cxx
+++ b/Framework/Core/src/TableTreeHelpers.cxx
@@ -407,34 +407,41 @@ void ColumnToBranch::nextChunk()
   accessChunk();
 }
 
-TableToTree::TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* treename)
+TableToRoot::TableToRoot(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* name)
+  : mTable{table.get()},
+    mFile{file},
+    mName{name}
 {
-  mTable = table.get();
-  mTree.reset(static_cast<TTree*>(file->Get(treename)));
-  if (mTree) {
-    return;
-  }
-  std::string treeName(treename);
-  auto pos = treeName.find_first_of('/');
-  if (pos != std::string::npos) {
-    file->cd(treeName.substr(0, pos).c_str());
-    treeName = treeName.substr(pos + 1, std::string::npos);
-  }
-  mTree = std::make_shared<TTree>(treeName.c_str(), treeName.c_str());
 }
 
-void TableToTree::addAllBranches()
+void TableToRoot::addAllColumns()
 {
   mRows = mTable->num_rows();
   auto columns = mTable->columns();
   auto fields = mTable->schema()->fields();
   assert(columns.size() == fields.size());
   for (auto i = 0u; i < columns.size(); ++i) {
-    addBranch(columns[i], fields[i]);
+    addColumn(columns[i], fields[i]);
+  }
+}
+
+TableToTree::TableToTree(std::shared_ptr<arrow::Table> const& table, TFile* file, const char* name)
+  : TableToRoot{table, file, name}
+{
+  mTree.reset(static_cast<TTree*>(mFile->Get(mName.c_str())));
+  if (mTree) {
+    return;
   }
+  std::string treeName(mName);
+  auto pos = treeName.find_first_of('/');
+  if (pos != std::string::npos) {
+    file->cd(treeName.substr(0, pos).c_str());
+    treeName = treeName.substr(pos + 1, std::string::npos);
+  }
+  mTree = std::make_shared<TTree>(treeName.c_str(), treeName.c_str());
 }
 
-void TableToTree::addBranch(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field)
+void TableToTree::addColumn(std::shared_ptr<arrow::ChunkedArray> const& column, std::shared_ptr<arrow::Field> const& field)
 {
   if (mRows == 0) {
     mRows = column->length();
@@ -444,13 +451,13 @@ void TableToTree::addBranch(std::shared_ptr<arrow::ChunkedArray> const& column,
   mColumnReaders.emplace_back(new ColumnToBranch{mTree.get(), column, field});
 }
 
-std::shared_ptr<TTree> TableToTree::process()
+void TableToTree::process()
 {
   int64_t row = 0;
   if (mTree->GetNbranches() == 0 || mRows == 0) {
     mTree->Write("", TObject::kOverwrite);
     mTree->SetDirectory(nullptr);
-    return mTree;
+    return;
   }
 
   for (auto& reader : mColumnReaders) {
@@ -468,7 +475,6 @@ std::shared_ptr<TTree> TableToTree::process()
   }
   mTree->Write("", TObject::kOverwrite);
   mTree->SetDirectory(nullptr);
-  return mTree;
 }
 
 TreeToTable::TreeToTable(arrow::MemoryPool* pool)
diff --git a/Framework/Core/test/benchmark_TableToTree.cxx b/Framework/Core/test/benchmark_TableToTree.cxx
index 0a7963518f00b..0ca4eb81112b8 100644
--- a/Framework/Core/test/benchmark_TableToTree.cxx
+++ b/Framework/Core/test/benchmark_TableToTree.cxx
@@ -62,7 +62,7 @@ static void BM_TableToTree(benchmark::State& state)
 
     // benchmark TableToTree
     TableToTree ta2tr(table, &fout, "table2tree");
-    ta2tr.addAllBranches();
+    ta2tr.addAllColumns();
     ta2tr.process();
 
     // clean up
diff --git a/Framework/Core/test/benchmark_TreeToTable.cxx b/Framework/Core/test/benchmark_TreeToTable.cxx
index 6eca853a90ce4..476e653acaf83 100644
--- a/Framework/Core/test/benchmark_TreeToTable.cxx
+++ b/Framework/Core/test/benchmark_TreeToTable.cxx
@@ -58,7 +58,7 @@ static void BM_TreeToTable(benchmark::State& state)
   // now convert the table to a tree
   TFile fout("tree2table.root", "RECREATE");
   TableToTree ta2tr(table, &fout, "tree2table");
-  ta2tr.addAllBranches();
+  ta2tr.addAllColumns();
   ta2tr.process();
   fout.Close();
 
diff --git a/Framework/Core/test/test_TreeToTable.cxx b/Framework/Core/test/test_TreeToTable.cxx
index 65569b72b9f35..32575a8caf3e0 100644
--- a/Framework/Core/test/test_TreeToTable.cxx
+++ b/Framework/Core/test/test_TreeToTable.cxx
@@ -143,9 +143,10 @@ TEST_CASE("TreeToTableConversion")
   // save table as tree
   TFile* f2 = TFile::Open("table2tree.root", "RECREATE");
   TableToTree ta2tr(table, f2, "mytree");
-  ta2tr.addAllBranches();
+  ta2tr.addAllColumns();
 
-  auto t2 = ta2tr.process();
+  ta2tr.process();
+  auto t2 = ta2tr.tree();
   auto br = (TBranch*)t2->GetBranch("ok");
   REQUIRE(t2->GetEntries() == ndp);
   REQUIRE(br->GetEntries() == ndp);
@@ -201,8 +202,8 @@ TEST_CASE("VariableLists")
 
   auto* f = TFile::Open("variable_lists.root", "RECREATE");
   TableToTree ta2tr(table, f, "lists");
-  ta2tr.addAllBranches();
-  auto tree = ta2tr.process();
+  ta2tr.addAllColumns();
+  ta2tr.process();
   f->Close();
 
   auto* f2 = TFile::Open("variable_lists.root", "READ");

From 552fc3cdad54e8e376fdfc8ad4fd2f5734ba90c0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 11:33:09 +0100
Subject: [PATCH 044/726] C++20 needs vector to be well defined

---
 Framework/Core/include/Framework/AnalysisHelpers.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Framework/Core/include/Framework/AnalysisHelpers.h b/Framework/Core/include/Framework/AnalysisHelpers.h
index b382d4f11b265..e5299c7a91b99 100644
--- a/Framework/Core/include/Framework/AnalysisHelpers.h
+++ b/Framework/Core/include/Framework/AnalysisHelpers.h
@@ -23,6 +23,7 @@
 #include "Framework/Output.h"
 #include "Framework/IndexBuilderHelpers.h"
 #include "Framework/Plugins.h"
+#include "Framework/ExpressionHelpers.h"
 
 #include <string>
 namespace o2::framework

From ea60f829ba6b7554dff7f92ce78fd0a513130f7b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 11:34:49 +0100
Subject: [PATCH 045/726] DPL: provide cast to string_view for
 header::Descriptor<16>

This will make some code compile under C++20, since C++20 is more aggressive in
doing implicit conversions where it currently uses the comparison operator.
---
 .../Headers/include/Headers/DataHeader.h        | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/DataFormats/Headers/include/Headers/DataHeader.h b/DataFormats/Headers/include/Headers/DataHeader.h
index eb640be82ed60..ed25d0d7f5082 100644
--- a/DataFormats/Headers/include/Headers/DataHeader.h
+++ b/DataFormats/Headers/include/Headers/DataHeader.h
@@ -224,14 +224,21 @@ struct Descriptor {
 
   // Note: don't need to define operator=(ItgType v) because the compiler
   // can use Descriptor(ItgType initializer) for conversion
+  using ImplicitConversion = std::conditional_t<(size <= 8), ItgType, std::string_view>;
 
   // type cast operator for simplified usage of the descriptor's integer member
-  // TODO: this is sort of a hack, takes the first element.
-  //       we should rethink these implicit conversions
-  operator ItgType() const
+  // in case it does not fit into the descriptor, the string representation is returned
+  operator ImplicitConversion() const
   {
-    static_assert(arraySize == 1, "casting Descriptor to ItgType only allowed for N<=8");
-    return itg[0];
+    if constexpr (std::is_same_v<ImplicitConversion, ItgType>) {
+      return itg[0];
+    } else {
+      size_t len = size;
+      while (len > 1 && str[len - 1] == 0) {
+        --len;
+      }
+      return std::string_view(str, len);
+    }
   }
 
   /// constructor from a compile-time string

From 51d7463e4fee5a546801740d60c850a8778e7465 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 11:36:02 +0100
Subject: [PATCH 046/726] C++20 disables aggregate initialisation if default
 constructors found

---
 DataFormats/Detectors/CTP/include/DataFormatsCTP/Digits.h | 2 --
 .../Detectors/CTP/include/DataFormatsCTP/LumiInfo.h       | 1 -
 .../include/ITSMFTReconstruction/Clusterer.h              | 1 -
 .../reconstruction/include/ITS3Reconstruction/Clusterer.h | 1 -
 Framework/Core/include/Framework/DeviceSpec.h             | 8 +++-----
 Framework/Core/src/DeviceSpecHelpers.cxx                  | 1 +
 6 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/DataFormats/Detectors/CTP/include/DataFormatsCTP/Digits.h b/DataFormats/Detectors/CTP/include/DataFormatsCTP/Digits.h
index c05a59599e707..e1ef4de73f222 100644
--- a/DataFormats/Detectors/CTP/include/DataFormatsCTP/Digits.h
+++ b/DataFormats/Detectors/CTP/include/DataFormatsCTP/Digits.h
@@ -51,7 +51,6 @@ struct CTPDigit {
   o2::InteractionRecord intRecord;
   std::bitset<CTP_NINPUTS> CTPInputMask;
   std::bitset<CTP_NCLASSES> CTPClassMask;
-  CTPDigit() = default;
   void printStream(std::ostream& stream) const;
   void setInputMask(gbtword80_t mask);
   void setClassMask(gbtword80_t mask);
@@ -72,7 +71,6 @@ struct CTPInputDigit {
   o2::InteractionRecord intRecord;
   std::bitset<CTP_MAXTRIGINPPERDET> inputsMask;
   o2::detectors::DetID::ID detector;
-  CTPInputDigit() = default;
   ClassDefNV(CTPInputDigit, 1)
 };
 } // namespace ctp
diff --git a/DataFormats/Detectors/CTP/include/DataFormatsCTP/LumiInfo.h b/DataFormats/Detectors/CTP/include/DataFormatsCTP/LumiInfo.h
index e9eb8ba497c17..c75fcc32ddaf4 100644
--- a/DataFormats/Detectors/CTP/include/DataFormatsCTP/LumiInfo.h
+++ b/DataFormats/Detectors/CTP/include/DataFormatsCTP/LumiInfo.h
@@ -22,7 +22,6 @@ namespace o2
 namespace ctp
 {
 struct LumiInfo {
-  LumiInfo() = default;
   uint32_t orbit = 0;       // orbit of TF when was updated
   uint32_t nHBFCounted = 0; // length of interval in HB
   uint32_t nHBFCountedFV0 = 0;
diff --git a/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/Clusterer.h b/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/Clusterer.h
index e1636c93d035a..d79eb4ac2b460 100644
--- a/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/Clusterer.h
+++ b/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/Clusterer.h
@@ -118,7 +118,6 @@ class Clusterer
     uint32_t firstPatt = 0;
     uint32_t nClus = 0;
     uint32_t nPatt = 0;
-    ThreadStat() = default;
   };
 
   struct ClustererThread {
diff --git a/Detectors/Upgrades/ITS3/reconstruction/include/ITS3Reconstruction/Clusterer.h b/Detectors/Upgrades/ITS3/reconstruction/include/ITS3Reconstruction/Clusterer.h
index 6eda6e98f0b52..4008dfb215dd2 100644
--- a/Detectors/Upgrades/ITS3/reconstruction/include/ITS3Reconstruction/Clusterer.h
+++ b/Detectors/Upgrades/ITS3/reconstruction/include/ITS3Reconstruction/Clusterer.h
@@ -118,7 +118,6 @@ class Clusterer
     uint32_t firstPatt = 0;
     uint32_t nClus = 0;
     uint32_t nPatt = 0;
-    ThreadStat() = default;
   };
 
   struct ClustererThread {
diff --git a/Framework/Core/include/Framework/DeviceSpec.h b/Framework/Core/include/Framework/DeviceSpec.h
index 6a11146283190..a22fc7eb1b7dc 100644
--- a/Framework/Core/include/Framework/DeviceSpec.h
+++ b/Framework/Core/include/Framework/DeviceSpec.h
@@ -46,13 +46,11 @@ namespace o2::framework
 ///
 /// auto &deviceSpec = ctx.services().get<DeviceSpec const>();
 struct DeviceSpec {
-  DeviceSpec(const DeviceSpec&) = delete;
-  DeviceSpec(DeviceSpec&&) = default;
   /// The name of the associated DataProcessorSpec
-  std::string name;
+  std::string name = "unknown";
   /// The id of the device, including time-pipelining and suffix
-  std::string id;
-  std::string channelPrefix;
+  std::string id = "unknown";
+  std::string channelPrefix = "";
   std::vector<InputChannelSpec> inputChannels;
   std::vector<OutputChannelSpec> outputChannels;
   std::vector<std::string> arguments;
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index f11f60171e451..969a15fa40674 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -534,6 +534,7 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
       .name = processor.name,
       .id = processor.maxInputTimeslices == 1 ? processor.name : processor.name + "_t" + std::to_string(edge.producerTimeIndex),
       .channelPrefix = channelPrefix,
+      .inputChannels = {},
       .options = processor.options,
       .services = ServiceSpecHelpers::filterDisabled(processor.requiredServices, overrideServices),
       .algorithm = processor.algorithm,

From d9c990eeda097de585154892ffa9a38467edaee3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 11:36:03 +0100
Subject: [PATCH 047/726] C++20 does not allow forward declaring classes which
 end up in a vector

This is due to the fact the destructor of std::vector is now constexpr.
---
 Detectors/MUON/MCH/Clustering/src/ClusterOriginal.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Detectors/MUON/MCH/Clustering/src/ClusterOriginal.h b/Detectors/MUON/MCH/Clustering/src/ClusterOriginal.h
index da47976b00a63..b9a66b07b2332 100644
--- a/Detectors/MUON/MCH/Clustering/src/ClusterOriginal.h
+++ b/Detectors/MUON/MCH/Clustering/src/ClusterOriginal.h
@@ -17,6 +17,7 @@
 #ifndef O2_MCH_CLUSTERORIGINAL_H_
 #define O2_MCH_CLUSTERORIGINAL_H_
 
+#include "PadOriginal.h"
 #include <utility>
 #include <vector>
 

From d15e8d8fb842932b8e0457487c3abb74aef7b424 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 11:36:03 +0100
Subject: [PATCH 048/726] C++20 now considers equality to be commutable (A ==
 B) === (B == A)

This means that the operator must be const so that swapping is allowed.
---
 Algorithm/test/pageparser.cxx     | 2 +-
 Framework/Core/test/test_ASoA.cxx | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Algorithm/test/pageparser.cxx b/Algorithm/test/pageparser.cxx
index 14b24c670cfd6..7551c32d9d864 100644
--- a/Algorithm/test/pageparser.cxx
+++ b/Algorithm/test/pageparser.cxx
@@ -50,7 +50,7 @@ struct ClusterData {
   {
   }
 
-  bool operator==(const ClusterData& rhs)
+  bool operator==(const ClusterData& rhs) const
   {
     return clusterid == rhs.clusterid && x == rhs.x && y == rhs.y && z == rhs.z && e == rhs.e;
   }
diff --git a/Framework/Core/test/test_ASoA.cxx b/Framework/Core/test/test_ASoA.cxx
index e6a537dab3dc9..467b15ec5bde0 100644
--- a/Framework/Core/test/test_ASoA.cxx
+++ b/Framework/Core/test/test_ASoA.cxx
@@ -147,10 +147,10 @@ TEST_CASE("TestTableIteration")
 
   b = tests2.begin();
   REQUIRE(b != e);
-  REQUIRE((b + 1) == (b + 1));
-  REQUIRE((b + 7) != b);
-  REQUIRE((b + 7) != e);
-  REQUIRE((b + 8) == e);
+  REQUIRE(((b + 1) == (b + 1)));
+  REQUIRE(((b + 7) != b));
+  REQUIRE(((b + 7) != e));
+  REQUIRE(((b + 8) == e));
 
   for (auto& t : tests2) {
     REQUIRE(t.x() == value / 4);

From c8992413849c8d284f6778ef7b1ecfc314b1bc80 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 11:36:04 +0100
Subject: [PATCH 049/726] Move to C++20

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ca431a8b95be2..5cbc12e72ade3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -37,7 +37,7 @@ set_property(GLOBAL PROPERTY JOB_POOLS analysis=${ANALYSIS_COMPILE_POOL})
 
 include(O2BuildSanityChecks)
 o2_build_sanity_checks()
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
 
 find_package(ONNXRuntime::ONNXRuntime CONFIG)

From 6ff3738cffa1cd2919f22668df7c52993ee8a593 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 11:33:09 +0100
Subject: [PATCH 050/726] C++20 needs vector to be well defined

---
 Detectors/MUON/MCH/Clustering/src/ClusterOriginal.h | 2 --
 Framework/Core/include/Framework/AnalysisHelpers.h  | 1 +
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/Detectors/MUON/MCH/Clustering/src/ClusterOriginal.h b/Detectors/MUON/MCH/Clustering/src/ClusterOriginal.h
index da47976b00a63..daca95e683365 100644
--- a/Detectors/MUON/MCH/Clustering/src/ClusterOriginal.h
+++ b/Detectors/MUON/MCH/Clustering/src/ClusterOriginal.h
@@ -27,8 +27,6 @@ namespace o2
 namespace mch
 {
 
-class PadOriginal;
-
 /// cluster for internal use
 class ClusterOriginal
 {
diff --git a/Framework/Core/include/Framework/AnalysisHelpers.h b/Framework/Core/include/Framework/AnalysisHelpers.h
index b382d4f11b265..e5299c7a91b99 100644
--- a/Framework/Core/include/Framework/AnalysisHelpers.h
+++ b/Framework/Core/include/Framework/AnalysisHelpers.h
@@ -23,6 +23,7 @@
 #include "Framework/Output.h"
 #include "Framework/IndexBuilderHelpers.h"
 #include "Framework/Plugins.h"
+#include "Framework/ExpressionHelpers.h"
 
 #include <string>
 namespace o2::framework

From 07342adb9775bdaa31a5ddb73876d565925db4b6 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 11:34:49 +0100
Subject: [PATCH 051/726] DPL: provide cast to string_view for
 header::Descriptor<16>

This will make some code compile under C++20, since C++20 is more aggressive in
doing implicit conversions where it currently uses the comparison operator.
---
 .../Headers/include/Headers/DataHeader.h        | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/DataFormats/Headers/include/Headers/DataHeader.h b/DataFormats/Headers/include/Headers/DataHeader.h
index eb640be82ed60..ed25d0d7f5082 100644
--- a/DataFormats/Headers/include/Headers/DataHeader.h
+++ b/DataFormats/Headers/include/Headers/DataHeader.h
@@ -224,14 +224,21 @@ struct Descriptor {
 
   // Note: don't need to define operator=(ItgType v) because the compiler
   // can use Descriptor(ItgType initializer) for conversion
+  using ImplicitConversion = std::conditional_t<(size <= 8), ItgType, std::string_view>;
 
   // type cast operator for simplified usage of the descriptor's integer member
-  // TODO: this is sort of a hack, takes the first element.
-  //       we should rethink these implicit conversions
-  operator ItgType() const
+  // in case it does not fit into the descriptor, the string representation is returned
+  operator ImplicitConversion() const
   {
-    static_assert(arraySize == 1, "casting Descriptor to ItgType only allowed for N<=8");
-    return itg[0];
+    if constexpr (std::is_same_v<ImplicitConversion, ItgType>) {
+      return itg[0];
+    } else {
+      size_t len = size;
+      while (len > 1 && str[len - 1] == 0) {
+        --len;
+      }
+      return std::string_view(str, len);
+    }
   }
 
   /// constructor from a compile-time string

From 31361e29c2957b10a82895f6a177517f0f0df99d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 11:36:03 +0100
Subject: [PATCH 052/726] C++20 now considers equality to be commutable (A ==
 B) === (B == A)

This means that the operator must be const so that swapping is allowed.
---
 Algorithm/test/pageparser.cxx     | 2 +-
 Framework/Core/test/test_ASoA.cxx | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Algorithm/test/pageparser.cxx b/Algorithm/test/pageparser.cxx
index 14b24c670cfd6..7551c32d9d864 100644
--- a/Algorithm/test/pageparser.cxx
+++ b/Algorithm/test/pageparser.cxx
@@ -50,7 +50,7 @@ struct ClusterData {
   {
   }
 
-  bool operator==(const ClusterData& rhs)
+  bool operator==(const ClusterData& rhs) const
   {
     return clusterid == rhs.clusterid && x == rhs.x && y == rhs.y && z == rhs.z && e == rhs.e;
   }
diff --git a/Framework/Core/test/test_ASoA.cxx b/Framework/Core/test/test_ASoA.cxx
index e6a537dab3dc9..467b15ec5bde0 100644
--- a/Framework/Core/test/test_ASoA.cxx
+++ b/Framework/Core/test/test_ASoA.cxx
@@ -147,10 +147,10 @@ TEST_CASE("TestTableIteration")
 
   b = tests2.begin();
   REQUIRE(b != e);
-  REQUIRE((b + 1) == (b + 1));
-  REQUIRE((b + 7) != b);
-  REQUIRE((b + 7) != e);
-  REQUIRE((b + 8) == e);
+  REQUIRE(((b + 1) == (b + 1)));
+  REQUIRE(((b + 7) != b));
+  REQUIRE(((b + 7) != e));
+  REQUIRE(((b + 8) == e));
 
   for (auto& t : tests2) {
     REQUIRE(t.x() == value / 4);

From ec28150efa735dfa1d176d39786b26c8ce881b91 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 11:36:02 +0100
Subject: [PATCH 053/726] C++20 disables aggregate initialisation if default
 constructors found

---
 DataFormats/Detectors/CTP/include/DataFormatsCTP/Digits.h | 2 --
 .../Detectors/CTP/include/DataFormatsCTP/LumiInfo.h       | 1 -
 .../include/ITSMFTReconstruction/Clusterer.h              | 1 -
 .../reconstruction/include/ITS3Reconstruction/Clusterer.h | 1 -
 Framework/Core/include/Framework/DeviceSpec.h             | 8 +++-----
 Framework/Core/src/DeviceSpecHelpers.cxx                  | 1 +
 6 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/DataFormats/Detectors/CTP/include/DataFormatsCTP/Digits.h b/DataFormats/Detectors/CTP/include/DataFormatsCTP/Digits.h
index c05a59599e707..e1ef4de73f222 100644
--- a/DataFormats/Detectors/CTP/include/DataFormatsCTP/Digits.h
+++ b/DataFormats/Detectors/CTP/include/DataFormatsCTP/Digits.h
@@ -51,7 +51,6 @@ struct CTPDigit {
   o2::InteractionRecord intRecord;
   std::bitset<CTP_NINPUTS> CTPInputMask;
   std::bitset<CTP_NCLASSES> CTPClassMask;
-  CTPDigit() = default;
   void printStream(std::ostream& stream) const;
   void setInputMask(gbtword80_t mask);
   void setClassMask(gbtword80_t mask);
@@ -72,7 +71,6 @@ struct CTPInputDigit {
   o2::InteractionRecord intRecord;
   std::bitset<CTP_MAXTRIGINPPERDET> inputsMask;
   o2::detectors::DetID::ID detector;
-  CTPInputDigit() = default;
   ClassDefNV(CTPInputDigit, 1)
 };
 } // namespace ctp
diff --git a/DataFormats/Detectors/CTP/include/DataFormatsCTP/LumiInfo.h b/DataFormats/Detectors/CTP/include/DataFormatsCTP/LumiInfo.h
index e9eb8ba497c17..c75fcc32ddaf4 100644
--- a/DataFormats/Detectors/CTP/include/DataFormatsCTP/LumiInfo.h
+++ b/DataFormats/Detectors/CTP/include/DataFormatsCTP/LumiInfo.h
@@ -22,7 +22,6 @@ namespace o2
 namespace ctp
 {
 struct LumiInfo {
-  LumiInfo() = default;
   uint32_t orbit = 0;       // orbit of TF when was updated
   uint32_t nHBFCounted = 0; // length of interval in HB
   uint32_t nHBFCountedFV0 = 0;
diff --git a/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/Clusterer.h b/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/Clusterer.h
index e1636c93d035a..d79eb4ac2b460 100644
--- a/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/Clusterer.h
+++ b/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/Clusterer.h
@@ -118,7 +118,6 @@ class Clusterer
     uint32_t firstPatt = 0;
     uint32_t nClus = 0;
     uint32_t nPatt = 0;
-    ThreadStat() = default;
   };
 
   struct ClustererThread {
diff --git a/Detectors/Upgrades/ITS3/reconstruction/include/ITS3Reconstruction/Clusterer.h b/Detectors/Upgrades/ITS3/reconstruction/include/ITS3Reconstruction/Clusterer.h
index 6eda6e98f0b52..4008dfb215dd2 100644
--- a/Detectors/Upgrades/ITS3/reconstruction/include/ITS3Reconstruction/Clusterer.h
+++ b/Detectors/Upgrades/ITS3/reconstruction/include/ITS3Reconstruction/Clusterer.h
@@ -118,7 +118,6 @@ class Clusterer
     uint32_t firstPatt = 0;
     uint32_t nClus = 0;
     uint32_t nPatt = 0;
-    ThreadStat() = default;
   };
 
   struct ClustererThread {
diff --git a/Framework/Core/include/Framework/DeviceSpec.h b/Framework/Core/include/Framework/DeviceSpec.h
index 6a11146283190..a22fc7eb1b7dc 100644
--- a/Framework/Core/include/Framework/DeviceSpec.h
+++ b/Framework/Core/include/Framework/DeviceSpec.h
@@ -46,13 +46,11 @@ namespace o2::framework
 ///
 /// auto &deviceSpec = ctx.services().get<DeviceSpec const>();
 struct DeviceSpec {
-  DeviceSpec(const DeviceSpec&) = delete;
-  DeviceSpec(DeviceSpec&&) = default;
   /// The name of the associated DataProcessorSpec
-  std::string name;
+  std::string name = "unknown";
   /// The id of the device, including time-pipelining and suffix
-  std::string id;
-  std::string channelPrefix;
+  std::string id = "unknown";
+  std::string channelPrefix = "";
   std::vector<InputChannelSpec> inputChannels;
   std::vector<OutputChannelSpec> outputChannels;
   std::vector<std::string> arguments;
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index f11f60171e451..969a15fa40674 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -534,6 +534,7 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
       .name = processor.name,
       .id = processor.maxInputTimeslices == 1 ? processor.name : processor.name + "_t" + std::to_string(edge.producerTimeIndex),
       .channelPrefix = channelPrefix,
+      .inputChannels = {},
       .options = processor.options,
       .services = ServiceSpecHelpers::filterDisabled(processor.requiredServices, overrideServices),
       .algorithm = processor.algorithm,

From 87d12c52eca4f237a47b3d925484c280518f2cc9 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 11:36:04 +0100
Subject: [PATCH 054/726] Move to C++20

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ca431a8b95be2..5cbc12e72ade3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -37,7 +37,7 @@ set_property(GLOBAL PROPERTY JOB_POOLS analysis=${ANALYSIS_COMPILE_POOL})
 
 include(O2BuildSanityChecks)
 o2_build_sanity_checks()
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
 
 find_package(ONNXRuntime::ONNXRuntime CONFIG)

From bf3ac496675aa4f3802a4f47343225e8e159f8ed Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 11:36:03 +0100
Subject: [PATCH 055/726] C++20 does not allow forward declaring classes which
 end up in a vector

This is due to the fact the destructor of std::vector is now constexpr.
---
 Detectors/MUON/MCH/Clustering/src/ClusterOriginal.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Detectors/MUON/MCH/Clustering/src/ClusterOriginal.h b/Detectors/MUON/MCH/Clustering/src/ClusterOriginal.h
index daca95e683365..730a116c942a9 100644
--- a/Detectors/MUON/MCH/Clustering/src/ClusterOriginal.h
+++ b/Detectors/MUON/MCH/Clustering/src/ClusterOriginal.h
@@ -17,6 +17,7 @@
 #ifndef O2_MCH_CLUSTERORIGINAL_H_
 #define O2_MCH_CLUSTERORIGINAL_H_
 
+#include "PadOriginal.h"
 #include <utility>
 #include <vector>
 

From a31c76bc551d2f616a525e3b997a4af45ea37813 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 11:36:03 +0100
Subject: [PATCH 056/726] C++20 now considers equality to be commutable (A ==
 B) === (B == A)

This means that the operator must be const so that swapping is allowed.
---
 Algorithm/test/pageparser.cxx                    | 2 +-
 DataFormats/Headers/include/Headers/DataHeader.h | 2 ++
 Framework/Core/test/test_ASoA.cxx                | 8 ++++----
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/Algorithm/test/pageparser.cxx b/Algorithm/test/pageparser.cxx
index 14b24c670cfd6..7551c32d9d864 100644
--- a/Algorithm/test/pageparser.cxx
+++ b/Algorithm/test/pageparser.cxx
@@ -50,7 +50,7 @@ struct ClusterData {
   {
   }
 
-  bool operator==(const ClusterData& rhs)
+  bool operator==(const ClusterData& rhs) const
   {
     return clusterid == rhs.clusterid && x == rhs.x && y == rhs.y && z == rhs.z && e == rhs.e;
   }
diff --git a/DataFormats/Headers/include/Headers/DataHeader.h b/DataFormats/Headers/include/Headers/DataHeader.h
index ed25d0d7f5082..628cb86dd7bc8 100644
--- a/DataFormats/Headers/include/Headers/DataHeader.h
+++ b/DataFormats/Headers/include/Headers/DataHeader.h
@@ -286,6 +286,8 @@ struct Descriptor {
   bool operator<(const Descriptor& other) const { return std::memcmp(this->str, other.str, N) < 0; }
   bool operator!=(const Descriptor& other) const { return not this->operator==(other); }
 
+  // Convesion operators for comparison with their implicitly convertible types
+  friend bool operator==(const Descriptor& lhs, ImplicitConversion rhs) { return static_cast<ImplicitConversion>(lhs)==rhs; }
   // explicitly forbid comparison with e.g. const char* strings
   // use: value == Descriptor<N>("DESC") for the appropriate
   // template instantiation instead
diff --git a/Framework/Core/test/test_ASoA.cxx b/Framework/Core/test/test_ASoA.cxx
index e6a537dab3dc9..467b15ec5bde0 100644
--- a/Framework/Core/test/test_ASoA.cxx
+++ b/Framework/Core/test/test_ASoA.cxx
@@ -147,10 +147,10 @@ TEST_CASE("TestTableIteration")
 
   b = tests2.begin();
   REQUIRE(b != e);
-  REQUIRE((b + 1) == (b + 1));
-  REQUIRE((b + 7) != b);
-  REQUIRE((b + 7) != e);
-  REQUIRE((b + 8) == e);
+  REQUIRE(((b + 1) == (b + 1)));
+  REQUIRE(((b + 7) != b));
+  REQUIRE(((b + 7) != e));
+  REQUIRE(((b + 8) == e));
 
   for (auto& t : tests2) {
     REQUIRE(t.x() == value / 4);

From ec5406fd7b65e1ee90a5af46ea8ba7615598264c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 11:36:04 +0100
Subject: [PATCH 057/726] Move to C++20

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ca431a8b95be2..5cbc12e72ade3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -37,7 +37,7 @@ set_property(GLOBAL PROPERTY JOB_POOLS analysis=${ANALYSIS_COMPILE_POOL})
 
 include(O2BuildSanityChecks)
 o2_build_sanity_checks()
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
 
 find_package(ONNXRuntime::ONNXRuntime CONFIG)

From 7eeaf87153d0f2ed6bb358292bc26b5b7e0eea86 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 13:12:45 +0100
Subject: [PATCH 058/726] C++20 now considers equality to be commutable (A ==
 B) === (B == A)

This means that the operator must be const so that swapping is allowed.
---
 Algorithm/test/pageparser.cxx                    | 2 +-
 DataFormats/Headers/include/Headers/DataHeader.h | 2 ++
 Framework/Core/test/test_ASoA.cxx                | 8 ++++----
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/Algorithm/test/pageparser.cxx b/Algorithm/test/pageparser.cxx
index 14b24c670cfd6..7551c32d9d864 100644
--- a/Algorithm/test/pageparser.cxx
+++ b/Algorithm/test/pageparser.cxx
@@ -50,7 +50,7 @@ struct ClusterData {
   {
   }
 
-  bool operator==(const ClusterData& rhs)
+  bool operator==(const ClusterData& rhs) const
   {
     return clusterid == rhs.clusterid && x == rhs.x && y == rhs.y && z == rhs.z && e == rhs.e;
   }
diff --git a/DataFormats/Headers/include/Headers/DataHeader.h b/DataFormats/Headers/include/Headers/DataHeader.h
index ed25d0d7f5082..a4d3abdba65c7 100644
--- a/DataFormats/Headers/include/Headers/DataHeader.h
+++ b/DataFormats/Headers/include/Headers/DataHeader.h
@@ -286,6 +286,8 @@ struct Descriptor {
   bool operator<(const Descriptor& other) const { return std::memcmp(this->str, other.str, N) < 0; }
   bool operator!=(const Descriptor& other) const { return not this->operator==(other); }
 
+  // Convesion operators for comparison with their implicitly convertible types
+  friend bool operator==(const Descriptor& lhs, ImplicitConversion rhs) { return static_cast<ImplicitConversion>(lhs) == rhs; }
   // explicitly forbid comparison with e.g. const char* strings
   // use: value == Descriptor<N>("DESC") for the appropriate
   // template instantiation instead
diff --git a/Framework/Core/test/test_ASoA.cxx b/Framework/Core/test/test_ASoA.cxx
index e6a537dab3dc9..467b15ec5bde0 100644
--- a/Framework/Core/test/test_ASoA.cxx
+++ b/Framework/Core/test/test_ASoA.cxx
@@ -147,10 +147,10 @@ TEST_CASE("TestTableIteration")
 
   b = tests2.begin();
   REQUIRE(b != e);
-  REQUIRE((b + 1) == (b + 1));
-  REQUIRE((b + 7) != b);
-  REQUIRE((b + 7) != e);
-  REQUIRE((b + 8) == e);
+  REQUIRE(((b + 1) == (b + 1)));
+  REQUIRE(((b + 7) != b));
+  REQUIRE(((b + 7) != e));
+  REQUIRE(((b + 8) == e));
 
   for (auto& t : tests2) {
     REQUIRE(t.x() == value / 4);

From d104b904a8281b65b5aa9f61f09b7524398ec2e1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 13:12:45 +0100
Subject: [PATCH 059/726] Move to C++20

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ca431a8b95be2..5cbc12e72ade3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -37,7 +37,7 @@ set_property(GLOBAL PROPERTY JOB_POOLS analysis=${ANALYSIS_COMPILE_POOL})
 
 include(O2BuildSanityChecks)
 o2_build_sanity_checks()
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
 
 find_package(ONNXRuntime::ONNXRuntime CONFIG)

From 06e0d43ead6aab27ad3fa4379c21b6fb3d8766c6 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 14:31:25 +0100
Subject: [PATCH 060/726] DPL: reset Oldest Possible Timeslice counters on new
 run

Centralise resetting of the Oldest Possible Timeslice counters now that we can.
---
 Framework/Core/src/CommonServices.cxx | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 44d63cd2ab1f5..fb0787b02da70 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -503,6 +503,17 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
         }
       }
       decongestion->lastTimeslice = oldestPossibleOutput.timeslice.value; },
+    .stop = [](ServiceRegistryRef services, void* service) {
+      auto* decongestion = (DecongestionService*)service;
+      decongestion->nextEnumerationTimeslice = 0;
+      decongestion->nextEnumerationTimesliceRewinded = false;
+      decongestion->lastTimeslice = 0;
+      decongestion->nextTimeslice = 0;
+      decongestion->oldestPossibleTimesliceTask = {0};
+      auto &state = services.get<DeviceState>();
+      for (auto &channel : state.inputChannelInfos) {
+        channel.oldestForChannel = {0};
+      } },
     .domainInfoUpdated = [](ServiceRegistryRef services, size_t oldestPossibleTimeslice, ChannelIndex channel) {
       auto& decongestion = services.get<DecongestionService>();
       auto& relayer = services.get<DataRelayer>();

From 80aa547753a2ca17ed9a28191e1ded1fb8cfb19c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 14:31:25 +0100
Subject: [PATCH 061/726] Fix coding convention

---
 Framework/Core/src/CommonServices.cxx | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index fb0787b02da70..4caddb9b98c78 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -941,8 +941,7 @@ o2::framework::ServiceSpec CommonServices::dataProcessingStats()
     .configure = noConfiguration(),
     .preProcessing = [](ProcessingContext& context, void* service) {
       auto* stats = (DataProcessingStats*)service;
-      flushMetrics(context.services(), *stats);
-    },
+      flushMetrics(context.services(), *stats); },
     .postProcessing = [](ProcessingContext& context, void* service) {
       auto* stats = (DataProcessingStats*)service;
       stats->updateStats({(short)ProcessingStatsId::PERFORMED_COMPUTATIONS, DataProcessingStats::Op::Add, 1});

From d9c94b33ac55294c50ef62b76154e0d4fe30fa18 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 14:32:03 +0100
Subject: [PATCH 062/726] Fix coding convention

---
 Framework/Core/src/CommonServices.cxx | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 44d63cd2ab1f5..dca98618895cd 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -930,8 +930,7 @@ o2::framework::ServiceSpec CommonServices::dataProcessingStats()
     .configure = noConfiguration(),
     .preProcessing = [](ProcessingContext& context, void* service) {
       auto* stats = (DataProcessingStats*)service;
-      flushMetrics(context.services(), *stats);
-    },
+      flushMetrics(context.services(), *stats); },
     .postProcessing = [](ProcessingContext& context, void* service) {
       auto* stats = (DataProcessingStats*)service;
       stats->updateStats({(short)ProcessingStatsId::PERFORMED_COMPUTATIONS, DataProcessingStats::Op::Add, 1});

From 57fbeb6546f329a62aab74a310de3f35f935b13c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 14:32:04 +0100
Subject: [PATCH 063/726] DPL: reset Oldest Possible Timeslice counters on new
 run

Centralise resetting of the Oldest Possible Timeslice counters now that we can.
---
 Framework/Core/src/CommonServices.cxx | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index dca98618895cd..4caddb9b98c78 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -503,6 +503,17 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
         }
       }
       decongestion->lastTimeslice = oldestPossibleOutput.timeslice.value; },
+    .stop = [](ServiceRegistryRef services, void* service) {
+      auto* decongestion = (DecongestionService*)service;
+      decongestion->nextEnumerationTimeslice = 0;
+      decongestion->nextEnumerationTimesliceRewinded = false;
+      decongestion->lastTimeslice = 0;
+      decongestion->nextTimeslice = 0;
+      decongestion->oldestPossibleTimesliceTask = {0};
+      auto &state = services.get<DeviceState>();
+      for (auto &channel : state.inputChannelInfos) {
+        channel.oldestForChannel = {0};
+      } },
     .domainInfoUpdated = [](ServiceRegistryRef services, size_t oldestPossibleTimeslice, ChannelIndex channel) {
       auto& decongestion = services.get<DecongestionService>();
       auto& relayer = services.get<DataRelayer>();

From ad0f5eab899fc340f89c5d8d4644d81d8598cd1c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 22:34:00 +0100
Subject: [PATCH 064/726] DPL GUI: do not hang if quit requested but some
 devices are stuck

---
 Framework/Core/src/runDataProcessing.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 61fdef070d874..49c5ebadb6460 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1073,7 +1073,7 @@ void gui_callback(uv_timer_s* ctx)
   if (gui->plugin->supportsDeferredClose()) {
     // For now, there is nothing for which we want to defer the close
     // so if the flag is set, we simply exit
-    if (*(gui->guiQuitRequested)) {
+    if (*(gui->guiQuitRequested) && *(gui->allChildrenGone)) {
       return;
     }
     void* draw_data = nullptr;

From 397c3b7d379316c947c28a6e8088f2c2d87f1a46 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 22:34:00 +0100
Subject: [PATCH 065/726] DPL GUI: add GUI to list devices preventing quitting

---
 Framework/Core/include/Framework/DebugGUI.h   |   4 +-
 Framework/Core/src/runDataProcessing.cxx      |   6 +-
 .../GUISupport/src/FrameworkGUIDebugger.cxx   | 110 ++++++++++++------
 .../GUISupport/src/FrameworkGUIDebugger.h     |   4 +-
 Framework/GUISupport/src/Plugin.cxx           |   5 +-
 5 files changed, 87 insertions(+), 42 deletions(-)

diff --git a/Framework/Core/include/Framework/DebugGUI.h b/Framework/Core/include/Framework/DebugGUI.h
index c7a611d50a1d4..109fd36e27511 100644
--- a/Framework/Core/include/Framework/DebugGUI.h
+++ b/Framework/Core/include/Framework/DebugGUI.h
@@ -26,6 +26,7 @@
 namespace o2::framework
 {
 struct ServiceRegistry;
+struct GuiCallbackContext;
 /// Plugin interface for DPL GUIs.
 struct DebugGUI {
   virtual std::function<void(void)> getGUIDebugger(std::vector<o2::framework::DeviceInfo> const& infos,
@@ -35,7 +36,8 @@ struct DebugGUI {
                                                    std::vector<o2::framework::DeviceMetricsInfo> const& metricsInfos,
                                                    o2::framework::DriverInfo const& driverInfo,
                                                    std::vector<o2::framework::DeviceControl>& controls,
-                                                   o2::framework::DriverControl& driverControl) = 0;
+                                                   o2::framework::DriverControl& driverControl,
+                                                   GuiCallbackContext& context) = 0;
   virtual void updateMousePos(float x, float y) = 0;
   virtual void updateMouseButton(bool isClicked) = 0;
   virtual void updateMouseWheel(int direction) = 0;
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 49c5ebadb6460..44a5c6b4ee94a 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1302,7 +1302,8 @@ int runStateMachine(DataProcessorSpecs const& workflow,
 
   DebugGUI* debugGUI = nullptr;
   void* window = nullptr;
-  decltype(debugGUI->getGUIDebugger(infos, runningWorkflow.devices, allStates, dataProcessorInfos, metricsInfos, driverInfo, controls, driverControl)) debugGUICallback;
+  GuiCallbackContext guiContext;
+  decltype(debugGUI->getGUIDebugger(infos, runningWorkflow.devices, allStates, dataProcessorInfos, metricsInfos, driverInfo, controls, driverControl, guiContext)) debugGUICallback;
 
   // An empty frameworkId means this is the driver, so we initialise the GUI
   auto initDebugGUI = []() -> DebugGUI* {
@@ -1389,7 +1390,6 @@ int runStateMachine(DataProcessorSpecs const& workflow,
   ServiceRegistryRef ref{serviceRegistry};
   ref.registerService(ServiceRegistryHelpers::handleForService<DevicesManager>(devicesManager));
 
-  GuiCallbackContext guiContext;
   guiContext.plugin = debugGUI;
   guiContext.frameLast = uv_hrtime();
   guiContext.frameLatency = &driverInfo.frameLatency;
@@ -1882,7 +1882,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
             uv_timer_stop(gui_timer);
           }
 
-          auto callback = debugGUI->getGUIDebugger(infos, runningWorkflow.devices, allStates, dataProcessorInfos, metricsInfos, driverInfo, controls, driverControl);
+          auto callback = debugGUI->getGUIDebugger(infos, runningWorkflow.devices, allStates, dataProcessorInfos, metricsInfos, driverInfo, controls, driverControl, guiContext);
           guiContext.callback = [&serviceRegistry, &driverServices, &debugGUI, &infos, &runningWorkflow, &dataProcessorInfos, &metricsInfos, &driverInfo, &controls, &driverControl, callback]() {
             callback();
             for (auto& service : driverServices) {
diff --git a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
index 35e0bc563a22e..375afef1007f9 100644
--- a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
@@ -16,6 +16,7 @@
 #include "DebugGUI/imgui_extras.h"
 #include "Framework/DriverControl.h"
 #include "Framework/DriverInfo.h"
+#include "Framework/GuiCallbackContext.h"
 #include "Framework/DeviceMetricsHelper.h"
 #include "Framework/DeviceMetricsInfo.h"
 #include "FrameworkGUIDeviceInspector.h"
@@ -506,7 +507,7 @@ void displayDeviceMetrics(const char* label,
     case MetricsDisplayStyle::Lines: {
       auto xAxisFlags = ImPlotAxisFlags_None;
       auto yAxisFlags = ImPlotAxisFlags_LockMin;
-      //ImPlot::FitNextPlotAxes(true, true, true, true);
+      // ImPlot::FitNextPlotAxes(true, true, true, true);
       if (ImPlot::BeginPlot("##Some plot", {-1, -1}, axisFlags)) {
         ImPlot::SetupAxes("time", "value", xAxisFlags, yAxisFlags);
         ImPlot::SetupAxisFormat(ImAxis_Y1, formatSI, nullptr);
@@ -1008,6 +1009,71 @@ void popWindowColorDueToStatus()
   ImGui::PopStyleVar(1);
 }
 
+int debugPID(int pid)
+{
+  std::string pidStr = std::to_string(pid);
+  setenv("O2DEBUGGEDPID", pidStr.c_str(), 1);
+#ifdef __APPLE__
+  std::string defaultAppleDebugCommand =
+    "osascript -e 'tell application \"Terminal\"'"
+    " -e 'activate'"
+    " -e 'do script \"lldb -p \" & (system attribute \"O2DEBUGGEDPID\") & \"; exit\"'"
+    " -e 'end tell'";
+  setenv("O2DPLDEBUG", defaultAppleDebugCommand.c_str(), 0);
+#else
+  setenv("O2DPLDEBUG", "xterm -hold -e gdb attach $O2DEBUGGEDPID &", 0);
+#endif
+  return system(getenv("O2DPLDEBUG"));
+}
+
+int profilePID(int pid)
+{
+  std::string pidStr = std::to_string(pid);
+  setenv("O2PROFILEDPID", pidStr.c_str(), 1);
+#ifdef __APPLE__
+  auto defaultAppleProfileCommand = fmt::format(
+    "osascript -e 'tell application \"Terminal\"'"
+    " -e 'activate'"
+    " -e 'do script \"xcrun xctrace record --output dpl-profile-{0}.trace"
+    " --instrument os_signpost --time-limit 30s --template Time\\\\ Profiler --attach {0} "
+    " && open dpl-profile-{0}.trace && exit\"'"
+    " -e 'end tell'",
+    pid);
+  std::cout << defaultAppleProfileCommand << std::endl;
+  setenv("O2DPLPROFILE", defaultAppleProfileCommand.c_str(), 0);
+#else
+  setenv("O2DPLPROFILE", "xterm -hold -e perf record -a -g -p $O2PROFILEDPID > perf-$O2PROFILEDPID.data &", 0);
+#endif
+  return system(getenv("O2DPLPROFILE"));
+}
+
+/// Show a window with the list of devices which have not yet exited
+/// and provide a few controls to interact with them.
+void displayShutdownDialog(std::vector<DeviceSpec> const& specs, std::vector<DeviceInfo> const& infos)
+{
+  for (size_t i = 0; i < infos.size(); ++i) {
+    auto& info = infos[i];
+    if (info.active == false) {
+      continue;
+    }
+    auto& spec = specs[i];
+    ImGui::Text("Unable to quit due to the following devices:");
+    ImGui::Text("%s (%d).", spec.name.c_str(), info.pid);
+    ImGui::SameLine();
+    if (ImGui::Button("Force quit")) {
+      kill(info.pid, SIGKILL);
+    }
+    ImGui::SameLine();
+    if (ImGui::Button("Debug")) {
+      debugPID(info.pid);
+    }
+    ImGui::SameLine();
+    if (ImGui::Button("Profile")) {
+      profilePID(info.pid);
+    }
+  }
+}
+
 /// Display information window about the driver
 /// and its state.
 void displayDriverInfo(DriverInfo const& driverInfo, DriverControl& driverControl)
@@ -1035,42 +1101,12 @@ void displayDriverInfo(DriverInfo const& driverInfo, DriverControl& driverContro
   }
   ImGui::SameLine();
   if (ImGui::Button("Debug driver")) {
-    std::string pidStr = std::to_string(pid);
-    setenv("O2DEBUGGEDPID", pidStr.c_str(), 1);
-#ifdef __APPLE__
-    std::string defaultAppleDebugCommand =
-      "osascript -e 'tell application \"Terminal\"'"
-      " -e 'activate'"
-      " -e 'do script \"lldb -p \" & (system attribute \"O2DEBUGGEDPID\") & \"; exit\"'"
-      " -e 'end tell'";
-    setenv("O2DPLDEBUG", defaultAppleDebugCommand.c_str(), 0);
-#else
-    setenv("O2DPLDEBUG", "xterm -hold -e gdb attach $O2DEBUGGEDPID &", 0);
-#endif
-    int retVal = system(getenv("O2DPLDEBUG"));
-    (void)retVal;
+    debugPID(pid);
   }
 
   ImGui::SameLine();
   if (ImGui::Button("Profile")) {
-    std::string pidStr = std::to_string(pid);
-    setenv("O2PROFILEDPID", pidStr.c_str(), 1);
-#ifdef __APPLE__
-    auto defaultAppleProfileCommand = fmt::format(
-      "osascript -e 'tell application \"Terminal\"'"
-      " -e 'activate'"
-      " -e 'do script \"xcrun xctrace record --output dpl-profile-{0}.trace"
-      " --instrument os_signpost --time-limit 30s --template Time\\\\ Profiler --attach {0} "
-      " && open dpl-profile-{0}.trace && exit\"'"
-      " -e 'end tell'",
-      pid);
-    std::cout << defaultAppleProfileCommand << std::endl;
-    setenv("O2DPLPROFILE", defaultAppleProfileCommand.c_str(), 0);
-#else
-    setenv("O2DPLPROFILE", "xterm -hold -e perf record -a -g -p $O2PROFILEDPID > perf-$O2PROFILEDPID.data &", 0);
-#endif
-    int retVal = system(getenv("O2DPLPROFILE"));
-    (void)retVal;
+    profilePID(pid);
   }
 
   // FIXME: this should really be a combo box
@@ -1110,7 +1146,8 @@ std::function<void(void)> getGUIDebugger(std::vector<DeviceInfo> const& infos,
                                          std::vector<DeviceMetricsInfo> const& metricsInfos,
                                          DriverInfo const& driverInfo,
                                          std::vector<DeviceControl>& controls,
-                                         DriverControl& driverControl)
+                                         DriverControl& driverControl,
+                                         GuiCallbackContext& gui)
 {
   static gui::WorkspaceGUIState globalGUIState;
   gui::WorkspaceGUIState& guiState = globalGUIState;
@@ -1141,7 +1178,7 @@ std::function<void(void)> getGUIDebugger(std::vector<DeviceInfo> const& infos,
   std::vector<TopologyNodeInfo> driverNodesInfos;
   driverNodesInfos.push_back(TopologyNodeInfo{"driver"});
 
-  return [&infos, &devices, &allStates, &metadata, &controls, &metricsInfos, &driverInfo, &driverControl, deviceNodesInfos, driverNodesInfos]() {
+  return [&infos, &devices, &allStates, &metadata, &controls, &metricsInfos, &driverInfo, &driverControl, &gui, deviceNodesInfos, driverNodesInfos]() {
     ImGuiStyle& style = ImGui::GetStyle();
     style.FrameRounding = 0.;
     style.WindowRounding = 0.;
@@ -1158,6 +1195,9 @@ std::function<void(void)> getGUIDebugger(std::vector<DeviceInfo> const& infos,
     metricsStore.specs[DRIVER_METRICS] = gsl::span(driverNodesInfos);
     displayMetrics(guiState, driverInfo, infos, metadata, controls, metricsStore);
     displayDriverInfo(driverInfo, driverControl);
+    if (*(gui.guiQuitRequested) && !*(gui.allChildrenGone)) {
+      displayShutdownDialog(devices, infos);
+    }
 
     int windowPosStepping = (ImGui::GetIO().DisplaySize.y - 500) / guiState.devices.size();
 
diff --git a/Framework/GUISupport/src/FrameworkGUIDebugger.h b/Framework/GUISupport/src/FrameworkGUIDebugger.h
index 5de5e350ddf8d..98bb50b07fd44 100644
--- a/Framework/GUISupport/src/FrameworkGUIDebugger.h
+++ b/Framework/GUISupport/src/FrameworkGUIDebugger.h
@@ -26,6 +26,7 @@ namespace o2::framework
 
 class DriverInfo;
 class DriverControl;
+class GuiCallbackContext;
 
 namespace gui
 {
@@ -37,7 +38,8 @@ std::function<void(void)> getGUIDebugger(std::vector<DeviceInfo> const& infos,
                                          std::vector<DeviceMetricsInfo> const& metricsInfos,
                                          DriverInfo const& driverInfo,
                                          std::vector<DeviceControl>& controls,
-                                         DriverControl& driverControl);
+                                         DriverControl& driverControl,
+                                         GuiCallbackContext& context);
 
 void updateMousePos(float x, float y);
 void updateMouseButton(bool clicked);
diff --git a/Framework/GUISupport/src/Plugin.cxx b/Framework/GUISupport/src/Plugin.cxx
index 7a356f6e55d9b..022b74174f632 100644
--- a/Framework/GUISupport/src/Plugin.cxx
+++ b/Framework/GUISupport/src/Plugin.cxx
@@ -36,9 +36,10 @@ struct ImGUIDebugGUI : o2::framework::DebugGUI {
                                            std::vector<DeviceMetricsInfo> const& metricsInfos,
                                            DriverInfo const& driverInfo,
                                            std::vector<DeviceControl>& controls,
-                                           DriverControl& driverControl) override
+                                           DriverControl& driverControl,
+                                           GuiCallbackContext& context) override
   {
-    return o2::framework::gui::getGUIDebugger(infos, devices, allStates, metadata, metricsInfos, driverInfo, controls, driverControl);
+    return o2::framework::gui::getGUIDebugger(infos, devices, allStates, metadata, metricsInfos, driverInfo, controls, driverControl, context);
   }
 
   void updateMousePos(float x, float y) override

From b7d30656d4d75df957c321694a25533e96a4bf9c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 Nov 2023 22:34:00 +0100
Subject: [PATCH 066/726] DPL GUI: allow GUI to move between different states

---
 Framework/Core/src/DataProcessingDevice.cxx   | 49 ++++++++++++++++++-
 Framework/Core/src/WSDriverClient.cxx         | 24 +++++++++
 Framework/Core/src/runDataProcessing.cxx      | 13 ++++-
 .../GUISupport/src/FrameworkGUIDebugger.cxx   |  2 +-
 .../src/FrameworkGUIDeviceInspector.cxx       | 27 +++++++++-
 .../src/FrameworkGUIDeviceInspector.h         |  5 +-
 .../src/FrameworkGUIDevicesGraph.cxx          |  5 +-
 .../GUISupport/src/FrameworkGUIDevicesGraph.h |  8 ++-
 8 files changed, 123 insertions(+), 10 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 5542be850fb70..3f78543bdc594 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -25,6 +25,7 @@
 #include "Framework/DispatchPolicy.h"
 #include "Framework/DispatchControl.h"
 #include "Framework/DanglingContext.h"
+#include "Framework/DriverInfo.h"
 #include "Framework/DomainInfoHeader.h"
 #include "Framework/DriverClient.h"
 #include "Framework/EndOfStreamContext.h"
@@ -159,16 +160,60 @@ DataProcessingDevice::DataProcessingDevice(RunningDeviceRef running, ServiceRegi
 
   std::function<void(const fair::mq::State)> stateWatcher = [this, &registry = mServiceRegistry](const fair::mq::State state) -> void {
     auto ref = ServiceRegistryRef{registry, ServiceRegistry::globalDeviceSalt()};
+    auto controlKind = this->GetConfig()->GetPropertyAsString("control");
     auto& deviceState = ref.get<DeviceState>();
     auto& control = ref.get<ControlService>();
     auto& callbacks = ref.get<CallbackService>();
-    control.notifyDeviceState(fair::mq::GetStateName(state));
+    auto stateName = fair::mq::GetStateName(state);
+    control.notifyDeviceState(stateName);
     callbacks.call<CallbackService::Id::DeviceStateChanged>(ServiceRegistryRef{ref}, (int)state);
+    LOG(detail) << "In state watcher callback " << stateName;
+
+    // If the termination policy is not to wait, we simply ignore all
+    // user imposed state changes and keep running until we are done.
+    if (controlKind != "gui") {
+      return;
+    }
+
+    static bool runningOnce = false;
 
     if (deviceState.nextFairMQState.empty() == false) {
+      LOG(detail) << "State change requested, changing state to " << deviceState.nextFairMQState.back();
       auto state = deviceState.nextFairMQState.back();
-      (void)this->ChangeState(state);
+      bool changed = this->ChangeState(state);
+      if (!changed) {
+        LOG(error) << "Failed to change state to " << state;
+      }
       deviceState.nextFairMQState.pop_back();
+    } else if (state == fair::mq::State::Running && deviceState.nextFairMQState.empty()) {
+      LOGP(detail, "Device is running and no transition expected. We are done.");
+      deviceState.transitionHandling = TransitionHandlingState::NoTransition;
+    } else {
+      while (runningOnce && deviceState.nextFairMQState.empty() && this->NewStatePending() == false) {
+        LOG(detail) << "No state change requested, waiting for next state change " << this->NewStatePending();
+        if (stateName == "EXITING") {
+          // Send ctrl c to ourselves. To bad FairMQ does not seem to exit when
+          // reaching the EXITING state.
+          kill(getpid(), SIGTERM);
+          return;
+        }
+        uv_run(deviceState.loop, UV_RUN_ONCE);
+        LOG(detail) << "Woke up from event loop";
+      }
+      if (runningOnce && deviceState.nextFairMQState.empty() == false) {
+        LOG(detail) << "State change requested, changing state to " << deviceState.nextFairMQState.back();
+        auto state = deviceState.nextFairMQState.back();
+        bool changed = this->ChangeState(state);
+        if (!changed) {
+          LOG(error) << "Failed to change state to " << state;
+        }
+        deviceState.nextFairMQState.pop_back();
+      }
+      LOG(detail) << "Exiting callback for state " << state;
+    }
+    if (runningOnce == false && state == fair::mq::State::Running) {
+      LOG(detail) << "First iteration, next time we start the event loop";
+      runningOnce = true;
     }
   };
 
diff --git a/Framework/Core/src/WSDriverClient.cxx b/Framework/Core/src/WSDriverClient.cxx
index d4ed77b9a004e..91f011e61cda3 100644
--- a/Framework/Core/src/WSDriverClient.cxx
+++ b/Framework/Core/src/WSDriverClient.cxx
@@ -15,6 +15,8 @@
 #include "Framework/ServiceRegistry.h"
 #include "Framework/DeviceSpec.h"
 #include "DriverClientContext.h"
+#include "Framework/RawDeviceService.h"
+#include "Device.h"
 #include "DPLWebSocket.h"
 #include <uv.h>
 #include <string_view>
@@ -134,6 +136,28 @@ void on_connect(uv_connect_t* connection, int status)
     state.nextFairMQState.emplace_back("STOP");
   });
 
+  client->observe("/shutdown", [ref = context->ref](std::string_view) {
+    auto currentStateName = ref.get<RawDeviceService>().device()->GetCurrentStateName();
+    LOGP(info, "Received shutdown request while in {}", currentStateName);
+
+    auto& state = ref.get<DeviceState>();
+    state.nextFairMQState.emplace_back("END");
+    if (currentStateName == "IDLE") {
+      return;
+    }
+    state.nextFairMQState.emplace_back("AUTO");
+    state.nextFairMQState.emplace_back("RESET DEVICE");
+    if (currentStateName == "DEVICE READY") {
+      return;
+    }
+    state.nextFairMQState.emplace_back("AUTO");
+    state.nextFairMQState.emplace_back("RESET TASK");
+    if (currentStateName == "READY") {
+      return;
+    }
+    state.nextFairMQState.emplace_back("STOP");
+  });
+
   client->observe("/trace", [ref = context->ref](std::string_view cmd) {
     auto& state = ref.get<DeviceState>();
     static constexpr int prefixSize = std::string_view{"/trace "}.size();
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 44a5c6b4ee94a..c7a57acd83971 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -29,6 +29,7 @@
 #include "Framework/DeviceMetricsInfo.h"
 #include "Framework/DeviceMetricsHelper.h"
 #include "Framework/DeviceConfigInfo.h"
+#include "Framework/DeviceController.h"
 #include "Framework/DeviceSpec.h"
 #include "Framework/DeviceState.h"
 #include "Framework/DeviceConfig.h"
@@ -2068,7 +2069,17 @@ int runStateMachine(DataProcessorSpecs const& workflow,
         // We send SIGCONT to make sure stopped children are resumed
         killChildren(infos, SIGCONT);
         // We send SIGTERM to make sure we do the STOP transition in FairMQ
-        killChildren(infos, SIGTERM);
+        if (driverInfo.processingPolicies.termination == TerminationPolicy::WAIT) {
+          for (size_t di = 0; di < infos.size(); ++di) {
+            auto& info = infos[di];
+            auto& control = controls[di];
+            if (info.active == true) {
+              control.controller->write("/shutdown", strlen("/shutdown"));
+            }
+          }
+        } else {
+          killChildren(infos, SIGTERM);
+        }
         // We have a timer to send SIGUSR1 to make sure we advance all devices
         // in a timely manner.
         force_step_timer.data = &infos;
diff --git a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
index 375afef1007f9..cc5180093ec94 100644
--- a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
@@ -1185,7 +1185,7 @@ std::function<void(void)> getGUIDebugger(std::vector<DeviceInfo> const& infos,
     style.Colors[ImGuiCol_WindowBg] = ImVec4(0x1b / 255.f, 0x1b / 255.f, 0x1b / 255.f, 1.00f);
     style.Colors[ImGuiCol_ScrollbarBg] = ImVec4(0x1b / 255.f, 0x1b / 255.f, 0x1b / 255.f, 1.00f);
 
-    showTopologyNodeGraph(guiState, infos, devices, allStates, metadata, controls, metricsInfos);
+    showTopologyNodeGraph(guiState, infos, devices, allStates, metadata, controls, metricsInfos, driverInfo.processingPolicies.termination);
 
     AllMetricsStore metricsStore;
 
diff --git a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
index a82753eb5af1f..9febb93858cb8 100644
--- a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
@@ -11,6 +11,7 @@
 
 #include "FrameworkGUIDeviceInspector.h"
 #include "Framework/DataProcessorInfo.h"
+#include "Framework/ProcessingPolicies.h"
 
 #include "Framework/DeviceControl.h"
 #include "Framework/DeviceSpec.h"
@@ -251,7 +252,8 @@ void displayDeviceInspector(DeviceSpec const& spec,
                             DataProcessingStates const& states,
                             DeviceMetricsInfo const& metrics,
                             DataProcessorInfo const& metadata,
-                            DeviceControl& control)
+                            DeviceControl& control,
+                            TerminationPolicy terminationPolicy)
 {
   ImGui::Text("Name: %s", spec.name.c_str());
   ImGui::Text("Executable: %s", metadata.executable.c_str());
@@ -340,12 +342,33 @@ void displayDeviceInspector(DeviceSpec const& spec,
     }
 
     if (control.requestedState > info.providedState) {
-      ImGui::Text(ICON_FA_CLOCK_O);
+      ImGui::TextUnformatted(ICON_FA_CLOCK_O "Requested transition in progress");
     } else {
+      // We only allow navigation if the termination policy is "WAIT"
+      ImGui::BeginDisabled(terminationPolicy == TerminationPolicy::QUIT);
       if (ImGui::Button("Restart")) {
         control.requestedState = info.providedState + 1;
         control.controller->write("/restart", strlen("/restart"));
       }
+      if (info.deviceState == "RUNNING") {
+        ImGui::SameLine();
+        if (ImGui::Button(ICON_FA_STOP)) {
+          control.requestedState = info.providedState + 1;
+          control.controller->write("/stop", strlen("/stop"));
+        }
+      } else if (info.deviceState == "READY") {
+        ImGui::SameLine();
+        if (ImGui::Button(ICON_FA_PLAY)) {
+          control.requestedState = info.providedState + 1;
+          control.controller->write("/start", strlen("/start"));
+        }
+        ImGui::SameLine();
+        if (ImGui::Button(ICON_FA_POWER_OFF)) {
+          control.requestedState = info.providedState + 1;
+          control.controller->write("/shutdown", strlen("/shutdown"));
+        }
+      }
+      ImGui::EndDisabled();
     }
   }
 
diff --git a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.h b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.h
index 8dac3204ea3ab..b69c6e9286215 100644
--- a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.h
+++ b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.h
@@ -18,12 +18,15 @@ struct DeviceInfo;
 struct DeviceMetricsInfo;
 struct DataProcessorInfo;
 struct DataProcessingStates;
+enum struct TerminationPolicy;
 
 namespace gui
 {
 
 /// Helper to display information about a device
-void displayDeviceInspector(DeviceSpec const& spec, DeviceInfo const& info, DataProcessingStates const& states, DeviceMetricsInfo const& metrics, DataProcessorInfo const& metadata, DeviceControl& control);
+void displayDeviceInspector(DeviceSpec const& spec, DeviceInfo const& info, DataProcessingStates const& states,
+                            DeviceMetricsInfo const& metrics, DataProcessorInfo const& metadata, DeviceControl& control,
+                            TerminationPolicy TerminationPolicy);
 
 } // namespace gui
 } // namespace o2::framework
diff --git a/Framework/GUISupport/src/FrameworkGUIDevicesGraph.cxx b/Framework/GUISupport/src/FrameworkGUIDevicesGraph.cxx
index 0b6ca07a9c86d..639122153e599 100644
--- a/Framework/GUISupport/src/FrameworkGUIDevicesGraph.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDevicesGraph.cxx
@@ -365,7 +365,8 @@ void showTopologyNodeGraph(WorkspaceGUIState& state,
                            std::vector<DataProcessingStates> const& allStates,
                            std::vector<DataProcessorInfo> const& metadata,
                            std::vector<DeviceControl>& controls,
-                           std::vector<DeviceMetricsInfo> const& metricsInfos)
+                           std::vector<DeviceMetricsInfo> const& metricsInfos,
+                           enum TerminationPolicy terminationPolicy)
 {
   ImGui::SetNextWindowPos(ImVec2(0, 0), 0);
   if (state.bottomPaneVisible) {
@@ -893,7 +894,7 @@ void showTopologyNodeGraph(WorkspaceGUIState& state,
       auto& metadatum = metadata[group.metadataId];
 
       if (state.rightPaneVisible) {
-        gui::displayDeviceInspector(spec, info, states, metrics, metadatum, control);
+        gui::displayDeviceInspector(spec, info, states, metrics, metadatum, control, terminationPolicy);
       }
     } else {
       ImGui::TextWrapped("Select a node in the topology to display information about it");
diff --git a/Framework/GUISupport/src/FrameworkGUIDevicesGraph.h b/Framework/GUISupport/src/FrameworkGUIDevicesGraph.h
index 7950f1893ebba..d53cdf26dd5a8 100644
--- a/Framework/GUISupport/src/FrameworkGUIDevicesGraph.h
+++ b/Framework/GUISupport/src/FrameworkGUIDevicesGraph.h
@@ -19,6 +19,11 @@
 
 #include <vector>
 
+namespace o2::framework
+{
+enum struct TerminationPolicy;
+} // namespace o2::framework
+
 namespace o2::framework::gui
 {
 
@@ -30,7 +35,8 @@ void showTopologyNodeGraph(WorkspaceGUIState& state,
                            std::vector<DataProcessingStates> const& allStates,
                            std::vector<DataProcessorInfo> const& metadata,
                            std::vector<DeviceControl>& controls,
-                           std::vector<DeviceMetricsInfo> const& metricsInfos);
+                           std::vector<DeviceMetricsInfo> const& metricsInfos,
+                           TerminationPolicy terminationPolicy);
 
 } // namespace o2::framework::gui
 

From 5d0a6197278c283d4adcd4c310f33d8c081caaec Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 8 Nov 2023 15:07:49 +0100
Subject: [PATCH 067/726] Support new fmt.

---
 Framework/Core/src/DataRelayer.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index c507d6eb308e3..266f5b054add1 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -570,7 +570,7 @@ DataRelayer::RelayChoice
       static std::atomic<size_t> obsoleteCount = 0;
       static std::atomic<size_t> mult = 1;
       if ((obsoleteCount++ % (1 * mult)) == 0) {
-        LOGP(warning, "Over {} incoming messages are already obsolete, not relaying.", obsoleteCount);
+        LOGP(warning, "Over {} incoming messages are already obsolete, not relaying.", obsoleteCount.load());
         if (obsoleteCount > mult * 10) {
           mult = mult * 10;
         }

From 0c1507c1e76a484ff08d1fd46f99f1e7a057998f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 9 Nov 2023 11:46:00 +0100
Subject: [PATCH 068/726] GPU: support fmt 10.1.1

---
 GPU/GPUTracking/Base/GPUReconstruction.cxx                | 2 +-
 GPU/GPUTracking/DataTypes/CalibdEdxTrackTopologyPol.cxx   | 6 +++---
 GPU/TPCFastTransformation/MultivariatePolynomial.h        | 8 ++++----
 .../MultivariatePolynomialHelper.cxx                      | 2 +-
 GPU/TPCFastTransformation/NDPiecewisePolynomials.h        | 8 ++++----
 5 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/GPU/GPUTracking/Base/GPUReconstruction.cxx b/GPU/GPUTracking/Base/GPUReconstruction.cxx
index 676ab0f6cf004..bf5ba46b6ab34 100644
--- a/GPU/GPUTracking/Base/GPUReconstruction.cxx
+++ b/GPU/GPUTracking/Base/GPUReconstruction.cxx
@@ -1215,7 +1215,7 @@ bool GPUReconstruction::CheckInstanceAvailable(DeviceType type)
   } else if (type == DeviceType::OCL2) {
     return sLibOCL2->LoadLibrary() == 0;
   } else {
-    GPUError("Error: Invalid device type %u", type);
+    GPUError("Error: Invalid device type %u", (unsigned)type);
     return false;
   }
 }
diff --git a/GPU/GPUTracking/DataTypes/CalibdEdxTrackTopologyPol.cxx b/GPU/GPUTracking/DataTypes/CalibdEdxTrackTopologyPol.cxx
index 435d2e3f56e35..aa57c8f85083e 100644
--- a/GPU/GPUTracking/DataTypes/CalibdEdxTrackTopologyPol.cxx
+++ b/GPU/GPUTracking/DataTypes/CalibdEdxTrackTopologyPol.cxx
@@ -180,13 +180,13 @@ void CalibdEdxTrackTopologyPol::writeToFile(TFile& outf, const char* name) const
 void CalibdEdxTrackTopologyPol::setFromContainer(const CalibdEdxTrackTopologyPolContainer& container)
 {
   if (2 * FFits != container.mCalibPols.size()) {
-    LOGP(warning, fmt::format("wrong number of polynomials stored! this {} container {}", 2 * FFits, container.mCalibPols.size()));
+    LOGP(warning, "wrong number of polynomials stored! this {} container {}", 2 * FFits, container.mCalibPols.size());
     return;
   }
 
   const auto nFacCont = container.mScalingFactorsqMax.size() + container.mScalingFactorsqTot.size();
   if (2 * FFits != nFacCont) {
-    LOGP(warning, fmt::format("wrong number of scaling factors stored! this {} container {}", 2 * FFits, nFacCont));
+    LOGP(warning, "wrong number of scaling factors stored! this {} container {}", 2 * FFits, nFacCont);
     return;
   }
 
@@ -214,7 +214,7 @@ void CalibdEdxTrackTopologyPol::loadFromFile(const char* fileName, const char* n
     setFromContainer(*polTmp);
     delete polTmp;
   } else {
-    LOGP(info, fmt::format("couldnt load object {} from input file", name));
+    LOGP(info, "couldnt load object {} from input file", name);
   }
 }
 
diff --git a/GPU/TPCFastTransformation/MultivariatePolynomial.h b/GPU/TPCFastTransformation/MultivariatePolynomial.h
index d2d80886a925c..bc980065f6263 100644
--- a/GPU/TPCFastTransformation/MultivariatePolynomial.h
+++ b/GPU/TPCFastTransformation/MultivariatePolynomial.h
@@ -178,7 +178,7 @@ void MultivariatePolynomial<Dim, Degree, InteractionOnly>::loadFromFile(TFile& i
     delete polTmp;
   } else {
 #ifndef GPUCA_ALIROOT_LIB
-    LOGP(info, fmt::format("couldnt load object {} from input file", name));
+    LOGP(info, "couldnt load object {} from input file", name);
 #endif
   }
 }
@@ -189,19 +189,19 @@ void MultivariatePolynomial<Dim, Degree, InteractionOnly>::setFromContainer(cons
   if constexpr (Dim > 0 && Degree > 0) {
     if (this->getDim() != container.mDim) {
 #ifndef GPUCA_ALIROOT_LIB
-      LOGP(info, fmt::format("wrong number of dimensions! this {} container {}", this->getDim(), container.mDim));
+      LOGP(info, "wrong number of dimensions! this {} container {}", this->getDim(), container.mDim);
 #endif
       return;
     }
     if (this->getDegree() != container.mDegree) {
 #ifndef GPUCA_ALIROOT_LIB
-      LOGP(info, fmt::format("wrong number of degrees! this {} container {}", this->getDegree(), container.mDegree));
+      LOGP(info, "wrong number of degrees! this {} container {}", this->getDegree(), container.mDegree);
 #endif
       return;
     }
     if (this->isInteractionOnly() != container.mInteractionOnly) {
 #ifndef GPUCA_ALIROOT_LIB
-      LOGP(info, fmt::format("InteractionOnly is set for this object to {}, but stored as {} in the container", this->isInteractionOnly(), container.mInteractionOnly));
+      LOGP(info, "InteractionOnly is set for this object to {}, but stored as {} in the container", this->isInteractionOnly(), container.mInteractionOnly);
 #endif
       return;
     }
diff --git a/GPU/TPCFastTransformation/MultivariatePolynomialHelper.cxx b/GPU/TPCFastTransformation/MultivariatePolynomialHelper.cxx
index 4d981293ca91f..1e40fcb51359e 100644
--- a/GPU/TPCFastTransformation/MultivariatePolynomialHelper.cxx
+++ b/GPU/TPCFastTransformation/MultivariatePolynomialHelper.cxx
@@ -26,7 +26,7 @@ using namespace GPUCA_NAMESPACE::gpu;
 void MultivariatePolynomialHelper<0, 0, false>::print() const
 {
 #ifndef GPUCA_NO_FMT
-  LOGP(info, getFormula().c_str());
+  LOGP(info, fmt::runtime(getFormula().c_str()));
 #endif
 }
 
diff --git a/GPU/TPCFastTransformation/NDPiecewisePolynomials.h b/GPU/TPCFastTransformation/NDPiecewisePolynomials.h
index caff727c36975..13ce49a142470 100644
--- a/GPU/TPCFastTransformation/NDPiecewisePolynomials.h
+++ b/GPU/TPCFastTransformation/NDPiecewisePolynomials.h
@@ -317,7 +317,7 @@ void NDPiecewisePolynomials<Dim, Degree, InteractionOnly>::loadFromFile(TFile& i
     delete gridTmp;
   } else {
 #ifndef GPUCA_ALIROOT_LIB
-    LOGP(info, fmt::format("couldnt load object {} from input file", name));
+    LOGP(info, "couldnt load object {} from input file", name);
 #endif
   }
 }
@@ -326,19 +326,19 @@ void NDPiecewisePolynomials<Dim, Degree, InteractionOnly>::setFromContainer(cons
 {
   if (Dim != container.mDim) {
 #ifndef GPUCA_ALIROOT_LIB
-    LOGP(info, fmt::format("wrong number of dimensions! this {} container {}", Dim, container.mDim));
+    LOGP(info, "wrong number of dimensions! this {} container {}", Dim, container.mDim);
 #endif
     return;
   }
   if (Degree != container.mDegree) {
 #ifndef GPUCA_ALIROOT_LIB
-    LOGP(info, fmt::format("wrong number of degrees! this {} container {}", Degree, container.mDegree));
+    LOGP(info, "wrong number of degrees! this {} container {}", Degree, container.mDegree);
 #endif
     return;
   }
   if (InteractionOnly != container.mInteractionOnly) {
 #ifndef GPUCA_ALIROOT_LIB
-    LOGP(info, fmt::format("InteractionOnly is set for this object to {}, but stored as {} in the container", InteractionOnly, container.mInteractionOnly));
+    LOGP(info, "InteractionOnly is set for this object to {}, but stored as {} in the container", InteractionOnly, container.mInteractionOnly);
 #endif
     return;
   }

From 99d75df2f37cc8df3647bac9d4df86e25563f87b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 9 Nov 2023 11:46:01 +0100
Subject: [PATCH 069/726] Framework: make sure we support fmt 10.1.1

---
 CCDB/src/CcdbApi.cxx                                     | 2 +-
 Common/Field/src/MagneticField.cxx                       | 2 +-
 Detectors/AOD/src/AODProducerWorkflowSpec.cxx            | 2 +-
 EventVisualisation/Workflow/src/FileProducer.cxx         | 2 +-
 Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx | 4 ++--
 Framework/Core/src/DPLWebSocket.cxx                      | 2 +-
 Framework/Core/src/DataOutputDirector.cxx                | 2 +-
 Framework/Core/src/DataProcessingContext.cxx             | 2 +-
 Framework/Core/src/HTTPParser.cxx                        | 4 ++--
 Framework/Core/src/runDataProcessing.cxx                 | 2 +-
 Framework/Utils/include/DPLUtils/RawParser.h             | 2 --
 Framework/Utils/src/RawParser.cxx                        | 8 ++++----
 12 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx
index aabc96cb779f8..904d979bfbd79 100644
--- a/CCDB/src/CcdbApi.cxx
+++ b/CCDB/src/CcdbApi.cxx
@@ -710,7 +710,7 @@ bool CcdbApi::retrieveBlob(std::string const& path, std::string const& targetdir
   try {
     o2::utils::createDirectoriesIfAbsent(fulltargetdir);
   } catch (std::exception e) {
-    LOGP(error, fmt::format("Could not create local snapshot cache directory {}, reason: {}", fulltargetdir, e.what()));
+    LOGP(error, "Could not create local snapshot cache directory {}, reason: {}", fulltargetdir, e.what());
     return false;
   }
 
diff --git a/Common/Field/src/MagneticField.cxx b/Common/Field/src/MagneticField.cxx
index 2e6ca6d647b11..5df6bbc0b0d34 100644
--- a/Common/Field/src/MagneticField.cxx
+++ b/Common/Field/src/MagneticField.cxx
@@ -539,7 +539,7 @@ void MagneticField::rescaleField(float l3Cur, float diCur, bool uniform, int con
   float sclL3 = l3Cur, sclDip = diCur;
   MagFieldParam::BMap_t map = getFieldMapScale(sclL3, sclDip, uniform);
   if (map != mMapType) {
-    LOGP(fatal, "Provided L3current={} DipCurrent={} uniform={} leads to map type {}, incompatible with loaded {}", l3Cur, diCur, uniform, map, mMapType);
+    LOGP(fatal, "Provided L3current={} DipCurrent={} uniform={} leads to map type {}, incompatible with loaded {}", l3Cur, diCur, uniform, (int)map, (int)mMapType);
   }
   setFactorSolenoid(sclL3);
   setFactorDipole(sclDip);
diff --git a/Detectors/AOD/src/AODProducerWorkflowSpec.cxx b/Detectors/AOD/src/AODProducerWorkflowSpec.cxx
index fa028535f379c..9b6151cb557b8 100644
--- a/Detectors/AOD/src/AODProducerWorkflowSpec.cxx
+++ b/Detectors/AOD/src/AODProducerWorkflowSpec.cxx
@@ -1221,7 +1221,7 @@ void AODProducerWorkflowDPL::fillSecondaryVertices(const o2::globaltracking::Rec
       if (item != mGIDToTableID.end()) {
         tableIdx[i] = item->second;
       } else {
-        LOG(warn) << fmt::format("Could not find a track index for prong ID {}", trIDs[i]);
+        LOG(warn) << fmt::format("Could not find a track index for prong ID {}", (int)trIDs[i]);
         missing = true;
       }
     }
diff --git a/EventVisualisation/Workflow/src/FileProducer.cxx b/EventVisualisation/Workflow/src/FileProducer.cxx
index 541afdefc4553..818ab546f3047 100644
--- a/EventVisualisation/Workflow/src/FileProducer.cxx
+++ b/EventVisualisation/Workflow/src/FileProducer.cxx
@@ -47,7 +47,7 @@ std::string FileProducer::newFileName() const
   gethostname(hostname, _POSIX_HOST_NAME_MAX);
 
   auto pid = getpid();
-  auto result = fmt::format(this->mName,
+  auto result = fmt::format(fmt::runtime(this->mName),
                             fmt::arg("hostname", hostname),
                             fmt::arg("pid", pid),
                             fmt::arg("timestamp", millisec_since_epoch),
diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 1e20f46c2e464..092857d585a02 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -246,11 +246,11 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
             // get first folder of next file
             ntf = 0;
             if (!didir->readTree(outputs, dh, fcnt, ntf, totalSizeCompressed, totalSizeUncompressed)) {
-              LOGP(fatal, "Can not retrieve tree for table {}: fileCounter {}, timeFrame {}", concrete.origin, fcnt, ntf);
+              LOGP(fatal, "Can not retrieve tree for table {}: fileCounter {}, timeFrame {}", concrete.origin.as<std::string>(), fcnt, ntf);
               throw std::runtime_error("Processing is stopped!");
             }
           } else {
-            LOGP(fatal, "Can not retrieve tree for table {}: fileCounter {}, timeFrame {}", concrete.origin, fcnt, ntf);
+            LOGP(fatal, "Can not retrieve tree for table {}: fileCounter {}, timeFrame {}", concrete.origin.as<std::string>(), fcnt, ntf);
             throw std::runtime_error("Processing is stopped!");
           }
         }
diff --git a/Framework/Core/src/DPLWebSocket.cxx b/Framework/Core/src/DPLWebSocket.cxx
index 97755727cbbd9..146a13206b55a 100644
--- a/Framework/Core/src/DPLWebSocket.cxx
+++ b/Framework/Core/src/DPLWebSocket.cxx
@@ -375,7 +375,7 @@ void WSDPLHandler::write(std::vector<uv_buf_t>& outputs)
 /// Helper to return an error
 void WSDPLHandler::error(int code, char const* message)
 {
-  static char const* errorFMT = "HTTP/1.1 {} {}\r\ncontent-type: text/plain\r\n\r\n{}: {}\r\n";
+  static constexpr auto errorFMT = "HTTP/1.1 {} {}\r\ncontent-type: text/plain\r\n\r\n{}: {}\r\n";
   std::string error = fmt::format(errorFMT, code, message, code, message);
   char* reply = strdup(error.data());
   uv_buf_t bfr = uv_buf_init(reply, error.size());
diff --git a/Framework/Core/src/DataOutputDirector.cxx b/Framework/Core/src/DataOutputDirector.cxx
index dcf7fe40b6214..078af891c9058 100644
--- a/Framework/Core/src/DataOutputDirector.cxx
+++ b/Framework/Core/src/DataOutputDirector.cxx
@@ -584,7 +584,7 @@ void DataOutputDirector::printOut()
 
   LOGP(info, "  File name bases      :");
   for (auto const& fb : mfilenameBases) {
-    LOGP(info, fb);
+    LOGP(info, "{}", fb);
   }
 }
 
diff --git a/Framework/Core/src/DataProcessingContext.cxx b/Framework/Core/src/DataProcessingContext.cxx
index 365975c706722..9c023e5021a9b 100644
--- a/Framework/Core/src/DataProcessingContext.cxx
+++ b/Framework/Core/src/DataProcessingContext.cxx
@@ -52,7 +52,7 @@ void DataProcessorContext::preDanglingCallbacks(DanglingContext& danglingContext
 void DataProcessorContext::postDanglingCallbacks(DanglingContext& danglingContext)
 {
   for (auto& handle : postDanglingHandles) {
-    LOGP(debug, "Invoking postDanglingCallback for service {} {}", handle.spec.name);
+    LOGP(debug, "Invoking postDanglingCallback for service {}", handle.spec.name);
     handle.callback(danglingContext, handle.service);
   }
 }
diff --git a/Framework/Core/src/HTTPParser.cxx b/Framework/Core/src/HTTPParser.cxx
index 137706c5f07c9..04ca6e8fdce55 100644
--- a/Framework/Core/src/HTTPParser.cxx
+++ b/Framework/Core/src/HTTPParser.cxx
@@ -189,7 +189,7 @@ void decode_websocket(char* start, size_t size, WebSocketHandler& handler)
 std::string encode_websocket_handshake_request(const char* endpoint, const char* protocol, int version, char const* nonce,
                                                std::vector<std::pair<std::string, std::string>> headers)
 {
-  char const* res =
+  constexpr auto res =
     "GET {} HTTP/1.1\r\n"
     "Upgrade: websocket\r\n"
     "Connection: Upgrade\r\n"
@@ -216,7 +216,7 @@ std::string HTTPParserHelpers::calculateAccept(const char* nonce)
 
 std::string encode_websocket_handshake_reply(char const* nonce)
 {
-  char const* res =
+  constexpr auto res =
     "HTTP/1.1 101 Switching Protocols\r\n"
     "Upgrade: websocket\r\n"
     "Connection: Upgrade\r\n"
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 61fdef070d874..86cc15299e8f3 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -696,7 +696,7 @@ void spawnDevice(uv_loop_t* loop,
   close(childFds[ref.index].childstdout[1]);
   if (varmap.count("post-fork-command")) {
     auto templateCmd = varmap["post-fork-command"];
-    auto cmd = fmt::format(templateCmd.as<std::string>(),
+    auto cmd = fmt::format(fmt::runtime(templateCmd.as<std::string>()),
                            fmt::arg("pid", id),
                            fmt::arg("id", spec.id),
                            fmt::arg("cpu", parentCPU),
diff --git a/Framework/Utils/include/DPLUtils/RawParser.h b/Framework/Utils/include/DPLUtils/RawParser.h
index 2c35c008983f0..c1ba1ef4802b2 100644
--- a/Framework/Utils/include/DPLUtils/RawParser.h
+++ b/Framework/Utils/include/DPLUtils/RawParser.h
@@ -58,14 +58,12 @@ struct RDHFormatter {
 template <>
 struct RDHFormatter<header::RAWDataHeaderV7> {
   using type = header::RAWDataHeaderV7;
-  static const char* sFormatString;
   static void apply(std::ostream&, type const&, FormatSpec, const char* = "");
 };
 
 template <>
 struct RDHFormatter<header::RAWDataHeaderV6> {
   using type = header::RAWDataHeaderV6;
-  static const char* sFormatString;
   static void apply(std::ostream&, type const&, FormatSpec, const char* = "");
 };
 
diff --git a/Framework/Utils/src/RawParser.cxx b/Framework/Utils/src/RawParser.cxx
index 2ec48c7f93d36..c7cbbf3d28880 100644
--- a/Framework/Utils/src/RawParser.cxx
+++ b/Framework/Utils/src/RawParser.cxx
@@ -51,9 +51,9 @@ void RawParserHelper::warnDeadBeef(const o2::header::DataHeader* dh)
   }
 }
 
-const char* RDHFormatter<V7>::sFormatString = "{:>5} {:>4} {:>4} {:>4} {:>4} {:>3} {:>3} {:>3}  {:>1} {:>2}";
 void RDHFormatter<V7>::apply(std::ostream& os, V7 const& header, FormatSpec choice, const char* delimiter)
 {
+  static constexpr const char* sFormatString = "{:>5} {:>4} {:>4} {:>4} {:>4} {:>3} {:>3} {:>3}  {:>1} {:>2}";
   if (choice == FormatSpec::Info) {
     os << "RDH v7";
   } else if (choice == FormatSpec::TableHeader) {
@@ -73,9 +73,9 @@ void RDHFormatter<V7>::apply(std::ostream& os, V7 const& header, FormatSpec choi
   }
   os << delimiter;
 }
-const char* RDHFormatter<V6>::sFormatString = "{:>5} {:>4} {:>4} {:>4} {:>4} {:>3} {:>3} {:>3}  {:>1}";
 void RDHFormatter<V6>::apply(std::ostream& os, V6 const& header, FormatSpec choice, const char* delimiter)
 {
+  static constexpr const char* sFormatString = "{:>5} {:>4} {:>4} {:>4} {:>4} {:>3} {:>3} {:>3}  {:>1}";
   if (choice == FormatSpec::Info) {
     os << "RDH v6";
   } else if (choice == FormatSpec::TableHeader) {
@@ -95,9 +95,9 @@ void RDHFormatter<V6>::apply(std::ostream& os, V6 const& header, FormatSpec choi
   os << delimiter;
 }
 
-const char* RDHFormatter<V5>::sFormatString = "{:>5} {:>4} {:>4} {:>4} {:>3} {:>3} {:>3}  {:>1}";
 void RDHFormatter<V5>::apply(std::ostream& os, V5 const& header, FormatSpec choice, const char* delimiter)
 {
+  static constexpr const char* sFormatString = "{:>5} {:>4} {:>4} {:>4} {:>3} {:>3} {:>3}  {:>1}";
   if (choice == FormatSpec::Info) {
     os << "RDH v5";
   } else if (choice == FormatSpec::TableHeader) {
@@ -116,9 +116,9 @@ void RDHFormatter<V5>::apply(std::ostream& os, V5 const& header, FormatSpec choi
   os << delimiter;
 }
 
-const char* RDHFormatter<V4>::sFormatString = "{:>5} {:>4} {:>4} {:>4} {:>3} {:>3} {:>3} {:>10} {:>5}  {:>1}";
 void RDHFormatter<V4>::apply(std::ostream& os, V4 const& header, FormatSpec choice, const char* delimiter)
 {
+  static constexpr const char* sFormatString = "{:>5} {:>4} {:>4} {:>4} {:>3} {:>3} {:>3} {:>10} {:>5}  {:>1}";
   if (choice == FormatSpec::Info) {
     os << "RDH v4";
   } else if (choice == FormatSpec::TableHeader) {

From dca52476b7b73168cd68570d8b35ccc4e5d6e287 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 9 Nov 2023 11:46:02 +0100
Subject: [PATCH 070/726] Detectors: make sure fmt 10.1.1 is supported

---
 .../include/DetectorsCommonDataFormats/EncodedBlocks.h      | 3 ++-
 .../RecoContainerCreateTracksVariadic.h                     | 4 ++--
 DataFormats/Parameters/src/GRPTool.cxx                      | 2 +-
 Detectors/CTF/workflow/src/ctf-reader-workflow.cxx          | 2 +-
 Detectors/DCS/src/AliasExpander.cxx                         | 4 ++--
 Detectors/EMCAL/base/src/RCUTrailer.cxx                     | 4 ++--
 Detectors/HMPID/calibration/src/HMPIDDCSProcessor.cxx       | 2 +-
 Detectors/ITSMFT/MFT/alignment/src/MillePede2.cxx           | 2 +-
 Detectors/ITSMFT/common/reconstruction/src/PixelData.cxx    | 2 +-
 Detectors/ITSMFT/common/reconstruction/src/RUInfo.cxx       | 2 +-
 Detectors/MUON/MCH/Calibration/src/PedestalData.cxx         | 2 +-
 .../MUON/MCH/DigitFiltering/src/DigitFilteringSpec.cxx      | 2 +-
 Detectors/MUON/MCH/Mapping/test/src/testPadIndices.cxx      | 2 +-
 .../MUON/MCH/PreClustering/src/PreClusterFinderSpec.cxx     | 4 ++--
 .../MCH/Raw/Decoder/src/testUserLogicEndpointDecoder.cxx    | 6 +++---
 Detectors/MUON/MCH/Raw/Encoder/Digit/DigitTreeReader.cxx    | 2 +-
 Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx            | 2 +-
 Detectors/MUON/MCH/Raw/test/testClosureCoDecDigit.cxx       | 2 +-
 Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx         | 2 +-
 Detectors/MUON/MCH/Workflow/src/ClusterFinderGEMSpec.cxx    | 2 +-
 .../MUON/MCH/Workflow/src/ClusterFinderOriginalSpec.cxx     | 2 +-
 Detectors/Raw/TFReaderDD/src/tf-reader-workflow.cxx         | 2 +-
 Detectors/Raw/src/RawDumpSpec.cxx                           | 4 ++--
 Detectors/Raw/src/rawfile-reader-workflow.cxx               | 2 +-
 Detectors/TPC/base/include/TPCBase/CalDet.h                 | 2 +-
 Detectors/TPC/base/src/CDBInterface.cxx                     | 2 +-
 Detectors/TPC/calibration/src/CalibTreeDump.cxx             | 2 +-
 Detectors/TPC/calibration/src/CalibdEdx.cxx                 | 3 ++-
 Detectors/TPC/calibration/src/VDriftHelper.cxx              | 3 ++-
 Detectors/TPC/dcs/src/DCSConfigSpec.cxx                     | 2 +-
 Detectors/TPC/dcs/src/DCSProcessor.cxx                      | 6 +++---
 Detectors/TPC/reconstruction/src/RawReaderCRU.cxx           | 2 +-
 .../TPC/workflow/include/TPCWorkflow/TPCFactorizeIDCSpec.h  | 2 +-
 .../include/TPCWorkflow/TPCFourierTransformAggregatorSpec.h | 2 +-
 Detectors/TPC/workflow/src/ClusterQCSpec.cxx                | 2 +-
 Detectors/TPC/workflow/src/IDCToVectorSpec.cxx              | 4 ++--
 Detectors/TRD/qc/src/Tracking.cxx                           | 2 +-
 37 files changed, 50 insertions(+), 47 deletions(-)

diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h
index a0853b8f14c73..6c024186b3085 100644
--- a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h
+++ b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h
@@ -402,7 +402,8 @@ class EncodedBlocks
       if (getANSHeader() == ANSVersionUnspecified) {
         rans::DenseHistogram<source_T> histogram{block.getDict(), block.getDict() + block.getNDict(), metadata.min};
         size_t renormingBits = rans::utils::sanitizeRenormingBitRange(metadata.probabilityBits);
-        LOG_IF(debug, renormingBits != metadata.probabilityBits) << fmt::format("While reading metadata from external dictionary, rANSV1 is rounding renorming precision from {} to {}");
+        LOG_IF(debug, renormingBits != metadata.probabilityBits)
+          << fmt::format("While reading metadata from external dictionary, rANSV1 is rounding renorming precision from {} to {}", metadata.probabilityBits, renormingBits);
         return rans::renorm(std::move(histogram), renormingBits, rans::RenormingPolicy::ForceIncompressible);
       } else {
         // dictionary is elias-delta coded inside the block
diff --git a/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h b/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h
index 4bbe355a781c7..4cec730c620a5 100644
--- a/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h
+++ b/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h
@@ -285,7 +285,7 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator, GTrackID
     if (srcSel[currentSource]) {
       if (matchesTPCTOF.size() && !tracksTPCTOF.size()) {
         throw std::runtime_error(fmt::format("TPC-TOF matched tracks ({}) require TPCTOF matches ({}) and TPCTOF tracks ({})",
-                                             matchesTPCTOF.size(), tracksTPCTOF.size()));
+                                             -1, matchesTPCTOF.size(), tracksTPCTOF.size()));
       }
       for (unsigned i = 0; i < matchesTPCTOF.size(); i++) {
         const auto& match = matchesTPCTOF[i];
@@ -321,7 +321,7 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator, GTrackID
     if (srcSel[currentSource]) {
       if (matchesMCHMID.size() && !tracksMCH.size()) {
         throw std::runtime_error(fmt::format("MCH-MID matched tracks ({}) require MCHMID matches ({}) and MCH tracks ({})",
-                                             matchesMCHMID.size(), tracksMCH.size()));
+                                             -1, matchesMCHMID.size(), tracksMCH.size()));
       }
       for (unsigned i = 0; i < matchesMCHMID.size(); i++) {
         const auto& match = matchesMCHMID[i];
diff --git a/DataFormats/Parameters/src/GRPTool.cxx b/DataFormats/Parameters/src/GRPTool.cxx
index 9b5078b60733c..413b08482d84a 100644
--- a/DataFormats/Parameters/src/GRPTool.cxx
+++ b/DataFormats/Parameters/src/GRPTool.cxx
@@ -205,7 +205,7 @@ bool publish(std::string const& filename, std::string const& path, std::string C
   try {
     o2::utils::createDirectoriesIfAbsent(targetdir);
   } catch (std::exception e) {
-    LOGP(error, fmt::format("Could not create local snapshot cache directory {}, reason: {}", targetdir, e.what()));
+    LOGP(error, "Could not create local snapshot cache directory {}, reason: {}", targetdir, e.what());
     return false;
   }
 
diff --git a/Detectors/CTF/workflow/src/ctf-reader-workflow.cxx b/Detectors/CTF/workflow/src/ctf-reader-workflow.cxx
index 3fcb4f69d0b8f..ef3a0f8d3c2c4 100644
--- a/Detectors/CTF/workflow/src/ctf-reader-workflow.cxx
+++ b/Detectors/CTF/workflow/src/ctf-reader-workflow.cxx
@@ -134,7 +134,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   int rateLimitingIPCID = std::stoi(configcontext.options().get<std::string>("timeframes-rate-limit-ipcid"));
   std::string chanFmt = configcontext.options().get<std::string>("metric-feedback-channel-format");
   if (rateLimitingIPCID > -1 && !chanFmt.empty()) {
-    ctfInput.metricChannel = fmt::format(chanFmt, o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
+    ctfInput.metricChannel = fmt::format(fmt::runtime(chanFmt), o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
   }
 
   specs.push_back(o2::ctf::getCTFReaderSpec(ctfInput));
diff --git a/Detectors/DCS/src/AliasExpander.cxx b/Detectors/DCS/src/AliasExpander.cxx
index b197d2fb78d67..41aaec8351f72 100644
--- a/Detectors/DCS/src/AliasExpander.cxx
+++ b/Detectors/DCS/src/AliasExpander.cxx
@@ -82,7 +82,7 @@ std::vector<std::string> extractRange(std::string range)
   std::vector<std::string> result;
 
   for (auto i = a; i <= b; i++) {
-    auto substituted = fmt::format(intFormat, i);
+    auto substituted = fmt::format(fmt::runtime(intFormat), i);
     result.push_back(substituted);
   }
   return result;
@@ -124,7 +124,7 @@ std::vector<std::string> expandAlias(const std::string& pattern)
   std::vector<std::string> result;
 
   for (auto r : range) {
-    auto substituted = fmt::format(newPattern, r);
+    auto substituted = fmt::format(fmt::runtime(newPattern), r);
     result.emplace_back(substituted);
   }
 
diff --git a/Detectors/EMCAL/base/src/RCUTrailer.cxx b/Detectors/EMCAL/base/src/RCUTrailer.cxx
index 6d10d0cb93c1d..24aac7b5bb25a 100644
--- a/Detectors/EMCAL/base/src/RCUTrailer.cxx
+++ b/Detectors/EMCAL/base/src/RCUTrailer.cxx
@@ -172,7 +172,7 @@ void RCUTrailer::setTimeSamplePhaseNS(uint64_t triggertime, uint64_t timesample)
       sample = 2;
       break;
     default:
-      throw Error(Error::ErrorType_t::SAMPLINGFREQ_INVALID, fmt::format("invalid time sample: {:f}", timesample).data());
+      throw Error(Error::ErrorType_t::SAMPLINGFREQ_INVALID, fmt::format(fmt::runtime("invalid time sample: {:f}"), timesample).data());
   };
   mAltroConfig.mSampleTime = sample;
   // calculate L1 phase
@@ -275,4 +275,4 @@ std::ostream& o2::emcal::operator<<(std::ostream& stream, const o2::emcal::RCUTr
 {
   trailer.printStream(stream);
   return stream;
-}
\ No newline at end of file
+}
diff --git a/Detectors/HMPID/calibration/src/HMPIDDCSProcessor.cxx b/Detectors/HMPID/calibration/src/HMPIDDCSProcessor.cxx
index 55bfe8fc101e7..7b3a88bf27385 100644
--- a/Detectors/HMPID/calibration/src/HMPIDDCSProcessor.cxx
+++ b/Detectors/HMPID/calibration/src/HMPIDDCSProcessor.cxx
@@ -395,7 +395,7 @@ double HMPIDDCSProcessor::calculatePhotonEnergy(int i)
   if (dp.id.get_type() == DeliveryType::DPVAL_DOUBLE) {
     lambda = o2::dcs::getValue<double>(dp);
   } else {
-    LOGP(warn, "DP type is {}", dp.id.get_type());
+    LOGP(warn, "DP type is {}", (int)dp.id.get_type());
     LOGP(warn, "Not correct datatype for HMP_TRANPLANT_MEASURE_{}_WAVELENGTH --> Default wavelength used for iteration procTrans{}", i, i);
     lambda = arrWaveLenDefault[i];
   }
diff --git a/Detectors/ITSMFT/MFT/alignment/src/MillePede2.cxx b/Detectors/ITSMFT/MFT/alignment/src/MillePede2.cxx
index af445c14f09c1..6220341f20054 100644
--- a/Detectors/ITSMFT/MFT/alignment/src/MillePede2.cxx
+++ b/Detectors/ITSMFT/MFT/alignment/src/MillePede2.cxx
@@ -1384,7 +1384,7 @@ int MillePede2::SolveGlobalMatEq()
     if (fgIterSol == MinResSolve::kSolFGMRes) {
       res = slv->SolveFGMRES(sol, fgMinResCondType, fgMinResMaxIter, fgMinResTol, fgNKrylovV);
     } else {
-      LOGF(warning, "MillePede2 - Undefined Iteritive Solver ID=%d, only %d are defined", fgIterSol, MinResSolve::kNSolvers);
+      LOGF(warning, "MillePede2 - Undefined Iteritive Solver ID=%d, only %d are defined", fgIterSol, (int)MinResSolve::kNSolvers);
     }
   }
 
diff --git a/Detectors/ITSMFT/common/reconstruction/src/PixelData.cxx b/Detectors/ITSMFT/common/reconstruction/src/PixelData.cxx
index a643f6fc4af1d..41b487786e90e 100644
--- a/Detectors/ITSMFT/common/reconstruction/src/PixelData.cxx
+++ b/Detectors/ITSMFT/common/reconstruction/src/PixelData.cxx
@@ -48,7 +48,7 @@ std::string ChipPixelData::getErrorDetails(int pos) const
     std::string rbuf = ": 0x<";
     int nc = getNBytesInRawBuff();
     for (int i = 0; i < nc; i++) {
-      rbuf += fmt::format(i ? " {:02x}" : "{:02x}", (int)getRawErrBuff()[i]);
+      rbuf += fmt::format(fmt::runtime(i ? " {:02x}" : "{:02x}"), (int)getRawErrBuff()[i]);
     }
     rbuf += '>';
     return rbuf;
diff --git a/Detectors/ITSMFT/common/reconstruction/src/RUInfo.cxx b/Detectors/ITSMFT/common/reconstruction/src/RUInfo.cxx
index 38c11d96ddd27..a05fd0504049d 100644
--- a/Detectors/ITSMFT/common/reconstruction/src/RUInfo.cxx
+++ b/Detectors/ITSMFT/common/reconstruction/src/RUInfo.cxx
@@ -35,5 +35,5 @@ std::string ChipInfo::asString() const
 
 void ChipInfo::print() const
 {
-  LOGP(info, asString());
+  LOGP(info, fmt::runtime(asString()));
 }
diff --git a/Detectors/MUON/MCH/Calibration/src/PedestalData.cxx b/Detectors/MUON/MCH/Calibration/src/PedestalData.cxx
index 0f5b377a4c92d..661bab7913b8e 100644
--- a/Detectors/MUON/MCH/Calibration/src/PedestalData.cxx
+++ b/Detectors/MUON/MCH/Calibration/src/PedestalData.cxx
@@ -120,7 +120,7 @@ void PedestalData::merge(const PedestalData* prev)
 void PedestalData::print() const
 {
   for (const auto& p : const_cast<PedestalData&>(*this)) {
-    LOGP(info, p.asString());
+    LOGP(info, fmt::runtime(p.asString()));
   }
 }
 
diff --git a/Detectors/MUON/MCH/DigitFiltering/src/DigitFilteringSpec.cxx b/Detectors/MUON/MCH/DigitFiltering/src/DigitFilteringSpec.cxx
index ea173bd1fc7cc..fe40659bc9265 100644
--- a/Detectors/MUON/MCH/DigitFiltering/src/DigitFilteringSpec.cxx
+++ b/Detectors/MUON/MCH/DigitFiltering/src/DigitFilteringSpec.cxx
@@ -88,7 +88,7 @@ class DigitFilteringTask
 
       if (!isOK(error)) {
         if (error.nofOutOfBounds > 0) {
-          LOGP(error, asString(error));
+          LOGP(error, "{}", asString(error));
           LOGP(error, "in a TF with {} rofs and {} digits", iRofs.size(), iDigits.size());
           abort = true;
         }
diff --git a/Detectors/MUON/MCH/Mapping/test/src/testPadIndices.cxx b/Detectors/MUON/MCH/Mapping/test/src/testPadIndices.cxx
index cc9d504d92e50..651a764820a1c 100644
--- a/Detectors/MUON/MCH/Mapping/test/src/testPadIndices.cxx
+++ b/Detectors/MUON/MCH/Mapping/test/src/testPadIndices.cxx
@@ -102,7 +102,7 @@ int main(int argc, char** argv)
   int nbad{0};
 
   for (auto de : deIds) {
-    std::string filepath(fmt::format(filePattern, de));
+    std::string filepath(fmt::format(fmt::runtime(filePattern), de));
     std::ifstream in(filepath);
     if (!in) {
       std::cout << "Cannot open " << filepath << "\n";
diff --git a/Detectors/MUON/MCH/PreClustering/src/PreClusterFinderSpec.cxx b/Detectors/MUON/MCH/PreClustering/src/PreClusterFinderSpec.cxx
index 2989401b55628..408ffda2ca9dc 100644
--- a/Detectors/MUON/MCH/PreClustering/src/PreClusterFinderSpec.cxx
+++ b/Detectors/MUON/MCH/PreClustering/src/PreClusterFinderSpec.cxx
@@ -83,7 +83,7 @@ class PreClusterFinderTask
       LOG(info) << "deinitializing preclusterizer in: "
                 << std::chrono::duration<double, std::milli>(tEnd - tStart).count() << " ms";
       mErrorMap.forEach([](Error error) {
-        LOGP(warning, error.asString());
+        LOGP(warning, "{}", error.asString());
       });
     };
     ic.services().get<CallbackService>().set<CallbackService::Id::Stop>(stop);
@@ -117,7 +117,7 @@ class PreClusterFinderTask
       if (!isOK(error)) {
         if (error.nofOutOfBounds > 0) {
           // FIXME: replace this error log with a counters' message ?
-          LOGP(error, asString(error));
+          LOGP(error, "{}", asString(error));
           LOGP(error, "in a TF with {} rofs and {} digits", digitROFs.size(), digits.size());
           abort = true;
         }
diff --git a/Detectors/MUON/MCH/Raw/Decoder/src/testUserLogicEndpointDecoder.cxx b/Detectors/MUON/MCH/Raw/Decoder/src/testUserLogicEndpointDecoder.cxx
index 43aefe520cb6e..519e56244aba2 100644
--- a/Detectors/MUON/MCH/Raw/Decoder/src/testUserLogicEndpointDecoder.cxx
+++ b/Detectors/MUON/MCH/Raw/Decoder/src/testUserLogicEndpointDecoder.cxx
@@ -369,7 +369,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(SyncInTheMiddleChargeSumModeTwoChannels, V, testTy
     5);
   std::string r2 = "S361-J6-DS2-ch-63-ts-345-q-123456-cs-789\n";
   r2 += "S361-J6-DS2-ch-63-ts-346-q-789012-cs-345\n";
-  r2 += fmt::format("S361-J6-DS2-chip-5-error-{}\n", ErrorUnexpectedSyncPacket);
+  r2 += fmt::format("S361-J6-DS2-chip-5-error-{}\n", (int)ErrorUnexpectedSyncPacket);
   BOOST_CHECK_EQUAL(r, r2);
 }
 
@@ -387,7 +387,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(TestCruPageBadClusterSize, V, testTypes)
   gsl::span<const uint64_t> page = CruPageBadClusterSize;
   std::string r = testPayloadDecodeCruPages<V::value>(page);
   BOOST_CHECK_EQUAL(r,
-                    fmt::format("S81-J0-DS0-chip-1-error-{}\nS81-J0-DS0-ch-42-ts-0-q-1\n", ErrorBadClusterSize));
+                    fmt::format("S81-J0-DS0-chip-1-error-{}\nS81-J0-DS0-ch-42-ts-0-q-1\n", (int)ErrorBadClusterSize));
 }
 
 BOOST_AUTO_TEST_CASE_TEMPLATE(TestCruPageBadN10bitWords, V, testTypes)
@@ -396,7 +396,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(TestCruPageBadN10bitWords, V, testTypes)
   std::string r = testPayloadDecodeCruPages<V::value>(page);
   std::string expected =
     fmt::format("S81-J0-DS0-ch-42-ts-87-q-2-1-0-0-1-0-0-0\nS81-J0-DS0-chip-1-error-{}\nS81-J0-DS0-ch-42-ts-0-q-1\n",
-                ErrorBadIncompleteWord);
+                (int)ErrorBadIncompleteWord);
   BOOST_CHECK_EQUAL(r, expected);
 }
 
diff --git a/Detectors/MUON/MCH/Raw/Encoder/Digit/DigitTreeReader.cxx b/Detectors/MUON/MCH/Raw/Encoder/Digit/DigitTreeReader.cxx
index 7b63ac2969e56..17f5f4463b16a 100644
--- a/Detectors/MUON/MCH/Raw/Encoder/Digit/DigitTreeReader.cxx
+++ b/Detectors/MUON/MCH/Raw/Encoder/Digit/DigitTreeReader.cxx
@@ -20,7 +20,7 @@ void AssertBranch(ROOT::Internal::TTreeReaderValueBase& value)
 {
   if (value.GetSetupStatus() < 0) {
     throw std::invalid_argument(fmt::format("Error {} setting up tree reader for branch {}",
-                                            value.GetSetupStatus(), value.GetBranchName()));
+                                            (int)value.GetSetupStatus(), value.GetBranchName()));
   }
 }
 
diff --git a/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx b/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
index e545c15025d8e..2407796773f57 100644
--- a/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
+++ b/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
@@ -104,7 +104,7 @@ std::vector<std::byte> paginate(gsl::span<const std::byte> buffer, const std::st
   return pages;
 }
 
-const char* sampaClusterFormat = "{}-CH{}-{}";
+constexpr const char* sampaClusterFormat = "{}-CH{}-{}";
 
 // Create a vector of SampaCluster from a string d
 // where d is of the form ts-#-bc-#-cs-#-q-# or
diff --git a/Detectors/MUON/MCH/Raw/test/testClosureCoDecDigit.cxx b/Detectors/MUON/MCH/Raw/test/testClosureCoDecDigit.cxx
index 661fd7741c954..aac72b144a80f 100644
--- a/Detectors/MUON/MCH/Raw/test/testClosureCoDecDigit.cxx
+++ b/Detectors/MUON/MCH/Raw/test/testClosureCoDecDigit.cxx
@@ -28,7 +28,7 @@
 
 using namespace o2::mch::raw;
 
-const char* sampaClusterFormat = "{}-CH{}-{}";
+constexpr const char* sampaClusterFormat = "{}-CH{}-{}";
 const bool useDummyElecMap = true;
 
 struct DePadId {
diff --git a/Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx b/Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx
index de920200bf4ff..c40df5d4b448e 100644
--- a/Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx
+++ b/Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx
@@ -105,7 +105,7 @@ class TrackFinderTask
       mTrackFinder.printTimers();
       LOG(info) << "tracking duration = " << mElapsedTime.count() << " s";
       mErrorMap.forEach([](Error error) {
-        LOGP(warning, error.asString());
+        LOGP(warning, "{}", error.asString());
       });
     };
     ic.services().get<CallbackService>().set<CallbackService::Id::Stop>(stop);
diff --git a/Detectors/MUON/MCH/Workflow/src/ClusterFinderGEMSpec.cxx b/Detectors/MUON/MCH/Workflow/src/ClusterFinderGEMSpec.cxx
index be6e415c74324..5158cc436b277 100644
--- a/Detectors/MUON/MCH/Workflow/src/ClusterFinderGEMSpec.cxx
+++ b/Detectors/MUON/MCH/Workflow/src/ClusterFinderGEMSpec.cxx
@@ -173,7 +173,7 @@ class ClusterFinderGEMTask
         mGEMDump = nullptr;
       }
       mErrorMap.forEach([](Error error) {
-        LOGP(warning, error.asString());
+        LOGP(warning, "{}", error.asString());
       });
     });
     auto stop = [this]() {
diff --git a/Detectors/MUON/MCH/Workflow/src/ClusterFinderOriginalSpec.cxx b/Detectors/MUON/MCH/Workflow/src/ClusterFinderOriginalSpec.cxx
index 2007f0634960f..8344d2837b814 100644
--- a/Detectors/MUON/MCH/Workflow/src/ClusterFinderOriginalSpec.cxx
+++ b/Detectors/MUON/MCH/Workflow/src/ClusterFinderOriginalSpec.cxx
@@ -73,7 +73,7 @@ class ClusterFinderOriginalTask
     ic.services().get<CallbackService>().set<CallbackService::Id::Stop>([this]() {
       LOG(info) << "cluster finder duration = " << mTimeClusterFinder.count() << " s";
       mErrorMap.forEach([](Error error) {
-        LOGP(warning, error.asString());
+        LOGP(warning, fmt::runtime(error.asString()));
       });
       this->mClusterFinder.deinit();
     });
diff --git a/Detectors/Raw/TFReaderDD/src/tf-reader-workflow.cxx b/Detectors/Raw/TFReaderDD/src/tf-reader-workflow.cxx
index facb8a880ba66..c468d1660fcc7 100644
--- a/Detectors/Raw/TFReaderDD/src/tf-reader-workflow.cxx
+++ b/Detectors/Raw/TFReaderDD/src/tf-reader-workflow.cxx
@@ -86,7 +86,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   int rateLimitingIPCID = std::stoi(configcontext.options().get<std::string>("timeframes-rate-limit-ipcid"));
   std::string chanFmt = configcontext.options().get<std::string>("metric-feedback-channel-format");
   if (rateLimitingIPCID > -1 && !chanFmt.empty()) {
-    rinp.metricChannel = fmt::format(chanFmt, o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
+    rinp.metricChannel = fmt::format(fmt::runtime(chanFmt), o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
   }
 
   WorkflowSpec specs;
diff --git a/Detectors/Raw/src/RawDumpSpec.cxx b/Detectors/Raw/src/RawDumpSpec.cxx
index 345eba5cefdc0..1cb667febe2a3 100644
--- a/Detectors/Raw/src/RawDumpSpec.cxx
+++ b/Detectors/Raw/src/RawDumpSpec.cxx
@@ -273,7 +273,7 @@ void RawDump::endOfStream(EndOfStreamContext& ec)
     auto fnm = fmt::format("{}{}{}raw.cfg", mOutDir, mOutDir.back() == '/' ? "" : "/", DetID::getName(id));
     auto fh = std::fopen(fnm.c_str(), "w");
     if (!fh) {
-      LOGP(fatal, "Failed to create configuration file {}");
+      LOGP(fatal, "Failed to create configuration file {}", fnm);
     }
     auto ws = std::fwrite(mConfigEntries[id].data(), 1, mConfigEntries[id].size(), fh);
     if (ws != mConfigEntries[id].size()) {
@@ -655,7 +655,7 @@ std::string RawDump::getBaseFileNameTRD(const header::RDHAny* rdh)
   int link = supermodule * 4 + side * 2 + ep, cru = link / 2;
   if (link >= NLinks) {
     auto flpname = fmt::format("flp-unknown_cru{}_ep{}_feeid0x{:05x}", cru, int(RDHUtils::getEndPointID(rdh)), RDHUtils::getFEEID(rdh));
-    LOGP(error, "Got wrong link {}, setting TRF file name to unrecognized flp {}", flpname);
+    LOGP(error, "Got wrong link {}, setting TRF file name to unrecognized flp {}", link, flpname);
     return flpname;
   }
   return fmt::format("alio2-cr1-flp{}_cru{}_{}", trdHWMap[cru].flpid, trdHWMap[cru].cruHWID, ep);
diff --git a/Detectors/Raw/src/rawfile-reader-workflow.cxx b/Detectors/Raw/src/rawfile-reader-workflow.cxx
index d8cc44fd5adbd..3a5440f80ed2d 100644
--- a/Detectors/Raw/src/rawfile-reader-workflow.cxx
+++ b/Detectors/Raw/src/rawfile-reader-workflow.cxx
@@ -90,7 +90,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   int rateLimitingIPCID = std::stoi(configcontext.options().get<std::string>("timeframes-rate-limit-ipcid"));
   std::string chanFmt = configcontext.options().get<std::string>("metric-feedback-channel-format");
   if (rateLimitingIPCID > -1 && !chanFmt.empty()) {
-    rinp.metricChannel = fmt::format(chanFmt, o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
+    rinp.metricChannel = fmt::format(fmt::runtime(chanFmt), o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
   }
   o2::conf::ConfigurableParam::updateFromString(configcontext.options().get<std::string>("configKeyValues"));
   auto hbfini = configcontext.options().get<std::string>("hbfutils-config");
diff --git a/Detectors/TPC/base/include/TPCBase/CalDet.h b/Detectors/TPC/base/include/TPCBase/CalDet.h
index d6b6b70119e4b..835e3638642df 100644
--- a/Detectors/TPC/base/include/TPCBase/CalDet.h
+++ b/Detectors/TPC/base/include/TPCBase/CalDet.h
@@ -463,7 +463,7 @@ void CalDet<T>::initData()
     if (!hasData) {
       mData.push_back(CalType(mPadSubset, i));
     }
-    mData[i].setName(fmt::format(frmt, mName, i));
+    mData[i].setName(fmt::format(fmt::runtime(frmt), mName, i));
   }
 }
 
diff --git a/Detectors/TPC/base/src/CDBInterface.cxx b/Detectors/TPC/base/src/CDBInterface.cxx
index 1180fd1cc8b1d..b359a04b385cb 100644
--- a/Detectors/TPC/base/src/CDBInterface.cxx
+++ b/Detectors/TPC/base/src/CDBInterface.cxx
@@ -633,5 +633,5 @@ void CDBStorage::printObjectSummary(std::string_view name, CDBType const type, M
     message += fmt::format("{:>20} = {}\n", key, value);
   }
 
-  LOGP(info, message);
+  LOGP(info, "{}", message);
 }
diff --git a/Detectors/TPC/calibration/src/CalibTreeDump.cxx b/Detectors/TPC/calibration/src/CalibTreeDump.cxx
index 3423be676566d..0748f1be55da5 100644
--- a/Detectors/TPC/calibration/src/CalibTreeDump.cxx
+++ b/Detectors/TPC/calibration/src/CalibTreeDump.cxx
@@ -223,7 +223,7 @@ void CalibTreeDump::addCalDetObjects(TTree* tree)
     std::string name = calDet.getName();
 
     if (name == "PadCalibrationObject" || name.size() == 0) {
-      name = fmt::format("calDet_{%02d}", iter);
+      name = fmt::format(fmt::runtime("calDet_{%02d}"), iter);
     }
 
     std::string meanName = fmt::format("{}_mean", name);
diff --git a/Detectors/TPC/calibration/src/CalibdEdx.cxx b/Detectors/TPC/calibration/src/CalibdEdx.cxx
index c012a4a2b3ec9..8c098cc04cf20 100644
--- a/Detectors/TPC/calibration/src/CalibdEdx.cxx
+++ b/Detectors/TPC/calibration/src/CalibdEdx.cxx
@@ -215,7 +215,8 @@ void fitHist(const Hist& hist, CalibdEdxCorrection& corr, TLinearFitter& fitter,
         corr.setChi2(id, charge, fitter.GetChisquare());
         corr.setEntries(id, charge, entries);
       }
-      LOGP(debug, "Sector: {}, gemType: {}, charge: {}, Fit pass: {} with {} % outliers in {} entries. Fitter Points: {}, mean fit: {}", id.sector, int(id.type), charge, fitPass, (float)outliers / (float)entries * 100, entries, fitter.GetNpoints(), params[0]);
+      LOGP(debug, "Sector: {}, gemType: {}, charge: {}, Fit pass: {} with {} % outliers in {} entries. Fitter Points: {}, mean fit: {}", 
+          id.sector, int(id.type), int(charge), fitPass, (float)outliers / (float)entries * 100, entries, fitter.GetNpoints(), params[0]);
     }
   }
 }
diff --git a/Detectors/TPC/calibration/src/VDriftHelper.cxx b/Detectors/TPC/calibration/src/VDriftHelper.cxx
index 034888998e567..b420c0392899b 100644
--- a/Detectors/TPC/calibration/src/VDriftHelper.cxx
+++ b/Detectors/TPC/calibration/src/VDriftHelper.cxx
@@ -43,7 +43,8 @@ VDriftHelper::VDriftHelper()
   if (o2::conf::ConfigurableParam::getProvenance("TPCDetParam.DriftTimeOffset") == o2::conf::ConfigurableParam::EParamProvenance::kRT) { // we stick to this value
     mVD.creationTime = std::numeric_limits<long>::max();
     mForceParamOffset = true;
-    LOGP(info, "TPC dridt time offset was set from command line to {} mus ({} TB}, will neglect update from CCDB", mVD.refTimeOffset, detpar.DriftTimeOffset);
+    LOGP(info, "TPC dridt time offset was set from command line to {} mus ({} TB), will neglect update from CCDB", 
+               mVD.refTimeOffset, detpar.DriftTimeOffset);
   }
 
   mUpdated = true;
diff --git a/Detectors/TPC/dcs/src/DCSConfigSpec.cxx b/Detectors/TPC/dcs/src/DCSConfigSpec.cxx
index 967b825d65a1d..34270cbca528d 100644
--- a/Detectors/TPC/dcs/src/DCSConfigSpec.cxx
+++ b/Detectors/TPC/dcs/src/DCSConfigSpec.cxx
@@ -289,7 +289,7 @@ void DCSConfigDevice::fillCRUConfig(gsl::span<const char> configBuff, bool updat
   }
 
   if (!update && (nLines != CRU::MaxCRU)) {
-    LOGP(error, "Full FEEConfig expected, but only {} / {} lines read for CRUConfig", nLines, CRU::MaxCRU);
+    LOGP(error, "Full FEEConfig expected, but only {} / {} lines read for CRUConfig", nLines, (int)CRU::MaxCRU);
   } else {
     LOGP(info, "updating CRUConfig for {} crus", nLines);
   }
diff --git a/Detectors/TPC/dcs/src/DCSProcessor.cxx b/Detectors/TPC/dcs/src/DCSProcessor.cxx
index 3a1572a6e6834..a26bce43e5c2e 100644
--- a/Detectors/TPC/dcs/src/DCSProcessor.cxx
+++ b/Detectors/TPC/dcs/src/DCSProcessor.cxx
@@ -69,7 +69,7 @@ float DCSProcessor::getValueF(const DPCOM& dp) const
   } else if (dp.id.get_type() == DeliveryType::DPVAL_DOUBLE) {
     return static_cast<float>(o2::dcs::getValue<double>(dp));
   } else {
-    LOGP(warning, "Unexpected delivery type for {}: {}", dp.id.get_alias(), dp.id.get_type());
+    LOGP(warning, "Unexpected delivery type for {}: {}", dp.id.get_alias(), (int)dp.id.get_type());
   }
 
   return 0.f;
@@ -94,7 +94,7 @@ void DCSProcessor::fillHV(const DPCOM& dp)
     // TODO: Remove once type is clear
     static bool statTypePrinted = false;
     if (!statTypePrinted) {
-      LOGP(info, "Delivery type for STATUS ({}): {}", alias, type);
+      LOGP(info, "Delivery type for STATUS ({}): {}", alias, (int)type);
       statTypePrinted = true;
     }
     if (type == DeliveryType::DPVAL_UINT) {
@@ -109,7 +109,7 @@ void DCSProcessor::fillHV(const DPCOM& dp)
     // TODO: Remove once type is clear
     static bool uiTypePrinted = false;
     if (!uiTypePrinted) {
-      LOGP(info, "Delivery type for current, voltage ({}): {}", alias, type);
+      LOGP(info, "Delivery type for current, voltage ({}): {}", alias, (int)type);
       uiTypePrinted = true;
     }
     const auto value = getValueF(dp);
diff --git a/Detectors/TPC/reconstruction/src/RawReaderCRU.cxx b/Detectors/TPC/reconstruction/src/RawReaderCRU.cxx
index 40063f23612ae..3a552ebcce82f 100644
--- a/Detectors/TPC/reconstruction/src/RawReaderCRU.cxx
+++ b/Detectors/TPC/reconstruction/src/RawReaderCRU.cxx
@@ -973,7 +973,7 @@ void RawReaderCRU::writeGBTDataPerLink(std::string_view outputDirectory, int max
 
       const int ep = iLink >= 12;
       const int link = iLink - (ep)*12;
-      auto outputFileName = fmt::format("{}/CRU_{:02}_EP_{}_Link_{:02}", outputDirectory.data(), mCRU, ep, link);
+      auto outputFileName = fmt::format("{}/CRU_{:02}_EP_{}_Link_{:02}", outputDirectory.data(), (int)mCRU, ep, link);
       std::ofstream outputFile(outputFileName, std::ios_base::binary | std::ios_base::app);
 
       for (auto packetNumber : linkInfo.PacketPositions) {
diff --git a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFactorizeIDCSpec.h b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFactorizeIDCSpec.h
index 69f90b2e71764..a4107690c941a 100644
--- a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFactorizeIDCSpec.h
+++ b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFactorizeIDCSpec.h
@@ -311,7 +311,7 @@ class TPCFactorizeIDCSpec : public o2::framework::Task
             o2::ccdb::CcdbObjectInfo ccdbInfoIDCDelta(CDBTypeMap.at(sideA ? CDBType::CalIDCDeltaA : CDBType::CalIDCDeltaC), std::string{}, std::string{}, std::map<std::string, std::string>{}, timeStampStartDelta, timeStampEndDelta);
 
             if (mDumpIDCDelta) {
-              mIDCGrouping.dumpToFile(fmt::format("{}DeltaAveraged_chunk{:02}_{:02}_side{}.root", getCurrentType(), iChunk, timeStampStartDelta, side).data());
+              mIDCGrouping.dumpToFile(fmt::format("{}DeltaAveraged_chunk{:02}_{:02}_side{}.root", getCurrentType(), iChunk, timeStampStartDelta, (int)side).data());
             }
 
             auto startCCDBIDCDelta = timer::now();
diff --git a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformAggregatorSpec.h b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformAggregatorSpec.h
index bb8334901b8d9..c24bbd1b245ce 100644
--- a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformAggregatorSpec.h
+++ b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformAggregatorSpec.h
@@ -125,7 +125,7 @@ class TPCFourierTransformAggregatorSpec : public o2::framework::Task
 
           if (mDumpFFT) {
             LOGP(info, "dumping FT to file");
-            mIDCFourierTransform[side].dumpToFile(fmt::format("FourierAGG_{:02}_side{}.root", processing_helpers::getCurrentTF(pc), side).data());
+            mIDCFourierTransform[side].dumpToFile(fmt::format("FourierAGG_{:02}_side{}.root", processing_helpers::getCurrentTF(pc), (int)side).data());
           }
 
           if (mSendOutDebug) {
diff --git a/Detectors/TPC/workflow/src/ClusterQCSpec.cxx b/Detectors/TPC/workflow/src/ClusterQCSpec.cxx
index 284085005ec15..8c93d2755b090 100644
--- a/Detectors/TPC/workflow/src/ClusterQCSpec.cxx
+++ b/Detectors/TPC/workflow/src/ClusterQCSpec.cxx
@@ -126,7 +126,7 @@ void ClusterQCDevice::endInterval()
   LOGP(info, "End interval for run: {}, TFs: {} - {}, creation: {} - {}, processed TFs: {}",
        mRunNumber, mFirstTF, mLastTF, mFirstCreation, mLastCreation, mClusterQC.getProcessedTFs());
 
-  const auto outputFileName = fmt::format(mOutputFileName, fmt::arg("run", mRunNumber),
+  const auto outputFileName = fmt::format(fmt::runtime(mOutputFileName), fmt::arg("run", mRunNumber),
                                           fmt::arg("firstTF", mFirstTF), fmt::arg("lastTF", mLastTF),
                                           fmt::arg("firstCreation", mFirstCreation), fmt::arg("lastCreation", mLastCreation));
   std::unique_ptr<TFile> f(TFile::Open(outputFileName.data(), "recreate"));
diff --git a/Detectors/TPC/workflow/src/IDCToVectorSpec.cxx b/Detectors/TPC/workflow/src/IDCToVectorSpec.cxx
index e9b26209a6fd1..a1b77b2774228 100644
--- a/Detectors/TPC/workflow/src/IDCToVectorSpec.cxx
+++ b/Detectors/TPC/workflow/src/IDCToVectorSpec.cxx
@@ -124,7 +124,7 @@ class IDCToVectorDevice : public o2::framework::Task
 
     // open files if necessary
     if ((mWriteDebug || mWriteDebugOnError) && !mDebugStream) {
-      const auto debugFileName = fmt::format(mDebugStreamFileName, fmt::arg("run", runNumber));
+      const auto debugFileName = fmt::format(fmt::runtime(mDebugStreamFileName), fmt::arg("run", runNumber));
       LOGP(info, "creating debug stream {}", debugFileName);
       mDebugStream = std::make_unique<o2::utils::TreeStreamRedirector>(debugFileName.data(), "recreate");
     }
@@ -134,7 +134,7 @@ class IDCToVectorDevice : public o2::framework::Task
       if (mRawDataType == 4) {
         rawType = "idc.raw";
       }
-      const auto rawFileName = fmt::format(mRawOutputFileName, fmt::arg("run", runNumber), fmt::arg("raw_type", rawType));
+      const auto rawFileName = fmt::format(fmt::runtime(mRawOutputFileName), fmt::arg("run", runNumber), fmt::arg("raw_type", rawType));
       LOGP(info, "creating raw debug file {}", rawFileName);
       mRawOutputFile.open(rawFileName, std::ios::binary);
     }
diff --git a/Detectors/TRD/qc/src/Tracking.cxx b/Detectors/TRD/qc/src/Tracking.cxx
index 98131ced41961..a827c3f5a1081 100644
--- a/Detectors/TRD/qc/src/Tracking.cxx
+++ b/Detectors/TRD/qc/src/Tracking.cxx
@@ -57,7 +57,7 @@ void Tracking::checkTrack(const TrackTRD& trkTrd, bool isTPCTRD)
   qcStruct.refGlobalTrackId = id;
   qcStruct.trackTRD = trkTrd;
 
-  LOGF(debug, "Got track with %i tracklets and ID %i", trkTrd.getNtracklets(), id);
+  LOGF(debug, "Got track with %i tracklets and ID %i", trkTrd.getNtracklets(), (int)id);
   o2::track::TrackParCov trk = isTPCTRD ? mTracksTPC[id].getParamOut() : mTracksITSTPC[id].getParamOut();
   qcStruct.trackSeed = trk;
   if (mPID) {

From eb7857f037b9893f38c421f57232f48c54f9b240 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 9 Nov 2023 11:46:02 +0100
Subject: [PATCH 071/726] Support ROOT v6.30.01

---
 Common/MathUtils/include/MathUtils/fit.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Common/MathUtils/include/MathUtils/fit.h b/Common/MathUtils/include/MathUtils/fit.h
index 2b1064b13fecb..00c39486a4ba0 100644
--- a/Common/MathUtils/include/MathUtils/fit.h
+++ b/Common/MathUtils/include/MathUtils/fit.h
@@ -59,7 +59,7 @@ template <typename T>
 TFitResultPtr fit(const size_t nBins, const T* arr, const T xMin, const T xMax, TF1& func, std::string_view option = "")
 {
   Foption_t fitOption;
-  ROOT::Fit::FitOptionsMake(ROOT::Fit::kHistogram, option.data(), fitOption);
+  ROOT::Fit::FitOptionsMake(ROOT::Fit::EFitObjectType::kHistogram, option.data(), fitOption);
 
   ROOT::Fit::DataRange range(xMin, xMax);
   ROOT::Fit::DataOptions opt;

From 456d9b0c503dc3ca31f720c3fd8c0d7f1b9c298c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 9 Nov 2023 11:46:03 +0100
Subject: [PATCH 072/726] C++20 does not allow aggregate initialisation if
 default ctor present

---
 Detectors/GlobalTracking/include/GlobalTracking/MatchTPCITS.h    | 1 -
 .../Vertexing/include/DetectorsVertexing/PVertexerHelpers.h      | 1 -
 2 files changed, 2 deletions(-)

diff --git a/Detectors/GlobalTracking/include/GlobalTracking/MatchTPCITS.h b/Detectors/GlobalTracking/include/GlobalTracking/MatchTPCITS.h
index f38195dbeb6ec..6f26fca6f4d03 100644
--- a/Detectors/GlobalTracking/include/GlobalTracking/MatchTPCITS.h
+++ b/Detectors/GlobalTracking/include/GlobalTracking/MatchTPCITS.h
@@ -288,7 +288,6 @@ struct InteractionCandidate : public o2::InteractionRecord {
   int rofITS;                                         // corresponding ITS ROF entry (in the ROFRecord vectors)
   uint32_t flag;                                      // origin, etc.
   o2::dataformats::RangeReference<int, int> seedsRef; // references to AB seeds
-  InteractionCandidate() = default;
   InteractionCandidate(const o2::InteractionRecord& ir, float t, float dt, int rof, uint32_t f = 0) : o2::InteractionRecord(ir), tBracket(t - dt, t + dt), rofITS(rof), flag(f) {}
 };
 
diff --git a/Detectors/Vertexing/include/DetectorsVertexing/PVertexerHelpers.h b/Detectors/Vertexing/include/DetectorsVertexing/PVertexerHelpers.h
index ec11c6e289c71..34c2d7c15a011 100644
--- a/Detectors/Vertexing/include/DetectorsVertexing/PVertexerHelpers.h
+++ b/Detectors/Vertexing/include/DetectorsVertexing/PVertexerHelpers.h
@@ -277,7 +277,6 @@ struct InteractionCandidate : public o2::InteractionRecord {
   float time = 0;
   float amplitude = 0;
   uint32_t flag = 0; // origin, etc.
-  InteractionCandidate() = default;
 };
 
 } // namespace vertexing

From 87c1722f6df43536e995726ecc944daa6e1b4a1a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 9 Nov 2023 11:46:04 +0100
Subject: [PATCH 073/726] DPL: adapt RCombinedDS to new ROOT API

---
 .../RootAnalysisHelpers/include/Framework/RCombinedDS.h   | 8 ++++----
 Framework/RootAnalysisHelpers/src/RCombinedDS.cxx         | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/Framework/RootAnalysisHelpers/include/Framework/RCombinedDS.h b/Framework/RootAnalysisHelpers/include/Framework/RCombinedDS.h
index b3b7979232467..c4dd7bce6a86f 100644
--- a/Framework/RootAnalysisHelpers/include/Framework/RCombinedDS.h
+++ b/Framework/RootAnalysisHelpers/include/Framework/RCombinedDS.h
@@ -341,14 +341,14 @@ class RCombinedDS final : public ROOT::RDF::RDataSource
     dummy += colName.data();
     throw std::runtime_error(dummy);
   }
-  const std::vector<std::string>& GetColumnNames() const override;
+  [[nodiscard]] const std::vector<std::string>& GetColumnNames() const override;
   std::vector<std::pair<ULong64_t, ULong64_t>> GetEntryRanges() override;
-  std::string GetTypeName(std::string_view colName) const override;
-  bool HasColumn(std::string_view colName) const override;
+  [[nodiscard]] std::string GetTypeName(std::string_view colName) const override;
+  [[nodiscard]] bool HasColumn(std::string_view colName) const override;
   bool SetEntry(unsigned int slot, ULong64_t entry) override;
   void InitSlot(unsigned int slot, ULong64_t firstEntry) override;
   void SetNSlots(unsigned int nSlots) override;
-  void Initialise() override;
+  virtual void Initialise();
 };
 
 ////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/Framework/RootAnalysisHelpers/src/RCombinedDS.cxx b/Framework/RootAnalysisHelpers/src/RCombinedDS.cxx
index f8ff71f8f6583..06958991cdf14 100644
--- a/Framework/RootAnalysisHelpers/src/RCombinedDS.cxx
+++ b/Framework/RootAnalysisHelpers/src/RCombinedDS.cxx
@@ -224,8 +224,8 @@ void RCombinedDS::Initialise()
 {
   fEntryRanges = fIndex->BuildIndex(fLeftDF, fRightDF);
 
-  fLeft->Initialise();
-  fRight->Initialise();
+  fLeft->Initialize();
+  fRight->Initialize();
 }
 
 /// Creates a RDataFrame using an arrow::Table as input.

From cdfce85eb5c22acfdcd5342a7e21c58780377e4a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 9 Nov 2023 11:55:58 +0100
Subject: [PATCH 074/726] Detectors: make sure fmt 10.1.1 is supported

---
 .../include/DetectorsCommonDataFormats/EncodedBlocks.h      | 3 ++-
 .../RecoContainerCreateTracksVariadic.h                     | 4 ++--
 DataFormats/Parameters/src/GRPTool.cxx                      | 2 +-
 Detectors/CTF/workflow/src/ctf-reader-workflow.cxx          | 2 +-
 Detectors/DCS/src/AliasExpander.cxx                         | 4 ++--
 Detectors/EMCAL/base/src/RCUTrailer.cxx                     | 4 ++--
 Detectors/HMPID/calibration/src/HMPIDDCSProcessor.cxx       | 2 +-
 Detectors/ITSMFT/MFT/alignment/src/MillePede2.cxx           | 2 +-
 Detectors/ITSMFT/common/reconstruction/src/PixelData.cxx    | 2 +-
 Detectors/ITSMFT/common/reconstruction/src/RUInfo.cxx       | 2 +-
 Detectors/MUON/MCH/Calibration/src/PedestalData.cxx         | 2 +-
 .../MUON/MCH/DigitFiltering/src/DigitFilteringSpec.cxx      | 2 +-
 Detectors/MUON/MCH/Mapping/test/src/testPadIndices.cxx      | 2 +-
 .../MUON/MCH/PreClustering/src/PreClusterFinderSpec.cxx     | 4 ++--
 .../MCH/Raw/Decoder/src/testUserLogicEndpointDecoder.cxx    | 6 +++---
 Detectors/MUON/MCH/Raw/Encoder/Digit/DigitTreeReader.cxx    | 2 +-
 Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx            | 2 +-
 Detectors/MUON/MCH/Raw/test/testClosureCoDecDigit.cxx       | 2 +-
 Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx         | 2 +-
 Detectors/MUON/MCH/Workflow/src/ClusterFinderGEMSpec.cxx    | 2 +-
 .../MUON/MCH/Workflow/src/ClusterFinderOriginalSpec.cxx     | 2 +-
 Detectors/Raw/TFReaderDD/src/tf-reader-workflow.cxx         | 2 +-
 Detectors/Raw/src/RawDumpSpec.cxx                           | 4 ++--
 Detectors/Raw/src/rawfile-reader-workflow.cxx               | 2 +-
 Detectors/TPC/base/include/TPCBase/CalDet.h                 | 2 +-
 Detectors/TPC/base/src/CDBInterface.cxx                     | 2 +-
 Detectors/TPC/calibration/src/CalibTreeDump.cxx             | 2 +-
 Detectors/TPC/calibration/src/CalibdEdx.cxx                 | 3 ++-
 Detectors/TPC/calibration/src/VDriftHelper.cxx              | 3 ++-
 Detectors/TPC/dcs/src/DCSConfigSpec.cxx                     | 2 +-
 Detectors/TPC/dcs/src/DCSProcessor.cxx                      | 6 +++---
 Detectors/TPC/reconstruction/src/RawReaderCRU.cxx           | 2 +-
 .../TPC/workflow/include/TPCWorkflow/TPCFactorizeIDCSpec.h  | 2 +-
 .../include/TPCWorkflow/TPCFourierTransformAggregatorSpec.h | 2 +-
 Detectors/TPC/workflow/src/ClusterQCSpec.cxx                | 2 +-
 Detectors/TPC/workflow/src/IDCToVectorSpec.cxx              | 4 ++--
 Detectors/TRD/qc/src/Tracking.cxx                           | 2 +-
 37 files changed, 50 insertions(+), 47 deletions(-)

diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h
index a0853b8f14c73..6c024186b3085 100644
--- a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h
+++ b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h
@@ -402,7 +402,8 @@ class EncodedBlocks
       if (getANSHeader() == ANSVersionUnspecified) {
         rans::DenseHistogram<source_T> histogram{block.getDict(), block.getDict() + block.getNDict(), metadata.min};
         size_t renormingBits = rans::utils::sanitizeRenormingBitRange(metadata.probabilityBits);
-        LOG_IF(debug, renormingBits != metadata.probabilityBits) << fmt::format("While reading metadata from external dictionary, rANSV1 is rounding renorming precision from {} to {}");
+        LOG_IF(debug, renormingBits != metadata.probabilityBits)
+          << fmt::format("While reading metadata from external dictionary, rANSV1 is rounding renorming precision from {} to {}", metadata.probabilityBits, renormingBits);
         return rans::renorm(std::move(histogram), renormingBits, rans::RenormingPolicy::ForceIncompressible);
       } else {
         // dictionary is elias-delta coded inside the block
diff --git a/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h b/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h
index 4bbe355a781c7..4cec730c620a5 100644
--- a/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h
+++ b/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h
@@ -285,7 +285,7 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator, GTrackID
     if (srcSel[currentSource]) {
       if (matchesTPCTOF.size() && !tracksTPCTOF.size()) {
         throw std::runtime_error(fmt::format("TPC-TOF matched tracks ({}) require TPCTOF matches ({}) and TPCTOF tracks ({})",
-                                             matchesTPCTOF.size(), tracksTPCTOF.size()));
+                                             -1, matchesTPCTOF.size(), tracksTPCTOF.size()));
       }
       for (unsigned i = 0; i < matchesTPCTOF.size(); i++) {
         const auto& match = matchesTPCTOF[i];
@@ -321,7 +321,7 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator, GTrackID
     if (srcSel[currentSource]) {
       if (matchesMCHMID.size() && !tracksMCH.size()) {
         throw std::runtime_error(fmt::format("MCH-MID matched tracks ({}) require MCHMID matches ({}) and MCH tracks ({})",
-                                             matchesMCHMID.size(), tracksMCH.size()));
+                                             -1, matchesMCHMID.size(), tracksMCH.size()));
       }
       for (unsigned i = 0; i < matchesMCHMID.size(); i++) {
         const auto& match = matchesMCHMID[i];
diff --git a/DataFormats/Parameters/src/GRPTool.cxx b/DataFormats/Parameters/src/GRPTool.cxx
index 9b5078b60733c..413b08482d84a 100644
--- a/DataFormats/Parameters/src/GRPTool.cxx
+++ b/DataFormats/Parameters/src/GRPTool.cxx
@@ -205,7 +205,7 @@ bool publish(std::string const& filename, std::string const& path, std::string C
   try {
     o2::utils::createDirectoriesIfAbsent(targetdir);
   } catch (std::exception e) {
-    LOGP(error, fmt::format("Could not create local snapshot cache directory {}, reason: {}", targetdir, e.what()));
+    LOGP(error, "Could not create local snapshot cache directory {}, reason: {}", targetdir, e.what());
     return false;
   }
 
diff --git a/Detectors/CTF/workflow/src/ctf-reader-workflow.cxx b/Detectors/CTF/workflow/src/ctf-reader-workflow.cxx
index 3fcb4f69d0b8f..ef3a0f8d3c2c4 100644
--- a/Detectors/CTF/workflow/src/ctf-reader-workflow.cxx
+++ b/Detectors/CTF/workflow/src/ctf-reader-workflow.cxx
@@ -134,7 +134,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   int rateLimitingIPCID = std::stoi(configcontext.options().get<std::string>("timeframes-rate-limit-ipcid"));
   std::string chanFmt = configcontext.options().get<std::string>("metric-feedback-channel-format");
   if (rateLimitingIPCID > -1 && !chanFmt.empty()) {
-    ctfInput.metricChannel = fmt::format(chanFmt, o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
+    ctfInput.metricChannel = fmt::format(fmt::runtime(chanFmt), o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
   }
 
   specs.push_back(o2::ctf::getCTFReaderSpec(ctfInput));
diff --git a/Detectors/DCS/src/AliasExpander.cxx b/Detectors/DCS/src/AliasExpander.cxx
index b197d2fb78d67..41aaec8351f72 100644
--- a/Detectors/DCS/src/AliasExpander.cxx
+++ b/Detectors/DCS/src/AliasExpander.cxx
@@ -82,7 +82,7 @@ std::vector<std::string> extractRange(std::string range)
   std::vector<std::string> result;
 
   for (auto i = a; i <= b; i++) {
-    auto substituted = fmt::format(intFormat, i);
+    auto substituted = fmt::format(fmt::runtime(intFormat), i);
     result.push_back(substituted);
   }
   return result;
@@ -124,7 +124,7 @@ std::vector<std::string> expandAlias(const std::string& pattern)
   std::vector<std::string> result;
 
   for (auto r : range) {
-    auto substituted = fmt::format(newPattern, r);
+    auto substituted = fmt::format(fmt::runtime(newPattern), r);
     result.emplace_back(substituted);
   }
 
diff --git a/Detectors/EMCAL/base/src/RCUTrailer.cxx b/Detectors/EMCAL/base/src/RCUTrailer.cxx
index 6d10d0cb93c1d..24aac7b5bb25a 100644
--- a/Detectors/EMCAL/base/src/RCUTrailer.cxx
+++ b/Detectors/EMCAL/base/src/RCUTrailer.cxx
@@ -172,7 +172,7 @@ void RCUTrailer::setTimeSamplePhaseNS(uint64_t triggertime, uint64_t timesample)
       sample = 2;
       break;
     default:
-      throw Error(Error::ErrorType_t::SAMPLINGFREQ_INVALID, fmt::format("invalid time sample: {:f}", timesample).data());
+      throw Error(Error::ErrorType_t::SAMPLINGFREQ_INVALID, fmt::format(fmt::runtime("invalid time sample: {:f}"), timesample).data());
   };
   mAltroConfig.mSampleTime = sample;
   // calculate L1 phase
@@ -275,4 +275,4 @@ std::ostream& o2::emcal::operator<<(std::ostream& stream, const o2::emcal::RCUTr
 {
   trailer.printStream(stream);
   return stream;
-}
\ No newline at end of file
+}
diff --git a/Detectors/HMPID/calibration/src/HMPIDDCSProcessor.cxx b/Detectors/HMPID/calibration/src/HMPIDDCSProcessor.cxx
index 55bfe8fc101e7..7b3a88bf27385 100644
--- a/Detectors/HMPID/calibration/src/HMPIDDCSProcessor.cxx
+++ b/Detectors/HMPID/calibration/src/HMPIDDCSProcessor.cxx
@@ -395,7 +395,7 @@ double HMPIDDCSProcessor::calculatePhotonEnergy(int i)
   if (dp.id.get_type() == DeliveryType::DPVAL_DOUBLE) {
     lambda = o2::dcs::getValue<double>(dp);
   } else {
-    LOGP(warn, "DP type is {}", dp.id.get_type());
+    LOGP(warn, "DP type is {}", (int)dp.id.get_type());
     LOGP(warn, "Not correct datatype for HMP_TRANPLANT_MEASURE_{}_WAVELENGTH --> Default wavelength used for iteration procTrans{}", i, i);
     lambda = arrWaveLenDefault[i];
   }
diff --git a/Detectors/ITSMFT/MFT/alignment/src/MillePede2.cxx b/Detectors/ITSMFT/MFT/alignment/src/MillePede2.cxx
index af445c14f09c1..6220341f20054 100644
--- a/Detectors/ITSMFT/MFT/alignment/src/MillePede2.cxx
+++ b/Detectors/ITSMFT/MFT/alignment/src/MillePede2.cxx
@@ -1384,7 +1384,7 @@ int MillePede2::SolveGlobalMatEq()
     if (fgIterSol == MinResSolve::kSolFGMRes) {
       res = slv->SolveFGMRES(sol, fgMinResCondType, fgMinResMaxIter, fgMinResTol, fgNKrylovV);
     } else {
-      LOGF(warning, "MillePede2 - Undefined Iteritive Solver ID=%d, only %d are defined", fgIterSol, MinResSolve::kNSolvers);
+      LOGF(warning, "MillePede2 - Undefined Iteritive Solver ID=%d, only %d are defined", fgIterSol, (int)MinResSolve::kNSolvers);
     }
   }
 
diff --git a/Detectors/ITSMFT/common/reconstruction/src/PixelData.cxx b/Detectors/ITSMFT/common/reconstruction/src/PixelData.cxx
index a643f6fc4af1d..41b487786e90e 100644
--- a/Detectors/ITSMFT/common/reconstruction/src/PixelData.cxx
+++ b/Detectors/ITSMFT/common/reconstruction/src/PixelData.cxx
@@ -48,7 +48,7 @@ std::string ChipPixelData::getErrorDetails(int pos) const
     std::string rbuf = ": 0x<";
     int nc = getNBytesInRawBuff();
     for (int i = 0; i < nc; i++) {
-      rbuf += fmt::format(i ? " {:02x}" : "{:02x}", (int)getRawErrBuff()[i]);
+      rbuf += fmt::format(fmt::runtime(i ? " {:02x}" : "{:02x}"), (int)getRawErrBuff()[i]);
     }
     rbuf += '>';
     return rbuf;
diff --git a/Detectors/ITSMFT/common/reconstruction/src/RUInfo.cxx b/Detectors/ITSMFT/common/reconstruction/src/RUInfo.cxx
index 38c11d96ddd27..a05fd0504049d 100644
--- a/Detectors/ITSMFT/common/reconstruction/src/RUInfo.cxx
+++ b/Detectors/ITSMFT/common/reconstruction/src/RUInfo.cxx
@@ -35,5 +35,5 @@ std::string ChipInfo::asString() const
 
 void ChipInfo::print() const
 {
-  LOGP(info, asString());
+  LOGP(info, fmt::runtime(asString()));
 }
diff --git a/Detectors/MUON/MCH/Calibration/src/PedestalData.cxx b/Detectors/MUON/MCH/Calibration/src/PedestalData.cxx
index 0f5b377a4c92d..661bab7913b8e 100644
--- a/Detectors/MUON/MCH/Calibration/src/PedestalData.cxx
+++ b/Detectors/MUON/MCH/Calibration/src/PedestalData.cxx
@@ -120,7 +120,7 @@ void PedestalData::merge(const PedestalData* prev)
 void PedestalData::print() const
 {
   for (const auto& p : const_cast<PedestalData&>(*this)) {
-    LOGP(info, p.asString());
+    LOGP(info, fmt::runtime(p.asString()));
   }
 }
 
diff --git a/Detectors/MUON/MCH/DigitFiltering/src/DigitFilteringSpec.cxx b/Detectors/MUON/MCH/DigitFiltering/src/DigitFilteringSpec.cxx
index ea173bd1fc7cc..fe40659bc9265 100644
--- a/Detectors/MUON/MCH/DigitFiltering/src/DigitFilteringSpec.cxx
+++ b/Detectors/MUON/MCH/DigitFiltering/src/DigitFilteringSpec.cxx
@@ -88,7 +88,7 @@ class DigitFilteringTask
 
       if (!isOK(error)) {
         if (error.nofOutOfBounds > 0) {
-          LOGP(error, asString(error));
+          LOGP(error, "{}", asString(error));
           LOGP(error, "in a TF with {} rofs and {} digits", iRofs.size(), iDigits.size());
           abort = true;
         }
diff --git a/Detectors/MUON/MCH/Mapping/test/src/testPadIndices.cxx b/Detectors/MUON/MCH/Mapping/test/src/testPadIndices.cxx
index cc9d504d92e50..651a764820a1c 100644
--- a/Detectors/MUON/MCH/Mapping/test/src/testPadIndices.cxx
+++ b/Detectors/MUON/MCH/Mapping/test/src/testPadIndices.cxx
@@ -102,7 +102,7 @@ int main(int argc, char** argv)
   int nbad{0};
 
   for (auto de : deIds) {
-    std::string filepath(fmt::format(filePattern, de));
+    std::string filepath(fmt::format(fmt::runtime(filePattern), de));
     std::ifstream in(filepath);
     if (!in) {
       std::cout << "Cannot open " << filepath << "\n";
diff --git a/Detectors/MUON/MCH/PreClustering/src/PreClusterFinderSpec.cxx b/Detectors/MUON/MCH/PreClustering/src/PreClusterFinderSpec.cxx
index 2989401b55628..408ffda2ca9dc 100644
--- a/Detectors/MUON/MCH/PreClustering/src/PreClusterFinderSpec.cxx
+++ b/Detectors/MUON/MCH/PreClustering/src/PreClusterFinderSpec.cxx
@@ -83,7 +83,7 @@ class PreClusterFinderTask
       LOG(info) << "deinitializing preclusterizer in: "
                 << std::chrono::duration<double, std::milli>(tEnd - tStart).count() << " ms";
       mErrorMap.forEach([](Error error) {
-        LOGP(warning, error.asString());
+        LOGP(warning, "{}", error.asString());
       });
     };
     ic.services().get<CallbackService>().set<CallbackService::Id::Stop>(stop);
@@ -117,7 +117,7 @@ class PreClusterFinderTask
       if (!isOK(error)) {
         if (error.nofOutOfBounds > 0) {
           // FIXME: replace this error log with a counters' message ?
-          LOGP(error, asString(error));
+          LOGP(error, "{}", asString(error));
           LOGP(error, "in a TF with {} rofs and {} digits", digitROFs.size(), digits.size());
           abort = true;
         }
diff --git a/Detectors/MUON/MCH/Raw/Decoder/src/testUserLogicEndpointDecoder.cxx b/Detectors/MUON/MCH/Raw/Decoder/src/testUserLogicEndpointDecoder.cxx
index 43aefe520cb6e..519e56244aba2 100644
--- a/Detectors/MUON/MCH/Raw/Decoder/src/testUserLogicEndpointDecoder.cxx
+++ b/Detectors/MUON/MCH/Raw/Decoder/src/testUserLogicEndpointDecoder.cxx
@@ -369,7 +369,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(SyncInTheMiddleChargeSumModeTwoChannels, V, testTy
     5);
   std::string r2 = "S361-J6-DS2-ch-63-ts-345-q-123456-cs-789\n";
   r2 += "S361-J6-DS2-ch-63-ts-346-q-789012-cs-345\n";
-  r2 += fmt::format("S361-J6-DS2-chip-5-error-{}\n", ErrorUnexpectedSyncPacket);
+  r2 += fmt::format("S361-J6-DS2-chip-5-error-{}\n", (int)ErrorUnexpectedSyncPacket);
   BOOST_CHECK_EQUAL(r, r2);
 }
 
@@ -387,7 +387,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(TestCruPageBadClusterSize, V, testTypes)
   gsl::span<const uint64_t> page = CruPageBadClusterSize;
   std::string r = testPayloadDecodeCruPages<V::value>(page);
   BOOST_CHECK_EQUAL(r,
-                    fmt::format("S81-J0-DS0-chip-1-error-{}\nS81-J0-DS0-ch-42-ts-0-q-1\n", ErrorBadClusterSize));
+                    fmt::format("S81-J0-DS0-chip-1-error-{}\nS81-J0-DS0-ch-42-ts-0-q-1\n", (int)ErrorBadClusterSize));
 }
 
 BOOST_AUTO_TEST_CASE_TEMPLATE(TestCruPageBadN10bitWords, V, testTypes)
@@ -396,7 +396,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(TestCruPageBadN10bitWords, V, testTypes)
   std::string r = testPayloadDecodeCruPages<V::value>(page);
   std::string expected =
     fmt::format("S81-J0-DS0-ch-42-ts-87-q-2-1-0-0-1-0-0-0\nS81-J0-DS0-chip-1-error-{}\nS81-J0-DS0-ch-42-ts-0-q-1\n",
-                ErrorBadIncompleteWord);
+                (int)ErrorBadIncompleteWord);
   BOOST_CHECK_EQUAL(r, expected);
 }
 
diff --git a/Detectors/MUON/MCH/Raw/Encoder/Digit/DigitTreeReader.cxx b/Detectors/MUON/MCH/Raw/Encoder/Digit/DigitTreeReader.cxx
index 7b63ac2969e56..17f5f4463b16a 100644
--- a/Detectors/MUON/MCH/Raw/Encoder/Digit/DigitTreeReader.cxx
+++ b/Detectors/MUON/MCH/Raw/Encoder/Digit/DigitTreeReader.cxx
@@ -20,7 +20,7 @@ void AssertBranch(ROOT::Internal::TTreeReaderValueBase& value)
 {
   if (value.GetSetupStatus() < 0) {
     throw std::invalid_argument(fmt::format("Error {} setting up tree reader for branch {}",
-                                            value.GetSetupStatus(), value.GetBranchName()));
+                                            (int)value.GetSetupStatus(), value.GetBranchName()));
   }
 }
 
diff --git a/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx b/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
index e545c15025d8e..2407796773f57 100644
--- a/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
+++ b/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
@@ -104,7 +104,7 @@ std::vector<std::byte> paginate(gsl::span<const std::byte> buffer, const std::st
   return pages;
 }
 
-const char* sampaClusterFormat = "{}-CH{}-{}";
+constexpr const char* sampaClusterFormat = "{}-CH{}-{}";
 
 // Create a vector of SampaCluster from a string d
 // where d is of the form ts-#-bc-#-cs-#-q-# or
diff --git a/Detectors/MUON/MCH/Raw/test/testClosureCoDecDigit.cxx b/Detectors/MUON/MCH/Raw/test/testClosureCoDecDigit.cxx
index 661fd7741c954..aac72b144a80f 100644
--- a/Detectors/MUON/MCH/Raw/test/testClosureCoDecDigit.cxx
+++ b/Detectors/MUON/MCH/Raw/test/testClosureCoDecDigit.cxx
@@ -28,7 +28,7 @@
 
 using namespace o2::mch::raw;
 
-const char* sampaClusterFormat = "{}-CH{}-{}";
+constexpr const char* sampaClusterFormat = "{}-CH{}-{}";
 const bool useDummyElecMap = true;
 
 struct DePadId {
diff --git a/Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx b/Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx
index de920200bf4ff..c40df5d4b448e 100644
--- a/Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx
+++ b/Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx
@@ -105,7 +105,7 @@ class TrackFinderTask
       mTrackFinder.printTimers();
       LOG(info) << "tracking duration = " << mElapsedTime.count() << " s";
       mErrorMap.forEach([](Error error) {
-        LOGP(warning, error.asString());
+        LOGP(warning, "{}", error.asString());
       });
     };
     ic.services().get<CallbackService>().set<CallbackService::Id::Stop>(stop);
diff --git a/Detectors/MUON/MCH/Workflow/src/ClusterFinderGEMSpec.cxx b/Detectors/MUON/MCH/Workflow/src/ClusterFinderGEMSpec.cxx
index be6e415c74324..5158cc436b277 100644
--- a/Detectors/MUON/MCH/Workflow/src/ClusterFinderGEMSpec.cxx
+++ b/Detectors/MUON/MCH/Workflow/src/ClusterFinderGEMSpec.cxx
@@ -173,7 +173,7 @@ class ClusterFinderGEMTask
         mGEMDump = nullptr;
       }
       mErrorMap.forEach([](Error error) {
-        LOGP(warning, error.asString());
+        LOGP(warning, "{}", error.asString());
       });
     });
     auto stop = [this]() {
diff --git a/Detectors/MUON/MCH/Workflow/src/ClusterFinderOriginalSpec.cxx b/Detectors/MUON/MCH/Workflow/src/ClusterFinderOriginalSpec.cxx
index 2007f0634960f..8344d2837b814 100644
--- a/Detectors/MUON/MCH/Workflow/src/ClusterFinderOriginalSpec.cxx
+++ b/Detectors/MUON/MCH/Workflow/src/ClusterFinderOriginalSpec.cxx
@@ -73,7 +73,7 @@ class ClusterFinderOriginalTask
     ic.services().get<CallbackService>().set<CallbackService::Id::Stop>([this]() {
       LOG(info) << "cluster finder duration = " << mTimeClusterFinder.count() << " s";
       mErrorMap.forEach([](Error error) {
-        LOGP(warning, error.asString());
+        LOGP(warning, fmt::runtime(error.asString()));
       });
       this->mClusterFinder.deinit();
     });
diff --git a/Detectors/Raw/TFReaderDD/src/tf-reader-workflow.cxx b/Detectors/Raw/TFReaderDD/src/tf-reader-workflow.cxx
index facb8a880ba66..c468d1660fcc7 100644
--- a/Detectors/Raw/TFReaderDD/src/tf-reader-workflow.cxx
+++ b/Detectors/Raw/TFReaderDD/src/tf-reader-workflow.cxx
@@ -86,7 +86,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   int rateLimitingIPCID = std::stoi(configcontext.options().get<std::string>("timeframes-rate-limit-ipcid"));
   std::string chanFmt = configcontext.options().get<std::string>("metric-feedback-channel-format");
   if (rateLimitingIPCID > -1 && !chanFmt.empty()) {
-    rinp.metricChannel = fmt::format(chanFmt, o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
+    rinp.metricChannel = fmt::format(fmt::runtime(chanFmt), o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
   }
 
   WorkflowSpec specs;
diff --git a/Detectors/Raw/src/RawDumpSpec.cxx b/Detectors/Raw/src/RawDumpSpec.cxx
index 345eba5cefdc0..1cb667febe2a3 100644
--- a/Detectors/Raw/src/RawDumpSpec.cxx
+++ b/Detectors/Raw/src/RawDumpSpec.cxx
@@ -273,7 +273,7 @@ void RawDump::endOfStream(EndOfStreamContext& ec)
     auto fnm = fmt::format("{}{}{}raw.cfg", mOutDir, mOutDir.back() == '/' ? "" : "/", DetID::getName(id));
     auto fh = std::fopen(fnm.c_str(), "w");
     if (!fh) {
-      LOGP(fatal, "Failed to create configuration file {}");
+      LOGP(fatal, "Failed to create configuration file {}", fnm);
     }
     auto ws = std::fwrite(mConfigEntries[id].data(), 1, mConfigEntries[id].size(), fh);
     if (ws != mConfigEntries[id].size()) {
@@ -655,7 +655,7 @@ std::string RawDump::getBaseFileNameTRD(const header::RDHAny* rdh)
   int link = supermodule * 4 + side * 2 + ep, cru = link / 2;
   if (link >= NLinks) {
     auto flpname = fmt::format("flp-unknown_cru{}_ep{}_feeid0x{:05x}", cru, int(RDHUtils::getEndPointID(rdh)), RDHUtils::getFEEID(rdh));
-    LOGP(error, "Got wrong link {}, setting TRF file name to unrecognized flp {}", flpname);
+    LOGP(error, "Got wrong link {}, setting TRF file name to unrecognized flp {}", link, flpname);
     return flpname;
   }
   return fmt::format("alio2-cr1-flp{}_cru{}_{}", trdHWMap[cru].flpid, trdHWMap[cru].cruHWID, ep);
diff --git a/Detectors/Raw/src/rawfile-reader-workflow.cxx b/Detectors/Raw/src/rawfile-reader-workflow.cxx
index d8cc44fd5adbd..3a5440f80ed2d 100644
--- a/Detectors/Raw/src/rawfile-reader-workflow.cxx
+++ b/Detectors/Raw/src/rawfile-reader-workflow.cxx
@@ -90,7 +90,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   int rateLimitingIPCID = std::stoi(configcontext.options().get<std::string>("timeframes-rate-limit-ipcid"));
   std::string chanFmt = configcontext.options().get<std::string>("metric-feedback-channel-format");
   if (rateLimitingIPCID > -1 && !chanFmt.empty()) {
-    rinp.metricChannel = fmt::format(chanFmt, o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
+    rinp.metricChannel = fmt::format(fmt::runtime(chanFmt), o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
   }
   o2::conf::ConfigurableParam::updateFromString(configcontext.options().get<std::string>("configKeyValues"));
   auto hbfini = configcontext.options().get<std::string>("hbfutils-config");
diff --git a/Detectors/TPC/base/include/TPCBase/CalDet.h b/Detectors/TPC/base/include/TPCBase/CalDet.h
index d6b6b70119e4b..835e3638642df 100644
--- a/Detectors/TPC/base/include/TPCBase/CalDet.h
+++ b/Detectors/TPC/base/include/TPCBase/CalDet.h
@@ -463,7 +463,7 @@ void CalDet<T>::initData()
     if (!hasData) {
       mData.push_back(CalType(mPadSubset, i));
     }
-    mData[i].setName(fmt::format(frmt, mName, i));
+    mData[i].setName(fmt::format(fmt::runtime(frmt), mName, i));
   }
 }
 
diff --git a/Detectors/TPC/base/src/CDBInterface.cxx b/Detectors/TPC/base/src/CDBInterface.cxx
index 1180fd1cc8b1d..b359a04b385cb 100644
--- a/Detectors/TPC/base/src/CDBInterface.cxx
+++ b/Detectors/TPC/base/src/CDBInterface.cxx
@@ -633,5 +633,5 @@ void CDBStorage::printObjectSummary(std::string_view name, CDBType const type, M
     message += fmt::format("{:>20} = {}\n", key, value);
   }
 
-  LOGP(info, message);
+  LOGP(info, "{}", message);
 }
diff --git a/Detectors/TPC/calibration/src/CalibTreeDump.cxx b/Detectors/TPC/calibration/src/CalibTreeDump.cxx
index 3423be676566d..0748f1be55da5 100644
--- a/Detectors/TPC/calibration/src/CalibTreeDump.cxx
+++ b/Detectors/TPC/calibration/src/CalibTreeDump.cxx
@@ -223,7 +223,7 @@ void CalibTreeDump::addCalDetObjects(TTree* tree)
     std::string name = calDet.getName();
 
     if (name == "PadCalibrationObject" || name.size() == 0) {
-      name = fmt::format("calDet_{%02d}", iter);
+      name = fmt::format(fmt::runtime("calDet_{%02d}"), iter);
     }
 
     std::string meanName = fmt::format("{}_mean", name);
diff --git a/Detectors/TPC/calibration/src/CalibdEdx.cxx b/Detectors/TPC/calibration/src/CalibdEdx.cxx
index c012a4a2b3ec9..6d6aa980b9680 100644
--- a/Detectors/TPC/calibration/src/CalibdEdx.cxx
+++ b/Detectors/TPC/calibration/src/CalibdEdx.cxx
@@ -215,7 +215,8 @@ void fitHist(const Hist& hist, CalibdEdxCorrection& corr, TLinearFitter& fitter,
         corr.setChi2(id, charge, fitter.GetChisquare());
         corr.setEntries(id, charge, entries);
       }
-      LOGP(debug, "Sector: {}, gemType: {}, charge: {}, Fit pass: {} with {} % outliers in {} entries. Fitter Points: {}, mean fit: {}", id.sector, int(id.type), charge, fitPass, (float)outliers / (float)entries * 100, entries, fitter.GetNpoints(), params[0]);
+      LOGP(debug, "Sector: {}, gemType: {}, charge: {}, Fit pass: {} with {} % outliers in {} entries. Fitter Points: {}, mean fit: {}",
+           id.sector, int(id.type), int(charge), fitPass, (float)outliers / (float)entries * 100, entries, fitter.GetNpoints(), params[0]);
     }
   }
 }
diff --git a/Detectors/TPC/calibration/src/VDriftHelper.cxx b/Detectors/TPC/calibration/src/VDriftHelper.cxx
index 034888998e567..0df353adba91f 100644
--- a/Detectors/TPC/calibration/src/VDriftHelper.cxx
+++ b/Detectors/TPC/calibration/src/VDriftHelper.cxx
@@ -43,7 +43,8 @@ VDriftHelper::VDriftHelper()
   if (o2::conf::ConfigurableParam::getProvenance("TPCDetParam.DriftTimeOffset") == o2::conf::ConfigurableParam::EParamProvenance::kRT) { // we stick to this value
     mVD.creationTime = std::numeric_limits<long>::max();
     mForceParamOffset = true;
-    LOGP(info, "TPC dridt time offset was set from command line to {} mus ({} TB}, will neglect update from CCDB", mVD.refTimeOffset, detpar.DriftTimeOffset);
+    LOGP(info, "TPC dridt time offset was set from command line to {} mus ({} TB), will neglect update from CCDB",
+         mVD.refTimeOffset, detpar.DriftTimeOffset);
   }
 
   mUpdated = true;
diff --git a/Detectors/TPC/dcs/src/DCSConfigSpec.cxx b/Detectors/TPC/dcs/src/DCSConfigSpec.cxx
index 967b825d65a1d..34270cbca528d 100644
--- a/Detectors/TPC/dcs/src/DCSConfigSpec.cxx
+++ b/Detectors/TPC/dcs/src/DCSConfigSpec.cxx
@@ -289,7 +289,7 @@ void DCSConfigDevice::fillCRUConfig(gsl::span<const char> configBuff, bool updat
   }
 
   if (!update && (nLines != CRU::MaxCRU)) {
-    LOGP(error, "Full FEEConfig expected, but only {} / {} lines read for CRUConfig", nLines, CRU::MaxCRU);
+    LOGP(error, "Full FEEConfig expected, but only {} / {} lines read for CRUConfig", nLines, (int)CRU::MaxCRU);
   } else {
     LOGP(info, "updating CRUConfig for {} crus", nLines);
   }
diff --git a/Detectors/TPC/dcs/src/DCSProcessor.cxx b/Detectors/TPC/dcs/src/DCSProcessor.cxx
index 3a1572a6e6834..a26bce43e5c2e 100644
--- a/Detectors/TPC/dcs/src/DCSProcessor.cxx
+++ b/Detectors/TPC/dcs/src/DCSProcessor.cxx
@@ -69,7 +69,7 @@ float DCSProcessor::getValueF(const DPCOM& dp) const
   } else if (dp.id.get_type() == DeliveryType::DPVAL_DOUBLE) {
     return static_cast<float>(o2::dcs::getValue<double>(dp));
   } else {
-    LOGP(warning, "Unexpected delivery type for {}: {}", dp.id.get_alias(), dp.id.get_type());
+    LOGP(warning, "Unexpected delivery type for {}: {}", dp.id.get_alias(), (int)dp.id.get_type());
   }
 
   return 0.f;
@@ -94,7 +94,7 @@ void DCSProcessor::fillHV(const DPCOM& dp)
     // TODO: Remove once type is clear
     static bool statTypePrinted = false;
     if (!statTypePrinted) {
-      LOGP(info, "Delivery type for STATUS ({}): {}", alias, type);
+      LOGP(info, "Delivery type for STATUS ({}): {}", alias, (int)type);
       statTypePrinted = true;
     }
     if (type == DeliveryType::DPVAL_UINT) {
@@ -109,7 +109,7 @@ void DCSProcessor::fillHV(const DPCOM& dp)
     // TODO: Remove once type is clear
     static bool uiTypePrinted = false;
     if (!uiTypePrinted) {
-      LOGP(info, "Delivery type for current, voltage ({}): {}", alias, type);
+      LOGP(info, "Delivery type for current, voltage ({}): {}", alias, (int)type);
       uiTypePrinted = true;
     }
     const auto value = getValueF(dp);
diff --git a/Detectors/TPC/reconstruction/src/RawReaderCRU.cxx b/Detectors/TPC/reconstruction/src/RawReaderCRU.cxx
index 40063f23612ae..3a552ebcce82f 100644
--- a/Detectors/TPC/reconstruction/src/RawReaderCRU.cxx
+++ b/Detectors/TPC/reconstruction/src/RawReaderCRU.cxx
@@ -973,7 +973,7 @@ void RawReaderCRU::writeGBTDataPerLink(std::string_view outputDirectory, int max
 
       const int ep = iLink >= 12;
       const int link = iLink - (ep)*12;
-      auto outputFileName = fmt::format("{}/CRU_{:02}_EP_{}_Link_{:02}", outputDirectory.data(), mCRU, ep, link);
+      auto outputFileName = fmt::format("{}/CRU_{:02}_EP_{}_Link_{:02}", outputDirectory.data(), (int)mCRU, ep, link);
       std::ofstream outputFile(outputFileName, std::ios_base::binary | std::ios_base::app);
 
       for (auto packetNumber : linkInfo.PacketPositions) {
diff --git a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFactorizeIDCSpec.h b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFactorizeIDCSpec.h
index 69f90b2e71764..a4107690c941a 100644
--- a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFactorizeIDCSpec.h
+++ b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFactorizeIDCSpec.h
@@ -311,7 +311,7 @@ class TPCFactorizeIDCSpec : public o2::framework::Task
             o2::ccdb::CcdbObjectInfo ccdbInfoIDCDelta(CDBTypeMap.at(sideA ? CDBType::CalIDCDeltaA : CDBType::CalIDCDeltaC), std::string{}, std::string{}, std::map<std::string, std::string>{}, timeStampStartDelta, timeStampEndDelta);
 
             if (mDumpIDCDelta) {
-              mIDCGrouping.dumpToFile(fmt::format("{}DeltaAveraged_chunk{:02}_{:02}_side{}.root", getCurrentType(), iChunk, timeStampStartDelta, side).data());
+              mIDCGrouping.dumpToFile(fmt::format("{}DeltaAveraged_chunk{:02}_{:02}_side{}.root", getCurrentType(), iChunk, timeStampStartDelta, (int)side).data());
             }
 
             auto startCCDBIDCDelta = timer::now();
diff --git a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformAggregatorSpec.h b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformAggregatorSpec.h
index bb8334901b8d9..c24bbd1b245ce 100644
--- a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformAggregatorSpec.h
+++ b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformAggregatorSpec.h
@@ -125,7 +125,7 @@ class TPCFourierTransformAggregatorSpec : public o2::framework::Task
 
           if (mDumpFFT) {
             LOGP(info, "dumping FT to file");
-            mIDCFourierTransform[side].dumpToFile(fmt::format("FourierAGG_{:02}_side{}.root", processing_helpers::getCurrentTF(pc), side).data());
+            mIDCFourierTransform[side].dumpToFile(fmt::format("FourierAGG_{:02}_side{}.root", processing_helpers::getCurrentTF(pc), (int)side).data());
           }
 
           if (mSendOutDebug) {
diff --git a/Detectors/TPC/workflow/src/ClusterQCSpec.cxx b/Detectors/TPC/workflow/src/ClusterQCSpec.cxx
index 284085005ec15..8c93d2755b090 100644
--- a/Detectors/TPC/workflow/src/ClusterQCSpec.cxx
+++ b/Detectors/TPC/workflow/src/ClusterQCSpec.cxx
@@ -126,7 +126,7 @@ void ClusterQCDevice::endInterval()
   LOGP(info, "End interval for run: {}, TFs: {} - {}, creation: {} - {}, processed TFs: {}",
        mRunNumber, mFirstTF, mLastTF, mFirstCreation, mLastCreation, mClusterQC.getProcessedTFs());
 
-  const auto outputFileName = fmt::format(mOutputFileName, fmt::arg("run", mRunNumber),
+  const auto outputFileName = fmt::format(fmt::runtime(mOutputFileName), fmt::arg("run", mRunNumber),
                                           fmt::arg("firstTF", mFirstTF), fmt::arg("lastTF", mLastTF),
                                           fmt::arg("firstCreation", mFirstCreation), fmt::arg("lastCreation", mLastCreation));
   std::unique_ptr<TFile> f(TFile::Open(outputFileName.data(), "recreate"));
diff --git a/Detectors/TPC/workflow/src/IDCToVectorSpec.cxx b/Detectors/TPC/workflow/src/IDCToVectorSpec.cxx
index e9b26209a6fd1..a1b77b2774228 100644
--- a/Detectors/TPC/workflow/src/IDCToVectorSpec.cxx
+++ b/Detectors/TPC/workflow/src/IDCToVectorSpec.cxx
@@ -124,7 +124,7 @@ class IDCToVectorDevice : public o2::framework::Task
 
     // open files if necessary
     if ((mWriteDebug || mWriteDebugOnError) && !mDebugStream) {
-      const auto debugFileName = fmt::format(mDebugStreamFileName, fmt::arg("run", runNumber));
+      const auto debugFileName = fmt::format(fmt::runtime(mDebugStreamFileName), fmt::arg("run", runNumber));
       LOGP(info, "creating debug stream {}", debugFileName);
       mDebugStream = std::make_unique<o2::utils::TreeStreamRedirector>(debugFileName.data(), "recreate");
     }
@@ -134,7 +134,7 @@ class IDCToVectorDevice : public o2::framework::Task
       if (mRawDataType == 4) {
         rawType = "idc.raw";
       }
-      const auto rawFileName = fmt::format(mRawOutputFileName, fmt::arg("run", runNumber), fmt::arg("raw_type", rawType));
+      const auto rawFileName = fmt::format(fmt::runtime(mRawOutputFileName), fmt::arg("run", runNumber), fmt::arg("raw_type", rawType));
       LOGP(info, "creating raw debug file {}", rawFileName);
       mRawOutputFile.open(rawFileName, std::ios::binary);
     }
diff --git a/Detectors/TRD/qc/src/Tracking.cxx b/Detectors/TRD/qc/src/Tracking.cxx
index 98131ced41961..a827c3f5a1081 100644
--- a/Detectors/TRD/qc/src/Tracking.cxx
+++ b/Detectors/TRD/qc/src/Tracking.cxx
@@ -57,7 +57,7 @@ void Tracking::checkTrack(const TrackTRD& trkTrd, bool isTPCTRD)
   qcStruct.refGlobalTrackId = id;
   qcStruct.trackTRD = trkTrd;
 
-  LOGF(debug, "Got track with %i tracklets and ID %i", trkTrd.getNtracklets(), id);
+  LOGF(debug, "Got track with %i tracklets and ID %i", trkTrd.getNtracklets(), (int)id);
   o2::track::TrackParCov trk = isTPCTRD ? mTracksTPC[id].getParamOut() : mTracksITSTPC[id].getParamOut();
   qcStruct.trackSeed = trk;
   if (mPID) {

From 5fe1995516e022abe732ad23bbbeb4658436dc4a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 9 Nov 2023 11:55:58 +0100
Subject: [PATCH 075/726] Support ROOT v6.30.01

---
 Common/MathUtils/include/MathUtils/fit.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Common/MathUtils/include/MathUtils/fit.h b/Common/MathUtils/include/MathUtils/fit.h
index 2b1064b13fecb..00c39486a4ba0 100644
--- a/Common/MathUtils/include/MathUtils/fit.h
+++ b/Common/MathUtils/include/MathUtils/fit.h
@@ -59,7 +59,7 @@ template <typename T>
 TFitResultPtr fit(const size_t nBins, const T* arr, const T xMin, const T xMax, TF1& func, std::string_view option = "")
 {
   Foption_t fitOption;
-  ROOT::Fit::FitOptionsMake(ROOT::Fit::kHistogram, option.data(), fitOption);
+  ROOT::Fit::FitOptionsMake(ROOT::Fit::EFitObjectType::kHistogram, option.data(), fitOption);
 
   ROOT::Fit::DataRange range(xMin, xMax);
   ROOT::Fit::DataOptions opt;

From ff8b346486f537914c67310fc31ad4e47a994d38 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 9 Nov 2023 11:55:58 +0100
Subject: [PATCH 076/726] C++20 does not allow aggregate initialisation if
 default ctor present

---
 Detectors/GlobalTracking/include/GlobalTracking/MatchTPCITS.h    | 1 -
 .../Vertexing/include/DetectorsVertexing/PVertexerHelpers.h      | 1 -
 2 files changed, 2 deletions(-)

diff --git a/Detectors/GlobalTracking/include/GlobalTracking/MatchTPCITS.h b/Detectors/GlobalTracking/include/GlobalTracking/MatchTPCITS.h
index f38195dbeb6ec..6f26fca6f4d03 100644
--- a/Detectors/GlobalTracking/include/GlobalTracking/MatchTPCITS.h
+++ b/Detectors/GlobalTracking/include/GlobalTracking/MatchTPCITS.h
@@ -288,7 +288,6 @@ struct InteractionCandidate : public o2::InteractionRecord {
   int rofITS;                                         // corresponding ITS ROF entry (in the ROFRecord vectors)
   uint32_t flag;                                      // origin, etc.
   o2::dataformats::RangeReference<int, int> seedsRef; // references to AB seeds
-  InteractionCandidate() = default;
   InteractionCandidate(const o2::InteractionRecord& ir, float t, float dt, int rof, uint32_t f = 0) : o2::InteractionRecord(ir), tBracket(t - dt, t + dt), rofITS(rof), flag(f) {}
 };
 
diff --git a/Detectors/Vertexing/include/DetectorsVertexing/PVertexerHelpers.h b/Detectors/Vertexing/include/DetectorsVertexing/PVertexerHelpers.h
index ec11c6e289c71..34c2d7c15a011 100644
--- a/Detectors/Vertexing/include/DetectorsVertexing/PVertexerHelpers.h
+++ b/Detectors/Vertexing/include/DetectorsVertexing/PVertexerHelpers.h
@@ -277,7 +277,6 @@ struct InteractionCandidate : public o2::InteractionRecord {
   float time = 0;
   float amplitude = 0;
   uint32_t flag = 0; // origin, etc.
-  InteractionCandidate() = default;
 };
 
 } // namespace vertexing

From 12a0e812fe0c9dd19e7f92cc06ed6e603c729a67 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 9 Nov 2023 11:55:58 +0100
Subject: [PATCH 077/726] DPL: adapt RCombinedDS to new ROOT API

---
 .../RootAnalysisHelpers/include/Framework/RCombinedDS.h   | 8 ++++----
 Framework/RootAnalysisHelpers/src/RCombinedDS.cxx         | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/Framework/RootAnalysisHelpers/include/Framework/RCombinedDS.h b/Framework/RootAnalysisHelpers/include/Framework/RCombinedDS.h
index b3b7979232467..c4dd7bce6a86f 100644
--- a/Framework/RootAnalysisHelpers/include/Framework/RCombinedDS.h
+++ b/Framework/RootAnalysisHelpers/include/Framework/RCombinedDS.h
@@ -341,14 +341,14 @@ class RCombinedDS final : public ROOT::RDF::RDataSource
     dummy += colName.data();
     throw std::runtime_error(dummy);
   }
-  const std::vector<std::string>& GetColumnNames() const override;
+  [[nodiscard]] const std::vector<std::string>& GetColumnNames() const override;
   std::vector<std::pair<ULong64_t, ULong64_t>> GetEntryRanges() override;
-  std::string GetTypeName(std::string_view colName) const override;
-  bool HasColumn(std::string_view colName) const override;
+  [[nodiscard]] std::string GetTypeName(std::string_view colName) const override;
+  [[nodiscard]] bool HasColumn(std::string_view colName) const override;
   bool SetEntry(unsigned int slot, ULong64_t entry) override;
   void InitSlot(unsigned int slot, ULong64_t firstEntry) override;
   void SetNSlots(unsigned int nSlots) override;
-  void Initialise() override;
+  virtual void Initialise();
 };
 
 ////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/Framework/RootAnalysisHelpers/src/RCombinedDS.cxx b/Framework/RootAnalysisHelpers/src/RCombinedDS.cxx
index f8ff71f8f6583..06958991cdf14 100644
--- a/Framework/RootAnalysisHelpers/src/RCombinedDS.cxx
+++ b/Framework/RootAnalysisHelpers/src/RCombinedDS.cxx
@@ -224,8 +224,8 @@ void RCombinedDS::Initialise()
 {
   fEntryRanges = fIndex->BuildIndex(fLeftDF, fRightDF);
 
-  fLeft->Initialise();
-  fRight->Initialise();
+  fLeft->Initialize();
+  fRight->Initialize();
 }
 
 /// Creates a RDataFrame using an arrow::Table as input.

From 6499173e550fb03dad9dc881c1fba1a91e9fe3b4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 9 Nov 2023 16:05:11 +0100
Subject: [PATCH 078/726] More race conditions for exit 128

---
 Framework/Core/src/DataProcessingDevice.cxx | 9 +++++++++
 Framework/Core/src/DeviceSpecHelpers.cxx    | 3 +++
 2 files changed, 12 insertions(+)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 5542be850fb70..851cb5e193d6f 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -982,6 +982,10 @@ void DataProcessingDevice::InitTask()
     uv_signal_init(state.loop, deviceContext.sigusr1Handle);
     uv_signal_start(deviceContext.sigusr1Handle, on_signal_callback, SIGUSR1);
   }
+  // If there is any signal, we want to make sure they are active
+  for (auto& handle : state.activeSignals) {
+    handle->data = &state;
+  }
   // When we start, we must make sure that we do listen to the signal
   deviceContext.sigusr1Handle->data = &mServiceRegistry;
 
@@ -1691,6 +1695,11 @@ void DataProcessingDevice::ResetTask()
   if (deviceContext.sigusr1Handle) {
     deviceContext.sigusr1Handle->data = nullptr;
   }
+  // Makes sure we do not have a working context on
+  // shutdown.
+  for (auto& handle : ref.get<DeviceState>().activeSignals) {
+    handle->data = nullptr;
+  }
 }
 
 struct WaitBackpressurePolicy {
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 969a15fa40674..c54dd6cec7a99 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -90,6 +90,9 @@ void signal_callback(uv_signal_t* handle, int)
 {
   // We simply wake up the event loop. Nothing to be done here.
   auto* state = (DeviceState*)handle->data;
+  if (!state) {
+    return;
+  }
   state->loopReason |= DeviceState::SIGNAL_ARRIVED;
   state->loopReason |= DeviceState::DATA_INCOMING;
 }

From 22192f9c1ca8f701bec165770a594b8d01a298eb Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 10 Nov 2023 08:37:45 +0100
Subject: [PATCH 079/726] GPU: support fmt 10.1.1

---
 GPU/GPUTracking/Base/GPUReconstruction.cxx                | 2 +-
 GPU/GPUTracking/DataTypes/CalibdEdxTrackTopologyPol.cxx   | 6 +++---
 GPU/TPCFastTransformation/MultivariatePolynomial.h        | 8 ++++----
 .../MultivariatePolynomialHelper.cxx                      | 2 +-
 GPU/TPCFastTransformation/NDPiecewisePolynomials.h        | 8 ++++----
 5 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/GPU/GPUTracking/Base/GPUReconstruction.cxx b/GPU/GPUTracking/Base/GPUReconstruction.cxx
index 676ab0f6cf004..bf5ba46b6ab34 100644
--- a/GPU/GPUTracking/Base/GPUReconstruction.cxx
+++ b/GPU/GPUTracking/Base/GPUReconstruction.cxx
@@ -1215,7 +1215,7 @@ bool GPUReconstruction::CheckInstanceAvailable(DeviceType type)
   } else if (type == DeviceType::OCL2) {
     return sLibOCL2->LoadLibrary() == 0;
   } else {
-    GPUError("Error: Invalid device type %u", type);
+    GPUError("Error: Invalid device type %u", (unsigned)type);
     return false;
   }
 }
diff --git a/GPU/GPUTracking/DataTypes/CalibdEdxTrackTopologyPol.cxx b/GPU/GPUTracking/DataTypes/CalibdEdxTrackTopologyPol.cxx
index 435d2e3f56e35..aa57c8f85083e 100644
--- a/GPU/GPUTracking/DataTypes/CalibdEdxTrackTopologyPol.cxx
+++ b/GPU/GPUTracking/DataTypes/CalibdEdxTrackTopologyPol.cxx
@@ -180,13 +180,13 @@ void CalibdEdxTrackTopologyPol::writeToFile(TFile& outf, const char* name) const
 void CalibdEdxTrackTopologyPol::setFromContainer(const CalibdEdxTrackTopologyPolContainer& container)
 {
   if (2 * FFits != container.mCalibPols.size()) {
-    LOGP(warning, fmt::format("wrong number of polynomials stored! this {} container {}", 2 * FFits, container.mCalibPols.size()));
+    LOGP(warning, "wrong number of polynomials stored! this {} container {}", 2 * FFits, container.mCalibPols.size());
     return;
   }
 
   const auto nFacCont = container.mScalingFactorsqMax.size() + container.mScalingFactorsqTot.size();
   if (2 * FFits != nFacCont) {
-    LOGP(warning, fmt::format("wrong number of scaling factors stored! this {} container {}", 2 * FFits, nFacCont));
+    LOGP(warning, "wrong number of scaling factors stored! this {} container {}", 2 * FFits, nFacCont);
     return;
   }
 
@@ -214,7 +214,7 @@ void CalibdEdxTrackTopologyPol::loadFromFile(const char* fileName, const char* n
     setFromContainer(*polTmp);
     delete polTmp;
   } else {
-    LOGP(info, fmt::format("couldnt load object {} from input file", name));
+    LOGP(info, "couldnt load object {} from input file", name);
   }
 }
 
diff --git a/GPU/TPCFastTransformation/MultivariatePolynomial.h b/GPU/TPCFastTransformation/MultivariatePolynomial.h
index d2d80886a925c..bc980065f6263 100644
--- a/GPU/TPCFastTransformation/MultivariatePolynomial.h
+++ b/GPU/TPCFastTransformation/MultivariatePolynomial.h
@@ -178,7 +178,7 @@ void MultivariatePolynomial<Dim, Degree, InteractionOnly>::loadFromFile(TFile& i
     delete polTmp;
   } else {
 #ifndef GPUCA_ALIROOT_LIB
-    LOGP(info, fmt::format("couldnt load object {} from input file", name));
+    LOGP(info, "couldnt load object {} from input file", name);
 #endif
   }
 }
@@ -189,19 +189,19 @@ void MultivariatePolynomial<Dim, Degree, InteractionOnly>::setFromContainer(cons
   if constexpr (Dim > 0 && Degree > 0) {
     if (this->getDim() != container.mDim) {
 #ifndef GPUCA_ALIROOT_LIB
-      LOGP(info, fmt::format("wrong number of dimensions! this {} container {}", this->getDim(), container.mDim));
+      LOGP(info, "wrong number of dimensions! this {} container {}", this->getDim(), container.mDim);
 #endif
       return;
     }
     if (this->getDegree() != container.mDegree) {
 #ifndef GPUCA_ALIROOT_LIB
-      LOGP(info, fmt::format("wrong number of degrees! this {} container {}", this->getDegree(), container.mDegree));
+      LOGP(info, "wrong number of degrees! this {} container {}", this->getDegree(), container.mDegree);
 #endif
       return;
     }
     if (this->isInteractionOnly() != container.mInteractionOnly) {
 #ifndef GPUCA_ALIROOT_LIB
-      LOGP(info, fmt::format("InteractionOnly is set for this object to {}, but stored as {} in the container", this->isInteractionOnly(), container.mInteractionOnly));
+      LOGP(info, "InteractionOnly is set for this object to {}, but stored as {} in the container", this->isInteractionOnly(), container.mInteractionOnly);
 #endif
       return;
     }
diff --git a/GPU/TPCFastTransformation/MultivariatePolynomialHelper.cxx b/GPU/TPCFastTransformation/MultivariatePolynomialHelper.cxx
index 4d981293ca91f..1e40fcb51359e 100644
--- a/GPU/TPCFastTransformation/MultivariatePolynomialHelper.cxx
+++ b/GPU/TPCFastTransformation/MultivariatePolynomialHelper.cxx
@@ -26,7 +26,7 @@ using namespace GPUCA_NAMESPACE::gpu;
 void MultivariatePolynomialHelper<0, 0, false>::print() const
 {
 #ifndef GPUCA_NO_FMT
-  LOGP(info, getFormula().c_str());
+  LOGP(info, fmt::runtime(getFormula().c_str()));
 #endif
 }
 
diff --git a/GPU/TPCFastTransformation/NDPiecewisePolynomials.h b/GPU/TPCFastTransformation/NDPiecewisePolynomials.h
index caff727c36975..13ce49a142470 100644
--- a/GPU/TPCFastTransformation/NDPiecewisePolynomials.h
+++ b/GPU/TPCFastTransformation/NDPiecewisePolynomials.h
@@ -317,7 +317,7 @@ void NDPiecewisePolynomials<Dim, Degree, InteractionOnly>::loadFromFile(TFile& i
     delete gridTmp;
   } else {
 #ifndef GPUCA_ALIROOT_LIB
-    LOGP(info, fmt::format("couldnt load object {} from input file", name));
+    LOGP(info, "couldnt load object {} from input file", name);
 #endif
   }
 }
@@ -326,19 +326,19 @@ void NDPiecewisePolynomials<Dim, Degree, InteractionOnly>::setFromContainer(cons
 {
   if (Dim != container.mDim) {
 #ifndef GPUCA_ALIROOT_LIB
-    LOGP(info, fmt::format("wrong number of dimensions! this {} container {}", Dim, container.mDim));
+    LOGP(info, "wrong number of dimensions! this {} container {}", Dim, container.mDim);
 #endif
     return;
   }
   if (Degree != container.mDegree) {
 #ifndef GPUCA_ALIROOT_LIB
-    LOGP(info, fmt::format("wrong number of degrees! this {} container {}", Degree, container.mDegree));
+    LOGP(info, "wrong number of degrees! this {} container {}", Degree, container.mDegree);
 #endif
     return;
   }
   if (InteractionOnly != container.mInteractionOnly) {
 #ifndef GPUCA_ALIROOT_LIB
-    LOGP(info, fmt::format("InteractionOnly is set for this object to {}, but stored as {} in the container", InteractionOnly, container.mInteractionOnly));
+    LOGP(info, "InteractionOnly is set for this object to {}, but stored as {} in the container", InteractionOnly, container.mInteractionOnly);
 #endif
     return;
   }

From 00127b7aa4d7dcc0638c47ef74d8281aafc7946c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 10 Nov 2023 08:37:45 +0100
Subject: [PATCH 080/726] Framework: make sure we support fmt 10.1.1

---
 CCDB/src/CcdbApi.cxx                                     | 2 +-
 Common/Field/src/MagneticField.cxx                       | 2 +-
 Detectors/AOD/src/AODProducerWorkflowSpec.cxx            | 2 +-
 EventVisualisation/Workflow/src/FileProducer.cxx         | 2 +-
 Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx | 4 ++--
 Framework/Core/src/DPLWebSocket.cxx                      | 2 +-
 Framework/Core/src/DataOutputDirector.cxx                | 2 +-
 Framework/Core/src/DataProcessingContext.cxx             | 2 +-
 Framework/Core/src/HTTPParser.cxx                        | 4 ++--
 Framework/Core/src/runDataProcessing.cxx                 | 2 +-
 Framework/Utils/include/DPLUtils/RawParser.h             | 2 --
 Framework/Utils/src/RawParser.cxx                        | 8 ++++----
 12 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx
index aabc96cb779f8..904d979bfbd79 100644
--- a/CCDB/src/CcdbApi.cxx
+++ b/CCDB/src/CcdbApi.cxx
@@ -710,7 +710,7 @@ bool CcdbApi::retrieveBlob(std::string const& path, std::string const& targetdir
   try {
     o2::utils::createDirectoriesIfAbsent(fulltargetdir);
   } catch (std::exception e) {
-    LOGP(error, fmt::format("Could not create local snapshot cache directory {}, reason: {}", fulltargetdir, e.what()));
+    LOGP(error, "Could not create local snapshot cache directory {}, reason: {}", fulltargetdir, e.what());
     return false;
   }
 
diff --git a/Common/Field/src/MagneticField.cxx b/Common/Field/src/MagneticField.cxx
index 2e6ca6d647b11..5df6bbc0b0d34 100644
--- a/Common/Field/src/MagneticField.cxx
+++ b/Common/Field/src/MagneticField.cxx
@@ -539,7 +539,7 @@ void MagneticField::rescaleField(float l3Cur, float diCur, bool uniform, int con
   float sclL3 = l3Cur, sclDip = diCur;
   MagFieldParam::BMap_t map = getFieldMapScale(sclL3, sclDip, uniform);
   if (map != mMapType) {
-    LOGP(fatal, "Provided L3current={} DipCurrent={} uniform={} leads to map type {}, incompatible with loaded {}", l3Cur, diCur, uniform, map, mMapType);
+    LOGP(fatal, "Provided L3current={} DipCurrent={} uniform={} leads to map type {}, incompatible with loaded {}", l3Cur, diCur, uniform, (int)map, (int)mMapType);
   }
   setFactorSolenoid(sclL3);
   setFactorDipole(sclDip);
diff --git a/Detectors/AOD/src/AODProducerWorkflowSpec.cxx b/Detectors/AOD/src/AODProducerWorkflowSpec.cxx
index fa028535f379c..9b6151cb557b8 100644
--- a/Detectors/AOD/src/AODProducerWorkflowSpec.cxx
+++ b/Detectors/AOD/src/AODProducerWorkflowSpec.cxx
@@ -1221,7 +1221,7 @@ void AODProducerWorkflowDPL::fillSecondaryVertices(const o2::globaltracking::Rec
       if (item != mGIDToTableID.end()) {
         tableIdx[i] = item->second;
       } else {
-        LOG(warn) << fmt::format("Could not find a track index for prong ID {}", trIDs[i]);
+        LOG(warn) << fmt::format("Could not find a track index for prong ID {}", (int)trIDs[i]);
         missing = true;
       }
     }
diff --git a/EventVisualisation/Workflow/src/FileProducer.cxx b/EventVisualisation/Workflow/src/FileProducer.cxx
index 541afdefc4553..818ab546f3047 100644
--- a/EventVisualisation/Workflow/src/FileProducer.cxx
+++ b/EventVisualisation/Workflow/src/FileProducer.cxx
@@ -47,7 +47,7 @@ std::string FileProducer::newFileName() const
   gethostname(hostname, _POSIX_HOST_NAME_MAX);
 
   auto pid = getpid();
-  auto result = fmt::format(this->mName,
+  auto result = fmt::format(fmt::runtime(this->mName),
                             fmt::arg("hostname", hostname),
                             fmt::arg("pid", pid),
                             fmt::arg("timestamp", millisec_since_epoch),
diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 1e20f46c2e464..092857d585a02 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -246,11 +246,11 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
             // get first folder of next file
             ntf = 0;
             if (!didir->readTree(outputs, dh, fcnt, ntf, totalSizeCompressed, totalSizeUncompressed)) {
-              LOGP(fatal, "Can not retrieve tree for table {}: fileCounter {}, timeFrame {}", concrete.origin, fcnt, ntf);
+              LOGP(fatal, "Can not retrieve tree for table {}: fileCounter {}, timeFrame {}", concrete.origin.as<std::string>(), fcnt, ntf);
               throw std::runtime_error("Processing is stopped!");
             }
           } else {
-            LOGP(fatal, "Can not retrieve tree for table {}: fileCounter {}, timeFrame {}", concrete.origin, fcnt, ntf);
+            LOGP(fatal, "Can not retrieve tree for table {}: fileCounter {}, timeFrame {}", concrete.origin.as<std::string>(), fcnt, ntf);
             throw std::runtime_error("Processing is stopped!");
           }
         }
diff --git a/Framework/Core/src/DPLWebSocket.cxx b/Framework/Core/src/DPLWebSocket.cxx
index 97755727cbbd9..146a13206b55a 100644
--- a/Framework/Core/src/DPLWebSocket.cxx
+++ b/Framework/Core/src/DPLWebSocket.cxx
@@ -375,7 +375,7 @@ void WSDPLHandler::write(std::vector<uv_buf_t>& outputs)
 /// Helper to return an error
 void WSDPLHandler::error(int code, char const* message)
 {
-  static char const* errorFMT = "HTTP/1.1 {} {}\r\ncontent-type: text/plain\r\n\r\n{}: {}\r\n";
+  static constexpr auto errorFMT = "HTTP/1.1 {} {}\r\ncontent-type: text/plain\r\n\r\n{}: {}\r\n";
   std::string error = fmt::format(errorFMT, code, message, code, message);
   char* reply = strdup(error.data());
   uv_buf_t bfr = uv_buf_init(reply, error.size());
diff --git a/Framework/Core/src/DataOutputDirector.cxx b/Framework/Core/src/DataOutputDirector.cxx
index dcf7fe40b6214..078af891c9058 100644
--- a/Framework/Core/src/DataOutputDirector.cxx
+++ b/Framework/Core/src/DataOutputDirector.cxx
@@ -584,7 +584,7 @@ void DataOutputDirector::printOut()
 
   LOGP(info, "  File name bases      :");
   for (auto const& fb : mfilenameBases) {
-    LOGP(info, fb);
+    LOGP(info, "{}", fb);
   }
 }
 
diff --git a/Framework/Core/src/DataProcessingContext.cxx b/Framework/Core/src/DataProcessingContext.cxx
index 365975c706722..9c023e5021a9b 100644
--- a/Framework/Core/src/DataProcessingContext.cxx
+++ b/Framework/Core/src/DataProcessingContext.cxx
@@ -52,7 +52,7 @@ void DataProcessorContext::preDanglingCallbacks(DanglingContext& danglingContext
 void DataProcessorContext::postDanglingCallbacks(DanglingContext& danglingContext)
 {
   for (auto& handle : postDanglingHandles) {
-    LOGP(debug, "Invoking postDanglingCallback for service {} {}", handle.spec.name);
+    LOGP(debug, "Invoking postDanglingCallback for service {}", handle.spec.name);
     handle.callback(danglingContext, handle.service);
   }
 }
diff --git a/Framework/Core/src/HTTPParser.cxx b/Framework/Core/src/HTTPParser.cxx
index 137706c5f07c9..04ca6e8fdce55 100644
--- a/Framework/Core/src/HTTPParser.cxx
+++ b/Framework/Core/src/HTTPParser.cxx
@@ -189,7 +189,7 @@ void decode_websocket(char* start, size_t size, WebSocketHandler& handler)
 std::string encode_websocket_handshake_request(const char* endpoint, const char* protocol, int version, char const* nonce,
                                                std::vector<std::pair<std::string, std::string>> headers)
 {
-  char const* res =
+  constexpr auto res =
     "GET {} HTTP/1.1\r\n"
     "Upgrade: websocket\r\n"
     "Connection: Upgrade\r\n"
@@ -216,7 +216,7 @@ std::string HTTPParserHelpers::calculateAccept(const char* nonce)
 
 std::string encode_websocket_handshake_reply(char const* nonce)
 {
-  char const* res =
+  constexpr auto res =
     "HTTP/1.1 101 Switching Protocols\r\n"
     "Upgrade: websocket\r\n"
     "Connection: Upgrade\r\n"
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 61fdef070d874..86cc15299e8f3 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -696,7 +696,7 @@ void spawnDevice(uv_loop_t* loop,
   close(childFds[ref.index].childstdout[1]);
   if (varmap.count("post-fork-command")) {
     auto templateCmd = varmap["post-fork-command"];
-    auto cmd = fmt::format(templateCmd.as<std::string>(),
+    auto cmd = fmt::format(fmt::runtime(templateCmd.as<std::string>()),
                            fmt::arg("pid", id),
                            fmt::arg("id", spec.id),
                            fmt::arg("cpu", parentCPU),
diff --git a/Framework/Utils/include/DPLUtils/RawParser.h b/Framework/Utils/include/DPLUtils/RawParser.h
index 2c35c008983f0..c1ba1ef4802b2 100644
--- a/Framework/Utils/include/DPLUtils/RawParser.h
+++ b/Framework/Utils/include/DPLUtils/RawParser.h
@@ -58,14 +58,12 @@ struct RDHFormatter {
 template <>
 struct RDHFormatter<header::RAWDataHeaderV7> {
   using type = header::RAWDataHeaderV7;
-  static const char* sFormatString;
   static void apply(std::ostream&, type const&, FormatSpec, const char* = "");
 };
 
 template <>
 struct RDHFormatter<header::RAWDataHeaderV6> {
   using type = header::RAWDataHeaderV6;
-  static const char* sFormatString;
   static void apply(std::ostream&, type const&, FormatSpec, const char* = "");
 };
 
diff --git a/Framework/Utils/src/RawParser.cxx b/Framework/Utils/src/RawParser.cxx
index 2ec48c7f93d36..c7cbbf3d28880 100644
--- a/Framework/Utils/src/RawParser.cxx
+++ b/Framework/Utils/src/RawParser.cxx
@@ -51,9 +51,9 @@ void RawParserHelper::warnDeadBeef(const o2::header::DataHeader* dh)
   }
 }
 
-const char* RDHFormatter<V7>::sFormatString = "{:>5} {:>4} {:>4} {:>4} {:>4} {:>3} {:>3} {:>3}  {:>1} {:>2}";
 void RDHFormatter<V7>::apply(std::ostream& os, V7 const& header, FormatSpec choice, const char* delimiter)
 {
+  static constexpr const char* sFormatString = "{:>5} {:>4} {:>4} {:>4} {:>4} {:>3} {:>3} {:>3}  {:>1} {:>2}";
   if (choice == FormatSpec::Info) {
     os << "RDH v7";
   } else if (choice == FormatSpec::TableHeader) {
@@ -73,9 +73,9 @@ void RDHFormatter<V7>::apply(std::ostream& os, V7 const& header, FormatSpec choi
   }
   os << delimiter;
 }
-const char* RDHFormatter<V6>::sFormatString = "{:>5} {:>4} {:>4} {:>4} {:>4} {:>3} {:>3} {:>3}  {:>1}";
 void RDHFormatter<V6>::apply(std::ostream& os, V6 const& header, FormatSpec choice, const char* delimiter)
 {
+  static constexpr const char* sFormatString = "{:>5} {:>4} {:>4} {:>4} {:>4} {:>3} {:>3} {:>3}  {:>1}";
   if (choice == FormatSpec::Info) {
     os << "RDH v6";
   } else if (choice == FormatSpec::TableHeader) {
@@ -95,9 +95,9 @@ void RDHFormatter<V6>::apply(std::ostream& os, V6 const& header, FormatSpec choi
   os << delimiter;
 }
 
-const char* RDHFormatter<V5>::sFormatString = "{:>5} {:>4} {:>4} {:>4} {:>3} {:>3} {:>3}  {:>1}";
 void RDHFormatter<V5>::apply(std::ostream& os, V5 const& header, FormatSpec choice, const char* delimiter)
 {
+  static constexpr const char* sFormatString = "{:>5} {:>4} {:>4} {:>4} {:>3} {:>3} {:>3}  {:>1}";
   if (choice == FormatSpec::Info) {
     os << "RDH v5";
   } else if (choice == FormatSpec::TableHeader) {
@@ -116,9 +116,9 @@ void RDHFormatter<V5>::apply(std::ostream& os, V5 const& header, FormatSpec choi
   os << delimiter;
 }
 
-const char* RDHFormatter<V4>::sFormatString = "{:>5} {:>4} {:>4} {:>4} {:>3} {:>3} {:>3} {:>10} {:>5}  {:>1}";
 void RDHFormatter<V4>::apply(std::ostream& os, V4 const& header, FormatSpec choice, const char* delimiter)
 {
+  static constexpr const char* sFormatString = "{:>5} {:>4} {:>4} {:>4} {:>3} {:>3} {:>3} {:>10} {:>5}  {:>1}";
   if (choice == FormatSpec::Info) {
     os << "RDH v4";
   } else if (choice == FormatSpec::TableHeader) {

From 22c06b25ef80e4738f36dcd87084e6cdd4f2eb25 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 10 Nov 2023 08:37:46 +0100
Subject: [PATCH 081/726] Detectors: make sure fmt 10.1.1 is supported

---
 .../include/DetectorsCommonDataFormats/EncodedBlocks.h      | 3 ++-
 .../RecoContainerCreateTracksVariadic.h                     | 4 ++--
 DataFormats/Parameters/src/GRPTool.cxx                      | 2 +-
 Detectors/CTF/workflow/src/ctf-reader-workflow.cxx          | 2 +-
 Detectors/DCS/src/AliasExpander.cxx                         | 4 ++--
 Detectors/EMCAL/base/src/RCUTrailer.cxx                     | 4 ++--
 Detectors/HMPID/calibration/src/HMPIDDCSProcessor.cxx       | 2 +-
 Detectors/ITSMFT/MFT/alignment/src/MillePede2.cxx           | 2 +-
 Detectors/ITSMFT/common/reconstruction/src/PixelData.cxx    | 2 +-
 Detectors/ITSMFT/common/reconstruction/src/RUInfo.cxx       | 2 +-
 Detectors/MUON/MCH/Calibration/src/PedestalData.cxx         | 2 +-
 .../MUON/MCH/DigitFiltering/src/DigitFilteringSpec.cxx      | 2 +-
 Detectors/MUON/MCH/Mapping/test/src/testPadIndices.cxx      | 2 +-
 .../MUON/MCH/PreClustering/src/PreClusterFinderSpec.cxx     | 4 ++--
 .../MCH/Raw/Decoder/src/testUserLogicEndpointDecoder.cxx    | 6 +++---
 Detectors/MUON/MCH/Raw/Encoder/Digit/DigitTreeReader.cxx    | 2 +-
 Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx            | 2 +-
 Detectors/MUON/MCH/Raw/test/testClosureCoDecDigit.cxx       | 2 +-
 Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx         | 2 +-
 Detectors/MUON/MCH/Workflow/src/ClusterFinderGEMSpec.cxx    | 2 +-
 .../MUON/MCH/Workflow/src/ClusterFinderOriginalSpec.cxx     | 2 +-
 Detectors/Raw/TFReaderDD/src/tf-reader-workflow.cxx         | 2 +-
 Detectors/Raw/src/RawDumpSpec.cxx                           | 4 ++--
 Detectors/Raw/src/rawfile-reader-workflow.cxx               | 2 +-
 Detectors/TPC/base/include/TPCBase/CalDet.h                 | 2 +-
 Detectors/TPC/base/src/CDBInterface.cxx                     | 2 +-
 Detectors/TPC/base/src/FEEConfig.cxx                        | 2 +-
 Detectors/TPC/calibration/src/CalibTreeDump.cxx             | 2 +-
 Detectors/TPC/calibration/src/CalibdEdx.cxx                 | 3 ++-
 Detectors/TPC/calibration/src/VDriftHelper.cxx              | 3 ++-
 Detectors/TPC/dcs/src/DCSConfigSpec.cxx                     | 2 +-
 Detectors/TPC/dcs/src/DCSProcessor.cxx                      | 6 +++---
 Detectors/TPC/reconstruction/src/RawReaderCRU.cxx           | 2 +-
 .../TPC/workflow/include/TPCWorkflow/TPCFactorizeIDCSpec.h  | 2 +-
 .../include/TPCWorkflow/TPCFourierTransformAggregatorSpec.h | 2 +-
 Detectors/TPC/workflow/src/ClusterQCSpec.cxx                | 2 +-
 Detectors/TPC/workflow/src/IDCToVectorSpec.cxx              | 4 ++--
 Detectors/TRD/qc/src/Tracking.cxx                           | 2 +-
 38 files changed, 51 insertions(+), 48 deletions(-)

diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h
index a0853b8f14c73..6c024186b3085 100644
--- a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h
+++ b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h
@@ -402,7 +402,8 @@ class EncodedBlocks
       if (getANSHeader() == ANSVersionUnspecified) {
         rans::DenseHistogram<source_T> histogram{block.getDict(), block.getDict() + block.getNDict(), metadata.min};
         size_t renormingBits = rans::utils::sanitizeRenormingBitRange(metadata.probabilityBits);
-        LOG_IF(debug, renormingBits != metadata.probabilityBits) << fmt::format("While reading metadata from external dictionary, rANSV1 is rounding renorming precision from {} to {}");
+        LOG_IF(debug, renormingBits != metadata.probabilityBits)
+          << fmt::format("While reading metadata from external dictionary, rANSV1 is rounding renorming precision from {} to {}", metadata.probabilityBits, renormingBits);
         return rans::renorm(std::move(histogram), renormingBits, rans::RenormingPolicy::ForceIncompressible);
       } else {
         // dictionary is elias-delta coded inside the block
diff --git a/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h b/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h
index 4bbe355a781c7..4cec730c620a5 100644
--- a/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h
+++ b/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h
@@ -285,7 +285,7 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator, GTrackID
     if (srcSel[currentSource]) {
       if (matchesTPCTOF.size() && !tracksTPCTOF.size()) {
         throw std::runtime_error(fmt::format("TPC-TOF matched tracks ({}) require TPCTOF matches ({}) and TPCTOF tracks ({})",
-                                             matchesTPCTOF.size(), tracksTPCTOF.size()));
+                                             -1, matchesTPCTOF.size(), tracksTPCTOF.size()));
       }
       for (unsigned i = 0; i < matchesTPCTOF.size(); i++) {
         const auto& match = matchesTPCTOF[i];
@@ -321,7 +321,7 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator, GTrackID
     if (srcSel[currentSource]) {
       if (matchesMCHMID.size() && !tracksMCH.size()) {
         throw std::runtime_error(fmt::format("MCH-MID matched tracks ({}) require MCHMID matches ({}) and MCH tracks ({})",
-                                             matchesMCHMID.size(), tracksMCH.size()));
+                                             -1, matchesMCHMID.size(), tracksMCH.size()));
       }
       for (unsigned i = 0; i < matchesMCHMID.size(); i++) {
         const auto& match = matchesMCHMID[i];
diff --git a/DataFormats/Parameters/src/GRPTool.cxx b/DataFormats/Parameters/src/GRPTool.cxx
index 9b5078b60733c..413b08482d84a 100644
--- a/DataFormats/Parameters/src/GRPTool.cxx
+++ b/DataFormats/Parameters/src/GRPTool.cxx
@@ -205,7 +205,7 @@ bool publish(std::string const& filename, std::string const& path, std::string C
   try {
     o2::utils::createDirectoriesIfAbsent(targetdir);
   } catch (std::exception e) {
-    LOGP(error, fmt::format("Could not create local snapshot cache directory {}, reason: {}", targetdir, e.what()));
+    LOGP(error, "Could not create local snapshot cache directory {}, reason: {}", targetdir, e.what());
     return false;
   }
 
diff --git a/Detectors/CTF/workflow/src/ctf-reader-workflow.cxx b/Detectors/CTF/workflow/src/ctf-reader-workflow.cxx
index 3fcb4f69d0b8f..ef3a0f8d3c2c4 100644
--- a/Detectors/CTF/workflow/src/ctf-reader-workflow.cxx
+++ b/Detectors/CTF/workflow/src/ctf-reader-workflow.cxx
@@ -134,7 +134,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   int rateLimitingIPCID = std::stoi(configcontext.options().get<std::string>("timeframes-rate-limit-ipcid"));
   std::string chanFmt = configcontext.options().get<std::string>("metric-feedback-channel-format");
   if (rateLimitingIPCID > -1 && !chanFmt.empty()) {
-    ctfInput.metricChannel = fmt::format(chanFmt, o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
+    ctfInput.metricChannel = fmt::format(fmt::runtime(chanFmt), o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
   }
 
   specs.push_back(o2::ctf::getCTFReaderSpec(ctfInput));
diff --git a/Detectors/DCS/src/AliasExpander.cxx b/Detectors/DCS/src/AliasExpander.cxx
index b197d2fb78d67..41aaec8351f72 100644
--- a/Detectors/DCS/src/AliasExpander.cxx
+++ b/Detectors/DCS/src/AliasExpander.cxx
@@ -82,7 +82,7 @@ std::vector<std::string> extractRange(std::string range)
   std::vector<std::string> result;
 
   for (auto i = a; i <= b; i++) {
-    auto substituted = fmt::format(intFormat, i);
+    auto substituted = fmt::format(fmt::runtime(intFormat), i);
     result.push_back(substituted);
   }
   return result;
@@ -124,7 +124,7 @@ std::vector<std::string> expandAlias(const std::string& pattern)
   std::vector<std::string> result;
 
   for (auto r : range) {
-    auto substituted = fmt::format(newPattern, r);
+    auto substituted = fmt::format(fmt::runtime(newPattern), r);
     result.emplace_back(substituted);
   }
 
diff --git a/Detectors/EMCAL/base/src/RCUTrailer.cxx b/Detectors/EMCAL/base/src/RCUTrailer.cxx
index 6d10d0cb93c1d..24aac7b5bb25a 100644
--- a/Detectors/EMCAL/base/src/RCUTrailer.cxx
+++ b/Detectors/EMCAL/base/src/RCUTrailer.cxx
@@ -172,7 +172,7 @@ void RCUTrailer::setTimeSamplePhaseNS(uint64_t triggertime, uint64_t timesample)
       sample = 2;
       break;
     default:
-      throw Error(Error::ErrorType_t::SAMPLINGFREQ_INVALID, fmt::format("invalid time sample: {:f}", timesample).data());
+      throw Error(Error::ErrorType_t::SAMPLINGFREQ_INVALID, fmt::format(fmt::runtime("invalid time sample: {:f}"), timesample).data());
   };
   mAltroConfig.mSampleTime = sample;
   // calculate L1 phase
@@ -275,4 +275,4 @@ std::ostream& o2::emcal::operator<<(std::ostream& stream, const o2::emcal::RCUTr
 {
   trailer.printStream(stream);
   return stream;
-}
\ No newline at end of file
+}
diff --git a/Detectors/HMPID/calibration/src/HMPIDDCSProcessor.cxx b/Detectors/HMPID/calibration/src/HMPIDDCSProcessor.cxx
index 55bfe8fc101e7..7b3a88bf27385 100644
--- a/Detectors/HMPID/calibration/src/HMPIDDCSProcessor.cxx
+++ b/Detectors/HMPID/calibration/src/HMPIDDCSProcessor.cxx
@@ -395,7 +395,7 @@ double HMPIDDCSProcessor::calculatePhotonEnergy(int i)
   if (dp.id.get_type() == DeliveryType::DPVAL_DOUBLE) {
     lambda = o2::dcs::getValue<double>(dp);
   } else {
-    LOGP(warn, "DP type is {}", dp.id.get_type());
+    LOGP(warn, "DP type is {}", (int)dp.id.get_type());
     LOGP(warn, "Not correct datatype for HMP_TRANPLANT_MEASURE_{}_WAVELENGTH --> Default wavelength used for iteration procTrans{}", i, i);
     lambda = arrWaveLenDefault[i];
   }
diff --git a/Detectors/ITSMFT/MFT/alignment/src/MillePede2.cxx b/Detectors/ITSMFT/MFT/alignment/src/MillePede2.cxx
index af445c14f09c1..6220341f20054 100644
--- a/Detectors/ITSMFT/MFT/alignment/src/MillePede2.cxx
+++ b/Detectors/ITSMFT/MFT/alignment/src/MillePede2.cxx
@@ -1384,7 +1384,7 @@ int MillePede2::SolveGlobalMatEq()
     if (fgIterSol == MinResSolve::kSolFGMRes) {
       res = slv->SolveFGMRES(sol, fgMinResCondType, fgMinResMaxIter, fgMinResTol, fgNKrylovV);
     } else {
-      LOGF(warning, "MillePede2 - Undefined Iteritive Solver ID=%d, only %d are defined", fgIterSol, MinResSolve::kNSolvers);
+      LOGF(warning, "MillePede2 - Undefined Iteritive Solver ID=%d, only %d are defined", fgIterSol, (int)MinResSolve::kNSolvers);
     }
   }
 
diff --git a/Detectors/ITSMFT/common/reconstruction/src/PixelData.cxx b/Detectors/ITSMFT/common/reconstruction/src/PixelData.cxx
index a643f6fc4af1d..41b487786e90e 100644
--- a/Detectors/ITSMFT/common/reconstruction/src/PixelData.cxx
+++ b/Detectors/ITSMFT/common/reconstruction/src/PixelData.cxx
@@ -48,7 +48,7 @@ std::string ChipPixelData::getErrorDetails(int pos) const
     std::string rbuf = ": 0x<";
     int nc = getNBytesInRawBuff();
     for (int i = 0; i < nc; i++) {
-      rbuf += fmt::format(i ? " {:02x}" : "{:02x}", (int)getRawErrBuff()[i]);
+      rbuf += fmt::format(fmt::runtime(i ? " {:02x}" : "{:02x}"), (int)getRawErrBuff()[i]);
     }
     rbuf += '>';
     return rbuf;
diff --git a/Detectors/ITSMFT/common/reconstruction/src/RUInfo.cxx b/Detectors/ITSMFT/common/reconstruction/src/RUInfo.cxx
index 38c11d96ddd27..a05fd0504049d 100644
--- a/Detectors/ITSMFT/common/reconstruction/src/RUInfo.cxx
+++ b/Detectors/ITSMFT/common/reconstruction/src/RUInfo.cxx
@@ -35,5 +35,5 @@ std::string ChipInfo::asString() const
 
 void ChipInfo::print() const
 {
-  LOGP(info, asString());
+  LOGP(info, fmt::runtime(asString()));
 }
diff --git a/Detectors/MUON/MCH/Calibration/src/PedestalData.cxx b/Detectors/MUON/MCH/Calibration/src/PedestalData.cxx
index 0f5b377a4c92d..661bab7913b8e 100644
--- a/Detectors/MUON/MCH/Calibration/src/PedestalData.cxx
+++ b/Detectors/MUON/MCH/Calibration/src/PedestalData.cxx
@@ -120,7 +120,7 @@ void PedestalData::merge(const PedestalData* prev)
 void PedestalData::print() const
 {
   for (const auto& p : const_cast<PedestalData&>(*this)) {
-    LOGP(info, p.asString());
+    LOGP(info, fmt::runtime(p.asString()));
   }
 }
 
diff --git a/Detectors/MUON/MCH/DigitFiltering/src/DigitFilteringSpec.cxx b/Detectors/MUON/MCH/DigitFiltering/src/DigitFilteringSpec.cxx
index ea173bd1fc7cc..fe40659bc9265 100644
--- a/Detectors/MUON/MCH/DigitFiltering/src/DigitFilteringSpec.cxx
+++ b/Detectors/MUON/MCH/DigitFiltering/src/DigitFilteringSpec.cxx
@@ -88,7 +88,7 @@ class DigitFilteringTask
 
       if (!isOK(error)) {
         if (error.nofOutOfBounds > 0) {
-          LOGP(error, asString(error));
+          LOGP(error, "{}", asString(error));
           LOGP(error, "in a TF with {} rofs and {} digits", iRofs.size(), iDigits.size());
           abort = true;
         }
diff --git a/Detectors/MUON/MCH/Mapping/test/src/testPadIndices.cxx b/Detectors/MUON/MCH/Mapping/test/src/testPadIndices.cxx
index cc9d504d92e50..651a764820a1c 100644
--- a/Detectors/MUON/MCH/Mapping/test/src/testPadIndices.cxx
+++ b/Detectors/MUON/MCH/Mapping/test/src/testPadIndices.cxx
@@ -102,7 +102,7 @@ int main(int argc, char** argv)
   int nbad{0};
 
   for (auto de : deIds) {
-    std::string filepath(fmt::format(filePattern, de));
+    std::string filepath(fmt::format(fmt::runtime(filePattern), de));
     std::ifstream in(filepath);
     if (!in) {
       std::cout << "Cannot open " << filepath << "\n";
diff --git a/Detectors/MUON/MCH/PreClustering/src/PreClusterFinderSpec.cxx b/Detectors/MUON/MCH/PreClustering/src/PreClusterFinderSpec.cxx
index 2989401b55628..408ffda2ca9dc 100644
--- a/Detectors/MUON/MCH/PreClustering/src/PreClusterFinderSpec.cxx
+++ b/Detectors/MUON/MCH/PreClustering/src/PreClusterFinderSpec.cxx
@@ -83,7 +83,7 @@ class PreClusterFinderTask
       LOG(info) << "deinitializing preclusterizer in: "
                 << std::chrono::duration<double, std::milli>(tEnd - tStart).count() << " ms";
       mErrorMap.forEach([](Error error) {
-        LOGP(warning, error.asString());
+        LOGP(warning, "{}", error.asString());
       });
     };
     ic.services().get<CallbackService>().set<CallbackService::Id::Stop>(stop);
@@ -117,7 +117,7 @@ class PreClusterFinderTask
       if (!isOK(error)) {
         if (error.nofOutOfBounds > 0) {
           // FIXME: replace this error log with a counters' message ?
-          LOGP(error, asString(error));
+          LOGP(error, "{}", asString(error));
           LOGP(error, "in a TF with {} rofs and {} digits", digitROFs.size(), digits.size());
           abort = true;
         }
diff --git a/Detectors/MUON/MCH/Raw/Decoder/src/testUserLogicEndpointDecoder.cxx b/Detectors/MUON/MCH/Raw/Decoder/src/testUserLogicEndpointDecoder.cxx
index 43aefe520cb6e..519e56244aba2 100644
--- a/Detectors/MUON/MCH/Raw/Decoder/src/testUserLogicEndpointDecoder.cxx
+++ b/Detectors/MUON/MCH/Raw/Decoder/src/testUserLogicEndpointDecoder.cxx
@@ -369,7 +369,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(SyncInTheMiddleChargeSumModeTwoChannels, V, testTy
     5);
   std::string r2 = "S361-J6-DS2-ch-63-ts-345-q-123456-cs-789\n";
   r2 += "S361-J6-DS2-ch-63-ts-346-q-789012-cs-345\n";
-  r2 += fmt::format("S361-J6-DS2-chip-5-error-{}\n", ErrorUnexpectedSyncPacket);
+  r2 += fmt::format("S361-J6-DS2-chip-5-error-{}\n", (int)ErrorUnexpectedSyncPacket);
   BOOST_CHECK_EQUAL(r, r2);
 }
 
@@ -387,7 +387,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(TestCruPageBadClusterSize, V, testTypes)
   gsl::span<const uint64_t> page = CruPageBadClusterSize;
   std::string r = testPayloadDecodeCruPages<V::value>(page);
   BOOST_CHECK_EQUAL(r,
-                    fmt::format("S81-J0-DS0-chip-1-error-{}\nS81-J0-DS0-ch-42-ts-0-q-1\n", ErrorBadClusterSize));
+                    fmt::format("S81-J0-DS0-chip-1-error-{}\nS81-J0-DS0-ch-42-ts-0-q-1\n", (int)ErrorBadClusterSize));
 }
 
 BOOST_AUTO_TEST_CASE_TEMPLATE(TestCruPageBadN10bitWords, V, testTypes)
@@ -396,7 +396,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(TestCruPageBadN10bitWords, V, testTypes)
   std::string r = testPayloadDecodeCruPages<V::value>(page);
   std::string expected =
     fmt::format("S81-J0-DS0-ch-42-ts-87-q-2-1-0-0-1-0-0-0\nS81-J0-DS0-chip-1-error-{}\nS81-J0-DS0-ch-42-ts-0-q-1\n",
-                ErrorBadIncompleteWord);
+                (int)ErrorBadIncompleteWord);
   BOOST_CHECK_EQUAL(r, expected);
 }
 
diff --git a/Detectors/MUON/MCH/Raw/Encoder/Digit/DigitTreeReader.cxx b/Detectors/MUON/MCH/Raw/Encoder/Digit/DigitTreeReader.cxx
index 7b63ac2969e56..17f5f4463b16a 100644
--- a/Detectors/MUON/MCH/Raw/Encoder/Digit/DigitTreeReader.cxx
+++ b/Detectors/MUON/MCH/Raw/Encoder/Digit/DigitTreeReader.cxx
@@ -20,7 +20,7 @@ void AssertBranch(ROOT::Internal::TTreeReaderValueBase& value)
 {
   if (value.GetSetupStatus() < 0) {
     throw std::invalid_argument(fmt::format("Error {} setting up tree reader for branch {}",
-                                            value.GetSetupStatus(), value.GetBranchName()));
+                                            (int)value.GetSetupStatus(), value.GetBranchName()));
   }
 }
 
diff --git a/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx b/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
index e545c15025d8e..2407796773f57 100644
--- a/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
+++ b/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
@@ -104,7 +104,7 @@ std::vector<std::byte> paginate(gsl::span<const std::byte> buffer, const std::st
   return pages;
 }
 
-const char* sampaClusterFormat = "{}-CH{}-{}";
+constexpr const char* sampaClusterFormat = "{}-CH{}-{}";
 
 // Create a vector of SampaCluster from a string d
 // where d is of the form ts-#-bc-#-cs-#-q-# or
diff --git a/Detectors/MUON/MCH/Raw/test/testClosureCoDecDigit.cxx b/Detectors/MUON/MCH/Raw/test/testClosureCoDecDigit.cxx
index 661fd7741c954..aac72b144a80f 100644
--- a/Detectors/MUON/MCH/Raw/test/testClosureCoDecDigit.cxx
+++ b/Detectors/MUON/MCH/Raw/test/testClosureCoDecDigit.cxx
@@ -28,7 +28,7 @@
 
 using namespace o2::mch::raw;
 
-const char* sampaClusterFormat = "{}-CH{}-{}";
+constexpr const char* sampaClusterFormat = "{}-CH{}-{}";
 const bool useDummyElecMap = true;
 
 struct DePadId {
diff --git a/Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx b/Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx
index de920200bf4ff..c40df5d4b448e 100644
--- a/Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx
+++ b/Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx
@@ -105,7 +105,7 @@ class TrackFinderTask
       mTrackFinder.printTimers();
       LOG(info) << "tracking duration = " << mElapsedTime.count() << " s";
       mErrorMap.forEach([](Error error) {
-        LOGP(warning, error.asString());
+        LOGP(warning, "{}", error.asString());
       });
     };
     ic.services().get<CallbackService>().set<CallbackService::Id::Stop>(stop);
diff --git a/Detectors/MUON/MCH/Workflow/src/ClusterFinderGEMSpec.cxx b/Detectors/MUON/MCH/Workflow/src/ClusterFinderGEMSpec.cxx
index be6e415c74324..5158cc436b277 100644
--- a/Detectors/MUON/MCH/Workflow/src/ClusterFinderGEMSpec.cxx
+++ b/Detectors/MUON/MCH/Workflow/src/ClusterFinderGEMSpec.cxx
@@ -173,7 +173,7 @@ class ClusterFinderGEMTask
         mGEMDump = nullptr;
       }
       mErrorMap.forEach([](Error error) {
-        LOGP(warning, error.asString());
+        LOGP(warning, "{}", error.asString());
       });
     });
     auto stop = [this]() {
diff --git a/Detectors/MUON/MCH/Workflow/src/ClusterFinderOriginalSpec.cxx b/Detectors/MUON/MCH/Workflow/src/ClusterFinderOriginalSpec.cxx
index 2007f0634960f..8344d2837b814 100644
--- a/Detectors/MUON/MCH/Workflow/src/ClusterFinderOriginalSpec.cxx
+++ b/Detectors/MUON/MCH/Workflow/src/ClusterFinderOriginalSpec.cxx
@@ -73,7 +73,7 @@ class ClusterFinderOriginalTask
     ic.services().get<CallbackService>().set<CallbackService::Id::Stop>([this]() {
       LOG(info) << "cluster finder duration = " << mTimeClusterFinder.count() << " s";
       mErrorMap.forEach([](Error error) {
-        LOGP(warning, error.asString());
+        LOGP(warning, fmt::runtime(error.asString()));
       });
       this->mClusterFinder.deinit();
     });
diff --git a/Detectors/Raw/TFReaderDD/src/tf-reader-workflow.cxx b/Detectors/Raw/TFReaderDD/src/tf-reader-workflow.cxx
index facb8a880ba66..c468d1660fcc7 100644
--- a/Detectors/Raw/TFReaderDD/src/tf-reader-workflow.cxx
+++ b/Detectors/Raw/TFReaderDD/src/tf-reader-workflow.cxx
@@ -86,7 +86,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   int rateLimitingIPCID = std::stoi(configcontext.options().get<std::string>("timeframes-rate-limit-ipcid"));
   std::string chanFmt = configcontext.options().get<std::string>("metric-feedback-channel-format");
   if (rateLimitingIPCID > -1 && !chanFmt.empty()) {
-    rinp.metricChannel = fmt::format(chanFmt, o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
+    rinp.metricChannel = fmt::format(fmt::runtime(chanFmt), o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
   }
 
   WorkflowSpec specs;
diff --git a/Detectors/Raw/src/RawDumpSpec.cxx b/Detectors/Raw/src/RawDumpSpec.cxx
index 345eba5cefdc0..1cb667febe2a3 100644
--- a/Detectors/Raw/src/RawDumpSpec.cxx
+++ b/Detectors/Raw/src/RawDumpSpec.cxx
@@ -273,7 +273,7 @@ void RawDump::endOfStream(EndOfStreamContext& ec)
     auto fnm = fmt::format("{}{}{}raw.cfg", mOutDir, mOutDir.back() == '/' ? "" : "/", DetID::getName(id));
     auto fh = std::fopen(fnm.c_str(), "w");
     if (!fh) {
-      LOGP(fatal, "Failed to create configuration file {}");
+      LOGP(fatal, "Failed to create configuration file {}", fnm);
     }
     auto ws = std::fwrite(mConfigEntries[id].data(), 1, mConfigEntries[id].size(), fh);
     if (ws != mConfigEntries[id].size()) {
@@ -655,7 +655,7 @@ std::string RawDump::getBaseFileNameTRD(const header::RDHAny* rdh)
   int link = supermodule * 4 + side * 2 + ep, cru = link / 2;
   if (link >= NLinks) {
     auto flpname = fmt::format("flp-unknown_cru{}_ep{}_feeid0x{:05x}", cru, int(RDHUtils::getEndPointID(rdh)), RDHUtils::getFEEID(rdh));
-    LOGP(error, "Got wrong link {}, setting TRF file name to unrecognized flp {}", flpname);
+    LOGP(error, "Got wrong link {}, setting TRF file name to unrecognized flp {}", link, flpname);
     return flpname;
   }
   return fmt::format("alio2-cr1-flp{}_cru{}_{}", trdHWMap[cru].flpid, trdHWMap[cru].cruHWID, ep);
diff --git a/Detectors/Raw/src/rawfile-reader-workflow.cxx b/Detectors/Raw/src/rawfile-reader-workflow.cxx
index d8cc44fd5adbd..3a5440f80ed2d 100644
--- a/Detectors/Raw/src/rawfile-reader-workflow.cxx
+++ b/Detectors/Raw/src/rawfile-reader-workflow.cxx
@@ -90,7 +90,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   int rateLimitingIPCID = std::stoi(configcontext.options().get<std::string>("timeframes-rate-limit-ipcid"));
   std::string chanFmt = configcontext.options().get<std::string>("metric-feedback-channel-format");
   if (rateLimitingIPCID > -1 && !chanFmt.empty()) {
-    rinp.metricChannel = fmt::format(chanFmt, o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
+    rinp.metricChannel = fmt::format(fmt::runtime(chanFmt), o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
   }
   o2::conf::ConfigurableParam::updateFromString(configcontext.options().get<std::string>("configKeyValues"));
   auto hbfini = configcontext.options().get<std::string>("hbfutils-config");
diff --git a/Detectors/TPC/base/include/TPCBase/CalDet.h b/Detectors/TPC/base/include/TPCBase/CalDet.h
index d6b6b70119e4b..835e3638642df 100644
--- a/Detectors/TPC/base/include/TPCBase/CalDet.h
+++ b/Detectors/TPC/base/include/TPCBase/CalDet.h
@@ -463,7 +463,7 @@ void CalDet<T>::initData()
     if (!hasData) {
       mData.push_back(CalType(mPadSubset, i));
     }
-    mData[i].setName(fmt::format(frmt, mName, i));
+    mData[i].setName(fmt::format(fmt::runtime(frmt), mName, i));
   }
 }
 
diff --git a/Detectors/TPC/base/src/CDBInterface.cxx b/Detectors/TPC/base/src/CDBInterface.cxx
index 1180fd1cc8b1d..b359a04b385cb 100644
--- a/Detectors/TPC/base/src/CDBInterface.cxx
+++ b/Detectors/TPC/base/src/CDBInterface.cxx
@@ -633,5 +633,5 @@ void CDBStorage::printObjectSummary(std::string_view name, CDBType const type, M
     message += fmt::format("{:>20} = {}\n", key, value);
   }
 
-  LOGP(info, message);
+  LOGP(info, "{}", message);
 }
diff --git a/Detectors/TPC/base/src/FEEConfig.cxx b/Detectors/TPC/base/src/FEEConfig.cxx
index 033bab55cd8d7..e3685928b3dca 100644
--- a/Detectors/TPC/base/src/FEEConfig.cxx
+++ b/Detectors/TPC/base/src/FEEConfig.cxx
@@ -133,7 +133,7 @@ void FEEConfig::print() const
   fmt::print("{0:=^{1}}\n", message, boxWidth);
 
   if (cruConfig.size() != CRU::MaxCRU) {
-    LOGP(error, "Unexpected size of cru config:{} != {}", cruConfig.size(), CRU::MaxCRU);
+    LOGP(error, "Unexpected size of cru config:{} != {}", cruConfig.size(), (int)CRU::MaxCRU);
   } else {
     for (int iCRU = 0; iCRU < cruConfig.size(); ++iCRU) {
       const auto& c = cruConfig.at(iCRU);
diff --git a/Detectors/TPC/calibration/src/CalibTreeDump.cxx b/Detectors/TPC/calibration/src/CalibTreeDump.cxx
index 3423be676566d..0748f1be55da5 100644
--- a/Detectors/TPC/calibration/src/CalibTreeDump.cxx
+++ b/Detectors/TPC/calibration/src/CalibTreeDump.cxx
@@ -223,7 +223,7 @@ void CalibTreeDump::addCalDetObjects(TTree* tree)
     std::string name = calDet.getName();
 
     if (name == "PadCalibrationObject" || name.size() == 0) {
-      name = fmt::format("calDet_{%02d}", iter);
+      name = fmt::format(fmt::runtime("calDet_{%02d}"), iter);
     }
 
     std::string meanName = fmt::format("{}_mean", name);
diff --git a/Detectors/TPC/calibration/src/CalibdEdx.cxx b/Detectors/TPC/calibration/src/CalibdEdx.cxx
index c012a4a2b3ec9..6d6aa980b9680 100644
--- a/Detectors/TPC/calibration/src/CalibdEdx.cxx
+++ b/Detectors/TPC/calibration/src/CalibdEdx.cxx
@@ -215,7 +215,8 @@ void fitHist(const Hist& hist, CalibdEdxCorrection& corr, TLinearFitter& fitter,
         corr.setChi2(id, charge, fitter.GetChisquare());
         corr.setEntries(id, charge, entries);
       }
-      LOGP(debug, "Sector: {}, gemType: {}, charge: {}, Fit pass: {} with {} % outliers in {} entries. Fitter Points: {}, mean fit: {}", id.sector, int(id.type), charge, fitPass, (float)outliers / (float)entries * 100, entries, fitter.GetNpoints(), params[0]);
+      LOGP(debug, "Sector: {}, gemType: {}, charge: {}, Fit pass: {} with {} % outliers in {} entries. Fitter Points: {}, mean fit: {}",
+           id.sector, int(id.type), int(charge), fitPass, (float)outliers / (float)entries * 100, entries, fitter.GetNpoints(), params[0]);
     }
   }
 }
diff --git a/Detectors/TPC/calibration/src/VDriftHelper.cxx b/Detectors/TPC/calibration/src/VDriftHelper.cxx
index 034888998e567..0df353adba91f 100644
--- a/Detectors/TPC/calibration/src/VDriftHelper.cxx
+++ b/Detectors/TPC/calibration/src/VDriftHelper.cxx
@@ -43,7 +43,8 @@ VDriftHelper::VDriftHelper()
   if (o2::conf::ConfigurableParam::getProvenance("TPCDetParam.DriftTimeOffset") == o2::conf::ConfigurableParam::EParamProvenance::kRT) { // we stick to this value
     mVD.creationTime = std::numeric_limits<long>::max();
     mForceParamOffset = true;
-    LOGP(info, "TPC dridt time offset was set from command line to {} mus ({} TB}, will neglect update from CCDB", mVD.refTimeOffset, detpar.DriftTimeOffset);
+    LOGP(info, "TPC dridt time offset was set from command line to {} mus ({} TB), will neglect update from CCDB",
+         mVD.refTimeOffset, detpar.DriftTimeOffset);
   }
 
   mUpdated = true;
diff --git a/Detectors/TPC/dcs/src/DCSConfigSpec.cxx b/Detectors/TPC/dcs/src/DCSConfigSpec.cxx
index 967b825d65a1d..34270cbca528d 100644
--- a/Detectors/TPC/dcs/src/DCSConfigSpec.cxx
+++ b/Detectors/TPC/dcs/src/DCSConfigSpec.cxx
@@ -289,7 +289,7 @@ void DCSConfigDevice::fillCRUConfig(gsl::span<const char> configBuff, bool updat
   }
 
   if (!update && (nLines != CRU::MaxCRU)) {
-    LOGP(error, "Full FEEConfig expected, but only {} / {} lines read for CRUConfig", nLines, CRU::MaxCRU);
+    LOGP(error, "Full FEEConfig expected, but only {} / {} lines read for CRUConfig", nLines, (int)CRU::MaxCRU);
   } else {
     LOGP(info, "updating CRUConfig for {} crus", nLines);
   }
diff --git a/Detectors/TPC/dcs/src/DCSProcessor.cxx b/Detectors/TPC/dcs/src/DCSProcessor.cxx
index 3a1572a6e6834..a26bce43e5c2e 100644
--- a/Detectors/TPC/dcs/src/DCSProcessor.cxx
+++ b/Detectors/TPC/dcs/src/DCSProcessor.cxx
@@ -69,7 +69,7 @@ float DCSProcessor::getValueF(const DPCOM& dp) const
   } else if (dp.id.get_type() == DeliveryType::DPVAL_DOUBLE) {
     return static_cast<float>(o2::dcs::getValue<double>(dp));
   } else {
-    LOGP(warning, "Unexpected delivery type for {}: {}", dp.id.get_alias(), dp.id.get_type());
+    LOGP(warning, "Unexpected delivery type for {}: {}", dp.id.get_alias(), (int)dp.id.get_type());
   }
 
   return 0.f;
@@ -94,7 +94,7 @@ void DCSProcessor::fillHV(const DPCOM& dp)
     // TODO: Remove once type is clear
     static bool statTypePrinted = false;
     if (!statTypePrinted) {
-      LOGP(info, "Delivery type for STATUS ({}): {}", alias, type);
+      LOGP(info, "Delivery type for STATUS ({}): {}", alias, (int)type);
       statTypePrinted = true;
     }
     if (type == DeliveryType::DPVAL_UINT) {
@@ -109,7 +109,7 @@ void DCSProcessor::fillHV(const DPCOM& dp)
     // TODO: Remove once type is clear
     static bool uiTypePrinted = false;
     if (!uiTypePrinted) {
-      LOGP(info, "Delivery type for current, voltage ({}): {}", alias, type);
+      LOGP(info, "Delivery type for current, voltage ({}): {}", alias, (int)type);
       uiTypePrinted = true;
     }
     const auto value = getValueF(dp);
diff --git a/Detectors/TPC/reconstruction/src/RawReaderCRU.cxx b/Detectors/TPC/reconstruction/src/RawReaderCRU.cxx
index 40063f23612ae..3a552ebcce82f 100644
--- a/Detectors/TPC/reconstruction/src/RawReaderCRU.cxx
+++ b/Detectors/TPC/reconstruction/src/RawReaderCRU.cxx
@@ -973,7 +973,7 @@ void RawReaderCRU::writeGBTDataPerLink(std::string_view outputDirectory, int max
 
       const int ep = iLink >= 12;
       const int link = iLink - (ep)*12;
-      auto outputFileName = fmt::format("{}/CRU_{:02}_EP_{}_Link_{:02}", outputDirectory.data(), mCRU, ep, link);
+      auto outputFileName = fmt::format("{}/CRU_{:02}_EP_{}_Link_{:02}", outputDirectory.data(), (int)mCRU, ep, link);
       std::ofstream outputFile(outputFileName, std::ios_base::binary | std::ios_base::app);
 
       for (auto packetNumber : linkInfo.PacketPositions) {
diff --git a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFactorizeIDCSpec.h b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFactorizeIDCSpec.h
index 69f90b2e71764..a4107690c941a 100644
--- a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFactorizeIDCSpec.h
+++ b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFactorizeIDCSpec.h
@@ -311,7 +311,7 @@ class TPCFactorizeIDCSpec : public o2::framework::Task
             o2::ccdb::CcdbObjectInfo ccdbInfoIDCDelta(CDBTypeMap.at(sideA ? CDBType::CalIDCDeltaA : CDBType::CalIDCDeltaC), std::string{}, std::string{}, std::map<std::string, std::string>{}, timeStampStartDelta, timeStampEndDelta);
 
             if (mDumpIDCDelta) {
-              mIDCGrouping.dumpToFile(fmt::format("{}DeltaAveraged_chunk{:02}_{:02}_side{}.root", getCurrentType(), iChunk, timeStampStartDelta, side).data());
+              mIDCGrouping.dumpToFile(fmt::format("{}DeltaAveraged_chunk{:02}_{:02}_side{}.root", getCurrentType(), iChunk, timeStampStartDelta, (int)side).data());
             }
 
             auto startCCDBIDCDelta = timer::now();
diff --git a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformAggregatorSpec.h b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformAggregatorSpec.h
index bb8334901b8d9..c24bbd1b245ce 100644
--- a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformAggregatorSpec.h
+++ b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformAggregatorSpec.h
@@ -125,7 +125,7 @@ class TPCFourierTransformAggregatorSpec : public o2::framework::Task
 
           if (mDumpFFT) {
             LOGP(info, "dumping FT to file");
-            mIDCFourierTransform[side].dumpToFile(fmt::format("FourierAGG_{:02}_side{}.root", processing_helpers::getCurrentTF(pc), side).data());
+            mIDCFourierTransform[side].dumpToFile(fmt::format("FourierAGG_{:02}_side{}.root", processing_helpers::getCurrentTF(pc), (int)side).data());
           }
 
           if (mSendOutDebug) {
diff --git a/Detectors/TPC/workflow/src/ClusterQCSpec.cxx b/Detectors/TPC/workflow/src/ClusterQCSpec.cxx
index 284085005ec15..8c93d2755b090 100644
--- a/Detectors/TPC/workflow/src/ClusterQCSpec.cxx
+++ b/Detectors/TPC/workflow/src/ClusterQCSpec.cxx
@@ -126,7 +126,7 @@ void ClusterQCDevice::endInterval()
   LOGP(info, "End interval for run: {}, TFs: {} - {}, creation: {} - {}, processed TFs: {}",
        mRunNumber, mFirstTF, mLastTF, mFirstCreation, mLastCreation, mClusterQC.getProcessedTFs());
 
-  const auto outputFileName = fmt::format(mOutputFileName, fmt::arg("run", mRunNumber),
+  const auto outputFileName = fmt::format(fmt::runtime(mOutputFileName), fmt::arg("run", mRunNumber),
                                           fmt::arg("firstTF", mFirstTF), fmt::arg("lastTF", mLastTF),
                                           fmt::arg("firstCreation", mFirstCreation), fmt::arg("lastCreation", mLastCreation));
   std::unique_ptr<TFile> f(TFile::Open(outputFileName.data(), "recreate"));
diff --git a/Detectors/TPC/workflow/src/IDCToVectorSpec.cxx b/Detectors/TPC/workflow/src/IDCToVectorSpec.cxx
index e9b26209a6fd1..a1b77b2774228 100644
--- a/Detectors/TPC/workflow/src/IDCToVectorSpec.cxx
+++ b/Detectors/TPC/workflow/src/IDCToVectorSpec.cxx
@@ -124,7 +124,7 @@ class IDCToVectorDevice : public o2::framework::Task
 
     // open files if necessary
     if ((mWriteDebug || mWriteDebugOnError) && !mDebugStream) {
-      const auto debugFileName = fmt::format(mDebugStreamFileName, fmt::arg("run", runNumber));
+      const auto debugFileName = fmt::format(fmt::runtime(mDebugStreamFileName), fmt::arg("run", runNumber));
       LOGP(info, "creating debug stream {}", debugFileName);
       mDebugStream = std::make_unique<o2::utils::TreeStreamRedirector>(debugFileName.data(), "recreate");
     }
@@ -134,7 +134,7 @@ class IDCToVectorDevice : public o2::framework::Task
       if (mRawDataType == 4) {
         rawType = "idc.raw";
       }
-      const auto rawFileName = fmt::format(mRawOutputFileName, fmt::arg("run", runNumber), fmt::arg("raw_type", rawType));
+      const auto rawFileName = fmt::format(fmt::runtime(mRawOutputFileName), fmt::arg("run", runNumber), fmt::arg("raw_type", rawType));
       LOGP(info, "creating raw debug file {}", rawFileName);
       mRawOutputFile.open(rawFileName, std::ios::binary);
     }
diff --git a/Detectors/TRD/qc/src/Tracking.cxx b/Detectors/TRD/qc/src/Tracking.cxx
index 98131ced41961..a827c3f5a1081 100644
--- a/Detectors/TRD/qc/src/Tracking.cxx
+++ b/Detectors/TRD/qc/src/Tracking.cxx
@@ -57,7 +57,7 @@ void Tracking::checkTrack(const TrackTRD& trkTrd, bool isTPCTRD)
   qcStruct.refGlobalTrackId = id;
   qcStruct.trackTRD = trkTrd;
 
-  LOGF(debug, "Got track with %i tracklets and ID %i", trkTrd.getNtracklets(), id);
+  LOGF(debug, "Got track with %i tracklets and ID %i", trkTrd.getNtracklets(), (int)id);
   o2::track::TrackParCov trk = isTPCTRD ? mTracksTPC[id].getParamOut() : mTracksITSTPC[id].getParamOut();
   qcStruct.trackSeed = trk;
   if (mPID) {

From 01391a1f67f1bc2632cd122af033824485249774 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 10 Nov 2023 08:37:47 +0100
Subject: [PATCH 082/726] Support ROOT v6.30.01

---
 Common/MathUtils/include/MathUtils/fit.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Common/MathUtils/include/MathUtils/fit.h b/Common/MathUtils/include/MathUtils/fit.h
index 2b1064b13fecb..00c39486a4ba0 100644
--- a/Common/MathUtils/include/MathUtils/fit.h
+++ b/Common/MathUtils/include/MathUtils/fit.h
@@ -59,7 +59,7 @@ template <typename T>
 TFitResultPtr fit(const size_t nBins, const T* arr, const T xMin, const T xMax, TF1& func, std::string_view option = "")
 {
   Foption_t fitOption;
-  ROOT::Fit::FitOptionsMake(ROOT::Fit::kHistogram, option.data(), fitOption);
+  ROOT::Fit::FitOptionsMake(ROOT::Fit::EFitObjectType::kHistogram, option.data(), fitOption);
 
   ROOT::Fit::DataRange range(xMin, xMax);
   ROOT::Fit::DataOptions opt;

From 0cb520d62df8b696232cd1d7b6855abbf4b77022 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 10 Nov 2023 08:37:47 +0100
Subject: [PATCH 083/726] C++20 does not allow aggregate initialisation if
 default ctor present

---
 Detectors/GlobalTracking/include/GlobalTracking/MatchTPCITS.h    | 1 -
 .../Vertexing/include/DetectorsVertexing/PVertexerHelpers.h      | 1 -
 2 files changed, 2 deletions(-)

diff --git a/Detectors/GlobalTracking/include/GlobalTracking/MatchTPCITS.h b/Detectors/GlobalTracking/include/GlobalTracking/MatchTPCITS.h
index f38195dbeb6ec..6f26fca6f4d03 100644
--- a/Detectors/GlobalTracking/include/GlobalTracking/MatchTPCITS.h
+++ b/Detectors/GlobalTracking/include/GlobalTracking/MatchTPCITS.h
@@ -288,7 +288,6 @@ struct InteractionCandidate : public o2::InteractionRecord {
   int rofITS;                                         // corresponding ITS ROF entry (in the ROFRecord vectors)
   uint32_t flag;                                      // origin, etc.
   o2::dataformats::RangeReference<int, int> seedsRef; // references to AB seeds
-  InteractionCandidate() = default;
   InteractionCandidate(const o2::InteractionRecord& ir, float t, float dt, int rof, uint32_t f = 0) : o2::InteractionRecord(ir), tBracket(t - dt, t + dt), rofITS(rof), flag(f) {}
 };
 
diff --git a/Detectors/Vertexing/include/DetectorsVertexing/PVertexerHelpers.h b/Detectors/Vertexing/include/DetectorsVertexing/PVertexerHelpers.h
index ec11c6e289c71..34c2d7c15a011 100644
--- a/Detectors/Vertexing/include/DetectorsVertexing/PVertexerHelpers.h
+++ b/Detectors/Vertexing/include/DetectorsVertexing/PVertexerHelpers.h
@@ -277,7 +277,6 @@ struct InteractionCandidate : public o2::InteractionRecord {
   float time = 0;
   float amplitude = 0;
   uint32_t flag = 0; // origin, etc.
-  InteractionCandidate() = default;
 };
 
 } // namespace vertexing

From ff6ba8c1c39a7228f6bbab79b547c124d1e05c0a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 10 Nov 2023 09:00:41 +0100
Subject: [PATCH 084/726] DPL: adapt RCombinedDS to new ROOT API

---
 .../include/Framework/RCombinedDS.h           |  8 ++++----
 .../RootAnalysisHelpers/src/RCombinedDS.cxx   | 20 +++++++++++++++++--
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/Framework/RootAnalysisHelpers/include/Framework/RCombinedDS.h b/Framework/RootAnalysisHelpers/include/Framework/RCombinedDS.h
index b3b7979232467..c65f53dfb1356 100644
--- a/Framework/RootAnalysisHelpers/include/Framework/RCombinedDS.h
+++ b/Framework/RootAnalysisHelpers/include/Framework/RCombinedDS.h
@@ -341,14 +341,14 @@ class RCombinedDS final : public ROOT::RDF::RDataSource
     dummy += colName.data();
     throw std::runtime_error(dummy);
   }
-  const std::vector<std::string>& GetColumnNames() const override;
+  [[nodiscard]] const std::vector<std::string>& GetColumnNames() const override;
   std::vector<std::pair<ULong64_t, ULong64_t>> GetEntryRanges() override;
-  std::string GetTypeName(std::string_view colName) const override;
-  bool HasColumn(std::string_view colName) const override;
+  [[nodiscard]] std::string GetTypeName(std::string_view colName) const override;
+  [[nodiscard]] bool HasColumn(std::string_view colName) const override;
   bool SetEntry(unsigned int slot, ULong64_t entry) override;
   void InitSlot(unsigned int slot, ULong64_t firstEntry) override;
   void SetNSlots(unsigned int nSlots) override;
-  void Initialise() override;
+  virtual void Initialise() override;
 };
 
 ////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/Framework/RootAnalysisHelpers/src/RCombinedDS.cxx b/Framework/RootAnalysisHelpers/src/RCombinedDS.cxx
index f8ff71f8f6583..4102f4775a251 100644
--- a/Framework/RootAnalysisHelpers/src/RCombinedDS.cxx
+++ b/Framework/RootAnalysisHelpers/src/RCombinedDS.cxx
@@ -220,12 +220,28 @@ std::vector<void*> RCombinedDS::GetColumnReadersImpl(std::string_view colName, c
   O2_BUILTIN_UNREACHABLE();
 }
 
+struct DetectInitialise {
+  template <typename U>
+  static auto call(U* u) -> decltype(std::declval<U>().Initialise())
+  {
+    u->Initialise();
+  }
+
+  template <typename U>
+  static auto call(U* u) -> decltype(std::declval<U>().Initialize())
+  {
+    u->Initialize();
+  }
+};
+
+}
+
 void RCombinedDS::Initialise()
 {
   fEntryRanges = fIndex->BuildIndex(fLeftDF, fRightDF);
 
-  fLeft->Initialise();
-  fRight->Initialise();
+  DetectInitialise::call(fLeft);
+  DetectInitialise::call(fRight);
 }
 
 /// Creates a RDataFrame using an arrow::Table as input.

From 999a2b05ac5538d3896e85794988ad226b28c8a9 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 10 Nov 2023 09:20:40 +0100
Subject: [PATCH 085/726] DPL: adapt RCombinedDS to new ROOT API

---
 Framework/CMakeLists.txt                      |  5 +++--
 .../include/Framework/RCombinedDS.h           |  8 ++++----
 .../RootAnalysisHelpers/src/RCombinedDS.cxx   |  6 +++---
 Framework/TestWorkflows/CMakeLists.txt        | 19 ++++++++++---------
 4 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/Framework/CMakeLists.txt b/Framework/CMakeLists.txt
index a007d6292f238..902713823c142 100644
--- a/Framework/CMakeLists.txt
+++ b/Framework/CMakeLists.txt
@@ -22,8 +22,9 @@ add_subdirectory(CCDBSupport)
 add_subdirectory(PhysicsSupport)
 add_subdirectory(DataTakingSupport)
 
-add_subdirectory(RootAnalysisHelpers)
-
+# FIXME: disable for now, since ROOT made a non backward compatible change
+# RCombinedDS. Will reenable once we move to ROOT 6.30.01
+#add_subdirectory(RootAnalysisHelpers)
 
 # Build the GUI support only if we have DebugGUI
 if (TARGET AliceO2::DebugGUI)
diff --git a/Framework/RootAnalysisHelpers/include/Framework/RCombinedDS.h b/Framework/RootAnalysisHelpers/include/Framework/RCombinedDS.h
index b3b7979232467..c712ab025ecb7 100644
--- a/Framework/RootAnalysisHelpers/include/Framework/RCombinedDS.h
+++ b/Framework/RootAnalysisHelpers/include/Framework/RCombinedDS.h
@@ -341,14 +341,14 @@ class RCombinedDS final : public ROOT::RDF::RDataSource
     dummy += colName.data();
     throw std::runtime_error(dummy);
   }
-  const std::vector<std::string>& GetColumnNames() const override;
+  [[nodiscard]] const std::vector<std::string>& GetColumnNames() const override;
   std::vector<std::pair<ULong64_t, ULong64_t>> GetEntryRanges() override;
-  std::string GetTypeName(std::string_view colName) const override;
-  bool HasColumn(std::string_view colName) const override;
+  [[nodiscard]] std::string GetTypeName(std::string_view colName) const override;
+  [[nodiscard]] bool HasColumn(std::string_view colName) const override;
   bool SetEntry(unsigned int slot, ULong64_t entry) override;
   void InitSlot(unsigned int slot, ULong64_t firstEntry) override;
   void SetNSlots(unsigned int nSlots) override;
-  void Initialise() override;
+  void Initialize() override;
 };
 
 ////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/Framework/RootAnalysisHelpers/src/RCombinedDS.cxx b/Framework/RootAnalysisHelpers/src/RCombinedDS.cxx
index f8ff71f8f6583..1b354f8450726 100644
--- a/Framework/RootAnalysisHelpers/src/RCombinedDS.cxx
+++ b/Framework/RootAnalysisHelpers/src/RCombinedDS.cxx
@@ -220,12 +220,12 @@ std::vector<void*> RCombinedDS::GetColumnReadersImpl(std::string_view colName, c
   O2_BUILTIN_UNREACHABLE();
 }
 
-void RCombinedDS::Initialise()
+void RCombinedDS::Initialize()
 {
   fEntryRanges = fIndex->BuildIndex(fLeftDF, fRightDF);
 
-  fLeft->Initialise();
-  fRight->Initialise();
+  fLeft->Initialize();
+  fRight->Initialize();
 }
 
 /// Creates a RDataFrame using an arrow::Table as input.
diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index 05c40f034b9d2..7a396c0f2c3a9 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -37,15 +37,16 @@ o2_add_dpl_workflow(sync-reconstruction-dummy
                   SOURCES src/o2SyncReconstructionDummy.cxx
                   COMPONENT_NAME TestWorkflows)
 
-o2_add_dpl_workflow(d0-analysis
-                  SOURCES src/o2D0Analysis.cxx
-                  PUBLIC_LINK_LIBRARIES O2::FrameworkRootAnalysisHelpers
-                  COMPONENT_NAME TestWorkflows)
-
-o2_add_dpl_workflow(simple-tracks-analysis
-                  SOURCES src/o2SimpleTracksAnalysis.cxx
-                  PUBLIC_LINK_LIBRARIES O2::FrameworkRootAnalysisHelpers
-                  COMPONENT_NAME TestWorkflows)
+# FIXME: renable when the new ROOT is used
+#o2_add_dpl_workflow(d0-analysis
+#                  SOURCES src/o2D0Analysis.cxx
+#                  PUBLIC_LINK_LIBRARIES O2::FrameworkRootAnalysisHelpers
+#                  COMPONENT_NAME TestWorkflows)
+#
+#o2_add_dpl_workflow(simple-tracks-analysis
+#                  SOURCES src/o2SimpleTracksAnalysis.cxx
+#                  PUBLIC_LINK_LIBRARIES O2::FrameworkRootAnalysisHelpers
+#                  COMPONENT_NAME TestWorkflows)
 
 o2_add_dpl_workflow(analysis-task-example
                   SOURCES src/o2AnalysisTaskExample.cxx

From f0b830d3596932fd68093aeec1c7e83702b61091 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 10 Nov 2023 11:46:19 +0100
Subject: [PATCH 086/726] Keep dropping data while in ready, unless a new state
 was requested

---
 Framework/Core/src/ExternalFairMQDeviceProxy.cxx | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
index 4d9d4c11df49f..50738ed96d116 100644
--- a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
+++ b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
@@ -694,10 +694,15 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
 
       // Continue iterating until all channels have seen a new state.
       while (std::all_of(lastNewStatePending.begin(), lastNewStatePending.end(), [](bool b) { return b; }) != true) {
-        if (uv_now(deviceState.loop) - start > 5000) {
+        // In case there is a new state pending, we break out of the loop after 5 seconds.
+        if (device->NewStatePending() && uv_now(deviceState.loop) - start > 5000) {
           LOGP(info, "Timeout while draining messages, going to next state anyway.");
           break;
         }
+        // Reset the start time if we have not seen a new state yet.
+        if (!device->NewStatePending()) {
+          start = uv_now(deviceState.loop);
+        }
         fair::mq::Parts parts;
         for (size_t ci = 0; ci < deviceState.inputChannelInfos.size(); ++ci) {
           auto& info = deviceState.inputChannelInfos[ci];

From 317aff77cb0fe2dd3b2768a8a649f26dd10a66bf Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 10 Nov 2023 14:46:45 +0100
Subject: [PATCH 087/726] DPL: keep dropping data while in ready, unless a new
 state was requested

For some reason, before we were dropping data only for 5 seconds.
To be seen if this was needed to prevent some timeout by ECS / ODC.

This has now changed to keep dropping data until the "cleanup" property
changes or until there is a NewStatePending().
---
 .../include/Framework/DataProcessingDevice.h  |  1 -
 .../Core/include/Framework/DeviceContext.h    |  3 ++
 .../Core/include/Framework/DeviceState.h      |  1 +
 Framework/Core/src/DataProcessingDevice.cxx   |  9 ++--
 .../Core/src/ExternalFairMQDeviceProxy.cxx    | 48 ++++---------------
 5 files changed, 19 insertions(+), 43 deletions(-)

diff --git a/Framework/Core/include/Framework/DataProcessingDevice.h b/Framework/Core/include/Framework/DataProcessingDevice.h
index 0cbecdd4e3172..779e1aeff5340 100644
--- a/Framework/Core/include/Framework/DataProcessingDevice.h
+++ b/Framework/Core/include/Framework/DataProcessingDevice.h
@@ -119,7 +119,6 @@ class DataProcessingDevice : public fair::mq::Device
   /// Handle to wake up the main loop from other threads
   /// e.g. when FairMQ notifies some callback in an asynchronous way
   uv_async_t* mAwakeHandle = nullptr;
-  int64_t mCleanupCount = -1;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/include/Framework/DeviceContext.h b/Framework/Core/include/Framework/DeviceContext.h
index 3777e7f608b75..dd0459548e3ad 100644
--- a/Framework/Core/include/Framework/DeviceContext.h
+++ b/Framework/Core/include/Framework/DeviceContext.h
@@ -15,6 +15,9 @@
 #ifndef O2_FRAMEWORK_DEVICECONTEXT_H_
 #define O2_FRAMEWORK_DEVICECONTEXT_H_
 
+#include <cstddef>
+#include <atomic>
+
 typedef struct uv_timer_s uv_timer_t;
 typedef struct uv_signal_s uv_signal_t;
 
diff --git a/Framework/Core/include/Framework/DeviceState.h b/Framework/Core/include/Framework/DeviceState.h
index 506abdc49fb24..ae3e6c8611fbc 100644
--- a/Framework/Core/include/Framework/DeviceState.h
+++ b/Framework/Core/include/Framework/DeviceState.h
@@ -59,6 +59,7 @@ struct DeviceState {
   std::vector<InputChannelInfo> inputChannelInfos;
   StreamingState streaming = StreamingState::Streaming;
   bool quitRequested = false;
+  std::atomic<int64_t> cleanupCount = -1;
 
   /// ComputingQuotaOffers which have not yet been
   /// evaluated by the ComputingQuotaEvaluator
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 5542be850fb70..cb17a78433267 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -136,15 +136,16 @@ DataProcessingDevice::DataProcessingDevice(RunningDeviceRef running, ServiceRegi
     mServiceRegistry{registry},
     mProcessingPolicies{policies}
 {
-  GetConfig()->Subscribe<std::string>("dpl", [&cleanupCount = mCleanupCount, &registry = mServiceRegistry](const std::string& key, std::string value) {
+  GetConfig()->Subscribe<std::string>("dpl", [&registry = mServiceRegistry](const std::string& key, std::string value) {
     if (key == "cleanup") {
+      auto ref = ServiceRegistryRef{registry, ServiceRegistry::globalDeviceSalt()};
+      auto& deviceState = ref.get<DeviceState>();
+      int64_t cleanupCount = deviceState.cleanupCount.load();
       int64_t newCleanupCount = std::stoll(value);
       if (newCleanupCount <= cleanupCount) {
         return;
       }
-      cleanupCount = newCleanupCount;
-      auto ref = ServiceRegistryRef{registry, ServiceRegistry::globalDeviceSalt()};
-      auto& deviceState = ref.get<DeviceState>();
+      deviceState.cleanupCount.store(newCleanupCount);
       for (auto& info : deviceState.inputChannelInfos) {
         fair::mq::Parts parts;
         while (info.channel->Receive(parts, 0)) {
diff --git a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
index 4d9d4c11df49f..5117c861b0615 100644
--- a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
+++ b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
@@ -687,53 +687,25 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
       if (fair::mq::State{state} != fair::mq::State::Ready) {
         return;
       }
-      // We keep track of whether or not all channels have seen a new state.
-      std::vector<bool> lastNewStatePending(deviceState.inputChannelInfos.size(), false);
       uv_update_time(deviceState.loop);
-      auto start = uv_now(deviceState.loop);
-
-      // Continue iterating until all channels have seen a new state.
-      while (std::all_of(lastNewStatePending.begin(), lastNewStatePending.end(), [](bool b) { return b; }) != true) {
-        if (uv_now(deviceState.loop) - start > 5000) {
-          LOGP(info, "Timeout while draining messages, going to next state anyway.");
-          break;
-        }
+      bool doDrain = true;
+      // Cleanup count is set by the cleanup property of the device.
+      // It is incremented every time the device is cleaned up.
+      // We use it to detect when the device is cleaned up.
+      int64_t cleanupCount = deviceState.cleanupCount.load();
+
+      // Continue iterating we saw the cleanup property being reset or
+      // the device state changing.
+      while (doDrain) {
+        doDrain = device->NewStatePending() || deviceState.cleanupCount == cleanupCount;
         fair::mq::Parts parts;
         for (size_t ci = 0; ci < deviceState.inputChannelInfos.size(); ++ci) {
           auto& info = deviceState.inputChannelInfos[ci];
           // We only care about rawfmq channels.
           if (info.channelType != ChannelAccountingType::RAWFMQ) {
-            lastNewStatePending[ci] = true;
-            continue;
-          }
-          // This means we have not set things up yet. I.e. the first iteration from
-          // ready to run has not happened yet.
-          if (info.channel == nullptr) {
-            lastNewStatePending[ci] = true;
             continue;
           }
           info.channel->Receive(parts, 10);
-          // Handle both cases of state changes:
-          //
-          // - The state has been changed from the outside and FairMQ knows about it.
-          // - The state has been changed from the GUI, and deviceState.nextFairMQState knows about it.
-          //
-          // This latter case is probably better handled from DPL itself, after all it's fair to
-          // assume we need to switch state as soon as the GUI notifies us.
-          // For now we keep it here to avoid side effects.
-          lastNewStatePending[ci] = device->NewStatePending() || (deviceState.nextFairMQState.empty() == false);
-          if (parts.Size() == 0) {
-            continue;
-          }
-          if (!lastNewStatePending[ci]) {
-            LOGP(warn, "Unexpected {} message on channel {} while in Ready state. Dropping.", parts.Size(), info.channel->GetName());
-          } else if (lastNewStatePending[ci]) {
-            LOGP(detail, "Some {} parts were received on channel {} while switching away from Ready. Keeping.", parts.Size(), info.channel->GetName());
-            for (int pi = 0; pi < parts.Size(); ++pi) {
-              info.parts.fParts.emplace_back(std::move(parts.At(pi)));
-            }
-            info.readPolled = true;
-          }
         }
         // Keep state transitions going also when running with the standalone GUI.
         uv_run(deviceState.loop, UV_RUN_NOWAIT);

From d6b639c617d6c9b8d3893e599e2600a51727efa9 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 10 Nov 2023 14:46:44 +0100
Subject: [PATCH 088/726] Follow coding conventions

---
 .../Core/include/Framework/DataProcessingDevice.h  | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/Framework/Core/include/Framework/DataProcessingDevice.h b/Framework/Core/include/Framework/DataProcessingDevice.h
index 2eade530edc3a..0cbecdd4e3172 100644
--- a/Framework/Core/include/Framework/DataProcessingDevice.h
+++ b/Framework/Core/include/Framework/DataProcessingDevice.h
@@ -107,15 +107,15 @@ class DataProcessingDevice : public fair::mq::Device
   std::unique_ptr<ConfigParamRegistry> mConfigRegistry;
   ServiceRegistry& mServiceRegistry;
 
-  uint64_t mLastSlowMetricSentTimestamp = 0;         /// The timestamp of the last time we sent slow metrics
-  uint64_t mLastMetricFlushedTimestamp = 0;          /// The timestamp of the last time we actually flushed metrics
-  uint64_t mBeginIterationTimestamp = 0;             /// The timestamp of when the current ConditionalRun was started
+  uint64_t mLastSlowMetricSentTimestamp = 0;             /// The timestamp of the last time we sent slow metrics
+  uint64_t mLastMetricFlushedTimestamp = 0;              /// The timestamp of the last time we actually flushed metrics
+  uint64_t mBeginIterationTimestamp = 0;                 /// The timestamp of when the current ConditionalRun was started
   std::vector<fair::mq::RegionInfo> mPendingRegionInfos; /// A list of the region infos not yet notified.
   std::mutex mRegionInfoMutex;
-  ProcessingPolicies mProcessingPolicies;                        /// User policies related to data processing
-  bool mWasActive = false;                                       /// Whether or not the device was active at last iteration.
-  std::vector<uv_work_t> mHandles;                               /// Handles to use to schedule work.
-  std::vector<TaskStreamInfo> mStreams;                          /// Information about the task running in the associated mHandle.
+  ProcessingPolicies mProcessingPolicies; /// User policies related to data processing
+  bool mWasActive = false;                /// Whether or not the device was active at last iteration.
+  std::vector<uv_work_t> mHandles;        /// Handles to use to schedule work.
+  std::vector<TaskStreamInfo> mStreams;   /// Information about the task running in the associated mHandle.
   /// Handle to wake up the main loop from other threads
   /// e.g. when FairMQ notifies some callback in an asynchronous way
   uv_async_t* mAwakeHandle = nullptr;

From 2d9ad4c6f4a76b3c3b3f7b8f1be039b17cc337f0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 10 Nov 2023 14:47:43 +0100
Subject: [PATCH 089/726] DPL: keep dropping data while in ready, unless a new
 state was requested

For some reason, before we were dropping data only for 5 seconds.
To be seen if this was needed to prevent some timeout by ECS / ODC.

This has now changed to keep dropping data until the "cleanup" property
changes or until there is a NewStatePending().
---
 .../include/Framework/DataProcessingDevice.h  |  1 -
 .../Core/include/Framework/DeviceState.h      |  1 +
 Framework/Core/src/DataProcessingDevice.cxx   |  9 ++--
 .../Core/src/ExternalFairMQDeviceProxy.cxx    | 48 ++++---------------
 4 files changed, 16 insertions(+), 43 deletions(-)

diff --git a/Framework/Core/include/Framework/DataProcessingDevice.h b/Framework/Core/include/Framework/DataProcessingDevice.h
index 0cbecdd4e3172..779e1aeff5340 100644
--- a/Framework/Core/include/Framework/DataProcessingDevice.h
+++ b/Framework/Core/include/Framework/DataProcessingDevice.h
@@ -119,7 +119,6 @@ class DataProcessingDevice : public fair::mq::Device
   /// Handle to wake up the main loop from other threads
   /// e.g. when FairMQ notifies some callback in an asynchronous way
   uv_async_t* mAwakeHandle = nullptr;
-  int64_t mCleanupCount = -1;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/include/Framework/DeviceState.h b/Framework/Core/include/Framework/DeviceState.h
index 506abdc49fb24..ae3e6c8611fbc 100644
--- a/Framework/Core/include/Framework/DeviceState.h
+++ b/Framework/Core/include/Framework/DeviceState.h
@@ -59,6 +59,7 @@ struct DeviceState {
   std::vector<InputChannelInfo> inputChannelInfos;
   StreamingState streaming = StreamingState::Streaming;
   bool quitRequested = false;
+  std::atomic<int64_t> cleanupCount = -1;
 
   /// ComputingQuotaOffers which have not yet been
   /// evaluated by the ComputingQuotaEvaluator
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 5542be850fb70..cb17a78433267 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -136,15 +136,16 @@ DataProcessingDevice::DataProcessingDevice(RunningDeviceRef running, ServiceRegi
     mServiceRegistry{registry},
     mProcessingPolicies{policies}
 {
-  GetConfig()->Subscribe<std::string>("dpl", [&cleanupCount = mCleanupCount, &registry = mServiceRegistry](const std::string& key, std::string value) {
+  GetConfig()->Subscribe<std::string>("dpl", [&registry = mServiceRegistry](const std::string& key, std::string value) {
     if (key == "cleanup") {
+      auto ref = ServiceRegistryRef{registry, ServiceRegistry::globalDeviceSalt()};
+      auto& deviceState = ref.get<DeviceState>();
+      int64_t cleanupCount = deviceState.cleanupCount.load();
       int64_t newCleanupCount = std::stoll(value);
       if (newCleanupCount <= cleanupCount) {
         return;
       }
-      cleanupCount = newCleanupCount;
-      auto ref = ServiceRegistryRef{registry, ServiceRegistry::globalDeviceSalt()};
-      auto& deviceState = ref.get<DeviceState>();
+      deviceState.cleanupCount.store(newCleanupCount);
       for (auto& info : deviceState.inputChannelInfos) {
         fair::mq::Parts parts;
         while (info.channel->Receive(parts, 0)) {
diff --git a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
index 4d9d4c11df49f..5117c861b0615 100644
--- a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
+++ b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
@@ -687,53 +687,25 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
       if (fair::mq::State{state} != fair::mq::State::Ready) {
         return;
       }
-      // We keep track of whether or not all channels have seen a new state.
-      std::vector<bool> lastNewStatePending(deviceState.inputChannelInfos.size(), false);
       uv_update_time(deviceState.loop);
-      auto start = uv_now(deviceState.loop);
-
-      // Continue iterating until all channels have seen a new state.
-      while (std::all_of(lastNewStatePending.begin(), lastNewStatePending.end(), [](bool b) { return b; }) != true) {
-        if (uv_now(deviceState.loop) - start > 5000) {
-          LOGP(info, "Timeout while draining messages, going to next state anyway.");
-          break;
-        }
+      bool doDrain = true;
+      // Cleanup count is set by the cleanup property of the device.
+      // It is incremented every time the device is cleaned up.
+      // We use it to detect when the device is cleaned up.
+      int64_t cleanupCount = deviceState.cleanupCount.load();
+
+      // Continue iterating we saw the cleanup property being reset or
+      // the device state changing.
+      while (doDrain) {
+        doDrain = device->NewStatePending() || deviceState.cleanupCount == cleanupCount;
         fair::mq::Parts parts;
         for (size_t ci = 0; ci < deviceState.inputChannelInfos.size(); ++ci) {
           auto& info = deviceState.inputChannelInfos[ci];
           // We only care about rawfmq channels.
           if (info.channelType != ChannelAccountingType::RAWFMQ) {
-            lastNewStatePending[ci] = true;
-            continue;
-          }
-          // This means we have not set things up yet. I.e. the first iteration from
-          // ready to run has not happened yet.
-          if (info.channel == nullptr) {
-            lastNewStatePending[ci] = true;
             continue;
           }
           info.channel->Receive(parts, 10);
-          // Handle both cases of state changes:
-          //
-          // - The state has been changed from the outside and FairMQ knows about it.
-          // - The state has been changed from the GUI, and deviceState.nextFairMQState knows about it.
-          //
-          // This latter case is probably better handled from DPL itself, after all it's fair to
-          // assume we need to switch state as soon as the GUI notifies us.
-          // For now we keep it here to avoid side effects.
-          lastNewStatePending[ci] = device->NewStatePending() || (deviceState.nextFairMQState.empty() == false);
-          if (parts.Size() == 0) {
-            continue;
-          }
-          if (!lastNewStatePending[ci]) {
-            LOGP(warn, "Unexpected {} message on channel {} while in Ready state. Dropping.", parts.Size(), info.channel->GetName());
-          } else if (lastNewStatePending[ci]) {
-            LOGP(detail, "Some {} parts were received on channel {} while switching away from Ready. Keeping.", parts.Size(), info.channel->GetName());
-            for (int pi = 0; pi < parts.Size(); ++pi) {
-              info.parts.fParts.emplace_back(std::move(parts.At(pi)));
-            }
-            info.readPolled = true;
-          }
         }
         // Keep state transitions going also when running with the standalone GUI.
         uv_run(deviceState.loop, UV_RUN_NOWAIT);

From b0ce3a22d9526f63145c05887c704ab0543868a2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 10 Nov 2023 15:19:22 +0100
Subject: [PATCH 090/726] Fix wrong index in test

---
 Detectors/CTF/test/test_ctf_io_zdc.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Detectors/CTF/test/test_ctf_io_zdc.cxx b/Detectors/CTF/test/test_ctf_io_zdc.cxx
index 87563f128a737..3ae6f5b991ad1 100644
--- a/Detectors/CTF/test/test_ctf_io_zdc.cxx
+++ b/Detectors/CTF/test/test_ctf_io_zdc.cxx
@@ -81,7 +81,7 @@ BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion)
     pedsdata[i].ir = irPed;
     for (int ic = 0; ic < NChannels; ic++) {
       pedsdata[i].data[ic] = gRandom->Integer(0xffff);
-      pedsdata[i].scaler[ic] = (i > 0 ? pedsdata[i].scaler[ic - 1] : 0) + gRandom->Integer(20);
+      pedsdata[i].scaler[ic] = (ic > 0 ? pedsdata[i].scaler[ic - 1] : 0) + gRandom->Integer(20);
     }
     irPed.orbit++;
   }

From 7cba8dcde3d6ff971f352b0f379b5e77fe4da752 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 10 Nov 2023 22:05:46 +0100
Subject: [PATCH 091/726] Follow coding conventions

---
 .../Core/include/Framework/DataProcessingDevice.h  | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/Framework/Core/include/Framework/DataProcessingDevice.h b/Framework/Core/include/Framework/DataProcessingDevice.h
index 2eade530edc3a..0cbecdd4e3172 100644
--- a/Framework/Core/include/Framework/DataProcessingDevice.h
+++ b/Framework/Core/include/Framework/DataProcessingDevice.h
@@ -107,15 +107,15 @@ class DataProcessingDevice : public fair::mq::Device
   std::unique_ptr<ConfigParamRegistry> mConfigRegistry;
   ServiceRegistry& mServiceRegistry;
 
-  uint64_t mLastSlowMetricSentTimestamp = 0;         /// The timestamp of the last time we sent slow metrics
-  uint64_t mLastMetricFlushedTimestamp = 0;          /// The timestamp of the last time we actually flushed metrics
-  uint64_t mBeginIterationTimestamp = 0;             /// The timestamp of when the current ConditionalRun was started
+  uint64_t mLastSlowMetricSentTimestamp = 0;             /// The timestamp of the last time we sent slow metrics
+  uint64_t mLastMetricFlushedTimestamp = 0;              /// The timestamp of the last time we actually flushed metrics
+  uint64_t mBeginIterationTimestamp = 0;                 /// The timestamp of when the current ConditionalRun was started
   std::vector<fair::mq::RegionInfo> mPendingRegionInfos; /// A list of the region infos not yet notified.
   std::mutex mRegionInfoMutex;
-  ProcessingPolicies mProcessingPolicies;                        /// User policies related to data processing
-  bool mWasActive = false;                                       /// Whether or not the device was active at last iteration.
-  std::vector<uv_work_t> mHandles;                               /// Handles to use to schedule work.
-  std::vector<TaskStreamInfo> mStreams;                          /// Information about the task running in the associated mHandle.
+  ProcessingPolicies mProcessingPolicies; /// User policies related to data processing
+  bool mWasActive = false;                /// Whether or not the device was active at last iteration.
+  std::vector<uv_work_t> mHandles;        /// Handles to use to schedule work.
+  std::vector<TaskStreamInfo> mStreams;   /// Information about the task running in the associated mHandle.
   /// Handle to wake up the main loop from other threads
   /// e.g. when FairMQ notifies some callback in an asynchronous way
   uv_async_t* mAwakeHandle = nullptr;

From 036b41a457987b3c01e6afb31a723d30556861f3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 10 Nov 2023 22:06:38 +0100
Subject: [PATCH 092/726] DPL: keep dropping data while in ready, unless a new
 state was requested

For some reason, before we were dropping data only for 5 seconds.
To be seen if this was needed to prevent some timeout by ECS / ODC.

This has now changed to keep dropping data until the "cleanup" property
changes or until there is a NewStatePending().
---
 .../include/Framework/DataProcessingDevice.h  |  1 -
 .../Core/include/Framework/DeviceState.h      |  2 +
 Framework/Core/src/DataProcessingDevice.cxx   |  9 ++--
 .../Core/src/ExternalFairMQDeviceProxy.cxx    | 48 ++++---------------
 4 files changed, 17 insertions(+), 43 deletions(-)

diff --git a/Framework/Core/include/Framework/DataProcessingDevice.h b/Framework/Core/include/Framework/DataProcessingDevice.h
index 0cbecdd4e3172..779e1aeff5340 100644
--- a/Framework/Core/include/Framework/DataProcessingDevice.h
+++ b/Framework/Core/include/Framework/DataProcessingDevice.h
@@ -119,7 +119,6 @@ class DataProcessingDevice : public fair::mq::Device
   /// Handle to wake up the main loop from other threads
   /// e.g. when FairMQ notifies some callback in an asynchronous way
   uv_async_t* mAwakeHandle = nullptr;
-  int64_t mCleanupCount = -1;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/include/Framework/DeviceState.h b/Framework/Core/include/Framework/DeviceState.h
index 506abdc49fb24..3be0302fd4113 100644
--- a/Framework/Core/include/Framework/DeviceState.h
+++ b/Framework/Core/include/Framework/DeviceState.h
@@ -19,6 +19,7 @@
 #include <string>
 #include <map>
 #include <utility>
+#include <atomic>
 
 typedef struct uv_loop_s uv_loop_t;
 typedef struct uv_timer_s uv_timer_t;
@@ -59,6 +60,7 @@ struct DeviceState {
   std::vector<InputChannelInfo> inputChannelInfos;
   StreamingState streaming = StreamingState::Streaming;
   bool quitRequested = false;
+  std::atomic<int64_t> cleanupCount = -1;
 
   /// ComputingQuotaOffers which have not yet been
   /// evaluated by the ComputingQuotaEvaluator
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 851cb5e193d6f..4d0cac7087611 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -136,15 +136,16 @@ DataProcessingDevice::DataProcessingDevice(RunningDeviceRef running, ServiceRegi
     mServiceRegistry{registry},
     mProcessingPolicies{policies}
 {
-  GetConfig()->Subscribe<std::string>("dpl", [&cleanupCount = mCleanupCount, &registry = mServiceRegistry](const std::string& key, std::string value) {
+  GetConfig()->Subscribe<std::string>("dpl", [&registry = mServiceRegistry](const std::string& key, std::string value) {
     if (key == "cleanup") {
+      auto ref = ServiceRegistryRef{registry, ServiceRegistry::globalDeviceSalt()};
+      auto& deviceState = ref.get<DeviceState>();
+      int64_t cleanupCount = deviceState.cleanupCount.load();
       int64_t newCleanupCount = std::stoll(value);
       if (newCleanupCount <= cleanupCount) {
         return;
       }
-      cleanupCount = newCleanupCount;
-      auto ref = ServiceRegistryRef{registry, ServiceRegistry::globalDeviceSalt()};
-      auto& deviceState = ref.get<DeviceState>();
+      deviceState.cleanupCount.store(newCleanupCount);
       for (auto& info : deviceState.inputChannelInfos) {
         fair::mq::Parts parts;
         while (info.channel->Receive(parts, 0)) {
diff --git a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
index 4d9d4c11df49f..5117c861b0615 100644
--- a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
+++ b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
@@ -687,53 +687,25 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
       if (fair::mq::State{state} != fair::mq::State::Ready) {
         return;
       }
-      // We keep track of whether or not all channels have seen a new state.
-      std::vector<bool> lastNewStatePending(deviceState.inputChannelInfos.size(), false);
       uv_update_time(deviceState.loop);
-      auto start = uv_now(deviceState.loop);
-
-      // Continue iterating until all channels have seen a new state.
-      while (std::all_of(lastNewStatePending.begin(), lastNewStatePending.end(), [](bool b) { return b; }) != true) {
-        if (uv_now(deviceState.loop) - start > 5000) {
-          LOGP(info, "Timeout while draining messages, going to next state anyway.");
-          break;
-        }
+      bool doDrain = true;
+      // Cleanup count is set by the cleanup property of the device.
+      // It is incremented every time the device is cleaned up.
+      // We use it to detect when the device is cleaned up.
+      int64_t cleanupCount = deviceState.cleanupCount.load();
+
+      // Continue iterating we saw the cleanup property being reset or
+      // the device state changing.
+      while (doDrain) {
+        doDrain = device->NewStatePending() || deviceState.cleanupCount == cleanupCount;
         fair::mq::Parts parts;
         for (size_t ci = 0; ci < deviceState.inputChannelInfos.size(); ++ci) {
           auto& info = deviceState.inputChannelInfos[ci];
           // We only care about rawfmq channels.
           if (info.channelType != ChannelAccountingType::RAWFMQ) {
-            lastNewStatePending[ci] = true;
-            continue;
-          }
-          // This means we have not set things up yet. I.e. the first iteration from
-          // ready to run has not happened yet.
-          if (info.channel == nullptr) {
-            lastNewStatePending[ci] = true;
             continue;
           }
           info.channel->Receive(parts, 10);
-          // Handle both cases of state changes:
-          //
-          // - The state has been changed from the outside and FairMQ knows about it.
-          // - The state has been changed from the GUI, and deviceState.nextFairMQState knows about it.
-          //
-          // This latter case is probably better handled from DPL itself, after all it's fair to
-          // assume we need to switch state as soon as the GUI notifies us.
-          // For now we keep it here to avoid side effects.
-          lastNewStatePending[ci] = device->NewStatePending() || (deviceState.nextFairMQState.empty() == false);
-          if (parts.Size() == 0) {
-            continue;
-          }
-          if (!lastNewStatePending[ci]) {
-            LOGP(warn, "Unexpected {} message on channel {} while in Ready state. Dropping.", parts.Size(), info.channel->GetName());
-          } else if (lastNewStatePending[ci]) {
-            LOGP(detail, "Some {} parts were received on channel {} while switching away from Ready. Keeping.", parts.Size(), info.channel->GetName());
-            for (int pi = 0; pi < parts.Size(); ++pi) {
-              info.parts.fParts.emplace_back(std::move(parts.At(pi)));
-            }
-            info.readPolled = true;
-          }
         }
         // Keep state transitions going also when running with the standalone GUI.
         uv_run(deviceState.loop, UV_RUN_NOWAIT);

From cd50cd9a0bc25c9fc58be8ed22e7b148d87c9cd2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 13 Nov 2023 12:16:04 +0100
Subject: [PATCH 093/726] DPL: fix logic in the READY state flushing

---
 Framework/Core/src/ExternalFairMQDeviceProxy.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
index 5117c861b0615..efd66b8499ab0 100644
--- a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
+++ b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
@@ -697,7 +697,7 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
       // Continue iterating we saw the cleanup property being reset or
       // the device state changing.
       while (doDrain) {
-        doDrain = device->NewStatePending() || deviceState.cleanupCount == cleanupCount;
+        doDrain = device->NewStatePending() == false && deviceState.cleanupCount == cleanupCount;
         fair::mq::Parts parts;
         for (size_t ci = 0; ci < deviceState.inputChannelInfos.size(); ++ci) {
           auto& info = deviceState.inputChannelInfos[ci];

From d080a1d0774b2c1746708e73abc85c7975c4e6ec Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 13 Nov 2023 12:39:54 +0100
Subject: [PATCH 094/726] DPL: print missing data when dropping timeframes

---
 Framework/Core/src/DataRelayer.cxx | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index 266f5b054add1..e9591a9cc76ea 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -297,11 +297,13 @@ void DataRelayer::setOldestPossibleInput(TimesliceId proposed, ChannelIndex chan
       continue;
     }
     mPruneOps.push_back(PruneOp{si});
+    bool didDrop = false;
     for (size_t mi = 0; mi < mInputs.size(); ++mi) {
       auto& input = mInputs[mi];
       auto& element = mCache[si * mInputs.size() + mi];
       if (element.size() != 0) {
         if (input.lifetime != Lifetime::Condition && mCompletionPolicy.name != "internal-dpl-injected-dummy-sink") {
+          didDrop = true;
           LOGP(error, "Dropping incomplete {} Lifetime::{} data in slot {} with timestamp {} < {} as it can never be completed.", DataSpecUtils::describe(input), input.lifetime, si, timestamp.value, newOldest.timeslice.value);
         } else {
           LOGP(debug,
@@ -312,6 +314,16 @@ void DataRelayer::setOldestPossibleInput(TimesliceId proposed, ChannelIndex chan
         }
       }
     }
+    // We did drop some data. Let's print what was missing.
+    if (didDrop) {
+      for (size_t mi = 0; mi < mInputs.size(); ++mi) {
+        auto& input = mInputs[mi];
+        auto& element = mCache[si * mInputs.size() + mi];
+        if (element.size() == 0) {
+          LOGP(error, "Missing {} (lifetime:{}) while dropping incomplete data in slot {} with timestamp {} < {}.", DataSpecUtils::describe(input), input.lifetime, si, timestamp.value, newOldest.timeslice.value);
+        }
+      }
+    }
   }
 }
 

From 060946d80e2842e6e36b15772de134453d932b1f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 13 Nov 2023 14:06:59 +0100
Subject: [PATCH 095/726] DPL: complain if inputs and outputs are incompatible

---
 Framework/Core/src/DeviceSpecHelpers.cxx |  2 ++
 Framework/Core/src/WorkflowHelpers.cxx   | 26 ++++++++++++++++++++++++
 Framework/Core/src/WorkflowHelpers.h     |  8 ++++++++
 3 files changed, 36 insertions(+)

diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index c54dd6cec7a99..45a76570dbf1b 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1072,6 +1072,8 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
 
   WorkflowHelpers::constructGraph(workflow, logicalEdges, outputs, availableForwardsInfo);
 
+  WorkflowHelpers::validateEdges(workflow, logicalEdges, outputs);
+
   // We need to instanciate one device per (me, timeIndex) in the
   // DeviceConnectionEdge. For each device we need one new binding
   // server per (me, other) -> port Moreover for each (me, other,
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index cd9c3913de228..68d6554ccd052 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -1233,4 +1233,30 @@ std::vector<InputSpec> WorkflowHelpers::computeDanglingOutputs(WorkflowSpec cons
 }
 
 #pragma diagnostic pop
+void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
+                                    std::vector<DeviceConnectionEdge> const& edges,
+                                    std::vector<OutputSpec> const& outputs)
+{
+  // Iterate over all the edges.
+  // Get the input lifetime and the output lifetime.
+  // Output lifetime must be Timeframe if the input lifetime is Timeframe.
+  std::string errors;
+  for (auto& edge : edges) {
+    auto& producer = workflow[edge.producer];
+    auto& consumer = workflow[edge.consumer];
+    auto& output = outputs[edge.outputGlobalIndex];
+    auto& input = consumer.inputs[edge.consumerInputIndex];
+    // We cannot simply say "!Timeframe" because we want to allow
+    // e.g. Enumeration => Timeframe
+    if (input.lifetime == Lifetime::Timeframe && output.lifetime == Lifetime::Sporadic) {
+      errors += fmt::format("Input {} of {} has lifetime Timeframe, but output {} of {} has lifetime Sporadic",
+                            DataSpecUtils::describe(input).c_str(), consumer.name,
+                            DataSpecUtils::describe(output).c_str(), producer.name);
+    }
+  }
+  if (!errors.empty()) {
+    throw std::runtime_error(errors);
+  }
+}
+
 } // namespace o2::framework
diff --git a/Framework/Core/src/WorkflowHelpers.h b/Framework/Core/src/WorkflowHelpers.h
index a167e99d4768c..67661dbdb979a 100644
--- a/Framework/Core/src/WorkflowHelpers.h
+++ b/Framework/Core/src/WorkflowHelpers.h
@@ -227,6 +227,14 @@ struct WorkflowHelpers {
 
   /// returns only dangling outputs
   static std::vector<InputSpec> computeDanglingOutputs(WorkflowSpec const& workflow);
+
+  /// Validate that the nodes at the ends of the edges of the graph
+  /// are actually compatible with each other.
+  /// For example we should make sure that Lifetime::Timeframe inputs of
+  /// one node is not connected to an Output of Lifetime::Sporadic of another node.
+  static void validateEdges(WorkflowSpec const& workflow,
+                            std::vector<DeviceConnectionEdge> const& edges,
+                            std::vector<OutputSpec> const& outputs);
 };
 
 } // namespace o2::framework

From 8c28b720075e211086cc03af6866280370c1dbca Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 13 Nov 2023 15:33:27 +0100
Subject: [PATCH 096/726] DPL: complain if inputs and outputs are incompatible

---
 Framework/Core/src/CommonDataProcessors.cxx |  4 +++-
 Framework/Core/src/DeviceSpecHelpers.cxx    |  2 ++
 Framework/Core/src/WorkflowHelpers.cxx      | 26 +++++++++++++++++++++
 Framework/Core/src/WorkflowHelpers.h        |  8 +++++++
 4 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 74e020595b345..e9cdc97e316ca 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -259,9 +259,11 @@ DataProcessorSpec CommonDataProcessors::getOutputObjHistSink(std::vector<OutputO
   };
 
   char const* name = "internal-dpl-aod-global-analysis-file-sink";
+  // Lifetime is sporadic because we do not ask each analysis task to send its
+  // results every timeframe.
   DataProcessorSpec spec{
     .name = name,
-    .inputs = {InputSpec("x", DataSpecUtils::dataDescriptorMatcherFrom(header::DataOrigin{"ATSK"}))},
+    .inputs = {InputSpec("x", DataSpecUtils::dataDescriptorMatcherFrom(header::DataOrigin{"ATSK"}), Lifetime::Sporadic)},
     .algorithm = {writerFunction},
   };
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index c54dd6cec7a99..45a76570dbf1b 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1072,6 +1072,8 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
 
   WorkflowHelpers::constructGraph(workflow, logicalEdges, outputs, availableForwardsInfo);
 
+  WorkflowHelpers::validateEdges(workflow, logicalEdges, outputs);
+
   // We need to instanciate one device per (me, timeIndex) in the
   // DeviceConnectionEdge. For each device we need one new binding
   // server per (me, other) -> port Moreover for each (me, other,
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index cd9c3913de228..68d6554ccd052 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -1233,4 +1233,30 @@ std::vector<InputSpec> WorkflowHelpers::computeDanglingOutputs(WorkflowSpec cons
 }
 
 #pragma diagnostic pop
+void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
+                                    std::vector<DeviceConnectionEdge> const& edges,
+                                    std::vector<OutputSpec> const& outputs)
+{
+  // Iterate over all the edges.
+  // Get the input lifetime and the output lifetime.
+  // Output lifetime must be Timeframe if the input lifetime is Timeframe.
+  std::string errors;
+  for (auto& edge : edges) {
+    auto& producer = workflow[edge.producer];
+    auto& consumer = workflow[edge.consumer];
+    auto& output = outputs[edge.outputGlobalIndex];
+    auto& input = consumer.inputs[edge.consumerInputIndex];
+    // We cannot simply say "!Timeframe" because we want to allow
+    // e.g. Enumeration => Timeframe
+    if (input.lifetime == Lifetime::Timeframe && output.lifetime == Lifetime::Sporadic) {
+      errors += fmt::format("Input {} of {} has lifetime Timeframe, but output {} of {} has lifetime Sporadic",
+                            DataSpecUtils::describe(input).c_str(), consumer.name,
+                            DataSpecUtils::describe(output).c_str(), producer.name);
+    }
+  }
+  if (!errors.empty()) {
+    throw std::runtime_error(errors);
+  }
+}
+
 } // namespace o2::framework
diff --git a/Framework/Core/src/WorkflowHelpers.h b/Framework/Core/src/WorkflowHelpers.h
index a167e99d4768c..67661dbdb979a 100644
--- a/Framework/Core/src/WorkflowHelpers.h
+++ b/Framework/Core/src/WorkflowHelpers.h
@@ -227,6 +227,14 @@ struct WorkflowHelpers {
 
   /// returns only dangling outputs
   static std::vector<InputSpec> computeDanglingOutputs(WorkflowSpec const& workflow);
+
+  /// Validate that the nodes at the ends of the edges of the graph
+  /// are actually compatible with each other.
+  /// For example we should make sure that Lifetime::Timeframe inputs of
+  /// one node is not connected to an Output of Lifetime::Sporadic of another node.
+  static void validateEdges(WorkflowSpec const& workflow,
+                            std::vector<DeviceConnectionEdge> const& edges,
+                            std::vector<OutputSpec> const& outputs);
 };
 
 } // namespace o2::framework

From cc9433c582436b75489b41cc058f14dae32b8d08 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 13 Nov 2023 16:52:38 +0100
Subject: [PATCH 097/726] Keep code checker happy

---
 Detectors/TRD/base/macros/OCDB2CCDB.C           | 14 +++++++-------
 Detectors/TRD/base/macros/OCDB2CCDBTrapConfig.C | 12 ++++++------
 Detectors/TRD/base/macros/PrintTrapConfig.C     | 12 ++++++------
 Detectors/TRD/base/macros/Readocdb.C            | 14 +++++++-------
 4 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/Detectors/TRD/base/macros/OCDB2CCDB.C b/Detectors/TRD/base/macros/OCDB2CCDB.C
index 0d7be94536bcd..3f023d6398f9f 100644
--- a/Detectors/TRD/base/macros/OCDB2CCDB.C
+++ b/Detectors/TRD/base/macros/OCDB2CCDB.C
@@ -83,8 +83,8 @@ using namespace o2::trd;
 // histograms used for extracting the mean and RMS of calibration parameters
 
 // global constants
-AliCDBStorage* storage = NULL;
-AliCDBManager* manager = NULL;
+AliCDBStorage* storage = nullptr;
+AliCDBManager* manager = nullptr;
 Int_t Run(0);
 void MakeRunListFromOCDB(const Char_t* directory, const Char_t* outfile, Bool_t fromAlien = kFALSE);
 AliCDBEntry* GetCDBentry(const Char_t* path, Bool_t owner = kTRUE);
@@ -184,7 +184,7 @@ void UnpackGainTable(std::string& gainkey, CalOnlineGainTables* gtbl)
   AliTRDCalOnlineGainTable* tbl = 0;
   AliTRDCalOnlineGainTableROC* tblroc = 0;
   AliTRDCalOnlineGainTableMCM* tblmcm = 0;
-  AliCDBEntry* entry = NULL;
+  AliCDBEntry* entry = nullptr;
   if ((entry = GetCDBentry(Form("TRD/Calib/%s", gainkey.c_str()), 0))) {
     tbl = (AliTRDCalOnlineGainTable*)entry->GetObject();
     for (int i = 0; i < 540; i++) {
@@ -239,7 +239,7 @@ void OCDB2CCDB(long timeStamp = -1, TString ccdbPath = "http://localhost:8080",
   manager->SetDefaultStorage(storageURI);
   manager->SetCacheFlag(kTRUE);
   storage = manager->GetDefaultStorage();
-  AliCDBEntry* entry = NULL;
+  AliCDBEntry* entry = nullptr;
   Run = run;
 
   std::string TRDCalBase = "TRD";
@@ -504,16 +504,16 @@ AliCDBEntry* GetCDBentry(const Char_t* path, Bool_t owner)
 {
   TString spath = path;
   //  ::Info("GetCDBentry", Form("QUERY RUN [%d] for \"%s\".", Run, spath.Data()));
-  AliCDBEntry* entry(NULL);
+  AliCDBEntry* entry(nullptr);
   storage->QueryCDB(Run, spath.Data());
   cout << spath.Data();
   if (!storage->GetQueryCDBList()->GetEntries()) {
     cout << "GetCDBentry" << Form("Missing \"%s\" in run %d.", spath.Data(), Run);
-    return NULL;
+    return nullptr;
   } else
     entry = manager->Get(spath.Data());
   if (!entry)
-    return NULL;
+    return nullptr;
 
   entry->SetOwner(owner);
   //  ::Info("GetCDBentry", Form("FOUND ENTRY @ [%p].", (void*)entry));
diff --git a/Detectors/TRD/base/macros/OCDB2CCDBTrapConfig.C b/Detectors/TRD/base/macros/OCDB2CCDBTrapConfig.C
index 36c8d7aa9aa1d..5ada93ea73f99 100644
--- a/Detectors/TRD/base/macros/OCDB2CCDBTrapConfig.C
+++ b/Detectors/TRD/base/macros/OCDB2CCDBTrapConfig.C
@@ -82,8 +82,8 @@ using namespace o2::trd;
 // histograms used for extracting the mean and RMS of calibration parameters
 
 // global constants
-AliCDBStorage* storage = NULL;
-AliCDBManager* manager = NULL;
+AliCDBStorage* storage = nullptr;
+AliCDBManager* manager = nullptr;
 Int_t Run(0);
 void MakeRunListFromOCDB(const Char_t* directory, const Char_t* outfile, Bool_t fromAlien = kFALSE);
 AliCDBEntry* GetCDBentry(const Char_t* path, Bool_t owner = kTRUE);
@@ -181,7 +181,7 @@ void OCDB2CCDBTrapConfig(TString ccdbPath = "http://localhost:8080", Int_t run =
   manager->SetDefaultStorage(storageURI);
   manager->SetCacheFlag(kTRUE);
   storage = manager->GetDefaultStorage();
-  AliCDBEntry* entry = NULL;
+  AliCDBEntry* entry = nullptr;
   Run = run;
 
   manager->SetRun(Run);
@@ -331,16 +331,16 @@ AliCDBEntry* GetCDBentry(const Char_t* path, Bool_t owner)
 {
   TString spath = path;
   //  ::Info("GetCDBentry", Form("QUERY RUN [%d] for \"%s\".", Run, spath.Data()));
-  AliCDBEntry* entry(NULL);
+  AliCDBEntry* entry(nullptr);
   storage->QueryCDB(Run, spath.Data());
   cout << spath.Data();
   if (!storage->GetQueryCDBList()->GetEntries()) {
     cout << "GetCDBentry" << Form("Missing \"%s\" in run %d.", spath.Data(), Run);
-    return NULL;
+    return nullptr;
   } else
     entry = manager->Get(spath.Data());
   if (!entry)
-    return NULL;
+    return nullptr;
 
   entry->SetOwner(owner);
   //  ::Info("GetCDBentry", Form("FOUND ENTRY @ [%p].", (void*)entry));
diff --git a/Detectors/TRD/base/macros/PrintTrapConfig.C b/Detectors/TRD/base/macros/PrintTrapConfig.C
index 949b8e4811b45..0fb39b52823b8 100644
--- a/Detectors/TRD/base/macros/PrintTrapConfig.C
+++ b/Detectors/TRD/base/macros/PrintTrapConfig.C
@@ -82,8 +82,8 @@ using namespace o2::trd;
 // histograms used for extracting the mean and RMS of calibration parameters
 
 // global constants
-AliCDBStorage* storage = NULL;
-AliCDBManager* manager = NULL;
+AliCDBStorage* storage = nullptr;
+AliCDBManager* manager = nullptr;
 Int_t Run(0);
 void MakeRunListFromOCDB(const Char_t* directory, const Char_t* outfile, Bool_t fromAlien = kFALSE);
 AliCDBEntry* GetCDBentry(const Char_t* path, Bool_t owner = kTRUE);
@@ -209,7 +209,7 @@ void PrintTrapConfig(Int_t run, const Char_t* storageURI = "alien://folder=/alic
   manager->SetDefaultStorage(storageURI);
   manager->SetCacheFlag(kTRUE);
   storage = manager->GetDefaultStorage();
-  AliCDBEntry* entry = NULL;
+  AliCDBEntry* entry = nullptr;
   Run = run;
 
   std::string TRDCalBase = "TRD";
@@ -365,16 +365,16 @@ AliCDBEntry* GetCDBentry(const Char_t* path, Bool_t owner)
 {
   TString spath = path;
   //  ::Info("GetCDBentry", Form("QUERY RUN [%d] for \"%s\".", Run, spath.Data()));
-  AliCDBEntry* entry(NULL);
+  AliCDBEntry* entry(nullptr);
   storage->QueryCDB(Run, spath.Data());
   cout << spath.Data();
   if (!storage->GetQueryCDBList()->GetEntries()) {
     cout << "GetCDBentry" << Form("Missing \"%s\" in run %d.", spath.Data(), Run);
-    return NULL;
+    return nullptr;
   } else
     entry = manager->Get(spath.Data());
   if (!entry)
-    return NULL;
+    return nullptr;
 
   entry->SetOwner(owner);
   //  ::Info("GetCDBentry", Form("FOUND ENTRY @ [%p].", (void*)entry));
diff --git a/Detectors/TRD/base/macros/Readocdb.C b/Detectors/TRD/base/macros/Readocdb.C
index c3e8c2978d789..12393fd8e3056 100644
--- a/Detectors/TRD/base/macros/Readocdb.C
+++ b/Detectors/TRD/base/macros/Readocdb.C
@@ -81,8 +81,8 @@ using namespace o2::trd;
 // histograms used for extracting the mean and RMS of calibration parameters
 
 // global constants
-AliCDBStorage* storage = NULL;
-AliCDBManager* manager = NULL;
+AliCDBStorage* storage = nullptr;
+AliCDBManager* manager = nullptr;
 Int_t Run(0);
 void MakeRunListFromOCDB(const Char_t* directory, const Char_t* outfile, Bool_t fromAlien = kFALSE);
 AliCDBEntry* GetCDBentry(const Char_t* path, Bool_t owner = kTRUE);
@@ -182,7 +182,7 @@ void UnpackGainTable(std::string& gainkey, CalOnlineGainTables* gtbl)
   AliTRDCalOnlineGainTable* tbl = 0;
   AliTRDCalOnlineGainTableROC* tblroc = 0;
   AliTRDCalOnlineGainTableMCM* tblmcm = 0;
-  AliCDBEntry* entry = NULL;
+  AliCDBEntry* entry = nullptr;
   if ((entry = GetCDBentry(Form("TRD/Calib/%s", gainkey.c_str()), 0))) {
     tbl = (AliTRDCalOnlineGainTable*)entry->GetObject();
     for (int i = 0; i < 540; i++) {
@@ -226,7 +226,7 @@ void Readocdb(Int_t run, const Char_t* storageURI = "alien://folder=/alice/data/
   manager->SetDefaultStorage(storageURI);
   manager->SetCacheFlag(kTRUE);
   storage = manager->GetDefaultStorage();
-  AliCDBEntry* entry = NULL;
+  AliCDBEntry* entry = nullptr;
   Run = run;
 
   manager->SetRun(Run);
@@ -275,16 +275,16 @@ AliCDBEntry* GetCDBentry(const Char_t* path, Bool_t owner)
 {
   TString spath = path;
   //  ::Info("GetCDBentry", Form("QUERY RUN [%d] for \"%s\".", Run, spath.Data()));
-  AliCDBEntry* entry(NULL);
+  AliCDBEntry* entry(nullptr);
   storage->QueryCDB(Run, spath.Data());
   cout << spath.Data();
   if (!storage->GetQueryCDBList()->GetEntries()) {
     cout << "GetCDBentry" << Form("Missing \"%s\" in run %d.", spath.Data(), Run);
-    return NULL;
+    return nullptr;
   } else
     entry = manager->Get(spath.Data());
   if (!entry)
-    return NULL;
+    return nullptr;
 
   entry->SetOwner(owner);
   //  ::Info("GetCDBentry", Form("FOUND ENTRY @ [%p].", (void*)entry));

From d66408e930a84c95c02818db722c65675a22f9c0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 14 Nov 2023 11:14:18 +0100
Subject: [PATCH 098/726] Do not expose AODToHepMC.h to ROOT

This is needed to fix macOS builds. Apparently ROOT injects
some arrow incompatible system headers in the chain.
---
 Generators/CMakeLists.txt | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Generators/CMakeLists.txt b/Generators/CMakeLists.txt
index 7112cada22686..2e5b5d022a0a5 100644
--- a/Generators/CMakeLists.txt
+++ b/Generators/CMakeLists.txt
@@ -104,13 +104,16 @@ if(pythia_FOUND)
               include/Generators/GeneratorFactory.h)
 endif()
 
+list(APPEND root_headers ${headers})
 if(HepMC3_FOUND)
   list(APPEND headers include/Generators/GeneratorHepMC.h)
   list(APPEND headers include/Generators/GeneratorHepMCParam.h)
+  list(APPEND root_headers include/Generators/GeneratorHepMC.h)
+  list(APPEND root_headers include/Generators/GeneratorHepMCParam.h)
   list(APPEND headers include/Generators/AODToHepMC.h)
 endif()
 
-o2_target_root_dictionary(Generators HEADERS ${headers})
+o2_target_root_dictionary(Generators HEADERS ${root_headers})
 
 o2_add_test_root_macro(share/external/extgen.C
                        PUBLIC_LINK_LIBRARIES O2::Generators FairRoot::Base

From ddf21d8b1556b4512786cbe9e120b31d28d74c38 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 14 Nov 2023 11:16:52 +0100
Subject: [PATCH 099/726] Fix deprecation warning about implicit conversion of
 anonymous enum

---
 GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx
index 497c63c5eb570..cfa3bb2707136 100644
--- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx
+++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx
@@ -77,7 +77,7 @@ GPUd() void GPUTPCCFCheckPadBaseline::Thread<0>(int nBlocks, int nThreads, int i
 
 #else // CPU CODE
 
-  constexpr size_t ElemsInTileRow = TilingLayout<GridSize<2>>::WidthInTiles * TimebinsPerCacheline * PadsPerCacheline;
+  constexpr size_t ElemsInTileRow = (size_t)TilingLayout<GridSize<2>>::WidthInTiles * TimebinsPerCacheline * PadsPerCacheline;
 
 #ifndef GPUCA_NO_VC
   using UShort8 = Vc::fixed_size_simd<unsigned short, PadsPerCacheline>;

From c61a5bb25b2db0234f3a8fcdd0a40f9c86652eb9 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 14 Nov 2023 11:16:53 +0100
Subject: [PATCH 100/726] DPL Proxies: add --sporadic-inputs /
 --sporadic-outputs option to proxies

---
 Framework/Utils/src/dpl-output-proxy.cxx | 11 +++++++++++
 Framework/Utils/src/raw-proxy.cxx        | 12 +++++++++---
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/Framework/Utils/src/dpl-output-proxy.cxx b/Framework/Utils/src/dpl-output-proxy.cxx
index 5a3e38572e1ac..fb70f7dfdf4ed 100644
--- a/Framework/Utils/src/dpl-output-proxy.cxx
+++ b/Framework/Utils/src/dpl-output-proxy.cxx
@@ -39,6 +39,10 @@ void customize(std::vector<ConfigParamSpec>& workflowOptions)
     ConfigParamSpec{
       "output-proxy-method", VariantType::String, "bind", {"proxy socket method: bind, connect"}});
 
+  workflowOptions.push_back(
+    ConfigParamSpec{
+      "sporadic-inputs", VariantType::Bool, false, {"consider all the inputs as sporadic"}});
+
   workflowOptions.push_back(
     ConfigParamSpec{
       "output-proxy-address", VariantType::String, "0.0.0.0", {"address to connect / bind to"}});
@@ -74,6 +78,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
   auto processorName = config.options().get<std::string>("proxy-name");
   auto inputConfig = config.options().get<std::string>("dataspec");
   int defaultPort = config.options().get<int>("default-port");
+  bool sporadicInputs = config.options().get<bool>("sporadic-inputs");
   auto defaultTransportConfig = config.options().get<std::string>("default-transport");
   if (defaultTransportConfig == "zeromq") {
     // nothing to do for the moment
@@ -87,6 +92,12 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
   if (inputs.size() == 0) {
     throw std::runtime_error("invalid dataspec '" + inputConfig + "'");
   }
+  // we need to set the lifetime of the inputs to sporadic if requested
+  if (sporadicInputs) {
+    for (auto& input : inputs) {
+      input.lifetime = Lifetime::Sporadic;
+    }
+  }
 
   // we build the default channel configuration from the binding of the first input
   // in order to have more than one we would need to possibility to have support for
diff --git a/Framework/Utils/src/raw-proxy.cxx b/Framework/Utils/src/raw-proxy.cxx
index 195e1bd61d081..e04e9879e1e0c 100644
--- a/Framework/Utils/src/raw-proxy.cxx
+++ b/Framework/Utils/src/raw-proxy.cxx
@@ -35,6 +35,10 @@ void customize(std::vector<ConfigParamSpec>& workflowOptions)
     ConfigParamSpec{
       "inject-missing-data", VariantType::Bool, false, {"inject missing data according to dataspec if not found in the input"}});
 
+  workflowOptions.push_back(
+    ConfigParamSpec{
+      "sporadic-outputs", VariantType::Bool, false, {"consider all the outputs as sporadic"}});
+
   workflowOptions.push_back(
     ConfigParamSpec{
       "print-input-sizes", VariantType::Int, 0, {"print statistics about sizes per input spec every n TFs"}});
@@ -52,16 +56,18 @@ void customize(std::vector<ConfigParamSpec>& workflowOptions)
 
 WorkflowSpec defineDataProcessing(ConfigContext const& config)
 {
-  std::string processorName = config.options().get<std::string>("proxy-name");
-  std::string outputconfig = config.options().get<std::string>("dataspec");
+  auto processorName = config.options().get<std::string>("proxy-name");
+  auto outputconfig = config.options().get<std::string>("dataspec");
   bool injectMissingData = config.options().get<bool>("inject-missing-data");
-  unsigned int printSizes = config.options().get<unsigned int>("print-input-sizes");
+  bool sporadicOutputs = config.options().get<bool>("sporadic-outputs");
+  auto printSizes = config.options().get<unsigned int>("print-input-sizes");
   bool throwOnUnmatched = config.options().get<bool>("throwOnUnmatched");
   uint64_t minSHM = std::stoul(config.options().get<std::string>("timeframes-shm-limit"));
   std::vector<InputSpec> matchers = select(outputconfig.c_str());
   Outputs readoutProxyOutput;
   for (auto const& matcher : matchers) {
     readoutProxyOutput.emplace_back(DataSpecUtils::asOutputSpec(matcher));
+    readoutProxyOutput.back().lifetime = sporadicOutputs ? Lifetime::Sporadic : Lifetime::Timeframe;
   }
 
   // we use the same specs as filters in the dpl adaptor

From 6af07a868e2d25a8731a47e253080c9ca9872e3f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 14 Nov 2023 15:00:39 +0100
Subject: [PATCH 101/726] DPL: reset the async queue on STOP

---
 Framework/Core/include/Framework/AsyncQueue.h | 3 +++
 Framework/Core/src/AsyncQueue.cxx             | 6 ++++++
 Framework/Core/src/CommonServices.cxx         | 4 ++++
 3 files changed, 13 insertions(+)

diff --git a/Framework/Core/include/Framework/AsyncQueue.h b/Framework/Core/include/Framework/AsyncQueue.h
index 543dfd6994152..37971fbe03bfb 100644
--- a/Framework/Core/include/Framework/AsyncQueue.h
+++ b/Framework/Core/include/Framework/AsyncQueue.h
@@ -59,6 +59,9 @@ struct AsyncQueueHelpers {
   /// 2. then priority
   /// 3. only execute the highest (timeslice, debounce) value
   static void run(AsyncQueue& queue, TimesliceId oldestPossibleTimeslice);
+
+  /// Reset the queue to its initial state
+  static void reset(AsyncQueue& queue);
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/AsyncQueue.cxx b/Framework/Core/src/AsyncQueue.cxx
index 36d72bf1b3dd7..44115d4985e91 100644
--- a/Framework/Core/src/AsyncQueue.cxx
+++ b/Framework/Core/src/AsyncQueue.cxx
@@ -107,4 +107,10 @@ auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> vo
                     queue.tasks.end());
 }
 
+auto AsyncQueueHelpers::reset(AsyncQueue& queue) -> void
+{
+  queue.tasks.clear();
+  queue.iteration = 0;
+}
+
 } // namespace o2::framework
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 4caddb9b98c78..66e0330841ea6 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -135,6 +135,10 @@ o2::framework::ServiceSpec CommonServices::asyncQueue()
     .name = "async-queue",
     .init = simpleServiceInit<AsyncQueue, AsyncQueue>(),
     .configure = noConfiguration(),
+    .stop = [](ServiceRegistryRef services, void* service) {
+      auto& queue = services.get<AsyncQueue>();
+      AsyncQueueHelpers::reset(queue);
+    },
     .kind = ServiceKind::Serial};
 }
 

From a510688914098b03730157d9ca0f37df11b76cd6 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 14 Nov 2023 15:07:34 +0100
Subject: [PATCH 102/726] DPL Utils: drop obsolete utils

In the end we never used generic devices for scatter/gather because
DPL provided internally such facility.
---
 Framework/Utils/CMakeLists.txt         |  25 ------
 Framework/Utils/src/DPLBroadcaster.cxx |  81 --------------------
 Framework/Utils/src/DPLGatherer.cxx    |  56 --------------
 Framework/Utils/src/DPLMerger.cxx      |  70 -----------------
 Framework/Utils/src/DPLRouter.cxx      |  54 -------------
 Framework/Utils/test/DPLOutputTest.cxx | 102 -------------------------
 6 files changed, 388 deletions(-)
 delete mode 100644 Framework/Utils/src/DPLBroadcaster.cxx
 delete mode 100644 Framework/Utils/src/DPLGatherer.cxx
 delete mode 100644 Framework/Utils/src/DPLMerger.cxx
 delete mode 100644 Framework/Utils/src/DPLRouter.cxx
 delete mode 100644 Framework/Utils/test/DPLOutputTest.cxx

diff --git a/Framework/Utils/CMakeLists.txt b/Framework/Utils/CMakeLists.txt
index db234537a0b6f..fcbc53ef0e6f0 100644
--- a/Framework/Utils/CMakeLists.txt
+++ b/Framework/Utils/CMakeLists.txt
@@ -11,13 +11,7 @@
 
 o2_add_library(DPLUtils
                SOURCES src/Utils.cxx
-                       src/DPLBroadcaster.cxx
-                       src/DPLGatherer.cxx
-                       src/DPLMerger.cxx
-                       src/DPLRouter.cxx
                        src/RawParser.cxx
-                       test/DPLBroadcasterMerger.cxx
-                       test/DPLOutputTest.cxx
                        test/RawPageTestData.cxx
                PUBLIC_LINK_LIBRARIES O2::Framework ROOT::Tree ROOT::TreePlayer O2::CommonUtils)
 
@@ -37,25 +31,6 @@ o2_add_executable(output-proxy
                   PUBLIC_LINK_LIBRARIES O2::Framework O2::DPLUtils)
 
 
-o2_add_test(DPLBroadcasterMerger
-            SOURCES test/test_DPLBroadcasterMerger.cxx src/Utils.cxx
-                    test/DPLBroadcasterMerger.cxx src/DPLMerger.cxx
-                    src/DPLBroadcaster.cxx
-            NO_BOOST_TEST
-            PUBLIC_LINK_LIBRARIES O2::DPLUtils
-            COMPONENT_NAME DPLUtils
-            LABELS dplutils
-            COMMAND_LINE_ARGS ${DPL_WORKFLOW_TESTS_EXTRA_OPTIONS} --run)
-
-o2_add_test(DPLOutput
-            NO_BOOST_TEST
-            SOURCES test/test_DPLOutputTest.cxx src/Utils.cxx
-                    test/DPLOutputTest.cxx
-            PUBLIC_LINK_LIBRARIES O2::DPLUtils
-            COMPONENT_NAME DPLUtils
-            LABELS long dplutils
-            COMMAND_LINE_ARGS ${DPL_WORKFLOW_TESTS_EXTRA_OPTIONS} --run)
-
 o2_add_test(RootTreeWriterWorkflow
             NO_BOOST_TEST
             SOURCES test/test_RootTreeWriterWorkflow.cxx
diff --git a/Framework/Utils/src/DPLBroadcaster.cxx b/Framework/Utils/src/DPLBroadcaster.cxx
deleted file mode 100644
index 4ea5e5747cd42..0000000000000
--- a/Framework/Utils/src/DPLBroadcaster.cxx
+++ /dev/null
@@ -1,81 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// \file DPLBroadcaster.cxx
-/// \brief Implementation of generic DPL broadcaster, v0.1
-///
-/// \author Gabriele Gaetano Fronzé, gfronze@cern.ch
-
-#include "DPLUtils/Utils.h"
-#include "Framework/DataProcessorSpec.h"
-#include "Framework/DataProcessingHeader.h"
-#include "Framework/DataRefUtils.h"
-#include "Headers/DataHeader.h"
-
-namespace o2f = o2::framework;
-
-namespace o2
-{
-namespace workflows
-{
-
-// This is a possible implementation of a DPL compliant and generic broadcaster.
-// Every other implementation should fall back to this one, after required translations.
-o2f::DataProcessorSpec defineBroadcaster(std::string devName, o2f::InputSpec usrInput, o2f::Outputs usrOutputs,
-                                         std::function<size_t(o2f::DataRef)> const func)
-{
-  return {devName,               // Device name from user
-          o2f::Inputs{usrInput}, // User defined input as a vector of one InputSpec
-          usrOutputs,            // user defined outputs as a vector of OutputSpecs
-
-          o2f::AlgorithmSpec{[usrOutputs, func](o2f::InitContext&) {
-            // Creating shared ptrs to useful parameters
-            auto outputsPtr = getOutputList(usrOutputs);
-            auto funcPtr = std::make_shared<std::function<size_t(o2f::DataRef)> const>(func);
-
-            // Defining the ProcessCallback as returned object of InitCallback
-            return [outputsPtr, funcPtr](o2f::ProcessingContext& ctx) {
-              // Getting original input message and getting his size using the provided function
-              auto inputMsg = ctx.inputs().getByPos(0);
-              // Getting message size using provided std::function
-              auto msgSize = (*funcPtr)(inputMsg);
-              // Iterating over the OutputSpecs to push the input message to all the output destinations
-              for (const auto& itOutputs : (*outputsPtr)) {
-                auto& fwdMsg = ctx.outputs().newChunk(itOutputs, msgSize);
-                std::memcpy(fwdMsg.data(), inputMsg.payload, msgSize);
-              }
-            };
-          }}};
-}
-
-// This is a shortcut for messages with fixed user-defined size
-o2f::DataProcessorSpec defineBroadcaster(std::string devName, o2f::InputSpec usrInput, o2f::Outputs usrOutputs,
-                                         size_t fixMsgSize)
-{
-  // This lambda returns a fixed message size
-  auto funcSize = [fixMsgSize](o2f::DataRef d) -> size_t { return fixMsgSize; };
-  // Callling complete implementation
-  return defineBroadcaster(devName, usrInput, usrOutputs, funcSize);
-}
-
-// This is an implementation which retrieves the message size using the API
-o2f::DataProcessorSpec defineBroadcaster(std::string devName, o2f::InputSpec usrInput, o2f::Outputs usrOutputs)
-{
-  // This lambda retrieves the message size using the API
-  auto funcSize = [](o2f::DataRef ref) -> size_t {
-    return o2::framework::DataRefUtils::getPayloadSize(ref);
-  };
-  // Callling complete implementation
-  return defineBroadcaster(devName, usrInput, usrOutputs, funcSize);
-}
-
-} // namespace workflows
-} // namespace o2
diff --git a/Framework/Utils/src/DPLGatherer.cxx b/Framework/Utils/src/DPLGatherer.cxx
deleted file mode 100644
index 039e6720980d1..0000000000000
--- a/Framework/Utils/src/DPLGatherer.cxx
+++ /dev/null
@@ -1,56 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// \file DPLGatherer.cxx
-/// \brief Implementation of generic DPL gatherer, v0.1
-///
-/// \author Gabriele Gaetano Fronzé, gfronze@cern.ch
-
-#include "DPLUtils/Utils.h"
-#include "Framework/DataProcessorSpec.h"
-#include "Framework/DataRefUtils.h"
-#include <vector>
-
-namespace o2f = o2::framework;
-
-namespace o2
-{
-namespace workflows
-{
-
-// This is a possible implementation of a DPL compliant and generic gatherer
-o2f::DataProcessorSpec defineGatherer(std::string devName, o2f::Inputs usrInputs, o2f::OutputSpec usrOutput)
-{
-  return {devName,                 // Device name from user
-          usrInputs,               // User defined input as a vector of one InputSpec
-          o2f::Outputs{usrOutput}, // user defined outputs as a vector of OutputSpecs
-
-          o2f::AlgorithmSpec{[usrOutput](o2f::InitContext&) {
-            // Creating shared ptrs to useful parameters
-            auto outputPtr = std::make_shared<o2f::Output>(getOutput(usrOutput));
-
-            // Defining the ProcessCallback as returned object of InitCallback
-            return [outputPtr](o2f::ProcessingContext& ctx) {
-              // Iterating over the Inputs to forward them on the same Output
-              for (const auto& itInputs : ctx.inputs()) {
-                // Retrieving message size from API
-                auto msgSize = o2::framework::DataRefUtils::getPayloadSize(itInputs);
-                // Allocating new chunk
-                auto& fwdMsg = ctx.outputs().newChunk((*outputPtr), msgSize);
-                // Moving the input to the output chunk
-                std::memmove(fwdMsg.data(), itInputs.payload, msgSize);
-              }
-            };
-          }}};
-}
-
-} // namespace workflows
-} // namespace o2
diff --git a/Framework/Utils/src/DPLMerger.cxx b/Framework/Utils/src/DPLMerger.cxx
deleted file mode 100644
index 6f91df32a4cca..0000000000000
--- a/Framework/Utils/src/DPLMerger.cxx
+++ /dev/null
@@ -1,70 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// \file DPLGatherer.cxx
-/// \brief Implementation of generic DPL gatherer, v0.1
-///
-/// \author Gabriele Gaetano Fronzé, gfronze@cern.ch
-
-#include "DPLUtils/Utils.h"
-#include "Framework/DataProcessorSpec.h"
-#include "Framework/DataRefUtils.h"
-#include <vector>
-
-namespace o2f = o2::framework;
-
-namespace o2
-{
-namespace workflows
-{
-
-// This is a possible implementation of a DPL compliant and generic gatherer
-// Every other implementation should fall back to this one, after required translations.
-o2f::DataProcessorSpec defineMerger(std::string devName, o2f::Inputs usrInputs, o2f::OutputSpec usrOutput,
-                                    std::function<void(OutputBuffer&, const o2f::DataRef)> const mergerFunc)
-{
-  return {devName,                 // Device name from user
-          usrInputs,               // User defined input as a vector of one InputSpec
-          o2f::Outputs{usrOutput}, // user defined outputs as a vector of OutputSpecs
-
-          o2f::AlgorithmSpec{[usrOutput, mergerFunc](o2f::InitContext&) {
-            // Creating shared ptrs to useful parameters
-            auto outputPtr = std::make_shared<o2f::Output>(getOutput(usrOutput));
-            auto mergerFuncPtr = std::make_shared<std::function<void(OutputBuffer&, o2f::DataRef)> const>(mergerFunc);
-
-            // Defining the ProcessCallback as returned object of InitCallback
-            return [outputPtr, mergerFuncPtr](o2f::ProcessingContext& ctx) {
-              OutputBuffer outputBuffer = ctx.outputs().makeVector<char>(*outputPtr);
-              // Iterating over the InputSpecs to aggregate msgs from the connected devices
-              for (const auto& itInputs : ctx.inputs()) {
-                (*mergerFuncPtr)(outputBuffer, itInputs);
-              }
-              // Adopting the buffer as new chunk
-              ctx.outputs().adoptContainer((*outputPtr), std::move(outputBuffer));
-            };
-          }}};
-}
-
-// This is a possible implementation of a DPL compliant and generic gatherer whit trivial messages concatenation
-o2f::DataProcessorSpec defineMerger(std::string devName, o2f::Inputs usrInputs, o2f::OutputSpec usrOutput)
-{
-  // This lambda retrieves the payload size through the API and back-inserts it on the output buffer
-  auto funcMerge = [](OutputBuffer& buf, const o2f::DataRef ref) {
-    auto msgSize = o2::framework::DataRefUtils::getPayloadSize(ref);
-    buf.resize(buf.size() + msgSize);
-    std::copy(&(ref.payload[0]), &(ref.payload[msgSize - 1]), std::back_inserter(buf));
-  };
-  // Callling complete implementation
-  return defineMerger(devName, usrInputs, usrOutput, funcMerge);
-}
-
-} // namespace workflows
-} // namespace o2
diff --git a/Framework/Utils/src/DPLRouter.cxx b/Framework/Utils/src/DPLRouter.cxx
deleted file mode 100644
index 7795cb37d4ea0..0000000000000
--- a/Framework/Utils/src/DPLRouter.cxx
+++ /dev/null
@@ -1,54 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// \file DPLGatherer.cxx
-/// \brief Implementation of generic DPL gatherer, v0.1
-///
-/// \author Gabriele Gaetano Fronzé, gfronze@cern.ch
-
-#include "DPLUtils/Utils.h"
-#include "Framework/DataProcessorSpec.h"
-#include "Framework/DataRefUtils.h"
-#include <vector>
-
-namespace o2f = o2::framework;
-
-namespace o2
-{
-namespace workflows
-{
-// This is a possible implementation of a DPL compliant and generic gatherer
-o2f::DataProcessorSpec defineRouter(std::string devName, o2f::Inputs usrInput, o2f::Outputs usrOutputs,
-                                    std::function<size_t(const o2f::DataRef)> const mappingFunc)
-{
-  return {devName,               // Device name from user
-          o2f::Inputs{usrInput}, // User defined input as a vector of one InputSpec
-          usrOutputs,            // user defined outputs as a vector of OutputSpecs
-
-          o2f::AlgorithmSpec{[usrOutputs, mappingFunc](o2f::InitContext&) {
-            // Creating shared ptrs to useful parameters
-            auto outputsPtr = getOutputList(usrOutputs);
-            auto mappingFuncPtr = std::make_shared<std::function<size_t(o2f::DataRef)> const>(mappingFunc);
-
-            // Defining the ProcessCallback as returned object of InitCallback
-            return [outputsPtr, mappingFuncPtr](o2f::ProcessingContext& ctx) {
-              auto inputMsg = ctx.inputs().getByPos(0);
-              auto msgSize = o2::framework::DataRefUtils::getPayloadSize(inputMsg);
-              auto& outputCh = (*outputsPtr)[(*mappingFuncPtr)(inputMsg)];
-
-              auto& fwdMsg = ctx.outputs().newChunk(outputCh, msgSize);
-              std::memcpy(fwdMsg.data(), inputMsg.payload, msgSize);
-            };
-          }}};
-}
-} // namespace workflows
-
-} // namespace o2
diff --git a/Framework/Utils/test/DPLOutputTest.cxx b/Framework/Utils/test/DPLOutputTest.cxx
deleted file mode 100644
index df0d69c8d6710..0000000000000
--- a/Framework/Utils/test/DPLOutputTest.cxx
+++ /dev/null
@@ -1,102 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// \author Gabriele Gaetano Fronzé, gfronze@cern.ch
-
-#include "DPLOutputTest.h"
-#include <fstream>
-#include "DPLUtils/Utils.h"
-#include "Framework/DataProcessorSpec.h"
-#include "random"
-#include "Framework/Logger.h"
-#include "Framework/ControlService.h"
-
-namespace o2f = o2::framework;
-
-namespace o2::workflows
-{
-
-o2f::DataProcessorSpec defineTestGenerator()
-{
-  return {"Generator",                                                  // Device name
-          {},                                                           // No inputs for a generator
-          o2f::Outputs{{"TST", "ToSink", 0, o2f::Lifetime::Timeframe}}, // One simple output
-
-          o2f::AlgorithmSpec{[](o2f::InitContext&) {
-            int msgCounter = 0;
-            auto msgCounter_shptr = std::make_shared<int>(msgCounter);
-
-            LOG(info) << ">>>>>>>>>>>>>> Generator initialised\n";
-
-            // Processing context in captured from return on InitCallback
-            return [msgCounter_shptr](o2f::ProcessingContext& ctx) {
-              int msgIndex = (*msgCounter_shptr)++;
-              if (msgIndex > 10) {
-                ctx.services().get<framework::ControlService>().endOfStream();
-              }
-              LOG(info) << ">>> MSG:" << msgIndex << "\n";
-
-              LOG(info) << ">>> Preparing MSG:" << msgIndex;
-
-              auto& outputMsg = ctx.outputs().newChunk({"TST", "ToSink", 0, o2f::Lifetime::Timeframe},
-                                                       (31 + 1) * sizeof(uint32_t) / sizeof(char));
-
-              LOG(info) << ">>> Preparing1 MSG:" << msgIndex;
-
-              auto payload = reinterpret_cast<uint32_t*>(outputMsg.data());
-
-              payload[0] = msgIndex;
-
-              LOG(info) << ">>> Preparing2 MSG:" << msgIndex;
-
-              for (int k = 0; k < 31; ++k) {
-                payload[k + 1] = (uint32_t)k;
-                LOG(info) << ">>>>\t" << payload[k + 1];
-              }
-
-              LOG(info) << ">>> Done MSG:" << msgIndex;
-            };
-          }}};
-}
-
-o2f::DataProcessorSpec defineTestSink()
-{
-  return {"Sink",                                                               // Device name
-          o2f::Inputs{{"input", "TST", "ToSink", 0, o2f::Lifetime::Transient}}, // No inputs, for the moment
-          {},
-
-          o2f::AlgorithmSpec{[](o2f::InitContext&) {
-            LOG(info) << ">>>>>>>>>>>>>> Sink initialised\n";
-
-            // Processing context in captured from return on InitCallback
-            return [](o2f::ProcessingContext& ctx) {
-              auto inputMsg = ctx.inputs().getByPos(0);
-              auto payload = reinterpret_cast<const uint32_t*>(inputMsg.payload);
-
-              LOG(info) << "Received message containing" << payload[0] << "elements\n";
-              for (int j = 0; j < payload[0]; ++j) {
-                LOG(info) << payload[j];
-              }
-            };
-          }}};
-}
-
-o2::framework::WorkflowSpec DPLOutputTest()
-{
-  auto lspec = o2f::WorkflowSpec();
-
-  // A generator of data
-  lspec.emplace_back(defineTestGenerator());
-  lspec.emplace_back(defineTestSink());
-  return std::move(lspec);
-}
-
-} // namespace o2::workflows

From 8b0a4b0c9253125bcdcd5f0d0e50ccc35cd9b602 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 14 Nov 2023 15:40:08 +0100
Subject: [PATCH 103/726] Do not expose AODToHepMC.h to ROOT

This is needed to fix macOS builds. Apparently ROOT injects
some arrow incompatible system headers in the chain.
---
 Generators/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Generators/CMakeLists.txt b/Generators/CMakeLists.txt
index 7112cada22686..587696bd52e53 100644
--- a/Generators/CMakeLists.txt
+++ b/Generators/CMakeLists.txt
@@ -107,10 +107,10 @@ endif()
 if(HepMC3_FOUND)
   list(APPEND headers include/Generators/GeneratorHepMC.h)
   list(APPEND headers include/Generators/GeneratorHepMCParam.h)
-  list(APPEND headers include/Generators/AODToHepMC.h)
 endif()
 
-o2_target_root_dictionary(Generators HEADERS ${headers})
+o2_target_root_dictionary(Generators HEADERS ${headers}
+                                             $<$<BOOL:${HepMC3_FOUND}>:include/Generators/AODToHepMC.h>)
 
 o2_add_test_root_macro(share/external/extgen.C
                        PUBLIC_LINK_LIBRARIES O2::Generators FairRoot::Base

From b6eba75a1f405b207c2a788db2d5c5075c3d1179 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 14 Nov 2023 16:16:03 +0100
Subject: [PATCH 104/726] Do not expose AODToHepMC.h to ROOT

This is needed to fix macOS builds. Apparently ROOT injects
some arrow incompatible system headers in the chain.
---
 Generators/CMakeLists.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Generators/CMakeLists.txt b/Generators/CMakeLists.txt
index 7112cada22686..f5ab384cef99a 100644
--- a/Generators/CMakeLists.txt
+++ b/Generators/CMakeLists.txt
@@ -107,7 +107,6 @@ endif()
 if(HepMC3_FOUND)
   list(APPEND headers include/Generators/GeneratorHepMC.h)
   list(APPEND headers include/Generators/GeneratorHepMCParam.h)
-  list(APPEND headers include/Generators/AODToHepMC.h)
 endif()
 
 o2_target_root_dictionary(Generators HEADERS ${headers})

From d629335c2717f2c338e3e374e30969e424e0c6bd Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 15 Nov 2023 09:45:15 +0100
Subject: [PATCH 105/726] DPL: complain if inputs and outputs are incompatible

---
 Framework/Core/src/CommonDataProcessors.cxx |  4 +++-
 Framework/Core/src/DeviceSpecHelpers.cxx    |  2 ++
 Framework/Core/src/WorkflowHelpers.cxx      | 26 +++++++++++++++++++++
 Framework/Core/src/WorkflowHelpers.h        |  8 +++++++
 4 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 74e020595b345..e9cdc97e316ca 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -259,9 +259,11 @@ DataProcessorSpec CommonDataProcessors::getOutputObjHistSink(std::vector<OutputO
   };
 
   char const* name = "internal-dpl-aod-global-analysis-file-sink";
+  // Lifetime is sporadic because we do not ask each analysis task to send its
+  // results every timeframe.
   DataProcessorSpec spec{
     .name = name,
-    .inputs = {InputSpec("x", DataSpecUtils::dataDescriptorMatcherFrom(header::DataOrigin{"ATSK"}))},
+    .inputs = {InputSpec("x", DataSpecUtils::dataDescriptorMatcherFrom(header::DataOrigin{"ATSK"}), Lifetime::Sporadic)},
     .algorithm = {writerFunction},
   };
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index c54dd6cec7a99..45a76570dbf1b 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1072,6 +1072,8 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
 
   WorkflowHelpers::constructGraph(workflow, logicalEdges, outputs, availableForwardsInfo);
 
+  WorkflowHelpers::validateEdges(workflow, logicalEdges, outputs);
+
   // We need to instanciate one device per (me, timeIndex) in the
   // DeviceConnectionEdge. For each device we need one new binding
   // server per (me, other) -> port Moreover for each (me, other,
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index cd9c3913de228..a731d7065cc4a 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -1233,4 +1233,30 @@ std::vector<InputSpec> WorkflowHelpers::computeDanglingOutputs(WorkflowSpec cons
 }
 
 #pragma diagnostic pop
+void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
+                                    std::vector<DeviceConnectionEdge> const& edges,
+                                    std::vector<OutputSpec> const& outputs)
+{
+  // Iterate over all the edges.
+  // Get the input lifetime and the output lifetime.
+  // Output lifetime must be Timeframe if the input lifetime is Timeframe.
+  std::string errors;
+  for (auto& edge : edges) {
+    auto& producer = workflow[edge.producer];
+    auto& consumer = workflow[edge.consumer];
+    auto& output = outputs[edge.outputGlobalIndex];
+    auto& input = consumer.inputs[edge.consumerInputIndex];
+    // We cannot simply say "!Timeframe" because we want to allow
+    // e.g. Enumeration => Timeframe
+    if (input.lifetime == Lifetime::Timeframe && output.lifetime == Lifetime::Sporadic) {
+      errors += fmt::format("Input {} of {} has lifetime Timeframe, but output {} of {} has lifetime Sporadic\n",
+                            DataSpecUtils::describe(input).c_str(), consumer.name,
+                            DataSpecUtils::describe(output).c_str(), producer.name);
+    }
+  }
+  if (!errors.empty()) {
+    throw std::runtime_error(errors);
+  }
+}
+
 } // namespace o2::framework
diff --git a/Framework/Core/src/WorkflowHelpers.h b/Framework/Core/src/WorkflowHelpers.h
index a167e99d4768c..67661dbdb979a 100644
--- a/Framework/Core/src/WorkflowHelpers.h
+++ b/Framework/Core/src/WorkflowHelpers.h
@@ -227,6 +227,14 @@ struct WorkflowHelpers {
 
   /// returns only dangling outputs
   static std::vector<InputSpec> computeDanglingOutputs(WorkflowSpec const& workflow);
+
+  /// Validate that the nodes at the ends of the edges of the graph
+  /// are actually compatible with each other.
+  /// For example we should make sure that Lifetime::Timeframe inputs of
+  /// one node is not connected to an Output of Lifetime::Sporadic of another node.
+  static void validateEdges(WorkflowSpec const& workflow,
+                            std::vector<DeviceConnectionEdge> const& edges,
+                            std::vector<OutputSpec> const& outputs);
 };
 
 } // namespace o2::framework

From fe23d4d49998845c84be7f54277b9733dbab6662 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 15 Nov 2023 10:32:03 +0100
Subject: [PATCH 106/726] DPL: fix typo in message

---
 Framework/Core/src/TimesliceIndex.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/src/TimesliceIndex.cxx b/Framework/Core/src/TimesliceIndex.cxx
index d9b4eec68f2af..267341eafd105 100644
--- a/Framework/Core/src/TimesliceIndex.cxx
+++ b/Framework/Core/src/TimesliceIndex.cxx
@@ -126,7 +126,7 @@ TimesliceIndex::OldestInputInfo TimesliceIndex::setOldestPossibleInput(Timeslice
 {
   // Each channel oldest possible input must be monotoically increasing.
   if (timestamp.value < mChannels[channel.value].oldestForChannel.value) {
-    LOG(error) << "Received bogus oldest possible timeslice " << timestamp.value << " for channel " << channel.value << " Excpected >= " << mChannels[channel.value].oldestForChannel.value;
+    LOG(error) << "Received bogus oldest possible timeslice " << timestamp.value << " for channel " << channel.value << ". Expected >= " << mChannels[channel.value].oldestForChannel.value;
   }
   mChannels[channel.value].oldestForChannel = timestamp;
   OldestInputInfo result{timestamp, channel};

From 4c2a683a141d499671a0ed5e42b474c36e145e6f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 16 Nov 2023 11:09:46 +0100
Subject: [PATCH 107/726] Make sure brace_constructible_size works in C++20

---
 Framework/Core/test/test_AnalysisTask.cxx     |  1 +
 .../include/Framework/StructToTuple.h         |  2 +-
 .../Foundation/test/test_StructToTuple.cxx    | 28 ++++++++++++++++++-
 3 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/test/test_AnalysisTask.cxx b/Framework/Core/test/test_AnalysisTask.cxx
index c166aec7f5b85..eae4b2cc36c16 100644
--- a/Framework/Core/test/test_AnalysisTask.cxx
+++ b/Framework/Core/test/test_AnalysisTask.cxx
@@ -153,6 +153,7 @@ TEST_CASE("AdaptorCompilation")
 {
   auto cfgc = makeEmptyConfigContext();
 
+  REQUIRE(brace_constructible_size<ATask>() == 1);
   auto task1 = adaptAnalysisTask<ATask>(*cfgc, TaskName{"test1"});
   REQUIRE(task1.inputs.size() == 2);
   REQUIRE(task1.outputs.size() == 1);
diff --git a/Framework/Foundation/include/Framework/StructToTuple.h b/Framework/Foundation/include/Framework/StructToTuple.h
index 4ea93e113249d..d94c0e3aaae03 100644
--- a/Framework/Foundation/include/Framework/StructToTuple.h
+++ b/Framework/Foundation/include/Framework/StructToTuple.h
@@ -281,7 +281,7 @@ auto homogeneous_apply_refs(L l, T&& object)
 {
   using type = std::decay_t<T>;
   constexpr int nesting = B ? 1 : 0;
-  constexpr unsigned long numElements = brace_constructible_size<T>() - nesting;
+  constexpr unsigned long numElements = brace_constructible_size<type>() - nesting;
   // clang-format off
   if DPL_HOMOGENEOUS_APPLY_ENTRY (9, 9)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (9, 8)
diff --git a/Framework/Foundation/test/test_StructToTuple.cxx b/Framework/Foundation/test/test_StructToTuple.cxx
index 06cf4b47a9f7c..59685a5f1d598 100644
--- a/Framework/Foundation/test/test_StructToTuple.cxx
+++ b/Framework/Foundation/test/test_StructToTuple.cxx
@@ -12,7 +12,11 @@
 #include <catch_amalgamated.hpp>
 #include "Framework/StructToTuple.h"
 
-struct Foo {
+struct Foo0 {
+};
+
+struct Foo1 {
+  int foo = 1;
 };
 
 // FIXME: this should really struct Bar : Foo, but a c++17 bug
@@ -23,6 +27,28 @@ struct Bar {
   int bar = 2;
 };
 
+TEST_CASE("SimpleDestructuring")
+{
+  Foo0 foo0;
+  auto t0 = o2::framework::homogeneous_apply_refs([](auto i) -> bool { return i > 1; }, foo0);
+  REQUIRE(t0.size() == 0);
+  Foo1 foo1;
+  auto t1 = o2::framework::homogeneous_apply_refs([](auto i) -> bool { return i > 1; }, foo1);
+
+  REQUIRE(t1.size() == 1);
+
+  // Should work with refs as well. When moving to C++20 this was
+  // not the case initially.
+  Foo1 const&& foo1ref = std::move(foo1);
+  auto t1ref = o2::framework::homogeneous_apply_refs([](auto i) -> bool { return i > 1; }, foo1ref);
+  REQUIRE(t1ref.size() == 1);
+
+  Bar bar;
+  auto t = o2::framework::homogeneous_apply_refs([](auto i) -> bool { return i > 1; }, bar);
+  REQUIRE(t[0] == false);
+  REQUIRE(t[1] == true);
+}
+
 /// Largest supported struct
 struct FooMax {
   int foo01 = 1;

From 73eddebedd8bd3dfb70df62c239046c0816d059c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 16 Nov 2023 11:53:27 +0100
Subject: [PATCH 108/726] DPL GUI: do not hang if quit requested but some
 devices are stuck

---
 Framework/Core/src/runDataProcessing.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 86cc15299e8f3..dbd626c32e568 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1073,7 +1073,7 @@ void gui_callback(uv_timer_s* ctx)
   if (gui->plugin->supportsDeferredClose()) {
     // For now, there is nothing for which we want to defer the close
     // so if the flag is set, we simply exit
-    if (*(gui->guiQuitRequested)) {
+    if (*(gui->guiQuitRequested) && *(gui->allChildrenGone)) {
       return;
     }
     void* draw_data = nullptr;

From 7f758d447a4f273ce395a2e4c03a68fee0f53a0b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 16 Nov 2023 11:53:27 +0100
Subject: [PATCH 109/726] DPL GUI: add GUI to list devices preventing quitting

---
 Framework/Core/include/Framework/DebugGUI.h   |   4 +-
 Framework/Core/src/runDataProcessing.cxx      |   6 +-
 .../GUISupport/src/FrameworkGUIDebugger.cxx   | 110 ++++++++++++------
 .../GUISupport/src/FrameworkGUIDebugger.h     |   4 +-
 Framework/GUISupport/src/Plugin.cxx           |   5 +-
 5 files changed, 87 insertions(+), 42 deletions(-)

diff --git a/Framework/Core/include/Framework/DebugGUI.h b/Framework/Core/include/Framework/DebugGUI.h
index c7a611d50a1d4..109fd36e27511 100644
--- a/Framework/Core/include/Framework/DebugGUI.h
+++ b/Framework/Core/include/Framework/DebugGUI.h
@@ -26,6 +26,7 @@
 namespace o2::framework
 {
 struct ServiceRegistry;
+struct GuiCallbackContext;
 /// Plugin interface for DPL GUIs.
 struct DebugGUI {
   virtual std::function<void(void)> getGUIDebugger(std::vector<o2::framework::DeviceInfo> const& infos,
@@ -35,7 +36,8 @@ struct DebugGUI {
                                                    std::vector<o2::framework::DeviceMetricsInfo> const& metricsInfos,
                                                    o2::framework::DriverInfo const& driverInfo,
                                                    std::vector<o2::framework::DeviceControl>& controls,
-                                                   o2::framework::DriverControl& driverControl) = 0;
+                                                   o2::framework::DriverControl& driverControl,
+                                                   GuiCallbackContext& context) = 0;
   virtual void updateMousePos(float x, float y) = 0;
   virtual void updateMouseButton(bool isClicked) = 0;
   virtual void updateMouseWheel(int direction) = 0;
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index dbd626c32e568..f31629d90416b 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1302,7 +1302,8 @@ int runStateMachine(DataProcessorSpecs const& workflow,
 
   DebugGUI* debugGUI = nullptr;
   void* window = nullptr;
-  decltype(debugGUI->getGUIDebugger(infos, runningWorkflow.devices, allStates, dataProcessorInfos, metricsInfos, driverInfo, controls, driverControl)) debugGUICallback;
+  GuiCallbackContext guiContext;
+  decltype(debugGUI->getGUIDebugger(infos, runningWorkflow.devices, allStates, dataProcessorInfos, metricsInfos, driverInfo, controls, driverControl, guiContext)) debugGUICallback;
 
   // An empty frameworkId means this is the driver, so we initialise the GUI
   auto initDebugGUI = []() -> DebugGUI* {
@@ -1389,7 +1390,6 @@ int runStateMachine(DataProcessorSpecs const& workflow,
   ServiceRegistryRef ref{serviceRegistry};
   ref.registerService(ServiceRegistryHelpers::handleForService<DevicesManager>(devicesManager));
 
-  GuiCallbackContext guiContext;
   guiContext.plugin = debugGUI;
   guiContext.frameLast = uv_hrtime();
   guiContext.frameLatency = &driverInfo.frameLatency;
@@ -1882,7 +1882,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
             uv_timer_stop(gui_timer);
           }
 
-          auto callback = debugGUI->getGUIDebugger(infos, runningWorkflow.devices, allStates, dataProcessorInfos, metricsInfos, driverInfo, controls, driverControl);
+          auto callback = debugGUI->getGUIDebugger(infos, runningWorkflow.devices, allStates, dataProcessorInfos, metricsInfos, driverInfo, controls, driverControl, guiContext);
           guiContext.callback = [&serviceRegistry, &driverServices, &debugGUI, &infos, &runningWorkflow, &dataProcessorInfos, &metricsInfos, &driverInfo, &controls, &driverControl, callback]() {
             callback();
             for (auto& service : driverServices) {
diff --git a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
index 35e0bc563a22e..375afef1007f9 100644
--- a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
@@ -16,6 +16,7 @@
 #include "DebugGUI/imgui_extras.h"
 #include "Framework/DriverControl.h"
 #include "Framework/DriverInfo.h"
+#include "Framework/GuiCallbackContext.h"
 #include "Framework/DeviceMetricsHelper.h"
 #include "Framework/DeviceMetricsInfo.h"
 #include "FrameworkGUIDeviceInspector.h"
@@ -506,7 +507,7 @@ void displayDeviceMetrics(const char* label,
     case MetricsDisplayStyle::Lines: {
       auto xAxisFlags = ImPlotAxisFlags_None;
       auto yAxisFlags = ImPlotAxisFlags_LockMin;
-      //ImPlot::FitNextPlotAxes(true, true, true, true);
+      // ImPlot::FitNextPlotAxes(true, true, true, true);
       if (ImPlot::BeginPlot("##Some plot", {-1, -1}, axisFlags)) {
         ImPlot::SetupAxes("time", "value", xAxisFlags, yAxisFlags);
         ImPlot::SetupAxisFormat(ImAxis_Y1, formatSI, nullptr);
@@ -1008,6 +1009,71 @@ void popWindowColorDueToStatus()
   ImGui::PopStyleVar(1);
 }
 
+int debugPID(int pid)
+{
+  std::string pidStr = std::to_string(pid);
+  setenv("O2DEBUGGEDPID", pidStr.c_str(), 1);
+#ifdef __APPLE__
+  std::string defaultAppleDebugCommand =
+    "osascript -e 'tell application \"Terminal\"'"
+    " -e 'activate'"
+    " -e 'do script \"lldb -p \" & (system attribute \"O2DEBUGGEDPID\") & \"; exit\"'"
+    " -e 'end tell'";
+  setenv("O2DPLDEBUG", defaultAppleDebugCommand.c_str(), 0);
+#else
+  setenv("O2DPLDEBUG", "xterm -hold -e gdb attach $O2DEBUGGEDPID &", 0);
+#endif
+  return system(getenv("O2DPLDEBUG"));
+}
+
+int profilePID(int pid)
+{
+  std::string pidStr = std::to_string(pid);
+  setenv("O2PROFILEDPID", pidStr.c_str(), 1);
+#ifdef __APPLE__
+  auto defaultAppleProfileCommand = fmt::format(
+    "osascript -e 'tell application \"Terminal\"'"
+    " -e 'activate'"
+    " -e 'do script \"xcrun xctrace record --output dpl-profile-{0}.trace"
+    " --instrument os_signpost --time-limit 30s --template Time\\\\ Profiler --attach {0} "
+    " && open dpl-profile-{0}.trace && exit\"'"
+    " -e 'end tell'",
+    pid);
+  std::cout << defaultAppleProfileCommand << std::endl;
+  setenv("O2DPLPROFILE", defaultAppleProfileCommand.c_str(), 0);
+#else
+  setenv("O2DPLPROFILE", "xterm -hold -e perf record -a -g -p $O2PROFILEDPID > perf-$O2PROFILEDPID.data &", 0);
+#endif
+  return system(getenv("O2DPLPROFILE"));
+}
+
+/// Show a window with the list of devices which have not yet exited
+/// and provide a few controls to interact with them.
+void displayShutdownDialog(std::vector<DeviceSpec> const& specs, std::vector<DeviceInfo> const& infos)
+{
+  for (size_t i = 0; i < infos.size(); ++i) {
+    auto& info = infos[i];
+    if (info.active == false) {
+      continue;
+    }
+    auto& spec = specs[i];
+    ImGui::Text("Unable to quit due to the following devices:");
+    ImGui::Text("%s (%d).", spec.name.c_str(), info.pid);
+    ImGui::SameLine();
+    if (ImGui::Button("Force quit")) {
+      kill(info.pid, SIGKILL);
+    }
+    ImGui::SameLine();
+    if (ImGui::Button("Debug")) {
+      debugPID(info.pid);
+    }
+    ImGui::SameLine();
+    if (ImGui::Button("Profile")) {
+      profilePID(info.pid);
+    }
+  }
+}
+
 /// Display information window about the driver
 /// and its state.
 void displayDriverInfo(DriverInfo const& driverInfo, DriverControl& driverControl)
@@ -1035,42 +1101,12 @@ void displayDriverInfo(DriverInfo const& driverInfo, DriverControl& driverContro
   }
   ImGui::SameLine();
   if (ImGui::Button("Debug driver")) {
-    std::string pidStr = std::to_string(pid);
-    setenv("O2DEBUGGEDPID", pidStr.c_str(), 1);
-#ifdef __APPLE__
-    std::string defaultAppleDebugCommand =
-      "osascript -e 'tell application \"Terminal\"'"
-      " -e 'activate'"
-      " -e 'do script \"lldb -p \" & (system attribute \"O2DEBUGGEDPID\") & \"; exit\"'"
-      " -e 'end tell'";
-    setenv("O2DPLDEBUG", defaultAppleDebugCommand.c_str(), 0);
-#else
-    setenv("O2DPLDEBUG", "xterm -hold -e gdb attach $O2DEBUGGEDPID &", 0);
-#endif
-    int retVal = system(getenv("O2DPLDEBUG"));
-    (void)retVal;
+    debugPID(pid);
   }
 
   ImGui::SameLine();
   if (ImGui::Button("Profile")) {
-    std::string pidStr = std::to_string(pid);
-    setenv("O2PROFILEDPID", pidStr.c_str(), 1);
-#ifdef __APPLE__
-    auto defaultAppleProfileCommand = fmt::format(
-      "osascript -e 'tell application \"Terminal\"'"
-      " -e 'activate'"
-      " -e 'do script \"xcrun xctrace record --output dpl-profile-{0}.trace"
-      " --instrument os_signpost --time-limit 30s --template Time\\\\ Profiler --attach {0} "
-      " && open dpl-profile-{0}.trace && exit\"'"
-      " -e 'end tell'",
-      pid);
-    std::cout << defaultAppleProfileCommand << std::endl;
-    setenv("O2DPLPROFILE", defaultAppleProfileCommand.c_str(), 0);
-#else
-    setenv("O2DPLPROFILE", "xterm -hold -e perf record -a -g -p $O2PROFILEDPID > perf-$O2PROFILEDPID.data &", 0);
-#endif
-    int retVal = system(getenv("O2DPLPROFILE"));
-    (void)retVal;
+    profilePID(pid);
   }
 
   // FIXME: this should really be a combo box
@@ -1110,7 +1146,8 @@ std::function<void(void)> getGUIDebugger(std::vector<DeviceInfo> const& infos,
                                          std::vector<DeviceMetricsInfo> const& metricsInfos,
                                          DriverInfo const& driverInfo,
                                          std::vector<DeviceControl>& controls,
-                                         DriverControl& driverControl)
+                                         DriverControl& driverControl,
+                                         GuiCallbackContext& gui)
 {
   static gui::WorkspaceGUIState globalGUIState;
   gui::WorkspaceGUIState& guiState = globalGUIState;
@@ -1141,7 +1178,7 @@ std::function<void(void)> getGUIDebugger(std::vector<DeviceInfo> const& infos,
   std::vector<TopologyNodeInfo> driverNodesInfos;
   driverNodesInfos.push_back(TopologyNodeInfo{"driver"});
 
-  return [&infos, &devices, &allStates, &metadata, &controls, &metricsInfos, &driverInfo, &driverControl, deviceNodesInfos, driverNodesInfos]() {
+  return [&infos, &devices, &allStates, &metadata, &controls, &metricsInfos, &driverInfo, &driverControl, &gui, deviceNodesInfos, driverNodesInfos]() {
     ImGuiStyle& style = ImGui::GetStyle();
     style.FrameRounding = 0.;
     style.WindowRounding = 0.;
@@ -1158,6 +1195,9 @@ std::function<void(void)> getGUIDebugger(std::vector<DeviceInfo> const& infos,
     metricsStore.specs[DRIVER_METRICS] = gsl::span(driverNodesInfos);
     displayMetrics(guiState, driverInfo, infos, metadata, controls, metricsStore);
     displayDriverInfo(driverInfo, driverControl);
+    if (*(gui.guiQuitRequested) && !*(gui.allChildrenGone)) {
+      displayShutdownDialog(devices, infos);
+    }
 
     int windowPosStepping = (ImGui::GetIO().DisplaySize.y - 500) / guiState.devices.size();
 
diff --git a/Framework/GUISupport/src/FrameworkGUIDebugger.h b/Framework/GUISupport/src/FrameworkGUIDebugger.h
index 5de5e350ddf8d..98bb50b07fd44 100644
--- a/Framework/GUISupport/src/FrameworkGUIDebugger.h
+++ b/Framework/GUISupport/src/FrameworkGUIDebugger.h
@@ -26,6 +26,7 @@ namespace o2::framework
 
 class DriverInfo;
 class DriverControl;
+class GuiCallbackContext;
 
 namespace gui
 {
@@ -37,7 +38,8 @@ std::function<void(void)> getGUIDebugger(std::vector<DeviceInfo> const& infos,
                                          std::vector<DeviceMetricsInfo> const& metricsInfos,
                                          DriverInfo const& driverInfo,
                                          std::vector<DeviceControl>& controls,
-                                         DriverControl& driverControl);
+                                         DriverControl& driverControl,
+                                         GuiCallbackContext& context);
 
 void updateMousePos(float x, float y);
 void updateMouseButton(bool clicked);
diff --git a/Framework/GUISupport/src/Plugin.cxx b/Framework/GUISupport/src/Plugin.cxx
index 7a356f6e55d9b..022b74174f632 100644
--- a/Framework/GUISupport/src/Plugin.cxx
+++ b/Framework/GUISupport/src/Plugin.cxx
@@ -36,9 +36,10 @@ struct ImGUIDebugGUI : o2::framework::DebugGUI {
                                            std::vector<DeviceMetricsInfo> const& metricsInfos,
                                            DriverInfo const& driverInfo,
                                            std::vector<DeviceControl>& controls,
-                                           DriverControl& driverControl) override
+                                           DriverControl& driverControl,
+                                           GuiCallbackContext& context) override
   {
-    return o2::framework::gui::getGUIDebugger(infos, devices, allStates, metadata, metricsInfos, driverInfo, controls, driverControl);
+    return o2::framework::gui::getGUIDebugger(infos, devices, allStates, metadata, metricsInfos, driverInfo, controls, driverControl, context);
   }
 
   void updateMousePos(float x, float y) override

From 0b08540a5f9c195dd0da911381cd817359f3a75e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 16 Nov 2023 11:53:27 +0100
Subject: [PATCH 110/726] DPL GUI: allow GUI to move between different states

---
 Framework/Core/src/DataProcessingDevice.cxx   | 49 ++++++++++++++++++-
 Framework/Core/src/WSDriverClient.cxx         | 24 +++++++++
 Framework/Core/src/runDataProcessing.cxx      | 13 ++++-
 .../GUISupport/src/FrameworkGUIDebugger.cxx   |  2 +-
 .../src/FrameworkGUIDeviceInspector.cxx       | 27 +++++++++-
 .../src/FrameworkGUIDeviceInspector.h         |  5 +-
 .../src/FrameworkGUIDevicesGraph.cxx          |  5 +-
 .../GUISupport/src/FrameworkGUIDevicesGraph.h |  8 ++-
 8 files changed, 123 insertions(+), 10 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 4d0cac7087611..c9a01d07449e4 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -25,6 +25,7 @@
 #include "Framework/DispatchPolicy.h"
 #include "Framework/DispatchControl.h"
 #include "Framework/DanglingContext.h"
+#include "Framework/DriverInfo.h"
 #include "Framework/DomainInfoHeader.h"
 #include "Framework/DriverClient.h"
 #include "Framework/EndOfStreamContext.h"
@@ -160,16 +161,60 @@ DataProcessingDevice::DataProcessingDevice(RunningDeviceRef running, ServiceRegi
 
   std::function<void(const fair::mq::State)> stateWatcher = [this, &registry = mServiceRegistry](const fair::mq::State state) -> void {
     auto ref = ServiceRegistryRef{registry, ServiceRegistry::globalDeviceSalt()};
+    auto controlKind = this->GetConfig()->GetPropertyAsString("control");
     auto& deviceState = ref.get<DeviceState>();
     auto& control = ref.get<ControlService>();
     auto& callbacks = ref.get<CallbackService>();
-    control.notifyDeviceState(fair::mq::GetStateName(state));
+    auto stateName = fair::mq::GetStateName(state);
+    control.notifyDeviceState(stateName);
     callbacks.call<CallbackService::Id::DeviceStateChanged>(ServiceRegistryRef{ref}, (int)state);
+    LOG(detail) << "In state watcher callback " << stateName;
+
+    // If the termination policy is not to wait, we simply ignore all
+    // user imposed state changes and keep running until we are done.
+    if (controlKind != "gui") {
+      return;
+    }
+
+    static bool runningOnce = false;
 
     if (deviceState.nextFairMQState.empty() == false) {
+      LOG(detail) << "State change requested, changing state to " << deviceState.nextFairMQState.back();
       auto state = deviceState.nextFairMQState.back();
-      (void)this->ChangeState(state);
+      bool changed = this->ChangeState(state);
+      if (!changed) {
+        LOG(error) << "Failed to change state to " << state;
+      }
       deviceState.nextFairMQState.pop_back();
+    } else if (state == fair::mq::State::Running && deviceState.nextFairMQState.empty()) {
+      LOGP(detail, "Device is running and no transition expected. We are done.");
+      deviceState.transitionHandling = TransitionHandlingState::NoTransition;
+    } else {
+      while (runningOnce && deviceState.nextFairMQState.empty() && this->NewStatePending() == false) {
+        LOG(detail) << "No state change requested, waiting for next state change " << this->NewStatePending();
+        if (stateName == "EXITING") {
+          // Send ctrl c to ourselves. To bad FairMQ does not seem to exit when
+          // reaching the EXITING state.
+          kill(getpid(), SIGTERM);
+          return;
+        }
+        uv_run(deviceState.loop, UV_RUN_ONCE);
+        LOG(detail) << "Woke up from event loop";
+      }
+      if (runningOnce && deviceState.nextFairMQState.empty() == false) {
+        LOG(detail) << "State change requested, changing state to " << deviceState.nextFairMQState.back();
+        auto state = deviceState.nextFairMQState.back();
+        bool changed = this->ChangeState(state);
+        if (!changed) {
+          LOG(error) << "Failed to change state to " << state;
+        }
+        deviceState.nextFairMQState.pop_back();
+      }
+      LOG(detail) << "Exiting callback for state " << state;
+    }
+    if (runningOnce == false && state == fair::mq::State::Running) {
+      LOG(detail) << "First iteration, next time we start the event loop";
+      runningOnce = true;
     }
   };
 
diff --git a/Framework/Core/src/WSDriverClient.cxx b/Framework/Core/src/WSDriverClient.cxx
index d4ed77b9a004e..91f011e61cda3 100644
--- a/Framework/Core/src/WSDriverClient.cxx
+++ b/Framework/Core/src/WSDriverClient.cxx
@@ -15,6 +15,8 @@
 #include "Framework/ServiceRegistry.h"
 #include "Framework/DeviceSpec.h"
 #include "DriverClientContext.h"
+#include "Framework/RawDeviceService.h"
+#include "Device.h"
 #include "DPLWebSocket.h"
 #include <uv.h>
 #include <string_view>
@@ -134,6 +136,28 @@ void on_connect(uv_connect_t* connection, int status)
     state.nextFairMQState.emplace_back("STOP");
   });
 
+  client->observe("/shutdown", [ref = context->ref](std::string_view) {
+    auto currentStateName = ref.get<RawDeviceService>().device()->GetCurrentStateName();
+    LOGP(info, "Received shutdown request while in {}", currentStateName);
+
+    auto& state = ref.get<DeviceState>();
+    state.nextFairMQState.emplace_back("END");
+    if (currentStateName == "IDLE") {
+      return;
+    }
+    state.nextFairMQState.emplace_back("AUTO");
+    state.nextFairMQState.emplace_back("RESET DEVICE");
+    if (currentStateName == "DEVICE READY") {
+      return;
+    }
+    state.nextFairMQState.emplace_back("AUTO");
+    state.nextFairMQState.emplace_back("RESET TASK");
+    if (currentStateName == "READY") {
+      return;
+    }
+    state.nextFairMQState.emplace_back("STOP");
+  });
+
   client->observe("/trace", [ref = context->ref](std::string_view cmd) {
     auto& state = ref.get<DeviceState>();
     static constexpr int prefixSize = std::string_view{"/trace "}.size();
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index f31629d90416b..f89b5f95693a6 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -29,6 +29,7 @@
 #include "Framework/DeviceMetricsInfo.h"
 #include "Framework/DeviceMetricsHelper.h"
 #include "Framework/DeviceConfigInfo.h"
+#include "Framework/DeviceController.h"
 #include "Framework/DeviceSpec.h"
 #include "Framework/DeviceState.h"
 #include "Framework/DeviceConfig.h"
@@ -2068,7 +2069,17 @@ int runStateMachine(DataProcessorSpecs const& workflow,
         // We send SIGCONT to make sure stopped children are resumed
         killChildren(infos, SIGCONT);
         // We send SIGTERM to make sure we do the STOP transition in FairMQ
-        killChildren(infos, SIGTERM);
+        if (driverInfo.processingPolicies.termination == TerminationPolicy::WAIT) {
+          for (size_t di = 0; di < infos.size(); ++di) {
+            auto& info = infos[di];
+            auto& control = controls[di];
+            if (info.active == true) {
+              control.controller->write("/shutdown", strlen("/shutdown"));
+            }
+          }
+        } else {
+          killChildren(infos, SIGTERM);
+        }
         // We have a timer to send SIGUSR1 to make sure we advance all devices
         // in a timely manner.
         force_step_timer.data = &infos;
diff --git a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
index 375afef1007f9..cc5180093ec94 100644
--- a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
@@ -1185,7 +1185,7 @@ std::function<void(void)> getGUIDebugger(std::vector<DeviceInfo> const& infos,
     style.Colors[ImGuiCol_WindowBg] = ImVec4(0x1b / 255.f, 0x1b / 255.f, 0x1b / 255.f, 1.00f);
     style.Colors[ImGuiCol_ScrollbarBg] = ImVec4(0x1b / 255.f, 0x1b / 255.f, 0x1b / 255.f, 1.00f);
 
-    showTopologyNodeGraph(guiState, infos, devices, allStates, metadata, controls, metricsInfos);
+    showTopologyNodeGraph(guiState, infos, devices, allStates, metadata, controls, metricsInfos, driverInfo.processingPolicies.termination);
 
     AllMetricsStore metricsStore;
 
diff --git a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
index a82753eb5af1f..9febb93858cb8 100644
--- a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
@@ -11,6 +11,7 @@
 
 #include "FrameworkGUIDeviceInspector.h"
 #include "Framework/DataProcessorInfo.h"
+#include "Framework/ProcessingPolicies.h"
 
 #include "Framework/DeviceControl.h"
 #include "Framework/DeviceSpec.h"
@@ -251,7 +252,8 @@ void displayDeviceInspector(DeviceSpec const& spec,
                             DataProcessingStates const& states,
                             DeviceMetricsInfo const& metrics,
                             DataProcessorInfo const& metadata,
-                            DeviceControl& control)
+                            DeviceControl& control,
+                            TerminationPolicy terminationPolicy)
 {
   ImGui::Text("Name: %s", spec.name.c_str());
   ImGui::Text("Executable: %s", metadata.executable.c_str());
@@ -340,12 +342,33 @@ void displayDeviceInspector(DeviceSpec const& spec,
     }
 
     if (control.requestedState > info.providedState) {
-      ImGui::Text(ICON_FA_CLOCK_O);
+      ImGui::TextUnformatted(ICON_FA_CLOCK_O "Requested transition in progress");
     } else {
+      // We only allow navigation if the termination policy is "WAIT"
+      ImGui::BeginDisabled(terminationPolicy == TerminationPolicy::QUIT);
       if (ImGui::Button("Restart")) {
         control.requestedState = info.providedState + 1;
         control.controller->write("/restart", strlen("/restart"));
       }
+      if (info.deviceState == "RUNNING") {
+        ImGui::SameLine();
+        if (ImGui::Button(ICON_FA_STOP)) {
+          control.requestedState = info.providedState + 1;
+          control.controller->write("/stop", strlen("/stop"));
+        }
+      } else if (info.deviceState == "READY") {
+        ImGui::SameLine();
+        if (ImGui::Button(ICON_FA_PLAY)) {
+          control.requestedState = info.providedState + 1;
+          control.controller->write("/start", strlen("/start"));
+        }
+        ImGui::SameLine();
+        if (ImGui::Button(ICON_FA_POWER_OFF)) {
+          control.requestedState = info.providedState + 1;
+          control.controller->write("/shutdown", strlen("/shutdown"));
+        }
+      }
+      ImGui::EndDisabled();
     }
   }
 
diff --git a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.h b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.h
index 8dac3204ea3ab..b69c6e9286215 100644
--- a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.h
+++ b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.h
@@ -18,12 +18,15 @@ struct DeviceInfo;
 struct DeviceMetricsInfo;
 struct DataProcessorInfo;
 struct DataProcessingStates;
+enum struct TerminationPolicy;
 
 namespace gui
 {
 
 /// Helper to display information about a device
-void displayDeviceInspector(DeviceSpec const& spec, DeviceInfo const& info, DataProcessingStates const& states, DeviceMetricsInfo const& metrics, DataProcessorInfo const& metadata, DeviceControl& control);
+void displayDeviceInspector(DeviceSpec const& spec, DeviceInfo const& info, DataProcessingStates const& states,
+                            DeviceMetricsInfo const& metrics, DataProcessorInfo const& metadata, DeviceControl& control,
+                            TerminationPolicy TerminationPolicy);
 
 } // namespace gui
 } // namespace o2::framework
diff --git a/Framework/GUISupport/src/FrameworkGUIDevicesGraph.cxx b/Framework/GUISupport/src/FrameworkGUIDevicesGraph.cxx
index 0b6ca07a9c86d..639122153e599 100644
--- a/Framework/GUISupport/src/FrameworkGUIDevicesGraph.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDevicesGraph.cxx
@@ -365,7 +365,8 @@ void showTopologyNodeGraph(WorkspaceGUIState& state,
                            std::vector<DataProcessingStates> const& allStates,
                            std::vector<DataProcessorInfo> const& metadata,
                            std::vector<DeviceControl>& controls,
-                           std::vector<DeviceMetricsInfo> const& metricsInfos)
+                           std::vector<DeviceMetricsInfo> const& metricsInfos,
+                           enum TerminationPolicy terminationPolicy)
 {
   ImGui::SetNextWindowPos(ImVec2(0, 0), 0);
   if (state.bottomPaneVisible) {
@@ -893,7 +894,7 @@ void showTopologyNodeGraph(WorkspaceGUIState& state,
       auto& metadatum = metadata[group.metadataId];
 
       if (state.rightPaneVisible) {
-        gui::displayDeviceInspector(spec, info, states, metrics, metadatum, control);
+        gui::displayDeviceInspector(spec, info, states, metrics, metadatum, control, terminationPolicy);
       }
     } else {
       ImGui::TextWrapped("Select a node in the topology to display information about it");
diff --git a/Framework/GUISupport/src/FrameworkGUIDevicesGraph.h b/Framework/GUISupport/src/FrameworkGUIDevicesGraph.h
index 7950f1893ebba..d53cdf26dd5a8 100644
--- a/Framework/GUISupport/src/FrameworkGUIDevicesGraph.h
+++ b/Framework/GUISupport/src/FrameworkGUIDevicesGraph.h
@@ -19,6 +19,11 @@
 
 #include <vector>
 
+namespace o2::framework
+{
+enum struct TerminationPolicy;
+} // namespace o2::framework
+
 namespace o2::framework::gui
 {
 
@@ -30,7 +35,8 @@ void showTopologyNodeGraph(WorkspaceGUIState& state,
                            std::vector<DataProcessingStates> const& allStates,
                            std::vector<DataProcessorInfo> const& metadata,
                            std::vector<DeviceControl>& controls,
-                           std::vector<DeviceMetricsInfo> const& metricsInfos);
+                           std::vector<DeviceMetricsInfo> const& metricsInfos,
+                           TerminationPolicy terminationPolicy);
 
 } // namespace o2::framework::gui
 

From 354819d1d8ad0954178d9aff29603ef23459b328 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 17 Nov 2023 11:01:54 +0100
Subject: [PATCH 111/726] Misc formatting cleanups

---
 .../MemoryResources/test/testMemoryResources.cxx   |  2 +-
 Framework/Core/test/test_FairMQ.cxx                | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/DataFormats/MemoryResources/test/testMemoryResources.cxx b/DataFormats/MemoryResources/test/testMemoryResources.cxx
index 592e76c8ef1f4..a77b291568701 100644
--- a/DataFormats/MemoryResources/test/testMemoryResources.cxx
+++ b/DataFormats/MemoryResources/test/testMemoryResources.cxx
@@ -165,7 +165,7 @@ BOOST_AUTO_TEST_CASE(adoptVector_test)
 
   testData::nconstructions = 0;
 
-  //Create a bogus message
+  // Create a bogus message
   auto message = factoryZMQ->CreateMessage(3 * sizeof(testData));
   auto messageAddr = message.get();
   testData tmpBuf[3] = {3, 2, 1};
diff --git a/Framework/Core/test/test_FairMQ.cxx b/Framework/Core/test/test_FairMQ.cxx
index 2919dd1faf0eb..a8487165739ec 100644
--- a/Framework/Core/test/test_FairMQ.cxx
+++ b/Framework/Core/test/test_FairMQ.cxx
@@ -44,7 +44,7 @@ bool addDataBlock(fair::mq::Parts& parts, o2::header::Stack&& inputStack, Contai
 template <typename ContainerT, typename std::enable_if<std::is_same<ContainerT, fair::mq::MessagePtr>::value, int>::type = 0>
 bool addDataBlock(fair::mq::Parts& parts, o2::header::Stack&& inputStack, ContainerT&& dataMessage, o2::pmr::FairMQMemoryResource* targetResource = nullptr)
 {
-  //make sure the payload size in DataHeader corresponds to message size
+  // make sure the payload size in DataHeader corresponds to message size
   using o2::header::DataHeader;
   auto* dataHeader = const_cast<DataHeader*>(o2::header::get<DataHeader*>(inputStack.data()));
   dataHeader->payloadSize = dataMessage->GetSize();
@@ -113,7 +113,7 @@ TEST_CASE("getMessage_Stack")
   auto allocSHM = getTransportAllocator(factorySHM.get());
   REQUIRE(allocSHM != nullptr);
   {
-    //check that a message is constructed properly with the default new_delete_resource
+    // check that a message is constructed properly with the default new_delete_resource
     Stack s1{DataHeader{gDataDescriptionInvalid, gDataOriginInvalid, DataHeader::SubSpecificationType{0}},
              NameHeader<9>{"somename"}};
 
@@ -128,7 +128,7 @@ TEST_CASE("getMessage_Stack")
     REQUIRE(message->GetType() == fair::mq::Transport::ZMQ);
   }
   {
-    //check that a message is constructed properly, cross resource
+    // check that a message is constructed properly, cross resource
     Stack s1{allocZMQ, DataHeader{gDataDescriptionInvalid, gDataOriginInvalid, DataHeader::SubSpecificationType{0}},
              NameHeader<9>{"somename"}};
     REQUIRE(allocZMQ->getNumberOfMessages() == 1);
@@ -159,7 +159,7 @@ TEST_CASE("addDataBlockForEach_test")
   REQUIRE(allocZMQ);
 
   {
-    //simple addition of a data block from an exisiting message
+    // simple addition of a data block from an exisiting message
     fair::mq::Parts message;
     auto simpleMessage = factoryZMQ->CreateMessage(10);
     addDataBlock(message,
@@ -190,7 +190,7 @@ TEST_CASE("addDataBlockForEach_test")
     REQUIRE(message[1].GetSize() == 2 * sizeof(elem));
     ; // check the size of the buffer is set correctly
 
-    //check contents
+    // check contents
     int sum{0};
     forEach(message, [&](auto header, auto data) {
       const int* numbers = reinterpret_cast<const int*>(data.data());
@@ -198,7 +198,7 @@ TEST_CASE("addDataBlockForEach_test")
     });
     REQUIRE(sum == 10);
 
-    //add one more data block and check total size using forEach;
+    // add one more data block and check total size using forEach;
     addDataBlock(message,
                  Stack{allocZMQ, DataHeader{gDataDescriptionInvalid, gDataOriginInvalid, DataHeader::SubSpecificationType{0}}},
                  factoryZMQ->CreateMessage(10));
@@ -206,7 +206,7 @@ TEST_CASE("addDataBlockForEach_test")
     forEach(message, [&](auto header, auto data) { size += header.size() + data.size(); });
     REQUIRE(size == sizeofDataHeader + 2 * sizeof(elem) + sizeofDataHeader + 10);
 
-    //check contents (headers)
+    // check contents (headers)
     int checkOK{0};
     forEach(message, [&](auto header, auto data) {
       auto dh = get<DataHeader*>(header.data());

From dcc2d884828c7f126207eefade9f257295211a48 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 17 Nov 2023 11:01:18 +0100
Subject: [PATCH 112/726] Avoid UUID conflicts when running with boost 1.83.0

Apparently something broke and now we get all the UUIDs to be the same.
---
 DataFormats/MemoryResources/test/testMemoryResources.cxx | 8 ++++----
 Framework/Core/test/test_FairMQ.cxx                      | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/DataFormats/MemoryResources/test/testMemoryResources.cxx b/DataFormats/MemoryResources/test/testMemoryResources.cxx
index a77b291568701..339f100a01108 100644
--- a/DataFormats/MemoryResources/test/testMemoryResources.cxx
+++ b/DataFormats/MemoryResources/test/testMemoryResources.cxx
@@ -47,7 +47,7 @@ int testData::nconstructions = 0;
 
 BOOST_AUTO_TEST_CASE(transportallocatormap_test)
 {
-  size_t session{fair::mq::tools::UuidHash()};
+  size_t session{(size_t)getpid() * 1000};
   fair::mq::ProgOptions config;
   config.SetProperty<std::string>("session", std::to_string(session));
 
@@ -64,7 +64,7 @@ using namespace boost::container::pmr;
 
 BOOST_AUTO_TEST_CASE(allocator_test)
 {
-  size_t session{fair::mq::tools::UuidHash()};
+  size_t session{(size_t)getpid() * 1000 + 1};
   fair::mq::ProgOptions config;
   config.SetProperty<std::string>("session", std::to_string(session));
 
@@ -102,7 +102,7 @@ BOOST_AUTO_TEST_CASE(allocator_test)
 
 BOOST_AUTO_TEST_CASE(getMessage_test)
 {
-  size_t session{fair::mq::tools::UuidHash()};
+  size_t session{(size_t)getpid() * 1000 + 2};
   fair::mq::ProgOptions config;
   config.SetProperty<std::string>("session", std::to_string(session));
 
@@ -154,7 +154,7 @@ BOOST_AUTO_TEST_CASE(getMessage_test)
 
 BOOST_AUTO_TEST_CASE(adoptVector_test)
 {
-  size_t session{fair::mq::tools::UuidHash()};
+  size_t session{(size_t)getpid() * 1000 + 3};
   fair::mq::ProgOptions config;
   config.SetProperty<std::string>("session", std::to_string(session));
 
diff --git a/Framework/Core/test/test_FairMQ.cxx b/Framework/Core/test/test_FairMQ.cxx
index a8487165739ec..883533855126d 100644
--- a/Framework/Core/test/test_FairMQ.cxx
+++ b/Framework/Core/test/test_FairMQ.cxx
@@ -100,7 +100,7 @@ auto forEach(fair::mq::Parts& parts, F&& function)
 
 TEST_CASE("getMessage_Stack")
 {
-  size_t session{fair::mq::tools::UuidHash()};
+  size_t session{(size_t)getpid() * 1000 + 0};
   fair::mq::ProgOptions config;
   config.SetProperty<std::string>("session", std::to_string(session));
 
@@ -149,7 +149,7 @@ TEST_CASE("getMessage_Stack")
 
 TEST_CASE("addDataBlockForEach_test")
 {
-  size_t session{fair::mq::tools::UuidHash()};
+  size_t session{(size_t)getpid() * 1000 + 1};
   fair::mq::ProgOptions config;
   config.SetProperty<std::string>("session", std::to_string(session));
 

From 9f1f86e762d577cd44984a7b010cb2c43442e354 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 20 Nov 2023 23:44:21 +0100
Subject: [PATCH 113/726] DPL: actually pass session when creating Transport

this might explain some sporadic crashes in the affected tests, due to session overllap.
---
 DataFormats/MemoryResources/test/testMemoryResources.cxx | 8 ++++----
 Framework/Core/test/test_FairMQ.cxx                      | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/DataFormats/MemoryResources/test/testMemoryResources.cxx b/DataFormats/MemoryResources/test/testMemoryResources.cxx
index 339f100a01108..264fe59b1caac 100644
--- a/DataFormats/MemoryResources/test/testMemoryResources.cxx
+++ b/DataFormats/MemoryResources/test/testMemoryResources.cxx
@@ -52,7 +52,7 @@ BOOST_AUTO_TEST_CASE(transportallocatormap_test)
   config.SetProperty<std::string>("session", std::to_string(session));
 
   auto factoryZMQ = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
-  auto factorySHM = fair::mq::TransportFactory::CreateTransportFactory("shmem");
+  auto factorySHM = fair::mq::TransportFactory::CreateTransportFactory("shmem", "transportallocatormap_test", &config);
   auto allocZMQ = getTransportAllocator(factoryZMQ.get());
   auto allocSHM = getTransportAllocator(factorySHM.get());
   BOOST_CHECK(allocZMQ != nullptr && allocSHM != allocZMQ);
@@ -69,7 +69,7 @@ BOOST_AUTO_TEST_CASE(allocator_test)
   config.SetProperty<std::string>("session", std::to_string(session));
 
   auto factoryZMQ = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
-  auto factorySHM = fair::mq::TransportFactory::CreateTransportFactory("shmem");
+  auto factorySHM = fair::mq::TransportFactory::CreateTransportFactory("shmem", "allocator_test", &config);
   auto allocZMQ = getTransportAllocator(factoryZMQ.get());
   auto allocSHM = getTransportAllocator(factorySHM.get());
 
@@ -107,7 +107,7 @@ BOOST_AUTO_TEST_CASE(getMessage_test)
   config.SetProperty<std::string>("session", std::to_string(session));
 
   auto factoryZMQ = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
-  auto factorySHM = fair::mq::TransportFactory::CreateTransportFactory("shmem");
+  auto factorySHM = fair::mq::TransportFactory::CreateTransportFactory("shmem", "getMessage_test", &config);
   auto allocZMQ = getTransportAllocator(factoryZMQ.get());
   auto allocSHM = getTransportAllocator(factorySHM.get());
 
@@ -159,7 +159,7 @@ BOOST_AUTO_TEST_CASE(adoptVector_test)
   config.SetProperty<std::string>("session", std::to_string(session));
 
   auto factoryZMQ = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
-  auto factorySHM = fair::mq::TransportFactory::CreateTransportFactory("shmem");
+  auto factorySHM = fair::mq::TransportFactory::CreateTransportFactory("shmem", "adoptVector_test", &config);
   auto allocZMQ = getTransportAllocator(factoryZMQ.get());
   auto allocSHM = getTransportAllocator(factorySHM.get());
 
diff --git a/Framework/Core/test/test_FairMQ.cxx b/Framework/Core/test/test_FairMQ.cxx
index 883533855126d..0ccfbf58f7576 100644
--- a/Framework/Core/test/test_FairMQ.cxx
+++ b/Framework/Core/test/test_FairMQ.cxx
@@ -105,7 +105,7 @@ TEST_CASE("getMessage_Stack")
   config.SetProperty<std::string>("session", std::to_string(session));
 
   auto factoryZMQ = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
-  auto factorySHM = fair::mq::TransportFactory::CreateTransportFactory("shmem");
+  auto factorySHM = fair::mq::TransportFactory::CreateTransportFactory("shmem", "getMessage_Stack", &config);
   REQUIRE(factorySHM != nullptr);
   REQUIRE(factoryZMQ != nullptr);
   auto allocZMQ = getTransportAllocator(factoryZMQ.get());

From a0df6d3ac79e2724c56429c2b355816484fd4b8d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 21 Nov 2023 21:34:36 +0100
Subject: [PATCH 114/726] Fix coding convention

---
 Generators/src/GeneratorPythia8.cxx | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Generators/src/GeneratorPythia8.cxx b/Generators/src/GeneratorPythia8.cxx
index f72a214223307..a2a83ca74b35e 100644
--- a/Generators/src/GeneratorPythia8.cxx
+++ b/Generators/src/GeneratorPythia8.cxx
@@ -456,10 +456,12 @@ void GeneratorPythia8::pruneEvent(Pythia8::Event& event, Select select)
         auto& otherMother = pruned[otherMotherIdx];
         int otherDaughter1 = otherMother.daughter1();
         int otherDaughter2 = otherMother.daughter2();
-        if (otherDaughter1 > 0)
+        if (otherDaughter1 > 0) {
           addId(allDaughters, otherDaughter1);
-        if (otherDaughter2 > 0)
+        }
+        if (otherDaughter2 > 0) {
           addId(allDaughters, otherDaughter2);
+        }
       }
       // At this point, we have added all mothers of current
       // daughter, and all daughters of those mothers.

From 169a476c5653964813d0ac4f65e800f2af1179cb Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 23 Nov 2023 10:49:42 +0100
Subject: [PATCH 115/726] DPL: add edge validation support after topology
 generation

---
 Framework/Core/src/CommonDataProcessors.cxx |  4 +++-
 Framework/Core/src/DeviceSpecHelpers.cxx    |  2 ++
 Framework/Core/src/WorkflowHelpers.cxx      | 26 ++++++++++++++++++++-
 Framework/Core/src/WorkflowHelpers.h        |  8 +++++++
 4 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 74e020595b345..e9cdc97e316ca 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -259,9 +259,11 @@ DataProcessorSpec CommonDataProcessors::getOutputObjHistSink(std::vector<OutputO
   };
 
   char const* name = "internal-dpl-aod-global-analysis-file-sink";
+  // Lifetime is sporadic because we do not ask each analysis task to send its
+  // results every timeframe.
   DataProcessorSpec spec{
     .name = name,
-    .inputs = {InputSpec("x", DataSpecUtils::dataDescriptorMatcherFrom(header::DataOrigin{"ATSK"}))},
+    .inputs = {InputSpec("x", DataSpecUtils::dataDescriptorMatcherFrom(header::DataOrigin{"ATSK"}), Lifetime::Sporadic)},
     .algorithm = {writerFunction},
   };
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index c54dd6cec7a99..45a76570dbf1b 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1072,6 +1072,8 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
 
   WorkflowHelpers::constructGraph(workflow, logicalEdges, outputs, availableForwardsInfo);
 
+  WorkflowHelpers::validateEdges(workflow, logicalEdges, outputs);
+
   // We need to instanciate one device per (me, timeIndex) in the
   // DeviceConnectionEdge. For each device we need one new binding
   // server per (me, other) -> port Moreover for each (me, other,
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index cd9c3913de228..fb9c4aa1db319 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -1232,5 +1232,29 @@ std::vector<InputSpec> WorkflowHelpers::computeDanglingOutputs(WorkflowSpec cons
   return results;
 }
 
-#pragma diagnostic pop
+using Validator = std::function<bool(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)>;
+void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
+                                    std::vector<DeviceConnectionEdge> const& edges,
+                                    std::vector<OutputSpec> const& outputs)
+{
+  std::vector<Validator> defaultValidators = {};
+  std::stringstream errors;
+  // Iterate over all the edges.
+  // Get the input lifetime and the output lifetime.
+  // Output lifetime must be Timeframe if the input lifetime is Timeframe.
+  bool hasErrors = false;
+  for (auto& edge : edges) {
+    DataProcessorSpec const& producer = workflow[edge.producer];
+    DataProcessorSpec const& consumer = workflow[edge.consumer];
+    OutputSpec const& output = outputs[edge.outputGlobalIndex];
+    InputSpec const& input = consumer.inputs[edge.consumerInputIndex];
+    for (auto& validator : defaultValidators) {
+      hasErrors |= validator(errors, producer, output, consumer, input);
+    }
+  }
+  if (!hasErrors) {
+    throw std::runtime_error(errors.str());
+  }
+}
+
 } // namespace o2::framework
diff --git a/Framework/Core/src/WorkflowHelpers.h b/Framework/Core/src/WorkflowHelpers.h
index a167e99d4768c..67661dbdb979a 100644
--- a/Framework/Core/src/WorkflowHelpers.h
+++ b/Framework/Core/src/WorkflowHelpers.h
@@ -227,6 +227,14 @@ struct WorkflowHelpers {
 
   /// returns only dangling outputs
   static std::vector<InputSpec> computeDanglingOutputs(WorkflowSpec const& workflow);
+
+  /// Validate that the nodes at the ends of the edges of the graph
+  /// are actually compatible with each other.
+  /// For example we should make sure that Lifetime::Timeframe inputs of
+  /// one node is not connected to an Output of Lifetime::Sporadic of another node.
+  static void validateEdges(WorkflowSpec const& workflow,
+                            std::vector<DeviceConnectionEdge> const& edges,
+                            std::vector<OutputSpec> const& outputs);
 };
 
 } // namespace o2::framework

From 00bb2fd7a1d64a10f62ba58cffcecb173af25042 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 23 Nov 2023 10:49:42 +0100
Subject: [PATCH 116/726] DPL: edge validator for Lifetime::Sporadic

---
 Framework/Core/src/WorkflowHelpers.cxx | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index fb46b295e17bc..a263d1befd32a 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -1232,6 +1232,17 @@ std::vector<InputSpec> WorkflowHelpers::computeDanglingOutputs(WorkflowSpec cons
   return results;
 }
 
+bool validateLifetime(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
+{
+  if (input.lifetime == Lifetime::Timeframe && output.lifetime == Lifetime::Sporadic) {
+    errors << fmt::format("Input {} of {} has lifetime Timeframe, but output {} of {} has lifetime Sporadic\n",
+                          DataSpecUtils::describe(input).c_str(), consumer.name,
+                          DataSpecUtils::describe(output).c_str(), producer.name);
+    return false;
+  }
+  return true;
+}
+
 bool validateExpendable(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
 {
   auto isExpendable = [](DataProcessorLabel const& label) {

From 1793e650827a9b96d13805aaf41e670a045c8c36 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 23 Nov 2023 10:49:42 +0100
Subject: [PATCH 117/726] DPL: add edge validator for expendable tasks

---
 Framework/Core/src/WorkflowHelpers.cxx | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index fb9c4aa1db319..fb46b295e17bc 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -1232,12 +1232,31 @@ std::vector<InputSpec> WorkflowHelpers::computeDanglingOutputs(WorkflowSpec cons
   return results;
 }
 
+bool validateExpendable(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
+{
+  auto isExpendable = [](DataProcessorLabel const& label) {
+    return label.value == "expendable";
+  };
+  auto isResilient = [](DataProcessorLabel const& label) {
+    return label.value == "expendable" || label.value == "resilient";
+  };
+  bool producerExpendable = std::find_if(producer.labels.begin(), producer.labels.end(), isExpendable) != producer.labels.end();
+  bool consumerCritical = std::find_if(consumer.labels.begin(), consumer.labels.end(), isResilient) == consumer.labels.end();
+  if (producerExpendable && consumerCritical) {
+    errors << fmt::format("Critical consumer {} depends on expendable producer {}\n",
+                          consumer.name,
+                          producer.name);
+    return false;
+  }
+  return true;
+}
+
 using Validator = std::function<bool(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)>;
 void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
                                     std::vector<DeviceConnectionEdge> const& edges,
                                     std::vector<OutputSpec> const& outputs)
 {
-  std::vector<Validator> defaultValidators = {};
+  std::vector<Validator> defaultValidators = {validateExpendable};
   std::stringstream errors;
   // Iterate over all the edges.
   // Get the input lifetime and the output lifetime.

From 66ef04a80514ea17f207dc8efb85fb0db42eca49 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 23 Nov 2023 10:52:59 +0100
Subject: [PATCH 118/726] DPL: edge validator for Lifetime::Sporadic

---
 Framework/Core/src/WorkflowHelpers.cxx | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index fb46b295e17bc..7823070c8be76 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -1232,6 +1232,17 @@ std::vector<InputSpec> WorkflowHelpers::computeDanglingOutputs(WorkflowSpec cons
   return results;
 }
 
+bool validateLifetime(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
+{
+  if (input.lifetime == Lifetime::Timeframe && output.lifetime == Lifetime::Sporadic) {
+    errors << fmt::format("Input {} of {} has lifetime Timeframe, but output {} of {} has lifetime Sporadic\n",
+                          DataSpecUtils::describe(input).c_str(), consumer.name,
+                          DataSpecUtils::describe(output).c_str(), producer.name);
+    return false;
+  }
+  return true;
+}
+
 bool validateExpendable(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
 {
   auto isExpendable = [](DataProcessorLabel const& label) {
@@ -1256,7 +1267,7 @@ void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
                                     std::vector<DeviceConnectionEdge> const& edges,
                                     std::vector<OutputSpec> const& outputs)
 {
-  std::vector<Validator> defaultValidators = {validateExpendable};
+  std::vector<Validator> defaultValidators = {validateExpendable, validateLifetime};
   std::stringstream errors;
   // Iterate over all the edges.
   // Get the input lifetime and the output lifetime.

From 81e010a2ee50ac1b9935c9fb6c59407422936776 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 23 Nov 2023 13:04:56 +0100
Subject: [PATCH 119/726] DPL: add edge validation support after topology
 generation

---
 Framework/Core/src/CommonDataProcessors.cxx |  4 +++-
 Framework/Core/src/DeviceSpecHelpers.cxx    |  2 ++
 Framework/Core/src/WorkflowHelpers.cxx      | 26 ++++++++++++++++++++-
 Framework/Core/src/WorkflowHelpers.h        |  8 +++++++
 4 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 74e020595b345..e9cdc97e316ca 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -259,9 +259,11 @@ DataProcessorSpec CommonDataProcessors::getOutputObjHistSink(std::vector<OutputO
   };
 
   char const* name = "internal-dpl-aod-global-analysis-file-sink";
+  // Lifetime is sporadic because we do not ask each analysis task to send its
+  // results every timeframe.
   DataProcessorSpec spec{
     .name = name,
-    .inputs = {InputSpec("x", DataSpecUtils::dataDescriptorMatcherFrom(header::DataOrigin{"ATSK"}))},
+    .inputs = {InputSpec("x", DataSpecUtils::dataDescriptorMatcherFrom(header::DataOrigin{"ATSK"}), Lifetime::Sporadic)},
     .algorithm = {writerFunction},
   };
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index c54dd6cec7a99..45a76570dbf1b 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1072,6 +1072,8 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
 
   WorkflowHelpers::constructGraph(workflow, logicalEdges, outputs, availableForwardsInfo);
 
+  WorkflowHelpers::validateEdges(workflow, logicalEdges, outputs);
+
   // We need to instanciate one device per (me, timeIndex) in the
   // DeviceConnectionEdge. For each device we need one new binding
   // server per (me, other) -> port Moreover for each (me, other,
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index cd9c3913de228..f8d6f351869dd 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -1232,5 +1232,29 @@ std::vector<InputSpec> WorkflowHelpers::computeDanglingOutputs(WorkflowSpec cons
   return results;
 }
 
-#pragma diagnostic pop
+using Validator = std::function<bool(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)>;
+void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
+                                    std::vector<DeviceConnectionEdge> const& edges,
+                                    std::vector<OutputSpec> const& outputs)
+{
+  std::vector<Validator> defaultValidators = {};
+  std::stringstream errors;
+  // Iterate over all the edges.
+  // Get the input lifetime and the output lifetime.
+  // Output lifetime must be Timeframe if the input lifetime is Timeframe.
+  bool hasErrors = false;
+  for (auto& edge : edges) {
+    DataProcessorSpec const& producer = workflow[edge.producer];
+    DataProcessorSpec const& consumer = workflow[edge.consumer];
+    OutputSpec const& output = outputs[edge.outputGlobalIndex];
+    InputSpec const& input = consumer.inputs[edge.consumerInputIndex];
+    for (auto& validator : defaultValidators) {
+      hasErrors |= validator(errors, producer, output, consumer, input);
+    }
+  }
+  if (hasErrors) {
+    throw std::runtime_error(errors.str());
+  }
+}
+
 } // namespace o2::framework
diff --git a/Framework/Core/src/WorkflowHelpers.h b/Framework/Core/src/WorkflowHelpers.h
index a167e99d4768c..67661dbdb979a 100644
--- a/Framework/Core/src/WorkflowHelpers.h
+++ b/Framework/Core/src/WorkflowHelpers.h
@@ -227,6 +227,14 @@ struct WorkflowHelpers {
 
   /// returns only dangling outputs
   static std::vector<InputSpec> computeDanglingOutputs(WorkflowSpec const& workflow);
+
+  /// Validate that the nodes at the ends of the edges of the graph
+  /// are actually compatible with each other.
+  /// For example we should make sure that Lifetime::Timeframe inputs of
+  /// one node is not connected to an Output of Lifetime::Sporadic of another node.
+  static void validateEdges(WorkflowSpec const& workflow,
+                            std::vector<DeviceConnectionEdge> const& edges,
+                            std::vector<OutputSpec> const& outputs);
 };
 
 } // namespace o2::framework

From 54354c89a31df88e486c83424a161223d3fccccc Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 23 Nov 2023 13:05:16 +0100
Subject: [PATCH 120/726] DPL: add edge validator for expendable tasks

---
 Framework/Core/src/WorkflowHelpers.cxx | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index f8d6f351869dd..6f198621d84a8 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -1232,12 +1232,31 @@ std::vector<InputSpec> WorkflowHelpers::computeDanglingOutputs(WorkflowSpec cons
   return results;
 }
 
+bool validateExpendable(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
+{
+  auto isExpendable = [](DataProcessorLabel const& label) {
+    return label.value == "expendable";
+  };
+  auto isResilient = [](DataProcessorLabel const& label) {
+    return label.value == "expendable" || label.value == "resilient";
+  };
+  bool producerExpendable = std::find_if(producer.labels.begin(), producer.labels.end(), isExpendable) != producer.labels.end();
+  bool consumerCritical = std::find_if(consumer.labels.begin(), consumer.labels.end(), isResilient) == consumer.labels.end();
+  if (producerExpendable && consumerCritical) {
+    errors << fmt::format("Critical consumer {} depends on expendable producer {}\n",
+                          consumer.name,
+                          producer.name);
+    return false;
+  }
+  return true;
+}
+
 using Validator = std::function<bool(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)>;
 void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
                                     std::vector<DeviceConnectionEdge> const& edges,
                                     std::vector<OutputSpec> const& outputs)
 {
-  std::vector<Validator> defaultValidators = {};
+  std::vector<Validator> defaultValidators = {validateExpendable};
   std::stringstream errors;
   // Iterate over all the edges.
   // Get the input lifetime and the output lifetime.

From 6fd39708f508963e83d5b3463608e6f35f3f5de8 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 23 Nov 2023 13:05:16 +0100
Subject: [PATCH 121/726] DPL: edge validator for Lifetime::Sporadic

---
 Framework/Core/src/WorkflowHelpers.cxx | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 6f198621d84a8..0864632d66e59 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -1232,6 +1232,17 @@ std::vector<InputSpec> WorkflowHelpers::computeDanglingOutputs(WorkflowSpec cons
   return results;
 }
 
+bool validateLifetime(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
+{
+  if (input.lifetime == Lifetime::Timeframe && output.lifetime == Lifetime::Sporadic) {
+    errors << fmt::format("Input {} of {} has lifetime Timeframe, but output {} of {} has lifetime Sporadic\n",
+                          DataSpecUtils::describe(input).c_str(), consumer.name,
+                          DataSpecUtils::describe(output).c_str(), producer.name);
+    return false;
+  }
+  return true;
+}
+
 bool validateExpendable(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
 {
   auto isExpendable = [](DataProcessorLabel const& label) {
@@ -1256,7 +1267,7 @@ void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
                                     std::vector<DeviceConnectionEdge> const& edges,
                                     std::vector<OutputSpec> const& outputs)
 {
-  std::vector<Validator> defaultValidators = {validateExpendable};
+  std::vector<Validator> defaultValidators = {validateExpendable, validateLifetime};
   std::stringstream errors;
   // Iterate over all the edges.
   // Get the input lifetime and the output lifetime.

From d85d40b58edea20b7b2c8e31156de0d7b55983e1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 23 Nov 2023 14:37:01 +0100
Subject: [PATCH 122/726] DPL: add edge validation support after topology
 generation

---
 Framework/Core/src/CommonDataProcessors.cxx |  4 +++-
 Framework/Core/src/DeviceSpecHelpers.cxx    |  2 ++
 Framework/Core/src/WorkflowHelpers.cxx      | 26 ++++++++++++++++++++-
 Framework/Core/src/WorkflowHelpers.h        |  8 +++++++
 4 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 74e020595b345..e9cdc97e316ca 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -259,9 +259,11 @@ DataProcessorSpec CommonDataProcessors::getOutputObjHistSink(std::vector<OutputO
   };
 
   char const* name = "internal-dpl-aod-global-analysis-file-sink";
+  // Lifetime is sporadic because we do not ask each analysis task to send its
+  // results every timeframe.
   DataProcessorSpec spec{
     .name = name,
-    .inputs = {InputSpec("x", DataSpecUtils::dataDescriptorMatcherFrom(header::DataOrigin{"ATSK"}))},
+    .inputs = {InputSpec("x", DataSpecUtils::dataDescriptorMatcherFrom(header::DataOrigin{"ATSK"}), Lifetime::Sporadic)},
     .algorithm = {writerFunction},
   };
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index c54dd6cec7a99..45a76570dbf1b 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1072,6 +1072,8 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
 
   WorkflowHelpers::constructGraph(workflow, logicalEdges, outputs, availableForwardsInfo);
 
+  WorkflowHelpers::validateEdges(workflow, logicalEdges, outputs);
+
   // We need to instanciate one device per (me, timeIndex) in the
   // DeviceConnectionEdge. For each device we need one new binding
   // server per (me, other) -> port Moreover for each (me, other,
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index cd9c3913de228..f320e788f5e07 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -1232,5 +1232,29 @@ std::vector<InputSpec> WorkflowHelpers::computeDanglingOutputs(WorkflowSpec cons
   return results;
 }
 
-#pragma diagnostic pop
+using Validator = std::function<bool(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)>;
+void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
+                                    std::vector<DeviceConnectionEdge> const& edges,
+                                    std::vector<OutputSpec> const& outputs)
+{
+  std::vector<Validator> defaultValidators = {};
+  std::stringstream errors;
+  // Iterate over all the edges.
+  // Get the input lifetime and the output lifetime.
+  // Output lifetime must be Timeframe if the input lifetime is Timeframe.
+  bool hasErrors = false;
+  for (auto& edge : edges) {
+    DataProcessorSpec const& producer = workflow[edge.producer];
+    DataProcessorSpec const& consumer = workflow[edge.consumer];
+    OutputSpec const& output = outputs[edge.outputGlobalIndex];
+    InputSpec const& input = consumer.inputs[edge.consumerInputIndex];
+    for (auto& validator : defaultValidators) {
+      hasErrors |= !validator(errors, producer, output, consumer, input);
+    }
+  }
+  if (hasErrors) {
+    throw std::runtime_error(errors.str());
+  }
+}
+
 } // namespace o2::framework
diff --git a/Framework/Core/src/WorkflowHelpers.h b/Framework/Core/src/WorkflowHelpers.h
index a167e99d4768c..67661dbdb979a 100644
--- a/Framework/Core/src/WorkflowHelpers.h
+++ b/Framework/Core/src/WorkflowHelpers.h
@@ -227,6 +227,14 @@ struct WorkflowHelpers {
 
   /// returns only dangling outputs
   static std::vector<InputSpec> computeDanglingOutputs(WorkflowSpec const& workflow);
+
+  /// Validate that the nodes at the ends of the edges of the graph
+  /// are actually compatible with each other.
+  /// For example we should make sure that Lifetime::Timeframe inputs of
+  /// one node is not connected to an Output of Lifetime::Sporadic of another node.
+  static void validateEdges(WorkflowSpec const& workflow,
+                            std::vector<DeviceConnectionEdge> const& edges,
+                            std::vector<OutputSpec> const& outputs);
 };
 
 } // namespace o2::framework

From 0be97542d80e2f4b0473ad38665285670c39f84b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 23 Nov 2023 14:37:01 +0100
Subject: [PATCH 123/726] DPL: add edge validator for expendable tasks

---
 Framework/Core/src/WorkflowHelpers.cxx | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index f320e788f5e07..ee5882ccc3f64 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -1232,12 +1232,31 @@ std::vector<InputSpec> WorkflowHelpers::computeDanglingOutputs(WorkflowSpec cons
   return results;
 }
 
+bool validateExpendable(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
+{
+  auto isExpendable = [](DataProcessorLabel const& label) {
+    return label.value == "expendable";
+  };
+  auto isResilient = [](DataProcessorLabel const& label) {
+    return label.value == "expendable" || label.value == "resilient";
+  };
+  bool producerExpendable = std::find_if(producer.labels.begin(), producer.labels.end(), isExpendable) != producer.labels.end();
+  bool consumerCritical = std::find_if(consumer.labels.begin(), consumer.labels.end(), isResilient) == consumer.labels.end();
+  if (producerExpendable && consumerCritical) {
+    errors << fmt::format("Critical consumer {} depends on expendable producer {}\n",
+                          consumer.name,
+                          producer.name);
+    return false;
+  }
+  return true;
+}
+
 using Validator = std::function<bool(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)>;
 void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
                                     std::vector<DeviceConnectionEdge> const& edges,
                                     std::vector<OutputSpec> const& outputs)
 {
-  std::vector<Validator> defaultValidators = {};
+  std::vector<Validator> defaultValidators = {validateExpendable};
   std::stringstream errors;
   // Iterate over all the edges.
   // Get the input lifetime and the output lifetime.

From 8a18c306b233605b8dbb925a26652b957b7c7de8 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 23 Nov 2023 14:37:01 +0100
Subject: [PATCH 124/726] DPL: edge validator for Lifetime::Sporadic

---
 Framework/Core/src/WorkflowHelpers.cxx | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index ee5882ccc3f64..11a24d1300e0a 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -1232,6 +1232,17 @@ std::vector<InputSpec> WorkflowHelpers::computeDanglingOutputs(WorkflowSpec cons
   return results;
 }
 
+bool validateLifetime(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
+{
+  if (input.lifetime == Lifetime::Timeframe && output.lifetime == Lifetime::Sporadic) {
+    errors << fmt::format("Input {} of {} has lifetime Timeframe, but output {} of {} has lifetime Sporadic\n",
+                          DataSpecUtils::describe(input).c_str(), consumer.name,
+                          DataSpecUtils::describe(output).c_str(), producer.name);
+    return false;
+  }
+  return true;
+}
+
 bool validateExpendable(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
 {
   auto isExpendable = [](DataProcessorLabel const& label) {
@@ -1256,7 +1267,7 @@ void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
                                     std::vector<DeviceConnectionEdge> const& edges,
                                     std::vector<OutputSpec> const& outputs)
 {
-  std::vector<Validator> defaultValidators = {validateExpendable};
+  std::vector<Validator> defaultValidators = {validateExpendable, validateLifetime};
   std::stringstream errors;
   // Iterate over all the edges.
   // Get the input lifetime and the output lifetime.

From 003d2f352bb7ebf3a644a92f7b053f99e30a32f0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 23 Nov 2023 22:58:04 +0100
Subject: [PATCH 125/726] DPL: edge validator for Lifetime::Sporadic

---
 Framework/Core/src/WorkflowHelpers.cxx | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index ee5882ccc3f64..11a24d1300e0a 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -1232,6 +1232,17 @@ std::vector<InputSpec> WorkflowHelpers::computeDanglingOutputs(WorkflowSpec cons
   return results;
 }
 
+bool validateLifetime(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
+{
+  if (input.lifetime == Lifetime::Timeframe && output.lifetime == Lifetime::Sporadic) {
+    errors << fmt::format("Input {} of {} has lifetime Timeframe, but output {} of {} has lifetime Sporadic\n",
+                          DataSpecUtils::describe(input).c_str(), consumer.name,
+                          DataSpecUtils::describe(output).c_str(), producer.name);
+    return false;
+  }
+  return true;
+}
+
 bool validateExpendable(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
 {
   auto isExpendable = [](DataProcessorLabel const& label) {
@@ -1256,7 +1267,7 @@ void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
                                     std::vector<DeviceConnectionEdge> const& edges,
                                     std::vector<OutputSpec> const& outputs)
 {
-  std::vector<Validator> defaultValidators = {validateExpendable};
+  std::vector<Validator> defaultValidators = {validateExpendable, validateLifetime};
   std::stringstream errors;
   // Iterate over all the edges.
   // Get the input lifetime and the output lifetime.

From 1c4c75d661c00e06c7af63172a4e5653911acf62 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 24 Nov 2023 10:17:01 +0100
Subject: [PATCH 126/726] DPL: introduce "resilient" tasks

Task with label "resilient" will not care about being sorted after
"expendable" tasks, even if they are not expendable themselves.
---
 Framework/Core/src/TopologyPolicy.cxx       | 38 +++++++++++++--------
 Utilities/DataSampling/src/DataSampling.cxx |  2 +-
 2 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/Framework/Core/src/TopologyPolicy.cxx b/Framework/Core/src/TopologyPolicy.cxx
index e0f7a208ed4e5..fb96eff5af2fc 100644
--- a/Framework/Core/src/TopologyPolicy.cxx
+++ b/Framework/Core/src/TopologyPolicy.cxx
@@ -82,33 +82,41 @@ bool expendableDataDeps(DataProcessorSpec const& a, DataProcessorSpec const& b)
   // If we are here we do not have any data dependency,
   // however we strill consider a dependent on b if
   // a has the "expendable" label and b does not.
-  bool isBExpendable = false;
-  bool isAExpendable = false;
-  for (auto const& label : b.labels) {
+  auto checkExpendable = [](DataProcessorLabel const& label) {
     if (label.value == "expendable") {
-      isBExpendable = true;
-      break;
+      return true;
     }
-  }
-  for (auto const& label : a.labels) {
-    if (label.value == "expendable") {
-      isAExpendable = true;
-      break;
+    return false;
+  };
+  // A task marked as expendable or resilient can be put after an expendable task
+  auto checkResilient = [](DataProcessorLabel const& label) {
+    if (label.value == "resilient") {
+      return true;
     }
-  }
-  // If none is expendable. We simply return false.
+    return false;
+  };
+  bool isBExpendable = std::find_if(b.labels.begin(), b.labels.end(), checkExpendable) != b.labels.end();
+  bool isAExpendable = std::find_if(a.labels.begin(), a.labels.end(), checkExpendable) != a.labels.end();
+  bool bResilient = std::find_if(b.labels.begin(), b.labels.end(), checkResilient) != b.labels.end();
+
+  // If none is expendable. We simply return false and sort as usual.
   if (!isAExpendable && !isBExpendable) {
-    LOGP(debug, "Neither {} nor {} are expendable. No dependency.", a.name, b.name);
+    LOGP(debug, "Neither {} nor {} are expendable. No dependency beyond data deps.", a.name, b.name);
     return false;
   }
-  // If both are expendable. We return false.
+  // If both are expendable. We return false and sort as usual.
   if (isAExpendable && isBExpendable) {
     LOGP(debug, "Both {} and {} are expendable. No dependency.", a.name, b.name);
     return false;
   }
 
+  // If b is expendable but b is resilient, we can keep the same order.
+  if (isAExpendable && bResilient) {
+    LOGP(debug, "{} is expendable but b is resilient, no need to add an unneeded dependency", a.name, a.name, b.name);
+    return false;
+  }
   // If a is expendable we consider it as if there was a dependency from a to b,
-  // but we still need to check if there is not one already from b to a.
+  // however we still need to check if there is not one already from b to a.
   if (isAExpendable) {
     LOGP(debug, "{} is expendable. Checking if there is a dependency from {} to {}.", a.name, b.name, a.name);
     return !dataDeps(b, a);
diff --git a/Utilities/DataSampling/src/DataSampling.cxx b/Utilities/DataSampling/src/DataSampling.cxx
index cd30759629c9d..fee4cf6e4ca10 100644
--- a/Utilities/DataSampling/src/DataSampling.cxx
+++ b/Utilities/DataSampling/src/DataSampling.cxx
@@ -95,7 +95,7 @@ void DataSampling::DoGenerateInfrastructure(Dispatcher& dispatcher, WorkflowSpec
     spec.inputs = dispatcher.getInputSpecs();
     spec.outputs = dispatcher.getOutputSpecs();
     spec.maxInputTimeslices = threads;
-    spec.labels = {{"DataSampling"}, {"Dispatcher"}};
+    spec.labels = {{"DataSampling"}, {"Dispatcher"}, {"resilient"}};
     spec.options = dispatcher.getOptions();
     spec.algorithm = adaptFromTask<Dispatcher>(std::move(dispatcher));
 

From fa52cc05eedd2a1c1f0561a7aa811f0677e0350d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 24 Nov 2023 10:17:01 +0100
Subject: [PATCH 127/726] DPL: Improve debugging of bad topologies.

Print dependency as graphviz digraph when topological sort fails.
---
 Framework/Core/src/runDataProcessing.cxx | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 86cc15299e8f3..d5da17d9b0f45 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -2590,6 +2590,11 @@ std::string debugTopoInfo(std::vector<DataProcessorSpec> const& specs,
   for (auto& d : specs) {
     out << "- " << d.name << std::endl;
   }
+  out << "digraph G {\n";
+  for (auto& e : edges) {
+    out << fmt::format("  \"{}\" -> \"{}\"\n", specs[e.first].name, specs[e.second].name);
+  }
+  out << "}\n";
   return out.str();
 }
 

From a6b2f4c260b63aeedd1822cb4f63e5487a142f8d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 24 Nov 2023 12:18:05 +0100
Subject: [PATCH 128/726] DPL: introduce "resilient" tasks

Task with label "resilient" will not care about being sorted after
"expendable" tasks, even if they are not expendable themselves.

In case a non-resilient / non-expendable task ends up depending
on an expendable one, complain appropriately.
---
 Framework/Core/src/TopologyPolicy.cxx    | 38 ++++++++++++++----------
 Framework/Core/src/runDataProcessing.cxx | 26 ++++++++++++++++
 2 files changed, 49 insertions(+), 15 deletions(-)

diff --git a/Framework/Core/src/TopologyPolicy.cxx b/Framework/Core/src/TopologyPolicy.cxx
index e0f7a208ed4e5..fb96eff5af2fc 100644
--- a/Framework/Core/src/TopologyPolicy.cxx
+++ b/Framework/Core/src/TopologyPolicy.cxx
@@ -82,33 +82,41 @@ bool expendableDataDeps(DataProcessorSpec const& a, DataProcessorSpec const& b)
   // If we are here we do not have any data dependency,
   // however we strill consider a dependent on b if
   // a has the "expendable" label and b does not.
-  bool isBExpendable = false;
-  bool isAExpendable = false;
-  for (auto const& label : b.labels) {
+  auto checkExpendable = [](DataProcessorLabel const& label) {
     if (label.value == "expendable") {
-      isBExpendable = true;
-      break;
+      return true;
     }
-  }
-  for (auto const& label : a.labels) {
-    if (label.value == "expendable") {
-      isAExpendable = true;
-      break;
+    return false;
+  };
+  // A task marked as expendable or resilient can be put after an expendable task
+  auto checkResilient = [](DataProcessorLabel const& label) {
+    if (label.value == "resilient") {
+      return true;
     }
-  }
-  // If none is expendable. We simply return false.
+    return false;
+  };
+  bool isBExpendable = std::find_if(b.labels.begin(), b.labels.end(), checkExpendable) != b.labels.end();
+  bool isAExpendable = std::find_if(a.labels.begin(), a.labels.end(), checkExpendable) != a.labels.end();
+  bool bResilient = std::find_if(b.labels.begin(), b.labels.end(), checkResilient) != b.labels.end();
+
+  // If none is expendable. We simply return false and sort as usual.
   if (!isAExpendable && !isBExpendable) {
-    LOGP(debug, "Neither {} nor {} are expendable. No dependency.", a.name, b.name);
+    LOGP(debug, "Neither {} nor {} are expendable. No dependency beyond data deps.", a.name, b.name);
     return false;
   }
-  // If both are expendable. We return false.
+  // If both are expendable. We return false and sort as usual.
   if (isAExpendable && isBExpendable) {
     LOGP(debug, "Both {} and {} are expendable. No dependency.", a.name, b.name);
     return false;
   }
 
+  // If b is expendable but b is resilient, we can keep the same order.
+  if (isAExpendable && bResilient) {
+    LOGP(debug, "{} is expendable but b is resilient, no need to add an unneeded dependency", a.name, a.name, b.name);
+    return false;
+  }
   // If a is expendable we consider it as if there was a dependency from a to b,
-  // but we still need to check if there is not one already from b to a.
+  // however we still need to check if there is not one already from b to a.
   if (isAExpendable) {
     LOGP(debug, "{} is expendable. Checking if there is a dependency from {} to {}.", a.name, b.name, a.name);
     return !dataDeps(b, a);
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index d5da17d9b0f45..724383ce501e6 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -2565,6 +2565,30 @@ void apply_permutation(
   }
 }
 
+// Check if the workflow is resiliant to failures
+void checkNonResiliency(std::vector<DataProcessorSpec> const& specs,
+                        std::vector<std::pair<int, int>> const& edges)
+{
+  auto checkExpendable = [](DataProcessorLabel const& label) {
+    return label.value == "expendable";
+  };
+  auto checkResilient = [](DataProcessorLabel const& label) {
+    return label.value == "resilient" || label.value == "expendable";
+  };
+
+  for (auto& edge : edges) {
+    auto& src = specs[edge.first];
+    auto& dst = specs[edge.second];
+    if (std::none_of(src.labels.begin(), src.labels.end(), checkExpendable)) {
+      continue;
+    }
+    if (std::any_of(dst.labels.begin(), dst.labels.end(), checkResilient)) {
+      continue;
+    }
+    throw std::runtime_error("Workflow is not resiliant to failures. Processor " + dst.name + " gets inputs from expendable devices, but is not marked as expendable or resilient itself.");
+  }
+}
+
 std::string debugTopoInfo(std::vector<DataProcessorSpec> const& specs,
                           std::vector<TopoIndexInfo> const& infos,
                           std::vector<std::pair<int, int>> const& edges)
@@ -2833,6 +2857,8 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
 
     auto topoInfos = WorkflowHelpers::topologicalSort(physicalWorkflow.size(), &edges[0].first, &edges[0].second, sizeof(std::pair<int, int>), edges.size());
     if (topoInfos.size() != physicalWorkflow.size()) {
+      // Check missing resilincy of one of the tasks
+      checkNonResiliency(physicalWorkflow, edges);
       throw std::runtime_error("Unable to do topological sort of the resulting workflow. Do you have loops?\n" + debugTopoInfo(physicalWorkflow, topoInfos, edges));
     }
     // Sort by layer and then by name, to ensure stability.

From cc7b6d3ee550aaeccb0a2852f686f06641ce10bf Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 24 Nov 2023 13:17:50 +0100
Subject: [PATCH 129/726] DPL: C++20 fixes

---
 Framework/Core/src/DataProcessingDevice.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 4d0cac7087611..b03904c5b3776 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -2234,7 +2234,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       buffer[ai] = record.isValid(ai) ? '2' : '0';
     }
     buffer[record.size()] = 0;
-    states.updateState({.id = short((int)ProcessingStateId::DATA_RELAYER_BASE + action.slot.index), (int)(record.size() + buffer - relayerSlotState), relayerSlotState});
+    states.updateState({.id = short((int)ProcessingStateId::DATA_RELAYER_BASE + action.slot.index), .size = (int)(record.size() + buffer - relayerSlotState), .data = relayerSlotState});
   };
 
   // This is the main dispatching loop

From f75ba9f4a966c8750e2cbc0cb786c92ab0b45e18 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 24 Nov 2023 14:49:38 +0100
Subject: [PATCH 130/726] Drop need to include ptree in ConfigurableParam.h

---
 Common/SimConfig/test/testSimCutParam.cxx     |  1 +
 Common/Utils/CMakeLists.txt                   |  3 +-
 .../include/CommonUtils/ConfigurableParam.h   | 62 ++++++---------
 .../CommonUtils/ConfigurableParamReaders.h    | 36 +++++++++
 Common/Utils/src/ConfigurableParam.cxx        | 70 +++++------------
 Common/Utils/src/ConfigurableParamReaders.cxx | 75 +++++++++++++++++++
 .../MUON/MCH/Raw/test/testClosureCoDec.cxx    |  1 +
 .../MCH/Simulation/test/testDigitizer.cxx     |  1 +
 .../MCH/Triggering/test/testEventFinder.cxx   |  1 +
 Detectors/TPC/base/test/testTPCParameters.cxx |  7 +-
 .../include/TPCCalibration/IDCAverageGroup.h  |  1 +
 .../include/TPCCalibration/IDCFactorization.h |  1 +
 .../include/TPCCalibration/SACFactorization.h |  1 +
 .../calibration/src/IDCGroupingParameter.cxx  |  2 +
 .../TPC/reconstruction/macro/makeIonTail.C    |  1 +
 Detectors/TPC/simulation/macro/toyCluster.C   |  1 +
 .../TPC/workflow/src/ApplyCCDBCalibSpec.cxx   |  1 +
 macro/o2sim.C                                 |  1 +
 18 files changed, 169 insertions(+), 97 deletions(-)
 create mode 100644 Common/Utils/include/CommonUtils/ConfigurableParamReaders.h
 create mode 100644 Common/Utils/src/ConfigurableParamReaders.cxx

diff --git a/Common/SimConfig/test/testSimCutParam.cxx b/Common/SimConfig/test/testSimCutParam.cxx
index 7a17ae9d1f2cf..468d76429cbba 100644
--- a/Common/SimConfig/test/testSimCutParam.cxx
+++ b/Common/SimConfig/test/testSimCutParam.cxx
@@ -15,6 +15,7 @@
 #include <boost/test/unit_test.hpp>
 #include "SimConfig/SimParams.h"
 #include "CommonUtils/ConfigurableParam.h"
+#include <boost/property_tree/ptree.hpp>
 
 using namespace o2::conf;
 
diff --git a/Common/Utils/CMakeLists.txt b/Common/Utils/CMakeLists.txt
index b0fa605cb6723..7f650b973990b 100644
--- a/Common/Utils/CMakeLists.txt
+++ b/Common/Utils/CMakeLists.txt
@@ -14,7 +14,7 @@ o2_add_library(CommonUtils
                        src/RootChain.cxx src/CompStream.cxx src/ShmManager.cxx
                        src/ValueMonitor.cxx
                        src/StringUtils.cxx
-                       src/ConfigurableParamHelper.cxx src/ConfigurableParam.cxx src/RootSerializableKeyValueStore.cxx
+                       src/ConfigurableParamReaders.cxx src/ConfigurableParamHelper.cxx src/ConfigurableParam.cxx src/RootSerializableKeyValueStore.cxx
                        src/KeyValParam.cxx
                        src/FileSystemUtils.cxx
                        src/FIFO.cxx
@@ -39,6 +39,7 @@ o2_target_root_dictionary(CommonUtils
                                   include/CommonUtils/MemFileHelper.h
                                   include/CommonUtils/ConfigurableParam.h
                                   include/CommonUtils/ConfigurableParamHelper.h
+                                  include/CommonUtils/ConfigurableParamReaders.h
                                   include/CommonUtils/ConfigurationMacroHelper.h
                                   include/CommonUtils/RootSerializableKeyValueStore.h
                                   include/CommonUtils/KeyValParam.h
diff --git a/Common/Utils/include/CommonUtils/ConfigurableParam.h b/Common/Utils/include/CommonUtils/ConfigurableParam.h
index 7099e37d5bc50..717a4c425fc82 100644
--- a/Common/Utils/include/CommonUtils/ConfigurableParam.h
+++ b/Common/Utils/include/CommonUtils/ConfigurableParam.h
@@ -18,7 +18,7 @@
 #include <cassert>
 #include <map>
 #include <unordered_map>
-#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/ptree_fwd.hpp>
 #include <typeinfo>
 #include <iostream>
 #include <array>
@@ -176,15 +176,10 @@ class ConfigurableParam
   static void printAllRegisteredParamNames();
   static void printAllKeyValuePairs(bool useLogger = false);
 
-  static const std::string& getInputDir() { return sInputDir; }
   static const std::string& getOutputDir() { return sOutputDir; }
 
-  static void setInputDir(const std::string& d) { sInputDir = d; }
   static void setOutputDir(const std::string& d) { sOutputDir = d; }
 
-  static boost::property_tree::ptree readINI(std::string const& filepath);
-  static boost::property_tree::ptree readJSON(std::string const& filepath);
-  static boost::property_tree::ptree readConfigFile(std::string const& filepath);
   static bool configFileExists(std::string const& filepath);
 
   // writes a human readable JSON file of all parameters
@@ -196,10 +191,12 @@ class ConfigurableParam
   template <typename T>
   static T getValueAs(std::string key)
   {
-    if (!sIsFullyInitialized) {
-      initialize();
-    }
-    return sPtree->get<T>(key);
+    return [](auto* tree, const std::string& key) -> T {
+      if (!sIsFullyInitialized) {
+        initialize();
+      }
+      return tree->template get<T>(key);
+    }(sPtree, key);
   }
 
   template <typename T>
@@ -208,19 +205,21 @@ class ConfigurableParam
     if (!sIsFullyInitialized) {
       initialize();
     }
-    assert(sPtree);
-    try {
-      auto key = mainkey + "." + subkey;
-      if (sPtree->get_optional<std::string>(key).is_initialized()) {
-        sPtree->put(key, x);
-        auto changed = updateThroughStorageMap(mainkey, subkey, typeid(T), (void*)&x);
-        if (changed != EParamUpdateStatus::Failed) {
-          sValueProvenanceMap->find(key)->second = kRT; // set to runtime
+    return [&subkey, &x, &mainkey](auto* tree) -> void {
+      assert(tree);
+      try {
+        auto key = mainkey + "." + subkey;
+        if (tree->template get_optional<std::string>(key).is_initialized()) {
+          tree->put(key, x);
+          auto changed = updateThroughStorageMap(mainkey, subkey, typeid(T), (void*)&x);
+          if (changed != EParamUpdateStatus::Failed) {
+            sValueProvenanceMap->find(key)->second = kRT; // set to runtime
+          }
         }
+      } catch (std::exception const& e) {
+        std::cerr << "Error in setValue (T) " << e.what() << "\n";
       }
-    } catch (std::exception const& e) {
-      std::cerr << "Error in setValue (T) " << e.what() << "\n";
-    }
+    }(sPtree);
   }
 
   static void setProvenance(std::string const& mainkey, std::string const& subkey, EParamProvenance p)
@@ -242,25 +241,7 @@ class ConfigurableParam
 
   // specialized for std::string
   // which means that the type will be converted internally
-  static void setValue(std::string const& key, std::string const& valuestring)
-  {
-    if (!sIsFullyInitialized) {
-      initialize();
-    }
-    assert(sPtree);
-    try {
-      if (sPtree->get_optional<std::string>(key).is_initialized()) {
-        sPtree->put(key, valuestring);
-        auto changed = updateThroughStorageMapWithConversion(key, valuestring);
-        if (changed != EParamUpdateStatus::Failed) {
-          sValueProvenanceMap->find(key)->second = kRT; // set to runtime
-        }
-      }
-    } catch (std::exception const& e) {
-      std::cerr << "Error in setValue (string) " << e.what() << "\n";
-    }
-  }
-
+  static void setValue(std::string const& key, std::string const& valuestring);
   static void setEnumValue(const std::string&, const std::string&);
   static void setArrayValue(const std::string&, const std::string&);
 
@@ -321,7 +302,6 @@ class ConfigurableParam
   // (stored as a vector of pairs <enumValueLabel, enumValueInt>)
   static EnumRegistry* sEnumRegistry;
 
-  static std::string sInputDir;
   static std::string sOutputDir;
 
   void setRegisterMode(bool b) { sRegisterMode = b; }
diff --git a/Common/Utils/include/CommonUtils/ConfigurableParamReaders.h b/Common/Utils/include/CommonUtils/ConfigurableParamReaders.h
new file mode 100644
index 0000000000000..d5ecd6cb97f7a
--- /dev/null
+++ b/Common/Utils/include/CommonUtils/ConfigurableParamReaders.h
@@ -0,0 +1,36 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef O2_COMMON_UTILS_CONFIGURABLEPARAMREADERS_H_
+#define O2_COMMON_UTILS_CONFIGURABLEPARAMREADERS_H_
+
+#include <boost/property_tree/ptree.hpp>
+#include <string>
+
+namespace o2::conf
+{
+
+// Helpers to read ConfigurableParam from different file formats
+class ConfigurableParamReaders
+{
+ public:
+  static void setInputDir(const std::string& d) { sInputDir = d; }
+  static const std::string& getInputDir() { return sInputDir; }
+
+  static boost::property_tree::ptree readINI(std::string const& filepath);
+  static boost::property_tree::ptree readJSON(std::string const& filepath);
+  static boost::property_tree::ptree readConfigFile(std::string const& filepath);
+
+ private:
+  static std::string sInputDir;
+};
+
+} // namespace o2::conf
+#endif // O2_COMMON_UTILS_CONF_CONFIGURABLEPARAMREADERS_H_
diff --git a/Common/Utils/src/ConfigurableParam.cxx b/Common/Utils/src/ConfigurableParam.cxx
index e581970fc81fb..0a3c9fe2d7595 100644
--- a/Common/Utils/src/ConfigurableParam.cxx
+++ b/Common/Utils/src/ConfigurableParam.cxx
@@ -9,11 +9,12 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
-//first version 8/2018, Sandro Wenzel
+// first version 8/2018, Sandro Wenzel
 
 #include "CommonUtils/ConfigurableParam.h"
 #include "CommonUtils/StringUtils.h"
 #include "CommonUtils/KeyValParam.h"
+#include "CommonUtils/ConfigurableParamReaders.h"
 #define BOOST_BIND_GLOBAL_PLACEHOLDERS
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/property_tree/ptree.hpp>
@@ -47,7 +48,6 @@ std::vector<ConfigurableParam*>* ConfigurableParam::sRegisteredParamClasses = nu
 boost::property_tree::ptree* ConfigurableParam::sPtree = nullptr;
 std::map<std::string, std::pair<std::type_info const&, void*>>* ConfigurableParam::sKeyToStorageMap = nullptr;
 std::map<std::string, ConfigurableParam::EParamProvenance>* ConfigurableParam::sValueProvenanceMap = nullptr;
-std::string ConfigurableParam::sInputDir = "";
 std::string ConfigurableParam::sOutputDir = "";
 EnumRegistry* ConfigurableParam::sEnumRegistry = nullptr;
 
@@ -194,60 +194,28 @@ void ConfigurableParam::writeINI(std::string const& filename, std::string const&
 
 bool ConfigurableParam::configFileExists(std::string const& filepath)
 {
-  return std::filesystem::exists(o2::utils::Str::concat_string(sInputDir, filepath));
+  return std::filesystem::exists(o2::utils::Str::concat_string(ConfigurableParamReaders::getInputDir(), filepath));
 }
 
 // ------------------------------------------------------------------
 
-boost::property_tree::ptree ConfigurableParam::readConfigFile(std::string const& filepath)
+void ConfigurableParam::setValue(std::string const& key, std::string const& valuestring)
 {
-  auto inpfilename = o2::utils::Str::concat_string(sInputDir, filepath);
-  if (!std::filesystem::exists(inpfilename)) {
-    LOG(fatal) << inpfilename << " : config file does not exist!";
-  }
-
-  boost::property_tree::ptree pt;
-
-  if (boost::iends_with(inpfilename, ".ini")) {
-    pt = readINI(inpfilename);
-  } else if (boost::iends_with(inpfilename, ".json")) {
-    pt = readJSON(inpfilename);
-  } else {
-    LOG(fatal) << "Configuration file must have either .ini or .json extension";
-  }
-
-  return pt;
-}
-
-// ------------------------------------------------------------------
-
-boost::property_tree::ptree ConfigurableParam::readINI(std::string const& filepath)
-{
-  boost::property_tree::ptree pt;
-  try {
-    boost::property_tree::read_ini(filepath, pt);
-  } catch (const boost::property_tree::ptree_error& e) {
-    LOG(fatal) << "Failed to read INI config file " << filepath << " (" << e.what() << ")";
-  } catch (...) {
-    LOG(fatal) << "Unknown error when reading INI config file ";
+  if (!sIsFullyInitialized) {
+    initialize();
   }
-
-  return pt;
-}
-
-// ------------------------------------------------------------------
-
-boost::property_tree::ptree ConfigurableParam::readJSON(std::string const& filepath)
-{
-  boost::property_tree::ptree pt;
-
+  assert(sPtree);
   try {
-    boost::property_tree::read_json(filepath, pt);
-  } catch (const boost::property_tree::ptree_error& e) {
-    LOG(fatal) << "Failed to read JSON config file " << filepath << " (" << e.what() << ")";
+    if (sPtree->get_optional<std::string>(key).is_initialized()) {
+      sPtree->put(key, valuestring);
+      auto changed = updateThroughStorageMapWithConversion(key, valuestring);
+      if (changed != EParamUpdateStatus::Failed) {
+        sValueProvenanceMap->find(key)->second = kRT; // set to runtime
+      }
+    }
+  } catch (std::exception const& e) {
+    std::cerr << "Error in setValue (string) " << e.what() << "\n";
   }
-
-  return pt;
 }
 
 // ------------------------------------------------------------------
@@ -258,7 +226,7 @@ void ConfigurableParam::writeJSON(std::string const& filename, std::string const
     LOG(info) << "ignoring writing of json file " << filename;
     return;
   }
-  initPropertyTree();     // update the boost tree before writing
+  initPropertyTree(); // update the boost tree before writing
   auto outfilename = o2::utils::Str::concat_string(sOutputDir, filename);
   if (!keyOnly.empty()) { // write ini for selected key only
     try {
@@ -409,7 +377,7 @@ void ConfigurableParam::updateFromFile(std::string const& configFile, std::strin
     return;
   }
 
-  boost::property_tree::ptree pt = readConfigFile(cfgfile);
+  boost::property_tree::ptree pt = ConfigurableParamReaders::readConfigFile(cfgfile);
 
   std::vector<std::pair<std::string, std::string>> keyValPairs;
   auto request = o2::utils::Str::tokenize(paramsList, ',', true);
@@ -522,7 +490,7 @@ void ConfigurableParam::updateFromString(std::string const& configString)
 
   const auto& kv = o2::conf::KeyValParam::Instance();
   if (getProvenance("keyval.input_dir") != kCODE) {
-    sInputDir = o2::utils::Str::concat_string(o2::utils::Str::rectifyDirectory(kv.input_dir));
+    ConfigurableParamReaders::setInputDir(o2::utils::Str::concat_string(o2::utils::Str::rectifyDirectory(kv.input_dir)));
   }
   if (getProvenance("keyval.output_dir") != kCODE) {
     if (kv.output_dir == "/dev/null") {
diff --git a/Common/Utils/src/ConfigurableParamReaders.cxx b/Common/Utils/src/ConfigurableParamReaders.cxx
new file mode 100644
index 0000000000000..fe45a4fccfc1a
--- /dev/null
+++ b/Common/Utils/src/ConfigurableParamReaders.cxx
@@ -0,0 +1,75 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "CommonUtils/ConfigurableParamReaders.h"
+#include "CommonUtils/StringUtils.h"
+#include <fairlogger/Logger.h>
+
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/property_tree/ini_parser.hpp>
+#include <boost/property_tree/json_parser.hpp>
+
+namespace o2::conf
+{
+// ------------------------------------------------------------------
+
+boost::property_tree::ptree ConfigurableParamReaders::readINI(std::string const& filepath)
+{
+  boost::property_tree::ptree pt;
+  try {
+    boost::property_tree::read_ini(filepath, pt);
+  } catch (const boost::property_tree::ptree_error& e) {
+    LOG(fatal) << "Failed to read INI config file " << filepath << " (" << e.what() << ")";
+  } catch (...) {
+    LOG(fatal) << "Unknown error when reading INI config file ";
+  }
+
+  return pt;
+}
+
+// ------------------------------------------------------------------
+
+boost::property_tree::ptree ConfigurableParamReaders::readJSON(std::string const& filepath)
+{
+  boost::property_tree::ptree pt;
+
+  try {
+    boost::property_tree::read_json(filepath, pt);
+  } catch (const boost::property_tree::ptree_error& e) {
+    LOG(fatal) << "Failed to read JSON config file " << filepath << " (" << e.what() << ")";
+  }
+
+  return pt;
+}
+
+boost::property_tree::ptree ConfigurableParamReaders::readConfigFile(std::string const& filepath)
+{
+  auto inpfilename = o2::utils::Str::concat_string(sInputDir, filepath);
+  if (!std::filesystem::exists(inpfilename)) {
+    LOG(fatal) << inpfilename << " : config file does not exist!";
+  }
+
+  boost::property_tree::ptree pt;
+
+  if (boost::iends_with(inpfilename, ".ini")) {
+    pt = ConfigurableParamReaders::readINI(inpfilename);
+  } else if (boost::iends_with(inpfilename, ".json")) {
+    pt = ConfigurableParamReaders::readJSON(inpfilename);
+  } else {
+    LOG(fatal) << "Configuration file must have either .ini or .json extension";
+  }
+
+  return pt;
+}
+
+std::string ConfigurableParamReaders::sInputDir = "";
+
+} // namespace o2::conf
diff --git a/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx b/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
index 2407796773f57..b18ffd1abee63 100644
--- a/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
+++ b/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
@@ -16,6 +16,7 @@
 #define BOOST_TEST_DYN_LINK
 
 #include <boost/test/unit_test.hpp>
+#include <boost/property_tree/ptree.hpp>
 #include "DetectorsRaw/HBFUtils.h"
 #include "DetectorsRaw/RawFileWriter.h"
 #include "Framework/Logger.h"
diff --git a/Detectors/MUON/MCH/Simulation/test/testDigitizer.cxx b/Detectors/MUON/MCH/Simulation/test/testDigitizer.cxx
index 791ab8e7a3909..2a550220bb137 100644
--- a/Detectors/MUON/MCH/Simulation/test/testDigitizer.cxx
+++ b/Detectors/MUON/MCH/Simulation/test/testDigitizer.cxx
@@ -31,6 +31,7 @@
 #include "TGeoManager.h"
 #include "boost/format.hpp"
 #include <boost/test/data/test_case.hpp>
+#include <boost/property_tree/ptree.hpp>
 #include <algorithm>
 #include <unordered_map>
 
diff --git a/Detectors/MUON/MCH/Triggering/test/testEventFinder.cxx b/Detectors/MUON/MCH/Triggering/test/testEventFinder.cxx
index eb786fc1bb068..60275f2cad2f4 100644
--- a/Detectors/MUON/MCH/Triggering/test/testEventFinder.cxx
+++ b/Detectors/MUON/MCH/Triggering/test/testEventFinder.cxx
@@ -27,6 +27,7 @@
 #include <vector>
 
 #include <boost/test/unit_test.hpp>
+#include <boost/property_tree/ptree.hpp>
 
 #include "CommonUtils/ConfigurableParam.h"
 #include "SimulationDataFormat/MCCompLabel.h"
diff --git a/Detectors/TPC/base/test/testTPCParameters.cxx b/Detectors/TPC/base/test/testTPCParameters.cxx
index 015a6abd964ec..b8baa1e2a74ac 100644
--- a/Detectors/TPC/base/test/testTPCParameters.cxx
+++ b/Detectors/TPC/base/test/testTPCParameters.cxx
@@ -23,9 +23,9 @@
 #include "TPCBase/ParameterGas.h"
 #include <CommonUtils/ConfigurableParam.h>
 #include <CommonUtils/ConfigurableParamHelper.h>
-namespace o2
-{
-namespace tpc
+#include <boost/property_tree/ptree.hpp>
+
+namespace o2::tpc
 {
 
 constexpr float NominalTimeBin = 8 * o2::constants::lhc::LHCBunchSpacingNS * 1e-3;
@@ -367,5 +367,4 @@ BOOST_AUTO_TEST_CASE(ParameterGEM_test2)
   BOOST_CHECK_CLOSE(o2::conf::ConfigurableParam::getValueAs<float>("TPCGEMParam.KappaStack"), 32, 1e-12);
   BOOST_CHECK_CLOSE(o2::conf::ConfigurableParam::getValueAs<float>("TPCGEMParam.EfficiencyStack"), 33, 1e-3);
 }
-} // namespace tpc
 } // namespace o2
diff --git a/Detectors/TPC/calibration/include/TPCCalibration/IDCAverageGroup.h b/Detectors/TPC/calibration/include/TPCCalibration/IDCAverageGroup.h
index fecbe9f14f18b..dd9a5fc257935 100644
--- a/Detectors/TPC/calibration/include/TPCCalibration/IDCAverageGroup.h
+++ b/Detectors/TPC/calibration/include/TPCCalibration/IDCAverageGroup.h
@@ -20,6 +20,7 @@
 #include "TPCCalibration/IDCAverageGroupBase.h"
 #include "TPCBase/Sector.h"
 #include "TPCBase/CalDet.h"
+#include <boost/property_tree/ptree.hpp>
 #include <vector>
 #include "Rtypes.h"
 
diff --git a/Detectors/TPC/calibration/include/TPCCalibration/IDCFactorization.h b/Detectors/TPC/calibration/include/TPCCalibration/IDCFactorization.h
index 13c9d0b98412f..4c7f119efcfe0 100644
--- a/Detectors/TPC/calibration/include/TPCCalibration/IDCFactorization.h
+++ b/Detectors/TPC/calibration/include/TPCCalibration/IDCFactorization.h
@@ -24,6 +24,7 @@
 #include "TPCCalibration/IDCContainer.h"
 #include "TPCCalibration/IDCGroupHelperSector.h"
 #include "DataFormatsTPC/Defs.h"
+#include <boost/property_tree/ptree.hpp>
 
 namespace o2::tpc
 {
diff --git a/Detectors/TPC/calibration/include/TPCCalibration/SACFactorization.h b/Detectors/TPC/calibration/include/TPCCalibration/SACFactorization.h
index b1b4430c11a00..7141750b8ce37 100644
--- a/Detectors/TPC/calibration/include/TPCCalibration/SACFactorization.h
+++ b/Detectors/TPC/calibration/include/TPCCalibration/SACFactorization.h
@@ -21,6 +21,7 @@
 #include "Rtypes.h"
 #include "TPCCalibration/IDCContainer.h"
 #include "DataFormatsTPC/Defs.h"
+#include <boost/property_tree/ptree.hpp>
 
 namespace o2::tpc
 {
diff --git a/Detectors/TPC/calibration/src/IDCGroupingParameter.cxx b/Detectors/TPC/calibration/src/IDCGroupingParameter.cxx
index bbecb4df8f8fd..895b6957d8a01 100644
--- a/Detectors/TPC/calibration/src/IDCGroupingParameter.cxx
+++ b/Detectors/TPC/calibration/src/IDCGroupingParameter.cxx
@@ -12,6 +12,8 @@
 #include "TPCCalibration/IDCGroupingParameter.h"
 #include "Framework/Logger.h"
 #include "Algorithm/RangeTokenizer.h"
+#include <boost/property_tree/ptree.hpp>
+#include <cassert>
 #include <stdexcept>
 
 using namespace o2::tpc;
diff --git a/Detectors/TPC/reconstruction/macro/makeIonTail.C b/Detectors/TPC/reconstruction/macro/makeIonTail.C
index 98db6540d20f7..d8d1be6830626 100644
--- a/Detectors/TPC/reconstruction/macro/makeIonTail.C
+++ b/Detectors/TPC/reconstruction/macro/makeIonTail.C
@@ -32,6 +32,7 @@
 #include "TPCBase/CRUCalibHelpers.h"
 #include "TPCBase/CRU.h"
 #include "CommonUtils/TreeStreamRedirector.h"
+#include <boost/property_tree/ptree.hpp>
 
 using namespace o2::tpc;
 size_t digitsInSaturateion(std::vector<Digit>& digits, bool correctCharge = false, CalPad* pedestals = nullptr, o2::utils::TreeStreamRedirector* stream = nullptr);
diff --git a/Detectors/TPC/simulation/macro/toyCluster.C b/Detectors/TPC/simulation/macro/toyCluster.C
index 257adaf5f79fc..2087a7773fe22 100644
--- a/Detectors/TPC/simulation/macro/toyCluster.C
+++ b/Detectors/TPC/simulation/macro/toyCluster.C
@@ -58,6 +58,7 @@
 #include "TPCReconstruction/HwClusterer.h"
 #include "TPCSimulation/GEMAmplification.h"
 #endif
+#include <boost/property_tree/ptree.hpp>
 
 using namespace o2::tpc;
 void fillTPCHits(const float theta, const float phi, const float dedx, std::vector<HitGroup>& hitGroupSector, std::pair<GlobalPosition3D, GlobalPosition3D>& trackInfo);
diff --git a/Detectors/TPC/workflow/src/ApplyCCDBCalibSpec.cxx b/Detectors/TPC/workflow/src/ApplyCCDBCalibSpec.cxx
index 37429a04c4b34..9c588ff50a45c 100644
--- a/Detectors/TPC/workflow/src/ApplyCCDBCalibSpec.cxx
+++ b/Detectors/TPC/workflow/src/ApplyCCDBCalibSpec.cxx
@@ -19,6 +19,7 @@
 #include "TPCBase/ParameterGas.h"
 #include "DataFormatsTPC/LtrCalibData.h"
 #include "TPCWorkflow/ApplyCCDBCalibSpec.h"
+#include <boost/property_tree/ptree.hpp>
 
 using namespace o2::framework;
 
diff --git a/macro/o2sim.C b/macro/o2sim.C
index 5257a225d3ed9..f610be730eeb7 100644
--- a/macro/o2sim.C
+++ b/macro/o2sim.C
@@ -39,6 +39,7 @@
 #include <sstream>
 #endif
 #include "migrateSimFiles.C"
+#include <boost/property_tree/ptree.hpp>
 
 void check_notransport()
 {

From 5eca1f19be52bd4817663c86dd5b945cea1bf428 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 24 Nov 2023 15:22:46 +0100
Subject: [PATCH 131/726] DPL: fixes for C++20

---
 .../test_FrameworkDataFlowToO2Control.cxx     | 42 ++++++++-----------
 1 file changed, 18 insertions(+), 24 deletions(-)

diff --git a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
index 4ef883f3b32de..7b0ffa462a9db 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
@@ -31,32 +31,26 @@ namespace
 {
 WorkflowSpec defineDataProcessing()
 {
-  return {{"A",                                                       //
-           Inputs{},                                                  //
-           Outputs{OutputSpec{"TST", "A1"}, OutputSpec{"TST", "A2"}}, // A1 will be consumed twice, A2 is dangling
-           AlgorithmSpec{},                                           //
-           {ConfigParamSpec{"channel-config", VariantType::String,    // raw input channel
-                            "name=into_dpl,type=pull,method=connect,address=ipc:///tmp/pipe-into-dpl,transport=shmem,rateLogging=10,rcvBufSize=789",
-                            {"Out-of-band channel config"}}}},
-          {"B", // producer, no inputs
-           Inputs{},
-           Outputs{OutputSpec{"TST", "B1"}},
+  return {{.name = "A",                                                                  //
+           .outputs = Outputs{OutputSpec{"TST", "A1"}, OutputSpec{"TST", "A2"}}, // A1 will be consumed twice, A2 is dangling
+           .algorithm = AlgorithmSpec{},                                         //
+           .options = {ConfigParamSpec{"channel-config", VariantType::String,    // raw input channel
+                                       "name=into_dpl,type=pull,method=connect,address=ipc:///tmp/pipe-into-dpl,transport=shmem,rateLogging=10,rcvBufSize=789",
+                                       {"Out-of-band channel config"}}}},
+          {.name = "B", // producer, no inputs
+           .outputs = Outputs{OutputSpec{"TST", "B1"}},
            .metadata = {{ecs::cpuKillThreshold, "3.0"}}},
-          {"C", // first consumer of A1, consumer of B1
-           {InputSpec{"y", "TST", "A1"}, InputSpec{"y", "TST", "B1"}},
-           Outputs{},
+          {.name = "C", // first consumer of A1, consumer of B1
+           .inputs = {InputSpec{"y", "TST", "A1"}, InputSpec{"y", "TST", "B1"}},
            .metadata = {{ecs::privateMemoryKillThresholdMB, "5000"}}},
-          {"D", // second consumer of A1
-           Inputs{
-             InputSpec{"x", "TST", "A1"}},
-           Outputs{},
-           AlgorithmSpec{},
-           {ConfigParamSpec{"a-param", VariantType::Int, 1, {"A parameter which should not be escaped"}},
-            ConfigParamSpec{"b-param", VariantType::String, "", {"a parameter which will be escaped"}},
-            ConfigParamSpec{"c-param", VariantType::String, "foo;bar", {"another parameter which will be escaped"}},
-            ConfigParamSpec{"channel-config", VariantType::String, // raw output channel
-                            "name=outta_dpl,type=push,method=bind,address=ipc:///tmp/pipe-outta-dpl,transport=shmem,rateLogging=10",
-                            {"Out-of-band channel config"}}}}};
+          {.name = "D", // second consumer of A1
+           .inputs = Inputs{InputSpec{"x", "TST", "A1"}},
+           .options = {ConfigParamSpec{"a-param", VariantType::Int, 1, {"A parameter which should not be escaped"}},
+                       ConfigParamSpec{"b-param", VariantType::String, "", {"a parameter which will be escaped"}},
+                       ConfigParamSpec{"c-param", VariantType::String, "foo;bar", {"another parameter which will be escaped"}},
+                       ConfigParamSpec{"channel-config", VariantType::String, // raw output channel
+                                       "name=outta_dpl,type=push,method=bind,address=ipc:///tmp/pipe-outta-dpl,transport=shmem,rateLogging=10",
+                                       {"Out-of-band channel config"}}}}};
 }
 
 char* strdiffchr(const char* s1, const char* s2)

From de8c022334e36fb0836ed8beb242861b9c515c48 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 24 Nov 2023 18:17:26 +0100
Subject: [PATCH 132/726] Drop need to include ptree in ConfigurableParam.h

---
 Common/SimConfig/test/testSimCutParam.cxx     |  1 +
 Common/Utils/CMakeLists.txt                   |  3 +-
 .../include/CommonUtils/ConfigurableParam.h   | 62 +++++----------
 .../CommonUtils/ConfigurableParamReaders.h    | 36 +++++++++
 Common/Utils/src/ConfigurableParam.cxx        | 70 +++++------------
 Common/Utils/src/ConfigurableParamReaders.cxx | 76 +++++++++++++++++++
 .../MUON/MCH/Raw/test/testClosureCoDec.cxx    |  1 +
 .../MCH/Simulation/test/testDigitizer.cxx     |  1 +
 .../MCH/Triggering/test/testEventFinder.cxx   |  1 +
 Detectors/TPC/base/test/testTPCParameters.cxx |  7 +-
 .../include/TPCCalibration/IDCAverageGroup.h  |  1 +
 .../include/TPCCalibration/IDCFactorization.h |  1 +
 .../include/TPCCalibration/SACFactorization.h |  1 +
 .../calibration/src/IDCGroupingParameter.cxx  |  2 +
 .../TPC/reconstruction/macro/makeIonTail.C    |  1 +
 Detectors/TPC/simulation/macro/toyCluster.C   |  1 +
 .../TPC/workflow/src/ApplyCCDBCalibSpec.cxx   |  1 +
 macro/o2sim.C                                 |  1 +
 18 files changed, 170 insertions(+), 97 deletions(-)
 create mode 100644 Common/Utils/include/CommonUtils/ConfigurableParamReaders.h
 create mode 100644 Common/Utils/src/ConfigurableParamReaders.cxx

diff --git a/Common/SimConfig/test/testSimCutParam.cxx b/Common/SimConfig/test/testSimCutParam.cxx
index 7a17ae9d1f2cf..468d76429cbba 100644
--- a/Common/SimConfig/test/testSimCutParam.cxx
+++ b/Common/SimConfig/test/testSimCutParam.cxx
@@ -15,6 +15,7 @@
 #include <boost/test/unit_test.hpp>
 #include "SimConfig/SimParams.h"
 #include "CommonUtils/ConfigurableParam.h"
+#include <boost/property_tree/ptree.hpp>
 
 using namespace o2::conf;
 
diff --git a/Common/Utils/CMakeLists.txt b/Common/Utils/CMakeLists.txt
index b0fa605cb6723..7f650b973990b 100644
--- a/Common/Utils/CMakeLists.txt
+++ b/Common/Utils/CMakeLists.txt
@@ -14,7 +14,7 @@ o2_add_library(CommonUtils
                        src/RootChain.cxx src/CompStream.cxx src/ShmManager.cxx
                        src/ValueMonitor.cxx
                        src/StringUtils.cxx
-                       src/ConfigurableParamHelper.cxx src/ConfigurableParam.cxx src/RootSerializableKeyValueStore.cxx
+                       src/ConfigurableParamReaders.cxx src/ConfigurableParamHelper.cxx src/ConfigurableParam.cxx src/RootSerializableKeyValueStore.cxx
                        src/KeyValParam.cxx
                        src/FileSystemUtils.cxx
                        src/FIFO.cxx
@@ -39,6 +39,7 @@ o2_target_root_dictionary(CommonUtils
                                   include/CommonUtils/MemFileHelper.h
                                   include/CommonUtils/ConfigurableParam.h
                                   include/CommonUtils/ConfigurableParamHelper.h
+                                  include/CommonUtils/ConfigurableParamReaders.h
                                   include/CommonUtils/ConfigurationMacroHelper.h
                                   include/CommonUtils/RootSerializableKeyValueStore.h
                                   include/CommonUtils/KeyValParam.h
diff --git a/Common/Utils/include/CommonUtils/ConfigurableParam.h b/Common/Utils/include/CommonUtils/ConfigurableParam.h
index 7099e37d5bc50..717a4c425fc82 100644
--- a/Common/Utils/include/CommonUtils/ConfigurableParam.h
+++ b/Common/Utils/include/CommonUtils/ConfigurableParam.h
@@ -18,7 +18,7 @@
 #include <cassert>
 #include <map>
 #include <unordered_map>
-#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/ptree_fwd.hpp>
 #include <typeinfo>
 #include <iostream>
 #include <array>
@@ -176,15 +176,10 @@ class ConfigurableParam
   static void printAllRegisteredParamNames();
   static void printAllKeyValuePairs(bool useLogger = false);
 
-  static const std::string& getInputDir() { return sInputDir; }
   static const std::string& getOutputDir() { return sOutputDir; }
 
-  static void setInputDir(const std::string& d) { sInputDir = d; }
   static void setOutputDir(const std::string& d) { sOutputDir = d; }
 
-  static boost::property_tree::ptree readINI(std::string const& filepath);
-  static boost::property_tree::ptree readJSON(std::string const& filepath);
-  static boost::property_tree::ptree readConfigFile(std::string const& filepath);
   static bool configFileExists(std::string const& filepath);
 
   // writes a human readable JSON file of all parameters
@@ -196,10 +191,12 @@ class ConfigurableParam
   template <typename T>
   static T getValueAs(std::string key)
   {
-    if (!sIsFullyInitialized) {
-      initialize();
-    }
-    return sPtree->get<T>(key);
+    return [](auto* tree, const std::string& key) -> T {
+      if (!sIsFullyInitialized) {
+        initialize();
+      }
+      return tree->template get<T>(key);
+    }(sPtree, key);
   }
 
   template <typename T>
@@ -208,19 +205,21 @@ class ConfigurableParam
     if (!sIsFullyInitialized) {
       initialize();
     }
-    assert(sPtree);
-    try {
-      auto key = mainkey + "." + subkey;
-      if (sPtree->get_optional<std::string>(key).is_initialized()) {
-        sPtree->put(key, x);
-        auto changed = updateThroughStorageMap(mainkey, subkey, typeid(T), (void*)&x);
-        if (changed != EParamUpdateStatus::Failed) {
-          sValueProvenanceMap->find(key)->second = kRT; // set to runtime
+    return [&subkey, &x, &mainkey](auto* tree) -> void {
+      assert(tree);
+      try {
+        auto key = mainkey + "." + subkey;
+        if (tree->template get_optional<std::string>(key).is_initialized()) {
+          tree->put(key, x);
+          auto changed = updateThroughStorageMap(mainkey, subkey, typeid(T), (void*)&x);
+          if (changed != EParamUpdateStatus::Failed) {
+            sValueProvenanceMap->find(key)->second = kRT; // set to runtime
+          }
         }
+      } catch (std::exception const& e) {
+        std::cerr << "Error in setValue (T) " << e.what() << "\n";
       }
-    } catch (std::exception const& e) {
-      std::cerr << "Error in setValue (T) " << e.what() << "\n";
-    }
+    }(sPtree);
   }
 
   static void setProvenance(std::string const& mainkey, std::string const& subkey, EParamProvenance p)
@@ -242,25 +241,7 @@ class ConfigurableParam
 
   // specialized for std::string
   // which means that the type will be converted internally
-  static void setValue(std::string const& key, std::string const& valuestring)
-  {
-    if (!sIsFullyInitialized) {
-      initialize();
-    }
-    assert(sPtree);
-    try {
-      if (sPtree->get_optional<std::string>(key).is_initialized()) {
-        sPtree->put(key, valuestring);
-        auto changed = updateThroughStorageMapWithConversion(key, valuestring);
-        if (changed != EParamUpdateStatus::Failed) {
-          sValueProvenanceMap->find(key)->second = kRT; // set to runtime
-        }
-      }
-    } catch (std::exception const& e) {
-      std::cerr << "Error in setValue (string) " << e.what() << "\n";
-    }
-  }
-
+  static void setValue(std::string const& key, std::string const& valuestring);
   static void setEnumValue(const std::string&, const std::string&);
   static void setArrayValue(const std::string&, const std::string&);
 
@@ -321,7 +302,6 @@ class ConfigurableParam
   // (stored as a vector of pairs <enumValueLabel, enumValueInt>)
   static EnumRegistry* sEnumRegistry;
 
-  static std::string sInputDir;
   static std::string sOutputDir;
 
   void setRegisterMode(bool b) { sRegisterMode = b; }
diff --git a/Common/Utils/include/CommonUtils/ConfigurableParamReaders.h b/Common/Utils/include/CommonUtils/ConfigurableParamReaders.h
new file mode 100644
index 0000000000000..d5ecd6cb97f7a
--- /dev/null
+++ b/Common/Utils/include/CommonUtils/ConfigurableParamReaders.h
@@ -0,0 +1,36 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef O2_COMMON_UTILS_CONFIGURABLEPARAMREADERS_H_
+#define O2_COMMON_UTILS_CONFIGURABLEPARAMREADERS_H_
+
+#include <boost/property_tree/ptree.hpp>
+#include <string>
+
+namespace o2::conf
+{
+
+// Helpers to read ConfigurableParam from different file formats
+class ConfigurableParamReaders
+{
+ public:
+  static void setInputDir(const std::string& d) { sInputDir = d; }
+  static const std::string& getInputDir() { return sInputDir; }
+
+  static boost::property_tree::ptree readINI(std::string const& filepath);
+  static boost::property_tree::ptree readJSON(std::string const& filepath);
+  static boost::property_tree::ptree readConfigFile(std::string const& filepath);
+
+ private:
+  static std::string sInputDir;
+};
+
+} // namespace o2::conf
+#endif // O2_COMMON_UTILS_CONF_CONFIGURABLEPARAMREADERS_H_
diff --git a/Common/Utils/src/ConfigurableParam.cxx b/Common/Utils/src/ConfigurableParam.cxx
index e581970fc81fb..0a3c9fe2d7595 100644
--- a/Common/Utils/src/ConfigurableParam.cxx
+++ b/Common/Utils/src/ConfigurableParam.cxx
@@ -9,11 +9,12 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
-//first version 8/2018, Sandro Wenzel
+// first version 8/2018, Sandro Wenzel
 
 #include "CommonUtils/ConfigurableParam.h"
 #include "CommonUtils/StringUtils.h"
 #include "CommonUtils/KeyValParam.h"
+#include "CommonUtils/ConfigurableParamReaders.h"
 #define BOOST_BIND_GLOBAL_PLACEHOLDERS
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/property_tree/ptree.hpp>
@@ -47,7 +48,6 @@ std::vector<ConfigurableParam*>* ConfigurableParam::sRegisteredParamClasses = nu
 boost::property_tree::ptree* ConfigurableParam::sPtree = nullptr;
 std::map<std::string, std::pair<std::type_info const&, void*>>* ConfigurableParam::sKeyToStorageMap = nullptr;
 std::map<std::string, ConfigurableParam::EParamProvenance>* ConfigurableParam::sValueProvenanceMap = nullptr;
-std::string ConfigurableParam::sInputDir = "";
 std::string ConfigurableParam::sOutputDir = "";
 EnumRegistry* ConfigurableParam::sEnumRegistry = nullptr;
 
@@ -194,60 +194,28 @@ void ConfigurableParam::writeINI(std::string const& filename, std::string const&
 
 bool ConfigurableParam::configFileExists(std::string const& filepath)
 {
-  return std::filesystem::exists(o2::utils::Str::concat_string(sInputDir, filepath));
+  return std::filesystem::exists(o2::utils::Str::concat_string(ConfigurableParamReaders::getInputDir(), filepath));
 }
 
 // ------------------------------------------------------------------
 
-boost::property_tree::ptree ConfigurableParam::readConfigFile(std::string const& filepath)
+void ConfigurableParam::setValue(std::string const& key, std::string const& valuestring)
 {
-  auto inpfilename = o2::utils::Str::concat_string(sInputDir, filepath);
-  if (!std::filesystem::exists(inpfilename)) {
-    LOG(fatal) << inpfilename << " : config file does not exist!";
-  }
-
-  boost::property_tree::ptree pt;
-
-  if (boost::iends_with(inpfilename, ".ini")) {
-    pt = readINI(inpfilename);
-  } else if (boost::iends_with(inpfilename, ".json")) {
-    pt = readJSON(inpfilename);
-  } else {
-    LOG(fatal) << "Configuration file must have either .ini or .json extension";
-  }
-
-  return pt;
-}
-
-// ------------------------------------------------------------------
-
-boost::property_tree::ptree ConfigurableParam::readINI(std::string const& filepath)
-{
-  boost::property_tree::ptree pt;
-  try {
-    boost::property_tree::read_ini(filepath, pt);
-  } catch (const boost::property_tree::ptree_error& e) {
-    LOG(fatal) << "Failed to read INI config file " << filepath << " (" << e.what() << ")";
-  } catch (...) {
-    LOG(fatal) << "Unknown error when reading INI config file ";
+  if (!sIsFullyInitialized) {
+    initialize();
   }
-
-  return pt;
-}
-
-// ------------------------------------------------------------------
-
-boost::property_tree::ptree ConfigurableParam::readJSON(std::string const& filepath)
-{
-  boost::property_tree::ptree pt;
-
+  assert(sPtree);
   try {
-    boost::property_tree::read_json(filepath, pt);
-  } catch (const boost::property_tree::ptree_error& e) {
-    LOG(fatal) << "Failed to read JSON config file " << filepath << " (" << e.what() << ")";
+    if (sPtree->get_optional<std::string>(key).is_initialized()) {
+      sPtree->put(key, valuestring);
+      auto changed = updateThroughStorageMapWithConversion(key, valuestring);
+      if (changed != EParamUpdateStatus::Failed) {
+        sValueProvenanceMap->find(key)->second = kRT; // set to runtime
+      }
+    }
+  } catch (std::exception const& e) {
+    std::cerr << "Error in setValue (string) " << e.what() << "\n";
   }
-
-  return pt;
 }
 
 // ------------------------------------------------------------------
@@ -258,7 +226,7 @@ void ConfigurableParam::writeJSON(std::string const& filename, std::string const
     LOG(info) << "ignoring writing of json file " << filename;
     return;
   }
-  initPropertyTree();     // update the boost tree before writing
+  initPropertyTree(); // update the boost tree before writing
   auto outfilename = o2::utils::Str::concat_string(sOutputDir, filename);
   if (!keyOnly.empty()) { // write ini for selected key only
     try {
@@ -409,7 +377,7 @@ void ConfigurableParam::updateFromFile(std::string const& configFile, std::strin
     return;
   }
 
-  boost::property_tree::ptree pt = readConfigFile(cfgfile);
+  boost::property_tree::ptree pt = ConfigurableParamReaders::readConfigFile(cfgfile);
 
   std::vector<std::pair<std::string, std::string>> keyValPairs;
   auto request = o2::utils::Str::tokenize(paramsList, ',', true);
@@ -522,7 +490,7 @@ void ConfigurableParam::updateFromString(std::string const& configString)
 
   const auto& kv = o2::conf::KeyValParam::Instance();
   if (getProvenance("keyval.input_dir") != kCODE) {
-    sInputDir = o2::utils::Str::concat_string(o2::utils::Str::rectifyDirectory(kv.input_dir));
+    ConfigurableParamReaders::setInputDir(o2::utils::Str::concat_string(o2::utils::Str::rectifyDirectory(kv.input_dir)));
   }
   if (getProvenance("keyval.output_dir") != kCODE) {
     if (kv.output_dir == "/dev/null") {
diff --git a/Common/Utils/src/ConfigurableParamReaders.cxx b/Common/Utils/src/ConfigurableParamReaders.cxx
new file mode 100644
index 0000000000000..2f36f8b27f5fa
--- /dev/null
+++ b/Common/Utils/src/ConfigurableParamReaders.cxx
@@ -0,0 +1,76 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "CommonUtils/ConfigurableParamReaders.h"
+#include "CommonUtils/StringUtils.h"
+#include <fairlogger/Logger.h>
+#include <filesystem>
+
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/property_tree/ini_parser.hpp>
+#include <boost/property_tree/json_parser.hpp>
+
+namespace o2::conf
+{
+// ------------------------------------------------------------------
+
+boost::property_tree::ptree ConfigurableParamReaders::readINI(std::string const& filepath)
+{
+  boost::property_tree::ptree pt;
+  try {
+    boost::property_tree::read_ini(filepath, pt);
+  } catch (const boost::property_tree::ptree_error& e) {
+    LOG(fatal) << "Failed to read INI config file " << filepath << " (" << e.what() << ")";
+  } catch (...) {
+    LOG(fatal) << "Unknown error when reading INI config file ";
+  }
+
+  return pt;
+}
+
+// ------------------------------------------------------------------
+
+boost::property_tree::ptree ConfigurableParamReaders::readJSON(std::string const& filepath)
+{
+  boost::property_tree::ptree pt;
+
+  try {
+    boost::property_tree::read_json(filepath, pt);
+  } catch (const boost::property_tree::ptree_error& e) {
+    LOG(fatal) << "Failed to read JSON config file " << filepath << " (" << e.what() << ")";
+  }
+
+  return pt;
+}
+
+boost::property_tree::ptree ConfigurableParamReaders::readConfigFile(std::string const& filepath)
+{
+  auto inpfilename = o2::utils::Str::concat_string(sInputDir, filepath);
+  if (!std::filesystem::exists(inpfilename)) {
+    LOG(fatal) << inpfilename << " : config file does not exist!";
+  }
+
+  boost::property_tree::ptree pt;
+
+  if (boost::iends_with(inpfilename, ".ini")) {
+    pt = ConfigurableParamReaders::readINI(inpfilename);
+  } else if (boost::iends_with(inpfilename, ".json")) {
+    pt = ConfigurableParamReaders::readJSON(inpfilename);
+  } else {
+    LOG(fatal) << "Configuration file must have either .ini or .json extension";
+  }
+
+  return pt;
+}
+
+std::string ConfigurableParamReaders::sInputDir = "";
+
+} // namespace o2::conf
diff --git a/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx b/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
index 2407796773f57..b18ffd1abee63 100644
--- a/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
+++ b/Detectors/MUON/MCH/Raw/test/testClosureCoDec.cxx
@@ -16,6 +16,7 @@
 #define BOOST_TEST_DYN_LINK
 
 #include <boost/test/unit_test.hpp>
+#include <boost/property_tree/ptree.hpp>
 #include "DetectorsRaw/HBFUtils.h"
 #include "DetectorsRaw/RawFileWriter.h"
 #include "Framework/Logger.h"
diff --git a/Detectors/MUON/MCH/Simulation/test/testDigitizer.cxx b/Detectors/MUON/MCH/Simulation/test/testDigitizer.cxx
index 791ab8e7a3909..2a550220bb137 100644
--- a/Detectors/MUON/MCH/Simulation/test/testDigitizer.cxx
+++ b/Detectors/MUON/MCH/Simulation/test/testDigitizer.cxx
@@ -31,6 +31,7 @@
 #include "TGeoManager.h"
 #include "boost/format.hpp"
 #include <boost/test/data/test_case.hpp>
+#include <boost/property_tree/ptree.hpp>
 #include <algorithm>
 #include <unordered_map>
 
diff --git a/Detectors/MUON/MCH/Triggering/test/testEventFinder.cxx b/Detectors/MUON/MCH/Triggering/test/testEventFinder.cxx
index eb786fc1bb068..60275f2cad2f4 100644
--- a/Detectors/MUON/MCH/Triggering/test/testEventFinder.cxx
+++ b/Detectors/MUON/MCH/Triggering/test/testEventFinder.cxx
@@ -27,6 +27,7 @@
 #include <vector>
 
 #include <boost/test/unit_test.hpp>
+#include <boost/property_tree/ptree.hpp>
 
 #include "CommonUtils/ConfigurableParam.h"
 #include "SimulationDataFormat/MCCompLabel.h"
diff --git a/Detectors/TPC/base/test/testTPCParameters.cxx b/Detectors/TPC/base/test/testTPCParameters.cxx
index 015a6abd964ec..b8baa1e2a74ac 100644
--- a/Detectors/TPC/base/test/testTPCParameters.cxx
+++ b/Detectors/TPC/base/test/testTPCParameters.cxx
@@ -23,9 +23,9 @@
 #include "TPCBase/ParameterGas.h"
 #include <CommonUtils/ConfigurableParam.h>
 #include <CommonUtils/ConfigurableParamHelper.h>
-namespace o2
-{
-namespace tpc
+#include <boost/property_tree/ptree.hpp>
+
+namespace o2::tpc
 {
 
 constexpr float NominalTimeBin = 8 * o2::constants::lhc::LHCBunchSpacingNS * 1e-3;
@@ -367,5 +367,4 @@ BOOST_AUTO_TEST_CASE(ParameterGEM_test2)
   BOOST_CHECK_CLOSE(o2::conf::ConfigurableParam::getValueAs<float>("TPCGEMParam.KappaStack"), 32, 1e-12);
   BOOST_CHECK_CLOSE(o2::conf::ConfigurableParam::getValueAs<float>("TPCGEMParam.EfficiencyStack"), 33, 1e-3);
 }
-} // namespace tpc
 } // namespace o2
diff --git a/Detectors/TPC/calibration/include/TPCCalibration/IDCAverageGroup.h b/Detectors/TPC/calibration/include/TPCCalibration/IDCAverageGroup.h
index fecbe9f14f18b..dd9a5fc257935 100644
--- a/Detectors/TPC/calibration/include/TPCCalibration/IDCAverageGroup.h
+++ b/Detectors/TPC/calibration/include/TPCCalibration/IDCAverageGroup.h
@@ -20,6 +20,7 @@
 #include "TPCCalibration/IDCAverageGroupBase.h"
 #include "TPCBase/Sector.h"
 #include "TPCBase/CalDet.h"
+#include <boost/property_tree/ptree.hpp>
 #include <vector>
 #include "Rtypes.h"
 
diff --git a/Detectors/TPC/calibration/include/TPCCalibration/IDCFactorization.h b/Detectors/TPC/calibration/include/TPCCalibration/IDCFactorization.h
index 13c9d0b98412f..4c7f119efcfe0 100644
--- a/Detectors/TPC/calibration/include/TPCCalibration/IDCFactorization.h
+++ b/Detectors/TPC/calibration/include/TPCCalibration/IDCFactorization.h
@@ -24,6 +24,7 @@
 #include "TPCCalibration/IDCContainer.h"
 #include "TPCCalibration/IDCGroupHelperSector.h"
 #include "DataFormatsTPC/Defs.h"
+#include <boost/property_tree/ptree.hpp>
 
 namespace o2::tpc
 {
diff --git a/Detectors/TPC/calibration/include/TPCCalibration/SACFactorization.h b/Detectors/TPC/calibration/include/TPCCalibration/SACFactorization.h
index b1b4430c11a00..7141750b8ce37 100644
--- a/Detectors/TPC/calibration/include/TPCCalibration/SACFactorization.h
+++ b/Detectors/TPC/calibration/include/TPCCalibration/SACFactorization.h
@@ -21,6 +21,7 @@
 #include "Rtypes.h"
 #include "TPCCalibration/IDCContainer.h"
 #include "DataFormatsTPC/Defs.h"
+#include <boost/property_tree/ptree.hpp>
 
 namespace o2::tpc
 {
diff --git a/Detectors/TPC/calibration/src/IDCGroupingParameter.cxx b/Detectors/TPC/calibration/src/IDCGroupingParameter.cxx
index bbecb4df8f8fd..895b6957d8a01 100644
--- a/Detectors/TPC/calibration/src/IDCGroupingParameter.cxx
+++ b/Detectors/TPC/calibration/src/IDCGroupingParameter.cxx
@@ -12,6 +12,8 @@
 #include "TPCCalibration/IDCGroupingParameter.h"
 #include "Framework/Logger.h"
 #include "Algorithm/RangeTokenizer.h"
+#include <boost/property_tree/ptree.hpp>
+#include <cassert>
 #include <stdexcept>
 
 using namespace o2::tpc;
diff --git a/Detectors/TPC/reconstruction/macro/makeIonTail.C b/Detectors/TPC/reconstruction/macro/makeIonTail.C
index 98db6540d20f7..d8d1be6830626 100644
--- a/Detectors/TPC/reconstruction/macro/makeIonTail.C
+++ b/Detectors/TPC/reconstruction/macro/makeIonTail.C
@@ -32,6 +32,7 @@
 #include "TPCBase/CRUCalibHelpers.h"
 #include "TPCBase/CRU.h"
 #include "CommonUtils/TreeStreamRedirector.h"
+#include <boost/property_tree/ptree.hpp>
 
 using namespace o2::tpc;
 size_t digitsInSaturateion(std::vector<Digit>& digits, bool correctCharge = false, CalPad* pedestals = nullptr, o2::utils::TreeStreamRedirector* stream = nullptr);
diff --git a/Detectors/TPC/simulation/macro/toyCluster.C b/Detectors/TPC/simulation/macro/toyCluster.C
index 257adaf5f79fc..2087a7773fe22 100644
--- a/Detectors/TPC/simulation/macro/toyCluster.C
+++ b/Detectors/TPC/simulation/macro/toyCluster.C
@@ -58,6 +58,7 @@
 #include "TPCReconstruction/HwClusterer.h"
 #include "TPCSimulation/GEMAmplification.h"
 #endif
+#include <boost/property_tree/ptree.hpp>
 
 using namespace o2::tpc;
 void fillTPCHits(const float theta, const float phi, const float dedx, std::vector<HitGroup>& hitGroupSector, std::pair<GlobalPosition3D, GlobalPosition3D>& trackInfo);
diff --git a/Detectors/TPC/workflow/src/ApplyCCDBCalibSpec.cxx b/Detectors/TPC/workflow/src/ApplyCCDBCalibSpec.cxx
index 37429a04c4b34..9c588ff50a45c 100644
--- a/Detectors/TPC/workflow/src/ApplyCCDBCalibSpec.cxx
+++ b/Detectors/TPC/workflow/src/ApplyCCDBCalibSpec.cxx
@@ -19,6 +19,7 @@
 #include "TPCBase/ParameterGas.h"
 #include "DataFormatsTPC/LtrCalibData.h"
 #include "TPCWorkflow/ApplyCCDBCalibSpec.h"
+#include <boost/property_tree/ptree.hpp>
 
 using namespace o2::framework;
 
diff --git a/macro/o2sim.C b/macro/o2sim.C
index 5257a225d3ed9..f610be730eeb7 100644
--- a/macro/o2sim.C
+++ b/macro/o2sim.C
@@ -39,6 +39,7 @@
 #include <sstream>
 #endif
 #include "migrateSimFiles.C"
+#include <boost/property_tree/ptree.hpp>
 
 void check_notransport()
 {

From f5f18ff8cc0764ec7422122d1889675a548670f0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 24 Nov 2023 23:12:49 +0100
Subject: [PATCH 133/726] DPL: more operator== ambiguities fixed

---
 Framework/Core/include/Framework/ASoA.h        | 8 ++++----
 Framework/Core/include/Framework/ASoAHelpers.h | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h
index a0a45a833efeb..898273b63d043 100644
--- a/Framework/Core/include/Framework/ASoA.h
+++ b/Framework/Core/include/Framework/ASoA.h
@@ -614,14 +614,14 @@ struct DefaultIndexPolicy : IndexPolicyBase {
     this->setCursor(mMaxRow);
   }
 
-  bool operator!=(DefaultIndexPolicy const& other) const
+  friend bool operator!=(DefaultIndexPolicy const& lh, DefaultIndexPolicy const& rh)
   {
-    return O2_BUILTIN_LIKELY(this->mRowIndex != other.mRowIndex);
+    return O2_BUILTIN_LIKELY(lh.mRowIndex != rh.mRowIndex);
   }
 
-  bool operator==(DefaultIndexPolicy const& other) const
+  friend bool operator==(DefaultIndexPolicy const& lh, DefaultIndexPolicy const& rh)
   {
-    return O2_BUILTIN_UNLIKELY(this->mRowIndex == other.mRowIndex);
+    return O2_BUILTIN_UNLIKELY(lh.mRowIndex == rh.mRowIndex);
   }
 
   bool operator!=(RowViewSentinel const& sentinel) const
diff --git a/Framework/Core/include/Framework/ASoAHelpers.h b/Framework/Core/include/Framework/ASoAHelpers.h
index 1573ce3abf235..932f4fd029298 100644
--- a/Framework/Core/include/Framework/ASoAHelpers.h
+++ b/Framework/Core/include/Framework/ASoAHelpers.h
@@ -1253,13 +1253,13 @@ struct CombinationsGenerator {
     {
       return this->mCurrent;
     }
-    bool operator==(const CombinationsIterator& rh)
+    friend bool operator==(const CombinationsIterator& lh, const CombinationsIterator& rh)
     {
-      return (this->mIsEnd && rh.mIsEnd) || (this->mCurrent == rh.mCurrent);
+      return (lh.mIsEnd && rh.mIsEnd) || (lh.mCurrent == rh.mCurrent);
     }
-    bool operator!=(const CombinationsIterator& rh)
+    friend bool operator!=(const CombinationsIterator& lh, const CombinationsIterator& rh)
     {
-      return !(*this == rh);
+      return !(lh == rh);
     }
   };
 

From 937d667c9db1e44385b3cac3db9d2cf086b683bc Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 25 Nov 2023 21:53:24 +0100
Subject: [PATCH 134/726] Fix more ambiguous operators

---
 Algorithm/include/Algorithm/PageParser.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Algorithm/include/Algorithm/PageParser.h b/Algorithm/include/Algorithm/PageParser.h
index e382fc318352e..3ca01d87bcba3 100644
--- a/Algorithm/include/Algorithm/PageParser.h
+++ b/Algorithm/include/Algorithm/PageParser.h
@@ -255,12 +255,12 @@ class PageParser
       return mElement;
     }
     // comparison
-    bool operator==(const SelfType& rh)
+    bool operator==(const SelfType& rh) const
     {
       return mPosition == rh.mPosition;
     }
     // comparison
-    bool operator!=(const SelfType& rh)
+    bool operator!=(const SelfType& rh) const
     {
       return mPosition != rh.mPosition;
     }

From 8e934a5bc36d570cd52cd42d0240db3d947b832a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 25 Nov 2023 21:56:57 +0100
Subject: [PATCH 135/726] DPL: fix ambiguous operator for C++20

---
 Framework/Core/include/Framework/GroupedCombinations.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/include/Framework/GroupedCombinations.h b/Framework/Core/include/Framework/GroupedCombinations.h
index 4d2d7678493dc..5904916613d2e 100644
--- a/Framework/Core/include/Framework/GroupedCombinations.h
+++ b/Framework/Core/include/Framework/GroupedCombinations.h
@@ -137,11 +137,11 @@ struct GroupedCombinationsGenerator {
     {
       return *mCurrentGrouped;
     }
-    bool operator==(const GroupedIterator& rh)
+    bool operator==(const GroupedIterator& rh) const
     {
       return (this->mIsEnd && rh.mIsEnd) || (this->mCurrent == rh.mCurrent);
     }
-    bool operator!=(const GroupedIterator& rh)
+    bool operator!=(const GroupedIterator& rh) const
     {
       return !(*this == rh);
     }

From 87656e97980c10a1a90551286264c3dbb461d42a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 27 Nov 2023 11:39:59 +0100
Subject: [PATCH 136/726] More methods which could be const made const

---
 Common/Field/src/MagFieldFast.cxx              | 2 +-
 Utilities/DataCompression/test/DataGenerator.h | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Common/Field/src/MagFieldFast.cxx b/Common/Field/src/MagFieldFast.cxx
index e652dffa396c5..5caad34d56dd4 100644
--- a/Common/Field/src/MagFieldFast.cxx
+++ b/Common/Field/src/MagFieldFast.cxx
@@ -238,7 +238,7 @@ bool MagFieldFast::Field(const math_utils::Point3D<double> xyz, double bxyz[3])
 bool MagFieldFast::GetSegment(float x, float y, float z, int& zSeg, int& rSeg, int& quadrant) const
 {
   // get segment of point location
-  const float zGridSpaceInv = 1.f / (kSolZMax * 2 / kNSolZRanges);
+  const float zGridSpaceInv = 1.f / (kSolZMax * 2 / (float)kNSolZRanges);
   zSeg = -1;
   if (z < kSolZMax) {
     if (z > -kSolZMax) {
diff --git a/Utilities/DataCompression/test/DataGenerator.h b/Utilities/DataCompression/test/DataGenerator.h
index 6c209f9007f6f..3fe0740ccbad8 100644
--- a/Utilities/DataCompression/test/DataGenerator.h
+++ b/Utilities/DataCompression/test/DataGenerator.h
@@ -154,8 +154,8 @@ class DataGenerator
     // pointer operator->() const {return &mValue;}
     // reference operator[](size_type n) const;
 
-    bool operator==(const self_type& other) { return mCount == other.mCount; }
-    bool operator!=(const self_type& other) { return not(*this == other); }
+    bool operator==(const self_type& other) const { return mCount == other.mCount; }
+    bool operator!=(const self_type& other) const { return not(*this == other); }
 
    private:
     const ContainerT& mParent;

From 41641d15b9c069420f689853e4b9a3c50b7a3b0e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 27 Nov 2023 12:36:55 +0100
Subject: [PATCH 137/726] Misc fixes for C++20

Apparently specifying the template specialization for trivial case is now an error.
---
 .../MUON/MCH/Tracking/src/TrackFinderSpec.cxx |  2 +-
 .../MUON/MID/Workflow/src/RawCheckerSpec.cxx  |  2 +-
 .../DataTypes/GPUTRDInterfaceO2Track.h        | 12 +++----
 .../TRDTracking/GPUTRDInterfaces.h            | 36 +++++++++----------
 4 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx b/Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx
index c40df5d4b448e..6239186309dc3 100644
--- a/Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx
+++ b/Detectors/MUON/MCH/Tracking/src/TrackFinderSpec.cxx
@@ -66,7 +66,7 @@ class TrackFinderTask
 {
  public:
   //_________________________________________________________________________________________________
-  TrackFinderTask<T>(bool computeTime, bool digits, std::shared_ptr<base::GRPGeomRequest> req)
+  TrackFinderTask(bool computeTime, bool digits, std::shared_ptr<base::GRPGeomRequest> req)
     : mComputeTime(computeTime), mDigits(digits), mCCDBRequest(req) {}
 
   //_________________________________________________________________________________________________
diff --git a/Detectors/MUON/MID/Workflow/src/RawCheckerSpec.cxx b/Detectors/MUON/MID/Workflow/src/RawCheckerSpec.cxx
index c2e9e3fc5c635..58774a6336294 100644
--- a/Detectors/MUON/MID/Workflow/src/RawCheckerSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/RawCheckerSpec.cxx
@@ -48,7 +48,7 @@ template <typename RAWCHECKER>
 class RawCheckerDeviceDPL
 {
  public:
-  RawCheckerDeviceDPL<RAWCHECKER>(const std::vector<uint16_t>& feeIds, const CrateMasks& crateMasks, const ElectronicsDelay& electronicsDelay) : mFeeIds(feeIds), mCrateMasks(crateMasks), mElectronicsDelay(electronicsDelay) {}
+  RawCheckerDeviceDPL(const std::vector<uint16_t>& feeIds, const CrateMasks& crateMasks, const ElectronicsDelay& electronicsDelay) : mFeeIds(feeIds), mCrateMasks(crateMasks), mElectronicsDelay(electronicsDelay) {}
 
   void init(o2::framework::InitContext& ic)
   {
diff --git a/GPU/GPUTracking/DataTypes/GPUTRDInterfaceO2Track.h b/GPU/GPUTracking/DataTypes/GPUTRDInterfaceO2Track.h
index 19cad3649fa42..fb8f6d1b47d52 100644
--- a/GPU/GPUTracking/DataTypes/GPUTRDInterfaceO2Track.h
+++ b/GPU/GPUTracking/DataTypes/GPUTRDInterfaceO2Track.h
@@ -47,10 +47,10 @@ template <>
 class trackInterface<o2::track::TrackParCov> : public o2::track::TrackParCov
 {
  public:
-  GPUdDefault() trackInterface<o2::track::TrackParCov>() = default;
-  trackInterface<o2::track::TrackParCov>(const o2::track::TrackParCov& param) = delete;
-  GPUd() trackInterface<o2::track::TrackParCov>(const o2::dataformats::TrackTPCITS& trkItsTpc) : o2::track::TrackParCov(trkItsTpc.getParamOut()) {}
-  GPUd() trackInterface<o2::track::TrackParCov>(const o2::tpc::TrackTPC& trkTpc) : o2::track::TrackParCov(trkTpc.getParamOut()) {}
+  GPUdDefault() trackInterface() = default;
+  trackInterface(const o2::track::TrackParCov& param) = delete;
+  GPUd() trackInterface(const o2::dataformats::TrackTPCITS& trkItsTpc) : o2::track::TrackParCov(trkItsTpc.getParamOut()) {}
+  GPUd() trackInterface(const o2::tpc::TrackTPC& trkTpc) : o2::track::TrackParCov(trkTpc.getParamOut()) {}
 
   GPUd() void set(float x, float alpha, const float* param, const float* cov)
   {
@@ -63,8 +63,8 @@ class trackInterface<o2::track::TrackParCov> : public o2::track::TrackParCov
       setCov(cov[i], i);
     }
   }
-  GPUd() trackInterface<o2::track::TrackParCov>(const GPUTPCGMMergedTrack& trk);
-  GPUd() trackInterface<o2::track::TrackParCov>(const gputpcgmmergertypes::GPUTPCOuterParam& param);
+  GPUd() trackInterface(const GPUTPCGMMergedTrack& trk);
+  GPUd() trackInterface(const gputpcgmmergertypes::GPUTPCOuterParam& param);
   GPUd() void updateCovZ2(float addZerror) { updateCov(addZerror, o2::track::CovLabels::kSigZ2); }
   GPUd() o2::track::TrackLTIntegral& getLTIntegralOut() { return mLTOut; }
   GPUd() const o2::track::TrackLTIntegral& getLTIntegralOut() const { return mLTOut; }
diff --git a/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h b/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h
index ab8db40e90e2e..6359f880e2860 100644
--- a/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h
+++ b/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h
@@ -101,9 +101,9 @@ class propagatorInterface<AliTrackerBase> : public AliTrackerBase
 
  public:
   typedef void propagatorParam;
-  propagatorInterface<AliTrackerBase>(const propagatorParam* = nullptr) : AliTrackerBase(), mParam(nullptr){};
-  propagatorInterface<AliTrackerBase>(const propagatorInterface<AliTrackerBase>&) CON_DELETE;
-  propagatorInterface<AliTrackerBase>& operator=(const propagatorInterface<AliTrackerBase>&) CON_DELETE;
+  propagatorInterface(const propagatorParam* = nullptr) : AliTrackerBase(), mParam(nullptr){};
+  propagatorInterface(const propagatorInterface<AliTrackerBase>&) CON_DELETE;
+  propagatorInterface& operator=(const propagatorInterface<AliTrackerBase>&) CON_DELETE;
 
   bool propagateToX(float x, float maxSnp, float maxStep) { return PropagateTrackToBxByBz(mParam, x, 0.13957, maxStep, false, maxSnp); }
   int getPropagatedYZ(float x, float& projY, float& projZ)
@@ -147,9 +147,9 @@ class propagatorInterface<o2::base::Propagator>
 {
  public:
   typedef o2::base::Propagator propagatorParam;
-  GPUd() propagatorInterface<o2::base::Propagator>(const propagatorParam* prop) : mProp(prop){};
-  GPUd() propagatorInterface<o2::base::Propagator>(const propagatorInterface<o2::base::Propagator>&) = delete;
-  GPUd() propagatorInterface<o2::base::Propagator>& operator=(const propagatorInterface<o2::base::Propagator>&) = delete;
+  GPUd() propagatorInterface(const propagatorParam* prop) : mProp(prop){};
+  GPUd() propagatorInterface(const propagatorInterface<o2::base::Propagator>&) = delete;
+  GPUd() propagatorInterface& operator=(const propagatorInterface<o2::base::Propagator>&) = delete;
 
   GPUdi() bool propagateToX(float x, float maxSnp, float maxStep) { return mProp->PropagateToXBxByBz(*mParam, x, maxSnp, maxStep); }
   GPUdi() int getPropagatedYZ(float x, float& projY, float& projZ) { return static_cast<int>(mParam->getYZAt(x, mProp->getNominalBz(), projY, projZ)); }
@@ -206,10 +206,10 @@ template <>
 class trackInterface<GPUTPCGMTrackParam> : public GPUTPCGMTrackParam
 {
  public:
-  GPUdDefault() trackInterface<GPUTPCGMTrackParam>() CON_DEFAULT;
-  GPUd() trackInterface<GPUTPCGMTrackParam>(const GPUTPCGMTrackParam& param) CON_DELETE;
-  GPUd() trackInterface<GPUTPCGMTrackParam>(const GPUTPCGMMergedTrack& trk) : GPUTPCGMTrackParam(trk.GetParam()), mAlpha(trk.GetAlpha()) {}
-  GPUd() trackInterface<GPUTPCGMTrackParam>(const gputpcgmmergertypes::GPUTPCOuterParam& param) : GPUTPCGMTrackParam(), mAlpha(param.alpha)
+  GPUdDefault() trackInterface() CON_DEFAULT;
+  GPUd() trackInterface(const GPUTPCGMTrackParam& param) CON_DELETE;
+  GPUd() trackInterface(const GPUTPCGMMergedTrack& trk) : GPUTPCGMTrackParam(trk.GetParam()), mAlpha(trk.GetAlpha()) {}
+  GPUd() trackInterface(const gputpcgmmergertypes::GPUTPCOuterParam& param) : GPUTPCGMTrackParam(), mAlpha(param.alpha)
   {
     SetX(param.X);
     for (int i = 0; i < 5; i++) {
@@ -220,11 +220,11 @@ class trackInterface<GPUTPCGMTrackParam> : public GPUTPCGMTrackParam
     }
   };
 #ifdef GPUCA_NOCOMPAT
-  GPUdDefault() trackInterface<GPUTPCGMTrackParam>(const trackInterface<GPUTPCGMTrackParam>& param) = default;
-  GPUdDefault() trackInterface<GPUTPCGMTrackParam>& operator=(const trackInterface<GPUTPCGMTrackParam>& param) = default;
+  GPUdDefault() trackInterface(const trackInterface<GPUTPCGMTrackParam>& param) = default;
+  GPUdDefault() trackInterface& operator=(const trackInterface<GPUTPCGMTrackParam>& param) = default;
 #endif
 #ifdef GPUCA_ALIROOT_LIB
-  trackInterface<GPUTPCGMTrackParam>(const AliHLTExternalTrackParam& param) : GPUTPCGMTrackParam(), mAlpha(param.fAlpha)
+  trackInterface(const AliHLTExternalTrackParam& param) : GPUTPCGMTrackParam(), mAlpha(param.fAlpha)
   {
     SetX(param.fX);
     SetPar(0, param.fY);
@@ -238,7 +238,7 @@ class trackInterface<GPUTPCGMTrackParam> : public GPUTPCGMTrackParam
   };
 #endif
 #if defined(GPUCA_HAVE_O2HEADERS)
-  GPUd() trackInterface<GPUTPCGMTrackParam>(const o2::dataformats::TrackTPCITS& param) : GPUTPCGMTrackParam(), mAlpha(param.getParamOut().getAlpha())
+  GPUd() trackInterface(const o2::dataformats::TrackTPCITS& param) : GPUTPCGMTrackParam(), mAlpha(param.getParamOut().getAlpha())
   {
     SetX(param.getParamOut().getX());
     SetPar(0, param.getParamOut().getY());
@@ -250,7 +250,7 @@ class trackInterface<GPUTPCGMTrackParam> : public GPUTPCGMTrackParam
       SetCov(i, param.getParamOut().getCov()[i]);
     }
   }
-  GPUd() trackInterface<GPUTPCGMTrackParam>(const o2::tpc::TrackTPC& param) : GPUTPCGMTrackParam(), mAlpha(param.getParamOut().getAlpha())
+  GPUd() trackInterface(const o2::tpc::TrackTPC& param) : GPUTPCGMTrackParam(), mAlpha(param.getParamOut().getAlpha())
   {
     SetX(param.getParamOut().getX());
     SetPar(0, param.getParamOut().getY());
@@ -307,7 +307,7 @@ class propagatorInterface<GPUTPCGMPropagator> : public GPUTPCGMPropagator
 {
  public:
   typedef GPUTPCGMPolynomialField propagatorParam;
-  GPUd() propagatorInterface<GPUTPCGMPropagator>(const propagatorParam* pField) : GPUTPCGMPropagator(), mTrack(nullptr)
+  GPUd() propagatorInterface(const propagatorParam* pField) : GPUTPCGMPropagator(), mTrack(nullptr)
   {
     this->SetMaterialTPC();
     this->SetPolynomialField(pField);
@@ -316,8 +316,8 @@ class propagatorInterface<GPUTPCGMPropagator> : public GPUTPCGMPropagator
     this->SetFitInProjections(0);
     this->SelectFieldRegion(GPUTPCGMPropagator::TRD);
   };
-  propagatorInterface<GPUTPCGMPropagator>(const propagatorInterface<GPUTPCGMPropagator>&) CON_DELETE;
-  propagatorInterface<GPUTPCGMPropagator>& operator=(const propagatorInterface<GPUTPCGMPropagator>&) CON_DELETE;
+  propagatorInterface(const propagatorInterface<GPUTPCGMPropagator>&) CON_DELETE;
+  propagatorInterface& operator=(const propagatorInterface<GPUTPCGMPropagator>&) CON_DELETE;
   GPUd() void setTrack(trackInterface<GPUTPCGMTrackParam>* trk)
   {
     SetTrack(trk, trk->getAlpha());

From 77a3db35b839e0b2ec50598d7976339fbc2e0527 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 28 Nov 2023 15:07:11 +0100
Subject: [PATCH 138/726] C++20 fixes

* Using "{T}" in a format string is not actually allowed because it's interpreted
  as the name of the placeholder.
* GlobalFwdTrack needs an explicit constructor from o2::track::TrackParCovFwd
---
 .../ReconstructionDataFormats/GlobalFwdTrack.h   |  1 +
 Detectors/TPC/qc/src/Tracks.cxx                  | 16 ++++++++--------
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/GlobalFwdTrack.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/GlobalFwdTrack.h
index 15d816b7b7727..5d13a216316ef 100644
--- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/GlobalFwdTrack.h
+++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/GlobalFwdTrack.h
@@ -31,6 +31,7 @@ class GlobalFwdTrack : public o2::track::TrackParCovFwd, public o2::dataformats:
  public:
   GlobalFwdTrack() = default;
   GlobalFwdTrack(const GlobalFwdTrack& t) = default;
+  GlobalFwdTrack(o2::track::TrackParCovFwd const& t) { *this = t; }
   ~GlobalFwdTrack() = default;
 
   SMatrix5 computeResiduals2Cov(const o2::track::TrackParCovFwd& t) const
diff --git a/Detectors/TPC/qc/src/Tracks.cxx b/Detectors/TPC/qc/src/Tracks.cxx
index 86f13d60a00fa..7e077487a9ef5 100644
--- a/Detectors/TPC/qc/src/Tracks.cxx
+++ b/Detectors/TPC/qc/src/Tracks.cxx
@@ -71,26 +71,26 @@ void Tracks::initializeHistograms()
   mMapHist["h2DQOverPtPhiAside"] = std::make_unique<TH2F>("h2DQOverPtPhiAside", "Charger over p_T vs. phi, A side;phi;q/p_T", 360, 0., 2 * M_PI, 400, -20., 20.);
   mMapHist["h2DQOverPtPhiCside"] = std::make_unique<TH2F>("h2DQOverPtPhiCside", "Charger over p_T vs. phi, C side;phi;q/p_T", 360, 0., 2 * M_PI, 400, -20., 20.);
   // eta vs pt and phi vs pt possitive and negative signs
-  mMapHist["hEtaVsPtPos"] = std::make_unique<TH2F>("hEtaVsPtPos", "#eta vs. p_{T} (Pos.);p_{T};eta", logPtBinning.size() - 1, logPtBinning.data(), 400, -2., 2.);
-  mMapHist["hEtaVsPtNeg"] = std::make_unique<TH2F>("hEtaVsPtNeg", "#eta vs. p_{T} (Neg.);p_{T};eta", logPtBinning.size() - 1, logPtBinning.data(), 400, -2., 2.);
-  mMapHist["hPhiVsPtPos"] = std::make_unique<TH2F>("hPhiVsPtPos", "#phi vs. p_{T} (Pos.);p_{T};phi", logPtBinning.size() - 1, logPtBinning.data(), 360, 0., 2 * M_PI);
-  mMapHist["hPhiVsPtNeg"] = std::make_unique<TH2F>("hPhiVsPtNeg", "#phi vs. p_{T} (Neg.);p_{T};phi", logPtBinning.size() - 1, logPtBinning.data(), 360, 0., 2 * M_PI);
+  mMapHist["hEtaVsPtPos"] = std::make_unique<TH2F>("hEtaVsPtPos", "#eta vs. p_{{T}} (Pos.);p_{{T}};eta", logPtBinning.size() - 1, logPtBinning.data(), 400, -2., 2.);
+  mMapHist["hEtaVsPtNeg"] = std::make_unique<TH2F>("hEtaVsPtNeg", "#eta vs. p_{{T}} (Neg.);p_{{T}};eta", logPtBinning.size() - 1, logPtBinning.data(), 400, -2., 2.);
+  mMapHist["hPhiVsPtPos"] = std::make_unique<TH2F>("hPhiVsPtPos", "#phi vs. p_{{T}} (Pos.);p_{{T}};phi", logPtBinning.size() - 1, logPtBinning.data(), 360, 0., 2 * M_PI);
+  mMapHist["hPhiVsPtNeg"] = std::make_unique<TH2F>("hPhiVsPtNeg", "#phi vs. p_{{T}} (Neg.);p_{{T}};phi", logPtBinning.size() - 1, logPtBinning.data(), 360, 0., 2 * M_PI);
 
   // 1d histograms
   mMapHist["hEtaRatio"] = std::make_unique<TH1F>("hEtaRatio", "Pseudorapidity, ratio neg./pos. ;eta", 400, -2., 2.);
   mMapHist["hPhiAsideRatio"] = std::make_unique<TH1F>("hPhiAsideRatio", "Azimuthal angle, A side, ratio neg./pos. ;phi", 360, 0., 2 * M_PI);
   mMapHist["hPhiCsideRatio"] = std::make_unique<TH1F>("hPhiCsideRatio", "Azimuthal angle, C side, ratio neg./pos. ;phi", 360, 0., 2 * M_PI);
-  mMapHist["hPtRatio"] = std::make_unique<TH1F>("hPtRatio", "Transverse momentum, ratio neg./pos. ;p_T", logPtBinning.size() - 1, logPtBinning.data());
+  mMapHist["hPtRatio"] = std::make_unique<TH1F>("hPtRatio", "Transverse momentum, ratio neg./pos. ;p_{{T}}", logPtBinning.size() - 1, logPtBinning.data());
 
   // DCA Histograms
   for (const auto type : types) {
     mMapHist[fmt::format("hDCAr_{}", type).data()] = std::make_unique<TH2F>(fmt::format("hDCAr_{}", type).data(), fmt::format("DCAr {};phi;DCAr (cm)", type).data(), 360, 0, o2::math_utils::twoPid(), 250, -10., 10.);
   }
   // DCA vs variables Histograms
-  mMapHist["hDCArVsPtPos"] = std::make_unique<TH2F>("hDCArVsPtPos", "DCAr Pos;#it{p}_{T}T (GeV/#it{c});DCAr (cm)", logPtBinning.size() - 1, logPtBinning.data(), 250, -10., 10.);
+  mMapHist["hDCArVsPtPos"] = std::make_unique<TH2F>("hDCArVsPtPos", "DCAr Pos;#it{p}_{{T}}T (GeV/#it{c});DCAr (cm)", logPtBinning.size() - 1, logPtBinning.data(), 250, -10., 10.);
   mMapHist["hDCArVsEtaPos"] = std::make_unique<TH2F>("hDCArVsEtaPos", "DCAr Pos;#eta;DCAr (cm)", 400, -2., 2., 250, -10., 10.);
   mMapHist["hDCArVsNClsPos"] = std::make_unique<TH2F>("hDCArVsNClsPos", "DCAr Pos;NClusters;DCAr (cm)", 400, -0.5, 399.5, 250, -10., 10.);
-  mMapHist["hDCArVsPtNeg"] = std::make_unique<TH2F>("hDCArVsPtNeg", "DCAr Neg;#it{p}_{T}T (GeV/#it{c});DCAr (cm)", logPtBinning.size() - 1, logPtBinning.data(), 250, -10., 10.);
+  mMapHist["hDCArVsPtNeg"] = std::make_unique<TH2F>("hDCArVsPtNeg", "DCAr Neg;#it{p}_{{T}}T (GeV/#it{c});DCAr (cm)", logPtBinning.size() - 1, logPtBinning.data(), 250, -10., 10.);
   mMapHist["hDCArVsEtaNeg"] = std::make_unique<TH2F>("hDCArVsEtaNeg", "DCAr Neg;#eta;DCAr (cm)", 400, -2., 2., 250, -10., 10.);
   mMapHist["hDCArVsNClsNeg"] = std::make_unique<TH2F>("hDCArVsNClsNeg", "DCAr Neg;NClusters;DCAr (cm)", 400, -0.5, 399.5, 250, -10., 10.);
 }
@@ -173,7 +173,7 @@ bool Tracks::processTrack(const o2::tpc::TrackTPC& track)
             reported = true;
           }
           dcaHist->SetTitle(fmt::format("DCAr {} o2::base::Propagator not properly initialized", types[type]).data());
-          dcaHistPT->SetTitle(fmt::format("DCAr p_{T} {} o2::base::Propagator not properly initialized", signType).data());
+          dcaHistPT->SetTitle(fmt::format("DCAr p_{{T}} {} o2::base::Propagator not properly initialized", signType).data());
           dcaHistEta->SetTitle(fmt::format("DCAr eta {} o2::base::Propagator not properly initialized", signType).data());
           dcaHistNCluster->SetTitle(fmt::format("DCAr nClusters {} o2::base::Propagator not properly initialized", signType).data());
         }

From 6bc6a196c45c177b1fdd4cb28a678cc4bac43319 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 28 Nov 2023 15:24:56 +0100
Subject: [PATCH 139/726] C++20 fixes

---
 Detectors/MUON/MCH/Raw/Decoder/src/UserLogicEndpointDecoder.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Detectors/MUON/MCH/Raw/Decoder/src/UserLogicEndpointDecoder.h b/Detectors/MUON/MCH/Raw/Decoder/src/UserLogicEndpointDecoder.h
index 8471497c18708..8b49a006aca0e 100644
--- a/Detectors/MUON/MCH/Raw/Decoder/src/UserLogicEndpointDecoder.h
+++ b/Detectors/MUON/MCH/Raw/Decoder/src/UserLogicEndpointDecoder.h
@@ -158,7 +158,7 @@ size_t UserLogicEndpointDecoder<CHARGESUM, VERSION>::append(Payload buffer)
       }
 
       mElinkDecoders.emplace(static_cast<uint16_t>(gbt),
-                             impl::makeArray<40>([=](size_t i) {
+                             impl::makeArray<40>([solarId, *this](size_t i) {
                                DsElecId dselec{solarId.value(), static_cast<uint8_t>(i / 5), static_cast<uint8_t>(i % 5)};
                                return ElinkDecoder(dselec, mDecodedDataHandlers);
                              }));

From c2b5ea091383108236823a55613ca83665bb7273 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 28 Nov 2023 15:54:02 +0100
Subject: [PATCH 140/726] More asymmetric operator==

---
 .../include/DataCompression/dc_primitives.h               | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Utilities/DataCompression/include/DataCompression/dc_primitives.h b/Utilities/DataCompression/include/DataCompression/dc_primitives.h
index fc0f941fe2358..0e43c07ee7e9c 100644
--- a/Utilities/DataCompression/include/DataCompression/dc_primitives.h
+++ b/Utilities/DataCompression/include/DataCompression/dc_primitives.h
@@ -170,9 +170,9 @@ class ExampleAlphabet
     // reference
     reference operator*();
     // comparison
-    bool operator==(const self_type& other);
+    bool operator==(const self_type& other) const;
     // comparison
-    bool operator!=(const self_type& other);
+    bool operator!=(const self_type& other) const;
 
    private:
   };
@@ -300,8 +300,8 @@ class ContiguousAlphabet
     // pointer operator->() const {return &mValue;}
     // reference operator[](size_type n) const;
 
-    bool operator==(const self_type& other) { return mValue == other.mValue && mIsEnd == other.mIsEnd; }
-    bool operator!=(const self_type& other) { return not(*this == other); }
+    bool operator==(const self_type& other) const { return mValue == other.mValue && mIsEnd == other.mIsEnd; }
+    bool operator!=(const self_type& other) const { return not(*this == other); }
 
    private:
     value_type mValue;

From fd3ec296323aa63e8ca93ca779fe81d8bcb8e41f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 28 Nov 2023 17:54:56 +0100
Subject: [PATCH 141/726] DPL: mark dummy sink as resilient to expendable task
 failures

---
 Framework/Core/src/CommonDataProcessors.cxx | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index e9cdc97e316ca..3c86f921c01b8 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -599,9 +599,8 @@ DataProcessorSpec CommonDataProcessors::getDummySink(std::vector<InputSpec> cons
     .options = !rateLimitingChannelConfig.empty() ? std::vector<ConfigParamSpec>{{"channel-config", VariantType::String, // raw input channel
                                                                                   rateLimitingChannelConfig,
                                                                                   {"Out-of-band channel config"}}}
-                                                  : std::vector<ConfigParamSpec>()
-
-  };
+                                                  : std::vector<ConfigParamSpec>(),
+    .labels = {{"resilient"}}};
 }
 
 AlgorithmSpec CommonDataProcessors::wrapWithRateLimiting(AlgorithmSpec spec)

From 65d05ab1ff55e03b6142bc42439a4c1a7ea15677 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 29 Nov 2023 10:05:51 +0100
Subject: [PATCH 142/726] Fix more enums used as int for C++20

---
 Detectors/ITSMFT/ITS/reconstruction/src/CookedTracker.cxx | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Detectors/ITSMFT/ITS/reconstruction/src/CookedTracker.cxx b/Detectors/ITSMFT/ITS/reconstruction/src/CookedTracker.cxx
index e68d26a4d12d3..5c804f6705dfd 100644
--- a/Detectors/ITSMFT/ITS/reconstruction/src/CookedTracker.cxx
+++ b/Detectors/ITSMFT/ITS/reconstruction/src/CookedTracker.cxx
@@ -739,7 +739,7 @@ void CookedTracker::Layer::init()
     Float_t phi = xyz.Phi();
     o2::math_utils::bringTo02Pi(phi);
     mPhi.push_back(phi);
-    Int_t s = phi * kNSectors / k2PI;
+    Int_t s = phi * (int)kNSectors / k2PI;
     mSectors[s < kNSectors ? s : kNSectors - 1].emplace_back(i, c->getZ());
   }
 
@@ -792,8 +792,8 @@ void CookedTracker::Layer::selectClusters(std::vector<Int_t>& selec, Float_t phi
 
   Float_t dphi = dy / mR;
 
-  int smin = (phi - dphi) / k2PI * kNSectors;
-  int ds = (phi + dphi) / k2PI * kNSectors - smin + 1;
+  int smin = (phi - dphi) / k2PI * (int)kNSectors;
+  int ds = (phi + dphi) / k2PI * (int)kNSectors - smin + 1;
 
   smin = (smin + kNSectors) % kNSectors;
 

From 0805af741cd7aa5f4c48386b25dcc7c05420b416 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 29 Nov 2023 14:24:57 +0100
Subject: [PATCH 143/726] Fix more enums used as integers

---
 .../GRP/calibration/include/GRPCalibration/GRPDCSDPsProcessor.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Detectors/GRP/calibration/include/GRPCalibration/GRPDCSDPsProcessor.h b/Detectors/GRP/calibration/include/GRPCalibration/GRPDCSDPsProcessor.h
index 0270136abbf77..6bb93f26066f7 100644
--- a/Detectors/GRP/calibration/include/GRPCalibration/GRPDCSDPsProcessor.h
+++ b/Detectors/GRP/calibration/include/GRPCalibration/GRPDCSDPsProcessor.h
@@ -174,7 +174,7 @@ struct GRPLHCInfo {
   static constexpr std::string_view bptxPhaseShiftAliases[NBPTXPhaseShiftAliases] = {"BPTX_Phase_Shift_B1", "BPTX_Phase_Shift_B2"};
   static constexpr std::string_view lumiAliases[NLumiAliases] = {"ALI_Lumi_Total_Inst"};
   static constexpr std::string_view lhcStringAliases[NLHCStringAliases] = {"ALI_Lumi_Source_Name", "BEAM_MODE", "MACHINE_MODE"};
-  static constexpr int nAliasesLHC = NCollimatorAliases + NBeamAliases + NBkgAliases + NBPTXAliases + NBPTXPhaseAliases + NBPTXPhaseRMSAliases + NBPTXPhaseShiftAliases + NLumiAliases + NLHCStringAliases;
+  static constexpr int nAliasesLHC = (int)NCollimatorAliases + (int)NBeamAliases + (int)NBkgAliases + (int)NBPTXAliases + (int)NBPTXPhaseAliases + (int)NBPTXPhaseRMSAliases + (int)NBPTXPhaseShiftAliases + (int)NLumiAliases + (int)NLHCStringAliases;
 
   std::array<std::vector<std::pair<uint64_t, double>>, 2> mIntensityBeam;
   std::array<std::vector<std::pair<uint64_t, double>>, 3> mBackground;

From 66d63444fb4d45253cc5bec5f4d5a915aad86343 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 30 Nov 2023 13:52:00 +0100
Subject: [PATCH 144/726] DPL: add support for Lifetime::sporadic to query
 builder

---
 Framework/Core/src/DataDescriptorQueryBuilder.cxx | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Framework/Core/src/DataDescriptorQueryBuilder.cxx b/Framework/Core/src/DataDescriptorQueryBuilder.cxx
index f79698ea2a08d..41a14d06f3acc 100644
--- a/Framework/Core/src/DataDescriptorQueryBuilder.cxx
+++ b/Framework/Core/src/DataDescriptorQueryBuilder.cxx
@@ -156,6 +156,9 @@ std::vector<InputSpec> DataDescriptorQueryBuilder::parse(char const* config)
       if (attribute.name == "lifetime" && attribute.defaultValue.get<std::string>() == "condition") {
         lifetime = Lifetime::Condition;
       }
+      if (attribute.name == "lifetime" && attribute.defaultValue.get<std::string>() == "sporadic") {
+        lifetime = Lifetime::Sporadic;
+      }
     }
     return InputSpec{binding, std::move(*lastMatcher.release()), lifetime, attributes};
   };

From 0217381a66bb0131a20f6e2c86d31c80ac2b640c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 30 Nov 2023 16:20:00 +0100
Subject: [PATCH 145/726] DPL: better message when child dies

---
 Framework/Core/src/runDataProcessing.cxx | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 724383ce501e6..0b29bfcdcb599 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -864,10 +864,9 @@ bool processSigChild(DeviceInfos& infos, DeviceSpecs& specs)
     int status;
     pid_t pid = waitpid((pid_t)(-1), &status, WNOHANG);
     if (pid > 0) {
+      // Normal exit
       int es = WEXITSTATUS(status);
-
       if (WIFEXITED(status) == false || es != 0) {
-        es = WIFEXITED(status) ? es : 128 + es;
         // Look for the name associated to the pid in the infos
         std::string id = "unknown";
         assert(specs.size() == infos.size());
@@ -882,10 +881,11 @@ bool processSigChild(DeviceInfos& infos, DeviceSpecs& specs)
         } else if (forceful_exit) {
           LOGP(error, "pid {} ({}) was forcefully terminated after being requested to quit", pid, id);
         } else {
-          if (es == 128) {
-            LOGP(error, "Workflow crashed - pid {} ({}) was killed abnormally with exit code {}, could be out of memory killer, segfault, unhandled exception, SIGKILL, etc...", pid, id, es);
+          if (WIFSIGNALED(status)) {
+            int exitSignal = WTERMSIG(status);
+            LOGP(error, "Workflow crashed - pid {} ({}) was killed abnormally with {} and exited with {}", pid, id, strsignal(exitSignal), es);
           } else {
-            LOGP(error, "pid {} ({}) crashed with or was killed with exit code {}", pid, id, es);
+            LOGP(error, "pid {} ({}) exited with {}", pid, id, es);
           }
         }
         hasError |= true;

From 88c53204fcd0ac78f1db69b6e7992baa1c245e6d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 30 Nov 2023 19:00:43 +0100
Subject: [PATCH 146/726] DPL: ignored inputs are "Sporadic" by construction.

---
 Framework/Core/src/WorkflowHelpers.cxx | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 2484f021ff0e2..0e26762829e14 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -732,10 +732,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
     std::vector<InputSpec> ignored = unmatched;
     ignored.insert(ignored.end(), redirectedOutputsInputs.begin(), redirectedOutputsInputs.end());
     for (auto& ignoredInput : ignored) {
-      if (ignoredInput.lifetime == Lifetime::OutOfBand) {
-        // FIXME: Use Lifetime::Dangling when fully working?
-        ignoredInput.lifetime = Lifetime::Timeframe;
-      }
+      ignoredInput.lifetime = Lifetime::Sporadic;
     }
 
     extraSpecs.push_back(CommonDataProcessors::getDummySink(ignored, rateLimitingChannelConfigOutput));

From 494b491ac315a5d7cc92458939b26ddd81d98355 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 4 Dec 2023 10:29:59 +0100
Subject: [PATCH 147/726] DPL: better message when child dies

---
 Framework/Core/src/runDataProcessing.cxx | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 724383ce501e6..54af8e9b4a25e 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -864,10 +864,9 @@ bool processSigChild(DeviceInfos& infos, DeviceSpecs& specs)
     int status;
     pid_t pid = waitpid((pid_t)(-1), &status, WNOHANG);
     if (pid > 0) {
+      // Normal exit
       int es = WEXITSTATUS(status);
-
       if (WIFEXITED(status) == false || es != 0) {
-        es = WIFEXITED(status) ? es : 128 + es;
         // Look for the name associated to the pid in the infos
         std::string id = "unknown";
         assert(specs.size() == infos.size());
@@ -882,8 +881,9 @@ bool processSigChild(DeviceInfos& infos, DeviceSpecs& specs)
         } else if (forceful_exit) {
           LOGP(error, "pid {} ({}) was forcefully terminated after being requested to quit", pid, id);
         } else {
-          if (es == 128) {
-            LOGP(error, "Workflow crashed - pid {} ({}) was killed abnormally with exit code {}, could be out of memory killer, segfault, unhandled exception, SIGKILL, etc...", pid, id, es);
+          if (WIFSIGNALED(status)) {
+            int exitSignal = WTERMSIG(status);
+            LOGP(error, "Workflow crashed - pid {} ({}) was killed abnormally with {} and exited with {}", pid, id, strsignal(exitSignal), es);
           } else {
             LOGP(error, "pid {} ({}) crashed with or was killed with exit code {}", pid, id, es);
           }

From 955d9432cde30f58f596723b75143e60fcd1a5f4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 4 Dec 2023 12:38:50 +0100
Subject: [PATCH 148/726] DPL: explicit cast enum to float

Required by C++20.
---
 GPU/GPUTracking/TRDTracking/GPUTRDTracker.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/GPU/GPUTracking/TRDTracking/GPUTRDTracker.cxx b/GPU/GPUTracking/TRDTracking/GPUTRDTracker.cxx
index 81c452926d7af..46479091d7c3a 100644
--- a/GPU/GPUTracking/TRDTracking/GPUTRDTracker.cxx
+++ b/GPU/GPUTracking/TRDTracking/GPUTRDTracker.cxx
@@ -998,7 +998,7 @@ GPUd() int GPUTRDTracker_t<TRDTRK, PROP>::GetSector(float alpha) const
   } else if (alpha >= 2.f * CAMath::Pi()) {
     alpha -= 2.f * CAMath::Pi();
   }
-  return (int)(alpha * kNSectors / (2.f * CAMath::Pi()));
+  return (int)(alpha * (float)kNSectors / (2.f * CAMath::Pi()));
 }
 
 template <class TRDTRK, class PROP>

From 917237204dbb86b0835f867c9150f17054dc5178 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 5 Dec 2023 11:53:34 +0100
Subject: [PATCH 149/726] Remove default Lifetime::Timeframe when instanciating
 an Output

This lifetime field will be removed soon, and this is unambiguous, because
it's in any case the default.
---
 .../AOD/src/AODMcProducerWorkflowSpec.cxx     |  8 +--
 Detectors/AOD/src/AODProducerWorkflowSpec.cxx |  8 +--
 .../src/CPVBadMapCalibDevice.cxx              |  2 +-
 .../src/CPVGainCalibDevice.cxx                |  2 +-
 .../src/CPVPedestalCalibDevice.cxx            |  2 +-
 .../CPV/workflow/src/ClusterReaderSpec.cxx    |  6 +--
 .../CPV/workflow/src/ClusterizerSpec.cxx      | 16 +++---
 .../CPV/workflow/src/DigitReaderSpec.cxx      |  6 +--
 .../CPV/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../workflow/src/RawToDigitConverterSpec.cxx  | 18 +++----
 .../CTP/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 Detectors/CTP/workflow/src/RawDecoderSpec.cxx |  8 +--
 .../CTP/workflowIO/src/DigitReaderSpec.cxx    |  4 +-
 .../testWorkflow/src/DCSDataReplaySpec.cxx    |  2 +-
 .../src/DCSRandomDataGeneratorSpec.cxx        |  2 +-
 .../include/EMCALWorkflow/PublisherSpec.h     | 11 ++--
 .../workflow/src/AnalysisClusterSpec.cxx      |  2 +-
 .../EMCAL/workflow/src/CellConverterSpec.cxx  |  6 +--
 .../workflow/src/CellRecalibratorSpec.cxx     | 10 ++--
 .../EMCAL/workflow/src/ClusterizerSpec.cxx    |  8 +--
 .../EMCAL/workflow/src/EMCALDigitizerSpec.cxx | 10 ++--
 .../EMCAL/workflow/src/EntropyEncoderSpec.cxx |  2 +-
 .../workflow/src/RawToCellConverterSpec.cxx   |  6 +--
 .../src/StandaloneAODProducerSpec.cxx         |  4 +-
 .../src/emc-channel-data-producer.cxx         |  6 +--
 .../include/FDDWorkflow/RawReaderFDD.h        |  4 +-
 .../FIT/FDD/workflow/src/DigitReaderSpec.cxx  |  8 +--
 .../FDD/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../FDD/workflow/src/RecPointReaderSpec.cxx   |  4 +-
 .../FDD/workflow/src/ReconstructorSpec.cxx    |  4 +-
 .../FT0CalibSlewingCollectorSpec.h            |  4 +-
 .../FT0TimeSpectraProcessor-Workflow.cxx      |  2 +-
 .../FIT/FT0/workflow/src/DigitReaderSpec.cxx  |  8 +--
 .../FT0/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../FT0/workflow/src/RecPointReaderSpec.cxx   |  4 +-
 .../FT0/workflow/src/ReconstructionSpec.cxx   |  4 +-
 .../FIT/FV0/workflow/src/DigitReaderSpec.cxx  |  8 +--
 .../FV0/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../FV0/workflow/src/RecPointReaderSpec.cxx   |  4 +-
 .../FV0/workflow/src/ReconstructionSpec.cxx   |  4 +-
 .../include/FITWorkflow/RawReaderFIT.h        | 14 ++---
 .../FOCAL/workflow/src/RawDecoderSpec.cxx     | 10 ++--
 .../Filtering/src/FilteredTFReaderSpec.cxx    | 14 ++---
 .../helpers/src/NoInpDummyOutSpec.cxx         |  2 +-
 .../qc/src/ITSTPCMatchingQCSpec.cxx           |  2 +-
 .../readers/src/GlobalFwdTrackReaderSpec.cxx  |  4 +-
 .../readers/src/IRFrameReaderSpec.cxx         |  2 +-
 .../readers/src/MatchedMFTMCHReaderSpec.cxx   |  2 +-
 .../readers/src/PrimaryVertexReaderSpec.cxx   |  8 +--
 .../readers/src/SecondaryVertexReaderSpec.cxx | 18 +++----
 .../src/StrangenessTrackingReaderSpec.cxx     |  6 +--
 .../readers/src/TrackCosmicsReaderSpec.cxx    |  4 +-
 .../readers/src/TrackTPCITSReaderSpec.cxx     | 10 ++--
 .../src/CosmicsMatchingSpec.cxx               |  4 +-
 .../src/GlobalFwdMatchingSpec.cxx             | 12 ++---
 .../src/HMPMatcherSpec.cxx                    |  4 +-
 .../src/PrimaryVertexingSpec.cxx              |  8 +--
 .../src/StrangenessTrackingSpec.cxx           |  6 +--
 .../src/TOFMatcherSpec.cxx                    | 54 +++++++++----------
 .../src/TPCITSMatchingSpec.cxx                | 12 ++---
 .../src/VertexTrackMatcherSpec.cxx            |  4 +-
 .../study/src/TPCDataFilter.cxx               | 10 ++--
 .../src/TPCInterpolationSpec.cxx              | 12 ++---
 .../src/TPCUnbinnedResidualReaderSpec.cxx     |  6 +--
 .../HMPID/workflow/src/ClustersReaderSpec.cxx |  4 +-
 .../HMPID/workflow/src/DataDecoderSpec.cxx    |  4 +-
 .../HMPID/workflow/src/DataDecoderSpec2.cxx   |  4 +-
 .../HMPID/workflow/src/DigitsReaderSpec.cxx   |  4 +-
 .../workflow/src/DigitsToClustersSpec.cxx     |  4 +-
 .../HMPID/workflow/src/EntropyEncoderSpec.cxx |  2 +-
 .../workflow/src/HMPMatchedReaderSpec.cxx     |  4 +-
 .../src/TestDataReader.cxx                    | 28 +++++-----
 .../ITSMFT/ITS/workflow/src/ClustererSpec.cxx | 10 ++--
 .../ITS/workflow/src/CookedTrackerSpec.cxx    | 16 +++---
 .../ITS/workflow/src/DCSGeneratorSpec.cxx     |  2 +-
 .../ITS/workflow/src/TrackReaderSpec.cxx      | 14 ++---
 .../ITSMFT/ITS/workflow/src/TrackerSpec.cxx   | 18 +++----
 .../ITS/workflow/src/VertexReaderSpec.cxx     |  4 +-
 .../ITSMFT/MFT/workflow/src/ClustererSpec.cxx | 10 ++--
 .../MFT/workflow/src/TrackReaderSpec.cxx      |  8 +--
 .../ITSMFT/MFT/workflow/src/TrackerSpec.cxx   | 10 ++--
 .../common/workflow/src/ClusterReaderSpec.cxx | 12 ++---
 .../common/workflow/src/DigitReaderSpec.cxx   | 12 ++---
 .../workflow/src/EntropyEncoderSpec.cxx       |  2 +-
 .../common/workflow/src/STFDecoderSpec.cxx    | 20 +++----
 Detectors/MUON/MCH/IO/src/DigitReaderSpec.cxx | 10 ++--
 .../MCH/Workflow/src/TrackAtVertexSpec.cxx    |  2 +-
 .../Workflow/src/entropy-encoder-workflow.cxx |  2 +-
 .../MUON/MID/Workflow/src/ClusterizerSpec.cxx |  8 +--
 .../MID/Workflow/src/ColumnDataSpecsUtils.cxx |  4 +-
 .../src/DecodedDataAggregatorSpec.cxx         |  4 +-
 .../MUON/MID/Workflow/src/DigitReaderSpec.cxx | 10 ++--
 .../MID/Workflow/src/EntropyDecoderSpec.cxx   |  4 +-
 .../MID/Workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../MUON/MID/Workflow/src/MaskMakerSpec.cxx   |  4 +-
 .../MID/Workflow/src/RawGBTDecoderSpec.cxx    |  4 +-
 .../MUON/MID/Workflow/src/TrackerSpec.cxx     | 12 ++---
 .../MID/Workflow/src/ZeroSuppressionSpec.cxx  |  4 +-
 .../PHOS/calib/src/PHOSBadMapCalibDevice.cxx  |  2 +-
 .../calib/src/PHOSHGLGRatioCalibDevice.cxx    |  2 +-
 .../calib/src/PHOSPedestalCalibDevice.cxx     |  2 +-
 .../PHOS/workflow/src/CellConverterSpec.cxx   | 12 ++---
 .../PHOS/workflow/src/CellReaderSpec.cxx      |  6 +--
 .../PHOS/workflow/src/ClusterizerSpec.cxx     | 16 +++---
 .../PHOS/workflow/src/DigitReaderSpec.cxx     |  6 +--
 .../PHOS/workflow/src/EntropyEncoderSpec.cxx  |  2 +-
 .../PHOS/workflow/src/EventBuilderSpec.cxx    |  4 +-
 .../workflow/src/RawToCellConverterSpec.cxx   | 12 ++---
 Detectors/PHOS/workflow/src/ReaderSpec.cxx    | 22 ++++----
 .../src/StandaloneAODProducerSpec.cxx         |  4 +-
 .../testWorkflow/TOFCalibCollectorSpec.h      |  4 +-
 .../workflow/src/CompressedDecodingTask.cxx   | 12 ++---
 .../TOF/workflow/src/EntropyDecoderSpec.cxx   |  2 +-
 .../TOF/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../TOF/workflow/src/TOFClusterizerSpec.cxx   | 14 ++---
 .../workflowIO/src/CalibClusReaderSpec.cxx    |  8 +--
 .../workflowIO/src/CalibInfoReaderSpec.cxx    |  4 +-
 .../TOF/workflowIO/src/ClusterReaderSpec.cxx  |  6 +--
 .../TOF/workflowIO/src/DigitReaderSpec.cxx    | 12 ++---
 .../workflowIO/src/TOFMatchedReaderSpec.cxx   |  6 +--
 .../include/TPCWorkflow/TPCFLPIDCSpec.h       |  6 +--
 .../TPCWorkflow/TPCFourierTransformEPNSpec.h  |  2 +-
 .../include/TPCWorkflow/TPCIntegrateIDCSpec.h |  4 +-
 .../include/TPCReaderWorkflow/PublisherSpec.h |  7 ++-
 .../workflow/readers/src/PublisherSpec.cxx    |  4 +-
 .../workflow/readers/src/TrackReaderSpec.cxx  |  6 +--
 .../readers/src/TriggerReaderSpec.cxx         |  2 +-
 .../workflow/src/ChunkedDigitPublisher.cxx    |  5 +-
 .../workflow/src/ClusterDecoderRawSpec.cxx    |  8 +--
 Detectors/TPC/workflow/src/ClustererSpec.cxx  |  8 +--
 .../TPC/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../TPC/workflow/src/KryptonClustererSpec.cxx |  2 +-
 .../TPC/workflow/src/KryptonRawFilterSpec.cxx |  2 +-
 .../TPC/workflow/src/LinkZSToDigitsSpec.cxx   |  2 +-
 .../TPC/workflow/src/MIPTrackFilterSpec.cxx   |  2 +-
 .../TPC/workflow/src/RawToDigitsSpec.cxx      |  4 +-
 .../TPC/workflow/src/SACProcessorSpec.cxx     |  4 +-
 Detectors/TPC/workflow/src/ZSSpec.cxx         |  6 +--
 .../workflow/test/test_ft_EPN_Aggregator.cxx  |  2 +-
 .../TRD/reconstruction/src/EventRecord.cxx    | 10 ++--
 .../include/TRDWorkflow/KrClustererSpec.h     |  4 +-
 .../TRDWorkflow/TRDGlobalTrackingQCSpec.h     |  2 +-
 .../include/TRDWorkflow/TRDPulseHeightSpec.h  |  4 +-
 .../workflow/io/src/TRDCalibReaderSpec.cxx    |  2 +-
 .../workflow/io/src/TRDDigitReaderSpec.cxx    |  6 +--
 .../TRD/workflow/io/src/TRDPHReaderSpec.cxx   |  2 +-
 .../workflow/io/src/TRDTrackReaderSpec.cxx    | 16 +++---
 .../workflow/io/src/TRDTrackletReaderSpec.cxx | 10 ++--
 .../TRD/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../TRD/workflow/src/TRDDigitizerSpec.cxx     |  8 +--
 .../workflow/src/TRDGlobalTrackingSpec.cxx    | 16 +++---
 .../src/TRDTrackletTransformerSpec.cxx        |  4 +-
 .../TRD/workflow/src/TRDTrapSimulatorSpec.cxx |  8 +--
 .../TRD/workflow/src/TrackBasedCalibSpec.cxx  |  4 +-
 .../ITS3/workflow/src/ClustererSpec.cxx       | 10 ++--
 .../ITS3/workflow/src/DigitReaderSpec.cxx     | 10 ++--
 .../ITS3/workflow/src/TrackReaderSpec.cxx     | 14 ++---
 .../ITS3/workflow/src/TrackerSpec.cxx         | 18 +++----
 .../ITS3/workflow/src/VertexReaderSpec.cxx    |  4 +-
 Detectors/Vertexing/src/SVertexer.cxx         | 24 ++++-----
 Detectors/ZDC/calib/src/InterCalibEPNSpec.cxx |  6 +--
 Detectors/ZDC/calib/src/TDCCalibEPNSpec.cxx   |  4 +-
 .../ZDC/calib/src/WaveformCalibEPNSpec.cxx    |  2 +-
 .../ZDC/raw/include/ZDCRaw/RawReaderZDC.h     |  6 +--
 .../ZDC/workflow/src/DigitReaderSpec.cxx      |  8 +--
 Detectors/ZDC/workflow/src/DigitRecoSpec.cxx  | 10 ++--
 .../ZDC/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../ZDC/workflow/src/RecEventReaderSpec.cxx   |  8 +--
 Detectors/ZDC/workflow/src/RecoReaderSpec.cxx |  8 +--
 GPU/Workflow/src/GPUWorkflowITS.cxx           | 18 +++----
 GPU/Workflow/src/GPUWorkflowPipeline.cxx      |  2 +-
 GPU/Workflow/src/GPUWorkflowSpec.cxx          | 14 ++---
 .../src/CPVDigitizerSpec.cxx                  |  8 +--
 .../src/CTPDigitizerSpec.cxx                  |  4 +-
 .../src/FDDDigitizerSpec.cxx                  | 10 ++--
 .../src/FT0DigitizerSpec.cxx                  | 10 ++--
 .../src/FV0DigitizerSpec.cxx                  | 10 ++--
 .../src/HMPIDDigitizerSpec.cxx                |  8 +--
 .../src/ITS3DigitizerSpec.cxx                 | 10 ++--
 .../src/ITSMFTDigitizerSpec.cxx               | 10 ++--
 .../src/MCHDigitizerSpec.cxx                  |  8 +--
 .../DigitizerWorkflow/src/MCTruthReaderSpec.h |  4 +-
 .../src/MCTruthSourceSpec.cxx                 |  4 +-
 .../src/MCTruthWriterSpec.cxx                 |  2 +-
 .../src/MIDDigitizerSpec.cxx                  |  8 +--
 .../src/PHOSDigitizerSpec.cxx                 |  8 +--
 .../src/TOFDigitizerSpec.cxx                  | 12 ++---
 .../src/TPCDigitizerSpec.cxx                  | 12 ++---
 .../src/ZDCDigitizerSpec.cxx                  | 10 ++--
 .../DataSampling/src/DataSamplingPolicy.cxx   |  2 +-
 Utilities/DataSampling/src/Dispatcher.cxx     |  1 -
 run/dpl_eventgen.cxx                          |  4 +-
 run/o2sim_hepmc_publisher.cxx                 |  4 +-
 run/o2sim_kine_publisher.cxx                  |  4 +-
 run/o2sim_mctracks_to_aod.cxx                 |  4 +-
 195 files changed, 680 insertions(+), 688 deletions(-)

diff --git a/Detectors/AOD/src/AODMcProducerWorkflowSpec.cxx b/Detectors/AOD/src/AODMcProducerWorkflowSpec.cxx
index 2e60b70c9855a..4c1623188e55c 100644
--- a/Detectors/AOD/src/AODMcProducerWorkflowSpec.cxx
+++ b/Detectors/AOD/src/AODMcProducerWorkflowSpec.cxx
@@ -296,13 +296,13 @@ void AODMcProducerWorkflowDPL::run(ProcessingContext& pc)
     TString ROOTVersion = ROOT_RELEASE;
     mMetaDataKeys = {"DataType", "Run", "O2Version", "ROOTVersion", "RecoPassName", "AnchorProduction", "AnchorPassName", "LPMProductionTag"};
     mMetaDataVals = {dataType, "3", O2Version, ROOTVersion, mRecoPass, mAnchorProd, mAnchorPass, mLPMProdTag};
-    pc.outputs().snapshot(Output{"AMD", "AODMetadataKeys", 0, Lifetime::Timeframe}, mMetaDataKeys);
-    pc.outputs().snapshot(Output{"AMD", "AODMetadataVals", 0, Lifetime::Timeframe}, mMetaDataVals);
+    pc.outputs().snapshot(Output{"AMD", "AODMetadataKeys", 0}, mMetaDataKeys);
+    pc.outputs().snapshot(Output{"AMD", "AODMetadataVals", 0}, mMetaDataVals);
     mIsMDSent = true;
   }
 
-  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0, Lifetime::Timeframe}, tfNumber);
-  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0, Lifetime::Timeframe}, "");
+  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0}, tfNumber);
+  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0}, "");
 
   pc.services().get<ControlService>().endOfStream();
   pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Detectors/AOD/src/AODProducerWorkflowSpec.cxx b/Detectors/AOD/src/AODProducerWorkflowSpec.cxx
index 323d4b9f7f41a..e97c90f14f64f 100644
--- a/Detectors/AOD/src/AODProducerWorkflowSpec.cxx
+++ b/Detectors/AOD/src/AODProducerWorkflowSpec.cxx
@@ -2241,11 +2241,11 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc)
   TString ROOTVersion = ROOT_RELEASE;
   mMetaDataKeys = {"DataType", "Run", "O2Version", "ROOTVersion", "RecoPassName", "AnchorProduction", "AnchorPassName", "LPMProductionTag"};
   mMetaDataVals = {dataType, "3", O2Version, ROOTVersion, mRecoPass, mAnchorProd, mAnchorPass, mLPMProdTag};
-  pc.outputs().snapshot(Output{"AMD", "AODMetadataKeys", 0, Lifetime::Timeframe}, mMetaDataKeys);
-  pc.outputs().snapshot(Output{"AMD", "AODMetadataVals", 0, Lifetime::Timeframe}, mMetaDataVals);
+  pc.outputs().snapshot(Output{"AMD", "AODMetadataKeys", 0}, mMetaDataKeys);
+  pc.outputs().snapshot(Output{"AMD", "AODMetadataVals", 0}, mMetaDataVals);
 
-  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0, Lifetime::Timeframe}, tfNumber);
-  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0, Lifetime::Timeframe}, "");
+  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0}, tfNumber);
+  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0}, "");
 
   mTimer.Stop();
 }
diff --git a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVBadMapCalibDevice.cxx b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVBadMapCalibDevice.cxx
index b1c61d25da9c8..f2a08e280f954 100644
--- a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVBadMapCalibDevice.cxx
+++ b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVBadMapCalibDevice.cxx
@@ -174,7 +174,7 @@ void CPVBadMapCalibDevice::sendOutput(DataAllocator& output)
     output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, "CPV_BadChanMap", subSpec}, info);
   }
 
-  output.snapshot(o2::framework::Output{"CPV", "BADMAPCHANGE", 0, o2::framework::Lifetime::Timeframe}, mMapDiff);
+  output.snapshot(o2::framework::Output{"CPV", "BADMAPCHANGE", 0}, mMapDiff);
 }
 
 bool CPVBadMapCalibDevice::differFromCurrent()
diff --git a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVGainCalibDevice.cxx b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVGainCalibDevice.cxx
index b5e98f00d021f..d7f187e3a88cc 100644
--- a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVGainCalibDevice.cxx
+++ b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVGainCalibDevice.cxx
@@ -143,7 +143,7 @@ void CPVGainCalibDevice::sendOutput(DataAllocator& output)
     fout.Close();
   }
   // Anyway send change to QC
-  output.snapshot(o2::framework::Output{"CPV", "GAINDIFF", 0, o2::framework::Lifetime::Timeframe}, mGainRatio);
+  output.snapshot(o2::framework::Output{"CPV", "GAINDIFF", 0}, mGainRatio);
 }
 
 void CPVGainCalibDevice::calculateGains()
diff --git a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVPedestalCalibDevice.cxx b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVPedestalCalibDevice.cxx
index 125e8e1bc8c40..5dd414ab7aeb8 100644
--- a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVPedestalCalibDevice.cxx
+++ b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVPedestalCalibDevice.cxx
@@ -118,7 +118,7 @@ void CPVPedestalCalibDevice::sendOutput(DataAllocator& output)
   }
   // Anyway send change to QC
   LOG(info) << "[CPVPedestalCalibDevice - run] Writing ";
-  output.snapshot(o2::framework::Output{"CPV", "PEDDIFF", 0, o2::framework::Lifetime::Timeframe}, mPedDiff);
+  output.snapshot(o2::framework::Output{"CPV", "PEDDIFF", 0}, mPedDiff);
 
   // Write pedestal distributions to calculate bad map
   std::string filename = mPath + "CPVPedestals.root";
diff --git a/Detectors/CPV/workflow/src/ClusterReaderSpec.cxx b/Detectors/CPV/workflow/src/ClusterReaderSpec.cxx
index f9d82eb23bf68..f9d0817325c36 100644
--- a/Detectors/CPV/workflow/src/ClusterReaderSpec.cxx
+++ b/Detectors/CPV/workflow/src/ClusterReaderSpec.cxx
@@ -44,10 +44,10 @@ void ClusterReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mClusters.size() << " Clusters in " << mTRs.size() << " TriggerRecords at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "CLUSTERS", 0, Lifetime::Timeframe}, mClusters);
-  pc.outputs().snapshot(Output{mOrigin, "CLUSTERTRIGRECS", 0, Lifetime::Timeframe}, mTRs);
+  pc.outputs().snapshot(Output{mOrigin, "CLUSTERS", 0}, mClusters);
+  pc.outputs().snapshot(Output{mOrigin, "CLUSTERTRIGRECS", 0}, mTRs);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{mOrigin, "CLUSTERTRUEMC", 0, Lifetime::Timeframe}, mMCTruth);
+    pc.outputs().snapshot(Output{mOrigin, "CLUSTERTRUEMC", 0}, mMCTruth);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/CPV/workflow/src/ClusterizerSpec.cxx b/Detectors/CPV/workflow/src/ClusterizerSpec.cxx
index 87d0b86a98f5c..e004c3cec8949 100644
--- a/Detectors/CPV/workflow/src/ClusterizerSpec.cxx
+++ b/Detectors/CPV/workflow/src/ClusterizerSpec.cxx
@@ -49,14 +49,14 @@ void ClusterizerSpec::run(framework::ProcessingContext& ctx)
   if (!digits.size()) { // nothing to process
     LOG(info) << "ClusterizerSpec::run() : no digits; moving on";
     mOutputClusters.clear();
-    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERS", 0, o2::framework::Lifetime::Timeframe}, mOutputClusters);
+    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERS", 0}, mOutputClusters);
     mOutputClusterTrigRecs.clear();
-    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRIGRECS", 0, o2::framework::Lifetime::Timeframe}, mOutputClusterTrigRecs);
+    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRIGRECS", 0}, mOutputClusterTrigRecs);
     mCalibDigits.clear();
-    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CALIBDIGITS", 0, o2::framework::Lifetime::Timeframe}, mCalibDigits);
+    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CALIBDIGITS", 0}, mCalibDigits);
     if (mPropagateMC) {
       mOutputTruthCont.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRUEMC", 0, o2::framework::Lifetime::Timeframe}, mOutputTruthCont);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRUEMC", 0}, mOutputTruthCont);
     }
     return;
   }
@@ -73,12 +73,12 @@ void ClusterizerSpec::run(framework::ProcessingContext& ctx)
 
   LOG(debug) << "CPVClusterizer::run() : Received " << digitsTR.size() << " TR, calling clusterizer ...";
 
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERS", 0, o2::framework::Lifetime::Timeframe}, mOutputClusters);
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRIGRECS", 0, o2::framework::Lifetime::Timeframe}, mOutputClusterTrigRecs);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERS", 0}, mOutputClusters);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRIGRECS", 0}, mOutputClusterTrigRecs);
   if (mPropagateMC) {
-    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRUEMC", 0, o2::framework::Lifetime::Timeframe}, mOutputTruthCont);
+    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRUEMC", 0}, mOutputTruthCont);
   }
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CALIBDIGITS", 0, o2::framework::Lifetime::Timeframe}, mCalibDigits);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CALIBDIGITS", 0}, mCalibDigits);
   LOG(info) << "Finished, wrote  " << mOutputClusters.size() << " clusters, " << mOutputClusterTrigRecs.size() << "TR and " << mOutputTruthCont.getIndexedSize() << " Labels";
 }
 o2::framework::DataProcessorSpec o2::cpv::reco_workflow::getClusterizerSpec(bool propagateMC)
diff --git a/Detectors/CPV/workflow/src/DigitReaderSpec.cxx b/Detectors/CPV/workflow/src/DigitReaderSpec.cxx
index ba74cb88b10d2..20fe497eb5d0c 100644
--- a/Detectors/CPV/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/CPV/workflow/src/DigitReaderSpec.cxx
@@ -44,10 +44,10 @@ void DigitReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mDigits.size() << " Digits in " << mTRs.size() << " TriggerRecords at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "DIGITS", 0, Lifetime::Timeframe}, mDigits);
-  pc.outputs().snapshot(Output{mOrigin, "DIGITTRIGREC", 0, Lifetime::Timeframe}, mTRs);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITS", 0}, mDigits);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITTRIGREC", 0}, mTRs);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{mOrigin, "DIGITSMCTR", 0, Lifetime::Timeframe}, mMCTruth);
+    pc.outputs().snapshot(Output{mOrigin, "DIGITSMCTR", 0}, mMCTruth);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/CPV/workflow/src/EntropyEncoderSpec.cxx b/Detectors/CPV/workflow/src/EntropyEncoderSpec.cxx
index 97f69fe928342..31ed720e66335 100644
--- a/Detectors/CPV/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/CPV/workflow/src/EntropyEncoderSpec.cxx
@@ -54,7 +54,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"CPV", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"CPV", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, triggers, clusters);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   if (mSelIR) {
diff --git a/Detectors/CPV/workflow/src/RawToDigitConverterSpec.cxx b/Detectors/CPV/workflow/src/RawToDigitConverterSpec.cxx
index fcdc75c38c67f..766902a2fdc95 100644
--- a/Detectors/CPV/workflow/src/RawToDigitConverterSpec.cxx
+++ b/Detectors/CPV/workflow/src/RawToDigitConverterSpec.cxx
@@ -136,11 +136,11 @@ void RawToDigitConverterSpec::run(framework::ProcessingContext& ctx)
              contDeadBeef == maxWarn ? fmt::format(". {} such inputs in row received, stopping reporting", contDeadBeef) : "");
       }
       mOutputDigits.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, mOutputDigits);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0}, mOutputDigits);
       mOutputTriggerRecords.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0, o2::framework::Lifetime::Timeframe}, mOutputTriggerRecords);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0}, mOutputTriggerRecords);
       mOutputHWErrors.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0, o2::framework::Lifetime::Timeframe}, mOutputHWErrors);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0}, mOutputHWErrors);
       return; // empty TF, nothing to process
     }
   }
@@ -272,11 +272,11 @@ void RawToDigitConverterSpec::run(framework::ProcessingContext& ctx)
     if (skipTF) {
       // Send no digits
       mOutputDigits.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, mOutputDigits);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0}, mOutputDigits);
       mOutputTriggerRecords.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0, o2::framework::Lifetime::Timeframe}, mOutputTriggerRecords);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0}, mOutputTriggerRecords);
       // Send errors
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0, o2::framework::Lifetime::Timeframe}, mOutputHWErrors);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0}, mOutputHWErrors);
       return;
     }
   }
@@ -305,9 +305,9 @@ void RawToDigitConverterSpec::run(framework::ProcessingContext& ctx)
   digitBuffer.clear();
 
   LOG(info) << "[CPVRawToDigitConverter - run] Sending " << mOutputDigits.size() << " digits in " << mOutputTriggerRecords.size() << "trigger records.";
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, mOutputDigits);
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0, o2::framework::Lifetime::Timeframe}, mOutputTriggerRecords);
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0, o2::framework::Lifetime::Timeframe}, mOutputHWErrors);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0}, mOutputDigits);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0}, mOutputTriggerRecords);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0}, mOutputHWErrors);
 }
 //_____________________________________________________________________________
 o2::framework::DataProcessorSpec o2::cpv::reco_workflow::getRawToDigitConverterSpec(bool askDISTSTF, bool isPedestal, bool useBadChannelMap, bool useGainCalibration)
diff --git a/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx b/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx
index 7ad4f821c12a9..ca16d10087fa6 100644
--- a/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx
@@ -64,7 +64,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
       lumi = lumiPrev;
     }
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"CTP", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"CTP", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, digits, lumi);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   mTimer.Stop();
diff --git a/Detectors/CTP/workflow/src/RawDecoderSpec.cxx b/Detectors/CTP/workflow/src/RawDecoderSpec.cxx
index 674c2f4208d8f..5f0cb3b759c48 100644
--- a/Detectors/CTP/workflow/src/RawDecoderSpec.cxx
+++ b/Detectors/CTP/workflow/src/RawDecoderSpec.cxx
@@ -80,10 +80,10 @@ void RawDecoderSpec::run(framework::ProcessingContext& ctx)
   auto& inputs = ctx.inputs();
   auto dummyOutput = [&ctx, this]() {
     if (this->mDoDigits) {
-      ctx.outputs().snapshot(o2::framework::Output{"CTP", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, this->mOutputDigits);
+      ctx.outputs().snapshot(o2::framework::Output{"CTP", "DIGITS", 0}, this->mOutputDigits);
     }
     if (this->mDoLumi) {
-      ctx.outputs().snapshot(o2::framework::Output{"CTP", "LUMI", 0, o2::framework::Lifetime::Timeframe}, this->mOutputLumiInfo);
+      ctx.outputs().snapshot(o2::framework::Output{"CTP", "LUMI", 0}, this->mOutputLumiInfo);
     }
   };
   // if we see requested data type input with 0xDEADBEEF subspec and 0 payload this means that the "delayed message"
@@ -117,7 +117,7 @@ void RawDecoderSpec::run(framework::ProcessingContext& ctx)
   }
   if (mDoDigits) {
     LOG(info) << "[CTPRawToDigitConverter - run] Writing " << mOutputDigits.size() << " digits. IR rejected:" << mDecoder.getIRRejected() << " TCR rejected:" << mDecoder.getTCRRejected();
-    ctx.outputs().snapshot(o2::framework::Output{"CTP", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, mOutputDigits);
+    ctx.outputs().snapshot(o2::framework::Output{"CTP", "DIGITS", 0}, mOutputDigits);
   }
   if (mDoLumi) {
     uint32_t tfCountsT = 0;
@@ -157,7 +157,7 @@ void RawDecoderSpec::run(framework::ProcessingContext& ctx)
       mOutputLumiInfo.printInputs();
       LOGP(info, "Orbit {}: {}/{} counts inp1/inp2 in {}/{} HBFs -> lumi_inp1 = {:.3e}+-{:.3e} lumi_inp2 = {:.3e}+-{:.3e}", mOutputLumiInfo.orbit, mCountsT, mCountsV, mNHBIntegratedT, mNHBIntegratedV, mOutputLumiInfo.getLumi(), mOutputLumiInfo.getLumiError(), mOutputLumiInfo.getLumiFV0(), mOutputLumiInfo.getLumiFV0Error());
     }
-    ctx.outputs().snapshot(o2::framework::Output{"CTP", "LUMI", 0, o2::framework::Lifetime::Timeframe}, mOutputLumiInfo);
+    ctx.outputs().snapshot(o2::framework::Output{"CTP", "LUMI", 0}, mOutputLumiInfo);
   }
 }
 o2::framework::DataProcessorSpec o2::ctp::reco_workflow::getRawDecoderSpec(bool askDISTSTF, bool digits, bool lumi)
diff --git a/Detectors/CTP/workflowIO/src/DigitReaderSpec.cxx b/Detectors/CTP/workflowIO/src/DigitReaderSpec.cxx
index 59f608bef2efb..ecf1c2e19b660 100644
--- a/Detectors/CTP/workflowIO/src/DigitReaderSpec.cxx
+++ b/Detectors/CTP/workflowIO/src/DigitReaderSpec.cxx
@@ -76,8 +76,8 @@ void DigitReader::run(ProcessingContext& pc)
 
   mTree->GetEntry(ent);
   LOG(info) << "DigitReader pushes " << mDigits.size() << " digits at entry " << ent;
-  pc.outputs().snapshot(Output{"CTP", "DIGITS", 0, Lifetime::Timeframe}, mDigits);
-  pc.outputs().snapshot(Output{"CTP", "LUMI", 0, Lifetime::Timeframe}, mLumi);
+  pc.outputs().snapshot(Output{"CTP", "DIGITS", 0}, mDigits);
+  pc.outputs().snapshot(Output{"CTP", "LUMI", 0}, mLumi);
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Detectors/DCS/testWorkflow/src/DCSDataReplaySpec.cxx b/Detectors/DCS/testWorkflow/src/DCSDataReplaySpec.cxx
index 05daf522f372a..783f6ae76e707 100644
--- a/Detectors/DCS/testWorkflow/src/DCSDataReplaySpec.cxx
+++ b/Detectors/DCS/testWorkflow/src/DCSDataReplaySpec.cxx
@@ -88,7 +88,7 @@ void DCSDataReplayer::run(o2::framework::ProcessingContext& pc)
 
   LOG(info)
     << "***************** TF " << tfid << " has generated " << dpcoms.size() << " DPs";
-  pc.outputs().snapshot(Output{"DCS", mDataDescription, 0, Lifetime::Timeframe}, dpcoms);
+  pc.outputs().snapshot(Output{"DCS", mDataDescription, 0}, dpcoms);
   mTFs++;
 }
 } // namespace
diff --git a/Detectors/DCS/testWorkflow/src/DCSRandomDataGeneratorSpec.cxx b/Detectors/DCS/testWorkflow/src/DCSRandomDataGeneratorSpec.cxx
index a82a5b4164e2a..899a4e110aea8 100644
--- a/Detectors/DCS/testWorkflow/src/DCSRandomDataGeneratorSpec.cxx
+++ b/Detectors/DCS/testWorkflow/src/DCSRandomDataGeneratorSpec.cxx
@@ -153,7 +153,7 @@ void DCSRandomDataGenerator::run(o2::framework::ProcessingContext& pc)
   auto timeNow = std::chrono::system_clock::now();
   timingInfo.creation = std::chrono::duration_cast<std::chrono::milliseconds>(timeNow.time_since_epoch()).count(); // in ms
 
-  pc.outputs().snapshot(Output{"DCS", mDataDescription, 0, Lifetime::Timeframe}, dpcoms);
+  pc.outputs().snapshot(Output{"DCS", mDataDescription, 0}, dpcoms);
   mTFs++;
 }
 } // namespace
diff --git a/Detectors/EMCAL/workflow/include/EMCALWorkflow/PublisherSpec.h b/Detectors/EMCAL/workflow/include/EMCALWorkflow/PublisherSpec.h
index f583653f1c3cb..468e66c43a0ab 100644
--- a/Detectors/EMCAL/workflow/include/EMCALWorkflow/PublisherSpec.h
+++ b/Detectors/EMCAL/workflow/include/EMCALWorkflow/PublisherSpec.h
@@ -60,23 +60,22 @@ framework::DataProcessorSpec getPublisherSpec(PublisherConf const& config, uint3
 
   // a creator callback for the actual reader instance
   auto creator = [dto, tro, mco, subspec, propagateMC](const char* treename, const char* filename, int nofEvents, Reader::PublishingMode publishingMode, const char* branchname, const char* triggerbranchname, const char* mcbranchname) {
-    constexpr auto persistency = o2::framework::Lifetime::Timeframe;
     if (propagateMC) {
       return std::make_shared<Reader>(treename,
                                       filename,
                                       nofEvents,
                                       publishingMode,
-                                      Output{mco.origin, mco.description, subspec, persistency},
+                                      Output{mco.origin, mco.description, subspec},
                                       mcbranchname,
-                                      Reader::BranchDefinition<T>{Output{dto.origin, dto.description, subspec, persistency}, branchname},
-                                      Reader::BranchDefinition<TriggerInputType>{Output{tro.origin, tro.description, subspec, persistency}, triggerbranchname});
+                                      Reader::BranchDefinition<T>{Output{dto.origin, dto.description, subspec}, branchname},
+                                      Reader::BranchDefinition<TriggerInputType>{Output{tro.origin, tro.description, subspec}, triggerbranchname});
     } else {
       return std::make_shared<Reader>(treename,
                                       filename,
                                       nofEvents,
                                       publishingMode,
-                                      Reader::BranchDefinition<T>{Output{dto.origin, dto.description, subspec, persistency}, branchname},
-                                      Reader::BranchDefinition<TriggerInputType>{Output{tro.origin, tro.description, subspec, persistency}, triggerbranchname});
+                                      Reader::BranchDefinition<T>{Output{dto.origin, dto.description, subspec}, branchname},
+                                      Reader::BranchDefinition<TriggerInputType>{Output{tro.origin, tro.description, subspec}, triggerbranchname});
     }
   };
 
diff --git a/Detectors/EMCAL/workflow/src/AnalysisClusterSpec.cxx b/Detectors/EMCAL/workflow/src/AnalysisClusterSpec.cxx
index 2c98a100ac17a..7b81fec681c14 100644
--- a/Detectors/EMCAL/workflow/src/AnalysisClusterSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/AnalysisClusterSpec.cxx
@@ -148,7 +148,7 @@ void AnalysisClusterSpec<InputType>::run(framework::ProcessingContext& ctx)
   }
 
   LOG(debug) << "[EMCALClusterizer - run] Writing " << mOutputAnaClusters->size() << " clusters ...";
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "ANALYSISCLUSTERS", 0, o2::framework::Lifetime::Timeframe}, *mOutputAnaClusters);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "ANALYSISCLUSTERS", 0}, *mOutputAnaClusters);
 }
 
 o2::framework::DataProcessorSpec o2::emcal::reco_workflow::getAnalysisClusterSpec(bool useDigits)
diff --git a/Detectors/EMCAL/workflow/src/CellConverterSpec.cxx b/Detectors/EMCAL/workflow/src/CellConverterSpec.cxx
index 489546a86eeaf..5fa7353e907e2 100644
--- a/Detectors/EMCAL/workflow/src/CellConverterSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/CellConverterSpec.cxx
@@ -161,10 +161,10 @@ void CellConverterSpec::run(framework::ProcessingContext& ctx)
     ncellsTrigger = 0;
   }
   LOG(debug) << "[EMCALCellConverter - run] Writing " << mOutputCells.size() << " cells ...";
-  ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLS", mSubspecificationOut, o2::framework::Lifetime::Timeframe}, mOutputCells);
-  ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLSTRGR", mSubspecificationOut, o2::framework::Lifetime::Timeframe}, mOutputTriggers);
+  ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLS", mSubspecificationOut}, mOutputCells);
+  ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLSTRGR", mSubspecificationOut}, mOutputTriggers);
   if (mPropagateMC) {
-    ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLSMCTR", mSubspecificationOut, o2::framework::Lifetime::Timeframe}, mOutputLabels);
+    ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLSMCTR", mSubspecificationOut}, mOutputLabels);
   }
 }
 
diff --git a/Detectors/EMCAL/workflow/src/CellRecalibratorSpec.cxx b/Detectors/EMCAL/workflow/src/CellRecalibratorSpec.cxx
index 493c205223161..3f19b49cd73bd 100644
--- a/Detectors/EMCAL/workflow/src/CellRecalibratorSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/CellRecalibratorSpec.cxx
@@ -113,15 +113,15 @@ void CellRecalibratorSpec::run(framework::ProcessingContext& ctx)
   }
 
   // send recalibrated objects
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLS", mOutputSubspec, o2::framework::Lifetime::Timeframe}, outputcells);
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLSTRGR", mOutputSubspec, o2::framework::Lifetime::Timeframe}, outputtriggers);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLS", mOutputSubspec}, outputcells);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLSTRGR", mOutputSubspec}, outputtriggers);
   if (outputMCLabels.has_value()) {
     LOG(info) << "Timeframe: " << inputMCLabels->getIndexedSize() << " label entries read, " << outputMCLabels->getIndexedSize() << " label entries kept";
-    ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLSMCTR", mOutputSubspec, o2::framework::Lifetime::Timeframe}, outputMCLabels.value());
+    ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLSMCTR", mOutputSubspec}, outputMCLabels.value());
   }
   if (mLEDsettings == LEDEventSettings::REDIRECT) {
-    ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLS", 10, o2::framework::Lifetime::Timeframe}, ledcells);
-    ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLSTRGR", 10, o2::framework::Lifetime::Timeframe}, ledtriggers);
+    ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLS", 10}, ledcells);
+    ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLSTRGR", 10}, ledtriggers);
   }
 }
 
diff --git a/Detectors/EMCAL/workflow/src/ClusterizerSpec.cxx b/Detectors/EMCAL/workflow/src/ClusterizerSpec.cxx
index eaa00d6d3d0b6..f938d02ce7e3f 100644
--- a/Detectors/EMCAL/workflow/src/ClusterizerSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/ClusterizerSpec.cxx
@@ -107,11 +107,11 @@ void ClusterizerSpec<InputType>::run(framework::ProcessingContext& ctx)
     currentStartIndices = mOutputCellDigitIndices->size();
   }
   LOG(debug) << "[EMCALClusterizer - run] Writing " << mOutputClusters->size() << " clusters ...";
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CLUSTERS", 0, o2::framework::Lifetime::Timeframe}, *mOutputClusters);
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "INDICES", 0, o2::framework::Lifetime::Timeframe}, *mOutputCellDigitIndices);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CLUSTERS", 0}, *mOutputClusters);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "INDICES", 0}, *mOutputCellDigitIndices);
 
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CLUSTERSTRGR", 0, o2::framework::Lifetime::Timeframe}, *mOutputTriggerRecord);
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "INDICESTRGR", 0, o2::framework::Lifetime::Timeframe}, *mOutputTriggerRecordIndices);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CLUSTERSTRGR", 0}, *mOutputTriggerRecord);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "INDICESTRGR", 0}, *mOutputTriggerRecordIndices);
   mTimer.Stop();
 }
 
diff --git a/Detectors/EMCAL/workflow/src/EMCALDigitizerSpec.cxx b/Detectors/EMCAL/workflow/src/EMCALDigitizerSpec.cxx
index 052e65af00bda..becf3c6cdcfc3 100644
--- a/Detectors/EMCAL/workflow/src/EMCALDigitizerSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/EMCALDigitizerSpec.cxx
@@ -140,15 +140,15 @@ void DigitizerSpec::run(framework::ProcessingContext& ctx)
   mDigitizer.finish();
 
   // here we have all digits and we can send them to consumer (aka snapshot it onto output)
-  ctx.outputs().snapshot(Output{"EMC", "DIGITS", 0, Lifetime::Timeframe}, mDigitizer.getDigits());
-  ctx.outputs().snapshot(Output{"EMC", "TRGRDIG", 0, Lifetime::Timeframe}, mDigitizer.getTriggerRecords());
+  ctx.outputs().snapshot(Output{"EMC", "DIGITS", 0}, mDigitizer.getDigits());
+  ctx.outputs().snapshot(Output{"EMC", "TRGRDIG", 0}, mDigitizer.getTriggerRecords());
   if (ctx.outputs().isAllowed({"EMC", "DIGITSMCTR", 0})) {
-    ctx.outputs().snapshot(Output{"EMC", "DIGITSMCTR", 0, Lifetime::Timeframe}, mDigitizer.getMCLabels());
+    ctx.outputs().snapshot(Output{"EMC", "DIGITSMCTR", 0}, mDigitizer.getMCLabels());
   }
   // EMCAL is always a triggering detector
   const o2::parameters::GRPObject::ROMode roMode = o2::parameters::GRPObject::TRIGGERING;
   LOG(info) << "EMCAL: Sending ROMode= " << roMode << " to GRPUpdater";
-  ctx.outputs().snapshot(Output{"EMC", "ROMode", 0, Lifetime::Timeframe}, roMode);
+  ctx.outputs().snapshot(Output{"EMC", "ROMode", 0}, roMode);
   // Create CTP digits
   std::vector<o2::ctp::CTPInputDigit> triggerinputs;
   for (auto& trg : mDigitizer.getTriggerRecords()) {
@@ -161,7 +161,7 @@ void DigitizerSpec::run(framework::ProcessingContext& ctx)
     nextdigit.inputsMask.set(0);
     triggerinputs.push_back(nextdigit);
   }
-  ctx.outputs().snapshot(Output{"EMC", "TRIGGERINPUT", 0, Lifetime::Timeframe}, triggerinputs);
+  ctx.outputs().snapshot(Output{"EMC", "TRIGGERINPUT", 0}, triggerinputs);
 
   timer.Stop();
   LOG(info) << "Digitization took " << timer.CpuTime() << "s";
diff --git a/Detectors/EMCAL/workflow/src/EntropyEncoderSpec.cxx b/Detectors/EMCAL/workflow/src/EntropyEncoderSpec.cxx
index da34fcfd8c534..773c4c65fc9fe 100644
--- a/Detectors/EMCAL/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/EntropyEncoderSpec.cxx
@@ -52,7 +52,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   auto triggers = pc.inputs().get<gsl::span<TriggerRecord>>("triggers");
   auto cells = pc.inputs().get<gsl::span<Cell>>("cells");
 
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"EMC", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"EMC", "CTFDATA", 0});
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
diff --git a/Detectors/EMCAL/workflow/src/RawToCellConverterSpec.cxx b/Detectors/EMCAL/workflow/src/RawToCellConverterSpec.cxx
index 178f4c4418b40..ac09b842d9ab8 100644
--- a/Detectors/EMCAL/workflow/src/RawToCellConverterSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/RawToCellConverterSpec.cxx
@@ -722,11 +722,11 @@ void RawToCellConverterSpec::handleMinorPageError(const RawReaderMemory::MinorEr
 void RawToCellConverterSpec::sendData(framework::ProcessingContext& ctx, const std::vector<o2::emcal::Cell>& cells, const std::vector<o2::emcal::TriggerRecord>& triggers, const std::vector<ErrorTypeFEE>& decodingErrors) const
 {
   constexpr auto originEMC = o2::header::gDataOriginEMC;
-  ctx.outputs().snapshot(framework::Output{originEMC, "CELLS", mSubspecification, framework::Lifetime::Timeframe}, cells);
-  ctx.outputs().snapshot(framework::Output{originEMC, "CELLSTRGR", mSubspecification, framework::Lifetime::Timeframe}, triggers);
+  ctx.outputs().snapshot(framework::Output{originEMC, "CELLS", mSubspecification}, cells);
+  ctx.outputs().snapshot(framework::Output{originEMC, "CELLSTRGR", mSubspecification}, triggers);
   if (mCreateRawDataErrors) {
     LOG(debug) << "Sending " << decodingErrors.size() << " decoding errors";
-    ctx.outputs().snapshot(framework::Output{originEMC, "DECODERERR", mSubspecification, framework::Lifetime::Timeframe}, decodingErrors);
+    ctx.outputs().snapshot(framework::Output{originEMC, "DECODERERR", mSubspecification}, decodingErrors);
   }
 }
 
diff --git a/Detectors/EMCAL/workflow/src/StandaloneAODProducerSpec.cxx b/Detectors/EMCAL/workflow/src/StandaloneAODProducerSpec.cxx
index 152e9dd54119d..73987ce6d1c1b 100644
--- a/Detectors/EMCAL/workflow/src/StandaloneAODProducerSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/StandaloneAODProducerSpec.cxx
@@ -136,8 +136,8 @@ void StandaloneAODProducerSpec::run(ProcessingContext& pc)
   } // end of event loop
   // std::cout << "Finished cell loop" << std::endl;
 
-  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0, Lifetime::Timeframe}, tfNumber);
-  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0, Lifetime::Timeframe}, "");
+  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0}, tfNumber);
+  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0}, "");
 
   mTimer.Stop();
 }
diff --git a/Detectors/EMCAL/workflow/src/emc-channel-data-producer.cxx b/Detectors/EMCAL/workflow/src/emc-channel-data-producer.cxx
index f878a933b2d8d..75e2cdbd9fce3 100644
--- a/Detectors/EMCAL/workflow/src/emc-channel-data-producer.cxx
+++ b/Detectors/EMCAL/workflow/src/emc-channel-data-producer.cxx
@@ -160,7 +160,7 @@ DataProcessorSpec generateData(const std::string nameRootFile, const std::string
         o2::pmr::vector<o2::emcal::TriggerRecord> TriggerOutput;
         TriggerOutput.emplace_back(0, 0, 0, CellOutput.size());
 
-        ctx.outputs().adoptContainer(Output{o2::header::gDataOriginEMC, "CELLS", 0, Lifetime::Timeframe}, std::move(CellOutput));
-        ctx.outputs().adoptContainer(Output{o2::header::gDataOriginEMC, "CELLSTRGR", 0, Lifetime::Timeframe}, std::move(TriggerOutput));
+        ctx.outputs().adoptContainer(Output{o2::header::gDataOriginEMC, "CELLS", 0}, std::move(CellOutput));
+        ctx.outputs().adoptContainer(Output{o2::header::gDataOriginEMC, "CELLSTRGR", 0}, std::move(TriggerOutput));
       }}};
-}
\ No newline at end of file
+}
diff --git a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawReaderFDD.h b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawReaderFDD.h
index e9612ae8c8bda..2d92c907c1d79 100644
--- a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawReaderFDD.h
+++ b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawReaderFDD.h
@@ -65,8 +65,8 @@ class RawReaderFDD : public RawReaderFDDBaseNorm
   }
   void makeSnapshot(o2::framework::ProcessingContext& pc)
   {
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFDD, "DIGITSBC", 0, o2::framework::Lifetime::Timeframe}, mVecDigits);
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFDD, "DIGITSCH", 0, o2::framework::Lifetime::Timeframe}, mVecChannelData);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFDD, "DIGITSBC", 0}, mVecDigits);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFDD, "DIGITSCH", 0}, mVecChannelData);
   }
   bool mDumpData;
   std::vector<Digit> mVecDigits;
diff --git a/Detectors/FIT/FDD/workflow/src/DigitReaderSpec.cxx b/Detectors/FIT/FDD/workflow/src/DigitReaderSpec.cxx
index 441c679a327e3..628a2160c6d0c 100644
--- a/Detectors/FIT/FDD/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/FIT/FDD/workflow/src/DigitReaderSpec.cxx
@@ -81,18 +81,18 @@ void DigitReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
 
   LOG(info) << "FDD DigitReader pushes " << digitsBC->size() << " digits";
-  pc.outputs().snapshot(Output{mOrigin, "DIGITSBC", 0, Lifetime::Timeframe}, *digitsBC);
-  pc.outputs().snapshot(Output{mOrigin, "DIGITSCH", 0, Lifetime::Timeframe}, *digitsCh);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITSBC", 0}, *digitsBC);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITSCH", 0}, *digitsCh);
 
   if (mUseMC) {
     // TODO: To be replaced with sending ConstMCTruthContainer as soon as reco workflow supports it
-    pc.outputs().snapshot(Output{mOrigin, "TRIGGERINPUT", 0, Lifetime::Timeframe}, *digitsTrig);
+    pc.outputs().snapshot(Output{mOrigin, "TRIGGERINPUT", 0}, *digitsTrig);
 
     std::vector<char> flatbuffer;
     mcTruthRootBuffer->copyandflatten(flatbuffer);
     o2::dataformats::MCTruthContainer<o2::fdd::MCLabel> mcTruth;
     mcTruth.restore_from(flatbuffer.data(), flatbuffer.size());
-    pc.outputs().snapshot(Output{mOrigin, "DIGITLBL", 0, Lifetime::Timeframe}, mcTruth);
+    pc.outputs().snapshot(Output{mOrigin, "DIGITLBL", 0}, mcTruth);
   }
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/FIT/FDD/workflow/src/EntropyEncoderSpec.cxx b/Detectors/FIT/FDD/workflow/src/EntropyEncoderSpec.cxx
index b79c033771743..abb2518e5ae0b 100644
--- a/Detectors/FIT/FDD/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/FIT/FDD/workflow/src/EntropyEncoderSpec.cxx
@@ -54,7 +54,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"FDD", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"FDD", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, digits, channels);
   if (mSelIR) {
     mCTFCoder.getIRFramesSelector().clear();
diff --git a/Detectors/FIT/FDD/workflow/src/RecPointReaderSpec.cxx b/Detectors/FIT/FDD/workflow/src/RecPointReaderSpec.cxx
index 3da258a672611..9b612c31d28e6 100644
--- a/Detectors/FIT/FDD/workflow/src/RecPointReaderSpec.cxx
+++ b/Detectors/FIT/FDD/workflow/src/RecPointReaderSpec.cxx
@@ -51,8 +51,8 @@ void RecPointReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
 
   LOG(info) << "FDD RecPointReader pushes " << mRecPoints->size() << " recpoints with " << mChannelData->size() << " channels at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, *mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, *mChannelData);
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, *mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, *mChannelData);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/FIT/FDD/workflow/src/ReconstructorSpec.cxx b/Detectors/FIT/FDD/workflow/src/ReconstructorSpec.cxx
index 8d76bc4ee45b5..b7a0b9876a2ee 100644
--- a/Detectors/FIT/FDD/workflow/src/ReconstructorSpec.cxx
+++ b/Detectors/FIT/FDD/workflow/src/ReconstructorSpec.cxx
@@ -54,8 +54,8 @@ void FDDReconstructorDPL::run(ProcessingContext& pc)
   }
   // do we ignore MC in this task?
   LOG(debug) << "FDD reconstruction pushes " << mRecPoints.size() << " RecPoints";
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, mRecChData);
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, mRecChData);
 }
 
 DataProcessorSpec getFDDReconstructorSpec(bool useMC)
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/FT0CalibSlewingCollectorSpec.h b/Detectors/FIT/FT0/calibration/testWorkflow/FT0CalibSlewingCollectorSpec.h
index e01fed6f20588..8a1a071a0a043 100644
--- a/Detectors/FIT/FT0/calibration/testWorkflow/FT0CalibSlewingCollectorSpec.h
+++ b/Detectors/FIT/FT0/calibration/testWorkflow/FT0CalibSlewingCollectorSpec.h
@@ -88,8 +88,8 @@ class FT0CalibCollectorDevice : public o2::framework::Task
       auto entries = collectedInfo.size();
       // this means that we are ready to send the output
       auto entriesPerChannel = mCollector->getEntriesPerChannel();
-      output.snapshot(Output{o2::header::gDataOriginFT0, "COLLECTEDINFO", 0, Lifetime::Timeframe}, collectedInfo);
-      output.snapshot(Output{o2::header::gDataOriginFT0, "ENTRIESCH", 0, Lifetime::Timeframe}, entriesPerChannel);
+      output.snapshot(Output{o2::header::gDataOriginFT0, "COLLECTEDINFO", 0}, collectedInfo);
+      output.snapshot(Output{o2::header::gDataOriginFT0, "ENTRIESCH", 0}, entriesPerChannel);
       mCollector->initOutput(); // reset the output for the next round
     }
   }
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/FT0TimeSpectraProcessor-Workflow.cxx b/Detectors/FIT/FT0/calibration/testWorkflow/FT0TimeSpectraProcessor-Workflow.cxx
index 541f5a9827a4a..f1dd64a250a3b 100644
--- a/Detectors/FIT/FT0/calibration/testWorkflow/FT0TimeSpectraProcessor-Workflow.cxx
+++ b/Detectors/FIT/FT0/calibration/testWorkflow/FT0TimeSpectraProcessor-Workflow.cxx
@@ -101,7 +101,7 @@ class FT0TimeSpectraProcessor final : public o2::framework::Task
       }
     }
 
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "TIME_SPECTRA", 0, o2::framework::Lifetime::Timeframe}, timeSpectraInfoObject.getBase());
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "TIME_SPECTRA", 0}, timeSpectraInfoObject.getBase());
   }
 };
 
diff --git a/Detectors/FIT/FT0/workflow/src/DigitReaderSpec.cxx b/Detectors/FIT/FT0/workflow/src/DigitReaderSpec.cxx
index e4f11d3f5bd1e..09586d778ac15 100644
--- a/Detectors/FIT/FT0/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/FIT/FT0/workflow/src/DigitReaderSpec.cxx
@@ -64,13 +64,13 @@ void DigitReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(debug) << "FT0DigitReader pushed " << channels.size() << " channels in " << digits.size() << " digits";
-  pc.outputs().snapshot(Output{"FT0", "DIGITSBC", 0, Lifetime::Timeframe}, digits);
-  pc.outputs().snapshot(Output{"FT0", "DIGITSCH", 0, Lifetime::Timeframe}, channels);
+  pc.outputs().snapshot(Output{"FT0", "DIGITSBC", 0}, digits);
+  pc.outputs().snapshot(Output{"FT0", "DIGITSCH", 0}, channels);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"FT0", "DIGITSMCTR", 0, Lifetime::Timeframe}, labels);
+    pc.outputs().snapshot(Output{"FT0", "DIGITSMCTR", 0}, labels);
   }
   if (mUseTrgInput) {
-    pc.outputs().snapshot(Output{"FT0", "TRIGGERINPUT", 0, Lifetime::Timeframe}, trgInput);
+    pc.outputs().snapshot(Output{"FT0", "TRIGGERINPUT", 0}, trgInput);
   }
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/FIT/FT0/workflow/src/EntropyEncoderSpec.cxx b/Detectors/FIT/FT0/workflow/src/EntropyEncoderSpec.cxx
index edf86b3d7da0d..81bdc2e729bb4 100644
--- a/Detectors/FIT/FT0/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/FIT/FT0/workflow/src/EntropyEncoderSpec.cxx
@@ -55,7 +55,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
 
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"FT0", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"FT0", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, digits, channels);
   if (mSelIR) {
     mCTFCoder.getIRFramesSelector().clear();
diff --git a/Detectors/FIT/FT0/workflow/src/RecPointReaderSpec.cxx b/Detectors/FIT/FT0/workflow/src/RecPointReaderSpec.cxx
index 69f21abc369c6..ba5ae4aa1356c 100644
--- a/Detectors/FIT/FT0/workflow/src/RecPointReaderSpec.cxx
+++ b/Detectors/FIT/FT0/workflow/src/RecPointReaderSpec.cxx
@@ -49,8 +49,8 @@ void RecPointReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
 
   LOG(debug) << "FT0 RecPointReader pushes " << mRecPoints->size() << " recpoints with " << mChannelData->size() << " channels at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, *mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, *mChannelData);
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, *mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, *mChannelData);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/FIT/FT0/workflow/src/ReconstructionSpec.cxx b/Detectors/FIT/FT0/workflow/src/ReconstructionSpec.cxx
index da5281d7e2ba1..40bc96ebca58e 100644
--- a/Detectors/FIT/FT0/workflow/src/ReconstructionSpec.cxx
+++ b/Detectors/FIT/FT0/workflow/src/ReconstructionSpec.cxx
@@ -74,8 +74,8 @@ void ReconstructionDPL::run(ProcessingContext& pc)
   mReco.processTF(digits, channels, mRecPoints, mRecChData);
   // do we ignore MC in this task?
   LOG(debug) << "FT0 reconstruction pushes " << mRecPoints.size() << " RecPoints";
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, mRecChData);
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, mRecChData);
 
   mTimer.Stop();
 }
diff --git a/Detectors/FIT/FV0/workflow/src/DigitReaderSpec.cxx b/Detectors/FIT/FV0/workflow/src/DigitReaderSpec.cxx
index 08a6c4cbb2e24..a49bda2cec18b 100644
--- a/Detectors/FIT/FV0/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/FIT/FV0/workflow/src/DigitReaderSpec.cxx
@@ -65,13 +65,13 @@ void DigitReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(debug) << "FV0DigitReader pushed " << channels.size() << " channels in " << digits.size() << " digits";
-  pc.outputs().snapshot(Output{"FV0", "DIGITSBC", 0, Lifetime::Timeframe}, digits);
-  pc.outputs().snapshot(Output{"FV0", "DIGITSCH", 0, Lifetime::Timeframe}, channels);
+  pc.outputs().snapshot(Output{"FV0", "DIGITSBC", 0}, digits);
+  pc.outputs().snapshot(Output{"FV0", "DIGITSCH", 0}, channels);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"FV0", "DIGITSMCTR", 0, Lifetime::Timeframe}, labels);
+    pc.outputs().snapshot(Output{"FV0", "DIGITSMCTR", 0}, labels);
   }
   if (mUseTrgInput) {
-    pc.outputs().snapshot(Output{"FV0", "TRIGGERINPUT", 0, Lifetime::Timeframe}, trgInput);
+    pc.outputs().snapshot(Output{"FV0", "TRIGGERINPUT", 0}, trgInput);
   }
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/FIT/FV0/workflow/src/EntropyEncoderSpec.cxx b/Detectors/FIT/FV0/workflow/src/EntropyEncoderSpec.cxx
index a115712a0c980..a25c16a5d697c 100644
--- a/Detectors/FIT/FV0/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/FIT/FV0/workflow/src/EntropyEncoderSpec.cxx
@@ -55,7 +55,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
 
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"FV0", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"FV0", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, digits, channels);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   if (mSelIR) {
diff --git a/Detectors/FIT/FV0/workflow/src/RecPointReaderSpec.cxx b/Detectors/FIT/FV0/workflow/src/RecPointReaderSpec.cxx
index 163beacf0c140..5997cac500ee6 100644
--- a/Detectors/FIT/FV0/workflow/src/RecPointReaderSpec.cxx
+++ b/Detectors/FIT/FV0/workflow/src/RecPointReaderSpec.cxx
@@ -49,8 +49,8 @@ void RecPointReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
 
   LOG(debug) << "FV0 RecPointReader pushes " << mRecPoints->size() << " recpoints with " << mChannelData->size() << " channels at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, *mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, *mChannelData);
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, *mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, *mChannelData);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/FIT/FV0/workflow/src/ReconstructionSpec.cxx b/Detectors/FIT/FV0/workflow/src/ReconstructionSpec.cxx
index d48b96fd4e626..520ac4dbaa563 100644
--- a/Detectors/FIT/FV0/workflow/src/ReconstructionSpec.cxx
+++ b/Detectors/FIT/FV0/workflow/src/ReconstructionSpec.cxx
@@ -68,8 +68,8 @@ void ReconstructionDPL::run(ProcessingContext& pc)
   }
 
   LOG(debug) << "FV0 reconstruction pushes " << mRecPoints.size() << " RecPoints";
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, mRecChData);
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, mRecChData);
 
   mTimer.Stop();
 }
diff --git a/Detectors/FIT/workflow/include/FITWorkflow/RawReaderFIT.h b/Detectors/FIT/workflow/include/FITWorkflow/RawReaderFIT.h
index 5c2cfce0a620c..cfe41e4bb474f 100644
--- a/Detectors/FIT/workflow/include/FITWorkflow/RawReaderFIT.h
+++ b/Detectors/FIT/workflow/include/FITWorkflow/RawReaderFIT.h
@@ -165,28 +165,28 @@ class RawReaderFIT : public RawReaderType
   }
   void makeSnapshot(o2::framework::ProcessingContext& pc) const
   {
-    pc.outputs().snapshot(o2::framework::Output{mDataOrigin, Digit_t::sChannelNameDPL, 0, o2::framework::Lifetime::Timeframe}, mVecDigit);
+    pc.outputs().snapshot(o2::framework::Output{mDataOrigin, Digit_t::sChannelNameDPL, 0}, mVecDigit);
     if constexpr (sSubDigitExists) {
       std::apply([&](const auto&... subDigit) {
-        ((pc.outputs().snapshot(o2::framework::Output{mDataOrigin, (std::decay<decltype(subDigit)>::type::value_type::sChannelNameDPL), 0, o2::framework::Lifetime::Timeframe}, subDigit)), ...);
+        ((pc.outputs().snapshot(o2::framework::Output{mDataOrigin, (std::decay<decltype(subDigit)>::type::value_type::sChannelNameDPL), 0}, subDigit)), ...);
       },
                  mVecSubDigit);
     }
     if constexpr (sSingleSubDigitExists) {
       std::apply([&](const auto&... singleSubDigit) {
-        ((pc.outputs().snapshot(o2::framework::Output{mDataOrigin, (std::decay<decltype(singleSubDigit)>::type::value_type::sChannelNameDPL), 0, o2::framework::Lifetime::Timeframe}, singleSubDigit)), ...);
+        ((pc.outputs().snapshot(o2::framework::Output{mDataOrigin, (std::decay<decltype(singleSubDigit)>::type::value_type::sChannelNameDPL), 0}, singleSubDigit)), ...);
       },
                  mVecSingleSubDigit);
     }
     if constexpr (sUseTrgInput) {
-      pc.outputs().snapshot(o2::framework::Output{mDataOrigin, DetTrigInput_t::sChannelNameDPL, 0, o2::framework::Lifetime::Timeframe}, mVecTrgInput);
+      pc.outputs().snapshot(o2::framework::Output{mDataOrigin, DetTrigInput_t::sChannelNameDPL, 0}, mVecTrgInput);
     }
-    pc.outputs().snapshot(o2::framework::Output{mDataOrigin, "RawDataMetric", 0, o2::framework::Lifetime::Timeframe}, mVecRawDataMetric);
+    pc.outputs().snapshot(o2::framework::Output{mDataOrigin, "RawDataMetric", 0}, mVecRawDataMetric);
   }
   template <typename VecDigitType>
   auto& getRefVec(o2::framework::ProcessingContext& pc)
   {
-    auto& refVec = pc.outputs().make<VecDigitType>(o2::framework::Output{mDataOrigin, VecDigitType::value_type::sChannelNameDPL, 0, o2::framework::Lifetime::Timeframe});
+    auto& refVec = pc.outputs().make<VecDigitType>(o2::framework::Output{mDataOrigin, VecDigitType::value_type::sChannelNameDPL, 0});
     return refVec;
   }
   void enableEmptyTFprotection()
@@ -204,4 +204,4 @@ class RawReaderFIT : public RawReaderType
 } // namespace fit
 } // namespace o2
 
-#endif
\ No newline at end of file
+#endif
diff --git a/Detectors/FOCAL/workflow/src/RawDecoderSpec.cxx b/Detectors/FOCAL/workflow/src/RawDecoderSpec.cxx
index d6004561f852d..c7260ae3c9815 100644
--- a/Detectors/FOCAL/workflow/src/RawDecoderSpec.cxx
+++ b/Detectors/FOCAL/workflow/src/RawDecoderSpec.cxx
@@ -274,10 +274,10 @@ void RawDecoderSpec::endOfStream(o2::framework::EndOfStreamContext& ec)
 
 void RawDecoderSpec::sendOutput(framework::ProcessingContext& ctx)
 {
-  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginFOC, "PADLAYERS", mOutputSubspec, framework::Lifetime::Timeframe}, mOutputPadLayers);
-  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginFOC, "PIXELHITS", mOutputSubspec, framework::Lifetime::Timeframe}, mOutputPixelHits);
-  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginFOC, "PIXELCHIPS", mOutputSubspec, framework::Lifetime::Timeframe}, mOutputPixelChips);
-  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginFOC, "TRIGGERS", mOutputSubspec, framework::Lifetime::Timeframe}, mOutputTriggerRecords);
+  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginFOC, "PADLAYERS", mOutputSubspec}, mOutputPadLayers);
+  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginFOC, "PIXELHITS", mOutputSubspec}, mOutputPixelHits);
+  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginFOC, "PIXELCHIPS", mOutputSubspec}, mOutputPixelChips);
+  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginFOC, "TRIGGERS", mOutputSubspec}, mOutputTriggerRecords);
 }
 
 void RawDecoderSpec::resetContainers()
@@ -689,4 +689,4 @@ o2::framework::DataProcessorSpec o2::focal::reco_workflow::getRawDecoderSpec(boo
                                             {"displayInconsistent", o2::framework::VariantType::Bool, false, {"Display information about inconsistent timeframes"}},
                                             {"pixeltype", o2::framework::VariantType::String, "OB", {"Pixel mapping type"}},
                                             {"pixelmapping", o2::framework::VariantType::String, "default", {"File with pixel mapping"}}}};
-}
\ No newline at end of file
+}
diff --git a/Detectors/Filtering/src/FilteredTFReaderSpec.cxx b/Detectors/Filtering/src/FilteredTFReaderSpec.cxx
index 3165208aef246..22fe1370040db 100644
--- a/Detectors/Filtering/src/FilteredTFReaderSpec.cxx
+++ b/Detectors/Filtering/src/FilteredTFReaderSpec.cxx
@@ -45,15 +45,15 @@ void FilteredTFReader::run(ProcessingContext& pc)
 
   LOG(info) << "Pushing filtered TF: " << mFiltTF.header.asString();
   // ITS
-  pc.outputs().snapshot(Output{"ITS", "ITSTrackROF", 0, Lifetime::Timeframe}, mFiltTF.ITSTrackROFs);
-  pc.outputs().snapshot(Output{"ITS", "TRACKS", 0, Lifetime::Timeframe}, mFiltTF.ITSTracks);
-  pc.outputs().snapshot(Output{"ITS", "TRACKCLSID", 0, Lifetime::Timeframe}, mFiltTF.ITSClusterIndices);
+  pc.outputs().snapshot(Output{"ITS", "ITSTrackROF", 0}, mFiltTF.ITSTrackROFs);
+  pc.outputs().snapshot(Output{"ITS", "TRACKS", 0}, mFiltTF.ITSTracks);
+  pc.outputs().snapshot(Output{"ITS", "TRACKCLSID", 0}, mFiltTF.ITSClusterIndices);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"ITS", "TRACKSMCTR", 0, Lifetime::Timeframe}, mFiltTF.ITSTrackMCTruth);
+    pc.outputs().snapshot(Output{"ITS", "TRACKSMCTR", 0}, mFiltTF.ITSTrackMCTruth);
   }
-  pc.outputs().snapshot(Output{"ITS", "CLUSTERSROF", 0, Lifetime::Timeframe}, mFiltTF.ITSClusterROFs);
-  pc.outputs().snapshot(Output{"ITS", "COMPCLUSTERS", 0, Lifetime::Timeframe}, mFiltTF.ITSClusters);
-  pc.outputs().snapshot(Output{"ITS", "PATTERNS", 0, Lifetime::Timeframe}, mFiltTF.ITSClusterPatterns);
+  pc.outputs().snapshot(Output{"ITS", "CLUSTERSROF", 0}, mFiltTF.ITSClusterROFs);
+  pc.outputs().snapshot(Output{"ITS", "COMPCLUSTERS", 0}, mFiltTF.ITSClusters);
+  pc.outputs().snapshot(Output{"ITS", "PATTERNS", 0}, mFiltTF.ITSClusterPatterns);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/GlobalTrackingWorkflow/helpers/src/NoInpDummyOutSpec.cxx b/Detectors/GlobalTrackingWorkflow/helpers/src/NoInpDummyOutSpec.cxx
index 34f445ed003da..17ad5f09027e1 100644
--- a/Detectors/GlobalTrackingWorkflow/helpers/src/NoInpDummyOutSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/helpers/src/NoInpDummyOutSpec.cxx
@@ -47,7 +47,7 @@ void NoInpDummyOut::run(ProcessingContext& pc)
 {
   static int counter = 0;
   // send just once dummy output to trigger the ccdb-fetcher
-  pc.outputs().make<std::vector<char>>(Output{"GLO", "DUMMY_OUT", 0, Lifetime::Timeframe});
+  pc.outputs().make<std::vector<char>>(Output{"GLO", "DUMMY_OUT", 0});
   if (mLoops >= 0 && ++counter >= mLoops) {
     pc.services().get<o2::framework::ControlService>().endOfStream();
     pc.services().get<o2::framework::ControlService>().readyToQuit(framework::QuitRequest::Me);
diff --git a/Detectors/GlobalTrackingWorkflow/qc/src/ITSTPCMatchingQCSpec.cxx b/Detectors/GlobalTrackingWorkflow/qc/src/ITSTPCMatchingQCSpec.cxx
index 3653fd8bcc90b..c8a402a48d659 100644
--- a/Detectors/GlobalTrackingWorkflow/qc/src/ITSTPCMatchingQCSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/qc/src/ITSTPCMatchingQCSpec.cxx
@@ -71,7 +71,7 @@ void ITSTPCMatchingQCDevice::sendOutput(DataAllocator& output)
 
   TObjArray objar;
   mMatchITSTPCQC->getHistos(objar);
-  output.snapshot(Output{"GLO", "ITSTPCMATCHQC", 0, Lifetime::Sporadic}, objar);
+  output.snapshot(Output{"GLO", "ITSTPCMATCHQC", 0}, objar);
 
   TFile* f = new TFile(Form("outITSTPCmatchingQC.root"), "RECREATE");
   objar.Write("ObjArray", TObject::kSingleKey);
diff --git a/Detectors/GlobalTrackingWorkflow/readers/src/GlobalFwdTrackReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/readers/src/GlobalFwdTrackReaderSpec.cxx
index 1fdc7b49616c3..11fa58333f89b 100644
--- a/Detectors/GlobalTrackingWorkflow/readers/src/GlobalFwdTrackReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/readers/src/GlobalFwdTrackReaderSpec.cxx
@@ -65,9 +65,9 @@ void GlobalFwdTrackReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mTracks.size() << " Global Forward tracks at entry " << ent;
 
-  pc.outputs().snapshot(Output{"GLO", "GLFWD", 0, Lifetime::Timeframe}, mTracks);
+  pc.outputs().snapshot(Output{"GLO", "GLFWD", 0}, mTracks);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"GLO", "GLFWD_MC", 0, Lifetime::Timeframe}, mLabels);
+    pc.outputs().snapshot(Output{"GLO", "GLFWD_MC", 0}, mLabels);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/GlobalTrackingWorkflow/readers/src/IRFrameReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/readers/src/IRFrameReaderSpec.cxx
index 012542e448d19..c1810a1deb743 100644
--- a/Detectors/GlobalTrackingWorkflow/readers/src/IRFrameReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/readers/src/IRFrameReaderSpec.cxx
@@ -63,7 +63,7 @@ void IRFrameReaderSpec::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(debug) << "Pushing " << mIRF.size() << " IR-frames in at entry " << ent;
-  pc.outputs().snapshot(Output{mDataOrigin, "IRFRAMES", mSubSpec, Lifetime::Timeframe}, mIRF);
+  pc.outputs().snapshot(Output{mDataOrigin, "IRFRAMES", mSubSpec}, mIRF);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/GlobalTrackingWorkflow/readers/src/MatchedMFTMCHReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/readers/src/MatchedMFTMCHReaderSpec.cxx
index 08c5c5871db84..5f02beebd1746 100644
--- a/Detectors/GlobalTrackingWorkflow/readers/src/MatchedMFTMCHReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/readers/src/MatchedMFTMCHReaderSpec.cxx
@@ -65,7 +65,7 @@ void MatchMFTMCHReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mTracks.size() << " MFTMCH matches at entry " << ent;
 
-  pc.outputs().snapshot(Output{"GLO", "MTC_MFTMCH", 0, Lifetime::Timeframe}, mTracks);
+  pc.outputs().snapshot(Output{"GLO", "MTC_MFTMCH", 0}, mTracks);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/GlobalTrackingWorkflow/readers/src/PrimaryVertexReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/readers/src/PrimaryVertexReaderSpec.cxx
index 06bdf8ff95a45..6e1aba8b2e1f3 100644
--- a/Detectors/GlobalTrackingWorkflow/readers/src/PrimaryVertexReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/readers/src/PrimaryVertexReaderSpec.cxx
@@ -84,12 +84,12 @@ void PrimaryVertexReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mVerticesPtr->size() << " vertices at entry " << ent;
 
-  pc.outputs().snapshot(Output{"GLO", "PVTX", 0, Lifetime::Timeframe}, mVertices);
-  pc.outputs().snapshot(Output{"GLO", "PVTX_TRMTC", 0, Lifetime::Timeframe}, mPV2MatchIdx);
-  pc.outputs().snapshot(Output{"GLO", "PVTX_TRMTCREFS", 0, Lifetime::Timeframe}, mPV2MatchIdxRef);
+  pc.outputs().snapshot(Output{"GLO", "PVTX", 0}, mVertices);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_TRMTC", 0}, mPV2MatchIdx);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_TRMTCREFS", 0}, mPV2MatchIdxRef);
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"GLO", "PVTX_MCTR", 0, Lifetime::Timeframe}, mLabels);
+    pc.outputs().snapshot(Output{"GLO", "PVTX_MCTR", 0}, mLabels);
   }
 
   if (mVerbose) {
diff --git a/Detectors/GlobalTrackingWorkflow/readers/src/SecondaryVertexReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/readers/src/SecondaryVertexReaderSpec.cxx
index e440a075d2558..4dd6324c83675 100644
--- a/Detectors/GlobalTrackingWorkflow/readers/src/SecondaryVertexReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/readers/src/SecondaryVertexReaderSpec.cxx
@@ -93,15 +93,15 @@ void SecondaryVertexReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mV0s.size() << " V0s and " << mCascs.size() << " cascades at entry " << ent;
 
-  pc.outputs().snapshot(Output{"GLO", "V0S_IDX", 0, Lifetime::Timeframe}, mV0sIdx);
-  pc.outputs().snapshot(Output{"GLO", "V0S", 0, Lifetime::Timeframe}, mV0s);
-  pc.outputs().snapshot(Output{"GLO", "PVTX_V0REFS", 0, Lifetime::Timeframe}, mPV2V0Ref);
-  pc.outputs().snapshot(Output{"GLO", "CASCS_IDX", 0, Lifetime::Timeframe}, mCascsIdx);
-  pc.outputs().snapshot(Output{"GLO", "CASCS", 0, Lifetime::Timeframe}, mCascs);
-  pc.outputs().snapshot(Output{"GLO", "PVTX_CASCREFS", 0, Lifetime::Timeframe}, mPV2CascRef);
-  pc.outputs().snapshot(Output{"GLO", "DECAYS3BODY_IDX", 0, Lifetime::Timeframe}, m3BodysIdx);
-  pc.outputs().snapshot(Output{"GLO", "DECAYS3BODY", 0, Lifetime::Timeframe}, m3Bodys);
-  pc.outputs().snapshot(Output{"GLO", "PVTX_3BODYREFS", 0, Lifetime::Timeframe}, mPV23BodyRef);
+  pc.outputs().snapshot(Output{"GLO", "V0S_IDX", 0}, mV0sIdx);
+  pc.outputs().snapshot(Output{"GLO", "V0S", 0}, mV0s);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_V0REFS", 0}, mPV2V0Ref);
+  pc.outputs().snapshot(Output{"GLO", "CASCS_IDX", 0}, mCascsIdx);
+  pc.outputs().snapshot(Output{"GLO", "CASCS", 0}, mCascs);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_CASCREFS", 0}, mPV2CascRef);
+  pc.outputs().snapshot(Output{"GLO", "DECAYS3BODY_IDX", 0}, m3BodysIdx);
+  pc.outputs().snapshot(Output{"GLO", "DECAYS3BODY", 0}, m3Bodys);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_3BODYREFS", 0}, mPV23BodyRef);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/GlobalTrackingWorkflow/readers/src/StrangenessTrackingReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/readers/src/StrangenessTrackingReaderSpec.cxx
index 23eff571105e6..8c7f87a720925 100644
--- a/Detectors/GlobalTrackingWorkflow/readers/src/StrangenessTrackingReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/readers/src/StrangenessTrackingReaderSpec.cxx
@@ -76,14 +76,14 @@ void StrangenessTrackingReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mStrangeTrack.size() << " strange tracks at entry " << ent;
-  pc.outputs().snapshot(Output{"GLO", "STRANGETRACKS", 0, Lifetime::Timeframe}, mStrangeTrack);
+  pc.outputs().snapshot(Output{"GLO", "STRANGETRACKS", 0}, mStrangeTrack);
 
   if (mUseMC) {
     LOG(info) << "Pushing " << mStrangeTrackMC.size() << " strange tracks MC labels at entry " << ent;
-    pc.outputs().snapshot(Output{"GLO", "STRANGETRACKS_MC", 0, Lifetime::Timeframe}, mStrangeTrackMC);
+    pc.outputs().snapshot(Output{"GLO", "STRANGETRACKS_MC", 0}, mStrangeTrackMC);
   }
 
-  // pc.outputs().snapshot(Output{"GLO", "PVTX_V0REFS", 0, Lifetime::Timeframe}, mPV2V0Ref);
+  // pc.outputs().snapshot(Output{"GLO", "PVTX_V0REFS", 0}, mPV2V0Ref);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/GlobalTrackingWorkflow/readers/src/TrackCosmicsReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/readers/src/TrackCosmicsReaderSpec.cxx
index 90db5d08acc58..7e3cdffd84a6d 100644
--- a/Detectors/GlobalTrackingWorkflow/readers/src/TrackCosmicsReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/readers/src/TrackCosmicsReaderSpec.cxx
@@ -41,9 +41,9 @@ void TrackCosmicsReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mTracks.size() << " Cosmic Tracks at entry " << ent;
 
-  pc.outputs().snapshot(Output{"GLO", "COSMICTRC", 0, Lifetime::Timeframe}, mTracks);
+  pc.outputs().snapshot(Output{"GLO", "COSMICTRC", 0}, mTracks);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"GLO", "COSMICTRC_MC", 0, Lifetime::Timeframe}, mLabels);
+    pc.outputs().snapshot(Output{"GLO", "COSMICTRC_MC", 0}, mLabels);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/GlobalTrackingWorkflow/readers/src/TrackTPCITSReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/readers/src/TrackTPCITSReaderSpec.cxx
index 8ae55e478603e..c7fd0d543ecf6 100644
--- a/Detectors/GlobalTrackingWorkflow/readers/src/TrackTPCITSReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/readers/src/TrackTPCITSReaderSpec.cxx
@@ -68,12 +68,12 @@ void TrackTPCITSReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mTracks.size() << " TPC-ITS matches at entry " << ent;
 
-  pc.outputs().snapshot(Output{"GLO", "TPCITS", 0, Lifetime::Timeframe}, mTracks);
-  pc.outputs().snapshot(Output{"GLO", "TPCITSAB_REFS", 0, Lifetime::Timeframe}, mABTrkClusRefs);
-  pc.outputs().snapshot(Output{"GLO", "TPCITSAB_CLID", 0, Lifetime::Timeframe}, mABTrkClIDs);
+  pc.outputs().snapshot(Output{"GLO", "TPCITS", 0}, mTracks);
+  pc.outputs().snapshot(Output{"GLO", "TPCITSAB_REFS", 0}, mABTrkClusRefs);
+  pc.outputs().snapshot(Output{"GLO", "TPCITSAB_CLID", 0}, mABTrkClIDs);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"GLO", "TPCITS_MC", 0, Lifetime::Timeframe}, mLabels);
-    pc.outputs().snapshot(Output{"GLO", "TPCITSAB_MC", 0, Lifetime::Timeframe}, mLabelsAB);
+    pc.outputs().snapshot(Output{"GLO", "TPCITS_MC", 0}, mLabels);
+    pc.outputs().snapshot(Output{"GLO", "TPCITSAB_MC", 0}, mLabelsAB);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/GlobalTrackingWorkflow/src/CosmicsMatchingSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/CosmicsMatchingSpec.cxx
index d8242fc2ccd79..a4c9d302019a5 100644
--- a/Detectors/GlobalTrackingWorkflow/src/CosmicsMatchingSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/CosmicsMatchingSpec.cxx
@@ -99,9 +99,9 @@ void CosmicsMatchingSpec::run(ProcessingContext& pc)
   updateTimeDependentParams(pc); // Make sure this is called after recoData.collectData, which may load some conditions
 
   mMatching.process(recoData);
-  pc.outputs().snapshot(Output{"GLO", "COSMICTRC", 0, Lifetime::Timeframe}, mMatching.getCosmicTracks());
+  pc.outputs().snapshot(Output{"GLO", "COSMICTRC", 0}, mMatching.getCosmicTracks());
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"GLO", "COSMICTRC_MC", 0, Lifetime::Timeframe}, mMatching.getCosmicTracksLbl());
+    pc.outputs().snapshot(Output{"GLO", "COSMICTRC_MC", 0}, mMatching.getCosmicTracksLbl());
   }
   mTimer.Stop();
 }
diff --git a/Detectors/GlobalTrackingWorkflow/src/GlobalFwdMatchingSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/GlobalFwdMatchingSpec.cxx
index 582f2be3c5f84..03dc823c62c42 100644
--- a/Detectors/GlobalTrackingWorkflow/src/GlobalFwdMatchingSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/GlobalFwdMatchingSpec.cxx
@@ -92,18 +92,18 @@ void GlobalFwdMatchingDPL::run(ProcessingContext& pc)
   const auto& matchingParam = GlobalFwdMatchingParam::Instance();
 
   if (matchingParam.saveMode == kSaveTrainingData) {
-    pc.outputs().snapshot(Output{"GLO", "GLFWDMFT", 0, Lifetime::Timeframe}, mMatching.getMFTMatchingPlaneParams());
-    pc.outputs().snapshot(Output{"GLO", "GLFWDMCH", 0, Lifetime::Timeframe}, mMatching.getMCHMatchingPlaneParams());
-    pc.outputs().snapshot(Output{"GLO", "GLFWDINF", 0, Lifetime::Timeframe}, mMatching.getMFTMCHMatchInfo());
+    pc.outputs().snapshot(Output{"GLO", "GLFWDMFT", 0}, mMatching.getMFTMatchingPlaneParams());
+    pc.outputs().snapshot(Output{"GLO", "GLFWDMCH", 0}, mMatching.getMCHMatchingPlaneParams());
+    pc.outputs().snapshot(Output{"GLO", "GLFWDINF", 0}, mMatching.getMFTMCHMatchInfo());
   } else {
-    pc.outputs().snapshot(Output{"GLO", "GLFWD", 0, Lifetime::Timeframe}, mMatching.getMatchedFwdTracks());
+    pc.outputs().snapshot(Output{"GLO", "GLFWD", 0}, mMatching.getMatchedFwdTracks());
   }
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"GLO", "GLFWD_MC", 0, Lifetime::Timeframe}, mMatching.getMatchLabels());
+    pc.outputs().snapshot(Output{"GLO", "GLFWD_MC", 0}, mMatching.getMatchLabels());
   }
   if (mMatchRootOutput) {
-    pc.outputs().snapshot(Output{"GLO", "MTC_MFTMCH", 0, Lifetime::Timeframe}, mMatching.getMFTMCHMatchInfo());
+    pc.outputs().snapshot(Output{"GLO", "MTC_MFTMCH", 0}, mMatching.getMFTMCHMatchInfo());
   }
   mTimer.Stop();
 }
diff --git a/Detectors/GlobalTrackingWorkflow/src/HMPMatcherSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/HMPMatcherSpec.cxx
index f8666b5561610..b0562fd4f3a1e 100644
--- a/Detectors/GlobalTrackingWorkflow/src/HMPMatcherSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/HMPMatcherSpec.cxx
@@ -128,9 +128,9 @@ void HMPMatcherSpec::run(ProcessingContext& pc)
 
   mMatcher.run(recoData);
 
-  pc.outputs().snapshot(Output{o2::header::gDataOriginHMP, "MATCHES", 0, Lifetime::Timeframe}, mMatcher.getMatchedTrackVector(o2::globaltracking::MatchHMP::trackType::CONSTR));
+  pc.outputs().snapshot(Output{o2::header::gDataOriginHMP, "MATCHES", 0}, mMatcher.getMatchedTrackVector(o2::globaltracking::MatchHMP::trackType::CONSTR));
   if (mUseMC) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginHMP, "MCLABELS", 0, Lifetime::Timeframe}, mMatcher.getMatchedHMPLabelsVector(o2::globaltracking::MatchHMP::trackType::CONSTR));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginHMP, "MCLABELS", 0}, mMatcher.getMatchedHMPLabelsVector(o2::globaltracking::MatchHMP::trackType::CONSTR));
   }
 
   mTimer.Stop();
diff --git a/Detectors/GlobalTrackingWorkflow/src/PrimaryVertexingSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/PrimaryVertexingSpec.cxx
index e70122e7a01b5..ff45790a95921 100644
--- a/Detectors/GlobalTrackingWorkflow/src/PrimaryVertexingSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/PrimaryVertexingSpec.cxx
@@ -152,12 +152,12 @@ void PrimaryVertexingSpec::run(ProcessingContext& pc)
     mVertexer.process(tracks, gids, ft0Data, vertices, vertexTrackIDs, v2tRefs, tracksMCInfo, lblVtx);
   }
 
-  pc.outputs().snapshot(Output{"GLO", "PVTX", 0, Lifetime::Timeframe}, vertices);
-  pc.outputs().snapshot(Output{"GLO", "PVTX_CONTIDREFS", 0, Lifetime::Timeframe}, v2tRefs);
-  pc.outputs().snapshot(Output{"GLO", "PVTX_CONTID", 0, Lifetime::Timeframe}, vertexTrackIDs);
+  pc.outputs().snapshot(Output{"GLO", "PVTX", 0}, vertices);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_CONTIDREFS"}, v2tRefs);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_CONTID", 0}, vertexTrackIDs);
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"GLO", "PVTX_MCTR", 0, Lifetime::Timeframe}, lblVtx);
+    pc.outputs().snapshot(Output{"GLO", "PVTX_MCTR", 0}, lblVtx);
   }
 
   mTimer.Stop();
diff --git a/Detectors/GlobalTrackingWorkflow/src/StrangenessTrackingSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/StrangenessTrackingSpec.cxx
index 46136ac6e2397..37b35df06ea72 100644
--- a/Detectors/GlobalTrackingWorkflow/src/StrangenessTrackingSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/StrangenessTrackingSpec.cxx
@@ -75,11 +75,11 @@ void StrangenessTrackerSpec::run(framework::ProcessingContext& pc)
   mTracker.loadData(recoData);
   mTracker.prepareITStracks();
   mTracker.process();
-  pc.outputs().snapshot(Output{"GLO", "STRANGETRACKS", 0, Lifetime::Timeframe}, mTracker.getStrangeTrackVec());
-  pc.outputs().snapshot(Output{"GLO", "CLUSUPDATES", 0, Lifetime::Timeframe}, mTracker.getClusAttachments());
+  pc.outputs().snapshot(Output{"GLO", "STRANGETRACKS", 0}, mTracker.getStrangeTrackVec());
+  pc.outputs().snapshot(Output{"GLO", "CLUSUPDATES", 0}, mTracker.getClusAttachments());
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"GLO", "STRANGETRACKS_MC", 0, Lifetime::Timeframe}, mTracker.getStrangeTrackLabels());
+    pc.outputs().snapshot(Output{"GLO", "STRANGETRACKS_MC", 0}, mTracker.getStrangeTrackLabels());
   }
 
   mTimer.Stop();
diff --git a/Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx
index 134ffbb75f341..ca4d3e2fc5dc8 100644
--- a/Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx
@@ -171,8 +171,8 @@ void TOFMatcherSpec::run(ProcessingContext& pc)
   static pmr::vector<o2::MCCompLabel> dummyMCLab;
 
   if (isTPCused) {
-    auto& mtcInfo = pc.outputs().make<std::vector<o2::dataformats::MatchInfoTOF>>(Output{o2::header::gDataOriginTOF, "MTC_TPC", ss, Lifetime::Timeframe});
-    auto& mclabels = mUseMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{o2::header::gDataOriginTOF, "MCMTC_TPC", ss, Lifetime::Timeframe}) : dummyMCLab;
+    auto& mtcInfo = pc.outputs().make<std::vector<o2::dataformats::MatchInfoTOF>>(Output{o2::header::gDataOriginTOF, "MTC_TPC", ss});
+    auto& mclabels = mUseMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{o2::header::gDataOriginTOF, "MCMTC_TPC", ss}) : dummyMCLab;
     auto& tracksTPCTOF = pc.outputs().make<std::vector<o2::dataformats::TrackTPCTOF>>(OutputRef{"tpctofTracks", ss});
     auto nmatch = mMatcher.getMatchedTrackVector(o2::dataformats::MatchInfoTOFReco::TrackType::TPC).size();
     LOG(debug) << (mDoTPCRefit ? "Refitting " : "Shifting Z for ") << nmatch << " matched TPC tracks with TOF time info";
@@ -180,48 +180,48 @@ void TOFMatcherSpec::run(ProcessingContext& pc)
   }
 
   if (isITSTPCused) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MTC_ITSTPC", 0, Lifetime::Timeframe}, mMatcher.getMatchedTrackVector(o2::dataformats::MatchInfoTOFReco::TrackType::ITSTPC));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MTC_ITSTPC", 0}, mMatcher.getMatchedTrackVector(o2::dataformats::MatchInfoTOFReco::TrackType::ITSTPC));
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MCMTC_ITSTPC", 0, Lifetime::Timeframe}, mMatcher.getMatchedTOFLabelsVector(o2::dataformats::MatchInfoTOFReco::TrackType::ITSTPC));
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MCMTC_ITSTPC", 0}, mMatcher.getMatchedTOFLabelsVector(o2::dataformats::MatchInfoTOFReco::TrackType::ITSTPC));
     }
   }
 
   if (isTPCTRDused) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MTC_TPCTRD", ss, Lifetime::Timeframe}, mMatcher.getMatchedTrackVector(o2::dataformats::MatchInfoTOFReco::TrackType::TPCTRD));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MTC_TPCTRD", ss}, mMatcher.getMatchedTrackVector(o2::dataformats::MatchInfoTOFReco::TrackType::TPCTRD));
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MCMTC_TPCTRD", ss, Lifetime::Timeframe}, mMatcher.getMatchedTOFLabelsVector(o2::dataformats::MatchInfoTOFReco::TrackType::TPCTRD));
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MCMTC_TPCTRD", ss}, mMatcher.getMatchedTOFLabelsVector(o2::dataformats::MatchInfoTOFReco::TrackType::TPCTRD));
     }
   }
 
   if (isITSTPCTRDused) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MTC_ITSTPCTRD", 0, Lifetime::Timeframe}, mMatcher.getMatchedTrackVector(o2::dataformats::MatchInfoTOFReco::TrackType::ITSTPCTRD));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MTC_ITSTPCTRD", 0}, mMatcher.getMatchedTrackVector(o2::dataformats::MatchInfoTOFReco::TrackType::ITSTPCTRD));
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MCMTC_ITSTPCTRD", 0, Lifetime::Timeframe}, mMatcher.getMatchedTOFLabelsVector(o2::dataformats::MatchInfoTOFReco::TrackType::ITSTPCTRD));
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MCMTC_ITSTPCTRD", 0}, mMatcher.getMatchedTOFLabelsVector(o2::dataformats::MatchInfoTOFReco::TrackType::ITSTPCTRD));
     }
   }
 
   // TODO: TRD-matched tracks
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CALIBDATA", 0, Lifetime::Timeframe}, mMatcher.getCalibVector());
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CALIBDATA", 0}, mMatcher.getCalibVector());
 
   if (mPushMatchable) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_0", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(0));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_1", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(1));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_2", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(2));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_3", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(3));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_4", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(4));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_5", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(5));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_6", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(6));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_7", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(7));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_8", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(8));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_9", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(9));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_10", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(10));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_11", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(11));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_12", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(12));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_13", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(13));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_14", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(14));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_15", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(15));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_16", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(16));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_17", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(17));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_0", 0}, mMatcher.getMatchedTracksPair(0));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_1", 0}, mMatcher.getMatchedTracksPair(1));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_2", 0}, mMatcher.getMatchedTracksPair(2));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_3", 0}, mMatcher.getMatchedTracksPair(3));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_4", 0}, mMatcher.getMatchedTracksPair(4));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_5", 0}, mMatcher.getMatchedTracksPair(5));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_6", 0}, mMatcher.getMatchedTracksPair(6));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_7", 0}, mMatcher.getMatchedTracksPair(7));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_8", 0}, mMatcher.getMatchedTracksPair(8));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_9", 0}, mMatcher.getMatchedTracksPair(9));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_10", 0}, mMatcher.getMatchedTracksPair(10));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_11", 0}, mMatcher.getMatchedTracksPair(11));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_12", 0}, mMatcher.getMatchedTracksPair(12));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_13", 0}, mMatcher.getMatchedTracksPair(13));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_14", 0}, mMatcher.getMatchedTracksPair(14));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_15", 0}, mMatcher.getMatchedTracksPair(15));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_16", 0}, mMatcher.getMatchedTracksPair(16));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_17", 0}, mMatcher.getMatchedTracksPair(17));
   }
 
   mTimer.Stop();
diff --git a/Detectors/GlobalTrackingWorkflow/src/TPCITSMatchingSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/TPCITSMatchingSpec.cxx
index 1a061ee9707f5..aebcd96c4faa7 100644
--- a/Detectors/GlobalTrackingWorkflow/src/TPCITSMatchingSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/TPCITSMatchingSpec.cxx
@@ -103,12 +103,12 @@ void TPCITSMatchingDPL::run(ProcessingContext& pc)
   static pmr::vector<o2::MCCompLabel> dummyMCLab, dummyMCLabAB;
   static pmr::vector<o2::dataformats::Triplet<float, float, float>> dummyCalib;
 
-  auto& matchedTracks = pc.outputs().make<std::vector<o2::dataformats::TrackTPCITS>>(Output{"GLO", "TPCITS", 0, Lifetime::Timeframe});
-  auto& ABTrackletRefs = pc.outputs().make<std::vector<o2::itsmft::TrkClusRef>>(Output{"GLO", "TPCITSAB_REFS", 0, Lifetime::Timeframe});
-  auto& ABTrackletClusterIDs = pc.outputs().make<std::vector<int>>(Output{"GLO", "TPCITSAB_CLID", 0, Lifetime::Timeframe});
-  auto& matchLabels = mUseMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"GLO", "TPCITS_MC", 0, Lifetime::Timeframe}) : dummyMCLab;
-  auto& ABTrackletLabels = mUseMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"GLO", "TPCITSAB_MC", 0, Lifetime::Timeframe}) : dummyMCLabAB;
-  auto& calib = mCalibMode ? pc.outputs().make<std::vector<o2::dataformats::Triplet<float, float, float>>>(Output{"GLO", "TPCITS_VDTGL", 0, Lifetime::Timeframe}) : dummyCalib;
+  auto& matchedTracks = pc.outputs().make<std::vector<o2::dataformats::TrackTPCITS>>(Output{"GLO", "TPCITS", 0});
+  auto& ABTrackletRefs = pc.outputs().make<std::vector<o2::itsmft::TrkClusRef>>(Output{"GLO", "TPCITSAB_REFS", 0});
+  auto& ABTrackletClusterIDs = pc.outputs().make<std::vector<int>>(Output{"GLO", "TPCITSAB_CLID", 0});
+  auto& matchLabels = mUseMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"GLO", "TPCITS_MC", 0}) : dummyMCLab;
+  auto& ABTrackletLabels = mUseMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"GLO", "TPCITSAB_MC", 0}) : dummyMCLabAB;
+  auto& calib = mCalibMode ? pc.outputs().make<std::vector<o2::dataformats::Triplet<float, float, float>>>(Output{"GLO", "TPCITS_VDTGL", 0}) : dummyCalib;
 
   mMatching.run(recoData, matchedTracks, ABTrackletRefs, ABTrackletClusterIDs, matchLabels, ABTrackletLabels, calib);
 
diff --git a/Detectors/GlobalTrackingWorkflow/src/VertexTrackMatcherSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/VertexTrackMatcherSpec.cxx
index 281fd06dea7eb..e5d2afb1ee700 100644
--- a/Detectors/GlobalTrackingWorkflow/src/VertexTrackMatcherSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/VertexTrackMatcherSpec.cxx
@@ -78,8 +78,8 @@ void VertexTrackMatcherSpec::run(ProcessingContext& pc)
 
   mMatcher.process(recoData, trackIndex, vtxRefs);
 
-  pc.outputs().snapshot(Output{"GLO", "PVTX_TRMTC", 0, Lifetime::Timeframe}, trackIndex);
-  pc.outputs().snapshot(Output{"GLO", "PVTX_TRMTCREFS", 0, Lifetime::Timeframe}, vtxRefs);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_TRMTC", 0}, trackIndex);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_TRMTCREFS", 0}, vtxRefs);
 
   mTimer.Stop();
   LOG(info) << "Made " << trackIndex.size() << " track associations for " << recoData.getPrimaryVertices().size()
diff --git a/Detectors/GlobalTrackingWorkflow/study/src/TPCDataFilter.cxx b/Detectors/GlobalTrackingWorkflow/study/src/TPCDataFilter.cxx
index ed06e014184b6..8a962686d743e 100644
--- a/Detectors/GlobalTrackingWorkflow/study/src/TPCDataFilter.cxx
+++ b/Detectors/GlobalTrackingWorkflow/study/src/TPCDataFilter.cxx
@@ -107,10 +107,10 @@ void TPCDataFilter::run(ProcessingContext& pc)
 void TPCDataFilter::sendOutput(ProcessingContext& pc)
 {
 
-  pc.outputs().snapshot(Output{"TPC", "TRACKSF", 0, Lifetime::Timeframe}, mTracksFiltered);
-  pc.outputs().snapshot(Output{"TPC", "CLUSREFSF", 0, Lifetime::Timeframe}, mTrackClusIdxFiltered);
+  pc.outputs().snapshot(Output{"TPC", "TRACKSF", 0}, mTracksFiltered);
+  pc.outputs().snapshot(Output{"TPC", "CLUSREFSF", 0}, mTrackClusIdxFiltered);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"TPC", "TRACKSMCLBLF", 0, Lifetime::Timeframe}, mTPCTrkLabelsFiltered);
+    pc.outputs().snapshot(Output{"TPC", "TRACKSMCLBLF", 0}, mTPCTrkLabelsFiltered);
   }
 
   o2::tpc::TPCSectorHeader clusterOutputSectorHeader{0};
@@ -118,7 +118,7 @@ void TPCDataFilter::sendOutput(ProcessingContext& pc)
   for (int i = 0; i < o2::tpc::constants::MAXSECTOR; i++) {
     clusterOutputSectorHeader.sectorBits = (1ul << i);
     o2::header::DataHeader::SubSpecificationType subspec = i;
-    char* buffer = pc.outputs().make<char>({o2::header::gDataOriginTPC, "CLUSTERNATIVEF", subspec, Lifetime::Timeframe, {clusterOutputSectorHeader}},
+    char* buffer = pc.outputs().make<char>({o2::header::gDataOriginTPC, "CLUSTERNATIVEF", subspec, {clusterOutputSectorHeader}},
                                            mClusFiltered.nClustersSector[i] * sizeof(*mClusFiltered.clustersLinear) + sizeof(o2::tpc::ClusterCountIndex))
                      .data();
     o2::tpc::ClusterCountIndex* outIndex = reinterpret_cast<o2::tpc::ClusterCountIndex*>(buffer);
@@ -138,7 +138,7 @@ void TPCDataFilter::sendOutput(ProcessingContext& pc)
       }
       o2::dataformats::ConstMCLabelContainer contflat;
       cont.flatten_to(contflat);
-      pc.outputs().snapshot({o2::header::gDataOriginTPC, "CLNATIVEMCLBLF", subspec, Lifetime::Timeframe, {clusterOutputSectorHeader}}, contflat);
+      pc.outputs().snapshot({o2::header::gDataOriginTPC, "CLNATIVEMCLBLF", subspec, {clusterOutputSectorHeader}}, contflat);
     }
   }
 }
diff --git a/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCInterpolationSpec.cxx b/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCInterpolationSpec.cxx
index 69554d3d495a4..89e3ae19cf65b 100644
--- a/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCInterpolationSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCInterpolationSpec.cxx
@@ -133,18 +133,18 @@ void TPCInterpolationDPL::run(ProcessingContext& pc)
   LOGF(info, "TPC interpolation timing: Cpu: %.3e Real: %.3e s", mTimer.CpuTime(), mTimer.RealTime());
   if (SpacePointsCalibConfParam::Instance().writeUnfiltered) {
     // these are the residuals and tracks before outlier rejection; they are not used in production
-    pc.outputs().snapshot(Output{"GLO", "TPCINT_RES", 0, Lifetime::Timeframe}, mInterpolation.getClusterResidualsUnfiltered());
+    pc.outputs().snapshot(Output{"GLO", "TPCINT_RES", 0}, mInterpolation.getClusterResidualsUnfiltered());
     if (mSendTrackData) {
-      pc.outputs().snapshot(Output{"GLO", "TPCINT_TRK", 0, Lifetime::Timeframe}, mInterpolation.getReferenceTracksUnfiltered());
+      pc.outputs().snapshot(Output{"GLO", "TPCINT_TRK", 0}, mInterpolation.getReferenceTracksUnfiltered());
     }
   }
-  pc.outputs().snapshot(Output{"GLO", "UNBINNEDRES", 0, Lifetime::Timeframe}, mInterpolation.getClusterResiduals());
-  pc.outputs().snapshot(Output{"GLO", "TRKREFS", 0, Lifetime::Timeframe}, mInterpolation.getTrackDataCompact());
+  pc.outputs().snapshot(Output{"GLO", "UNBINNEDRES", 0}, mInterpolation.getClusterResiduals());
+  pc.outputs().snapshot(Output{"GLO", "TRKREFS", 0}, mInterpolation.getTrackDataCompact());
   if (mSendTrackData) {
-    pc.outputs().snapshot(Output{"GLO", "TRKDATA", 0, Lifetime::Timeframe}, mInterpolation.getReferenceTracks());
+    pc.outputs().snapshot(Output{"GLO", "TRKDATA", 0}, mInterpolation.getReferenceTracks());
   }
   if (mDebugOutput) {
-    pc.outputs().snapshot(Output{"GLO", "TRKDATAEXT", 0, Lifetime::Timeframe}, mInterpolation.getTrackDataExtended());
+    pc.outputs().snapshot(Output{"GLO", "TRKDATAEXT", 0}, mInterpolation.getTrackDataExtended());
   }
   mInterpolation.reset();
 }
diff --git a/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCUnbinnedResidualReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCUnbinnedResidualReaderSpec.cxx
index af50b32bc8bba..55da5a5e71e44 100644
--- a/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCUnbinnedResidualReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCUnbinnedResidualReaderSpec.cxx
@@ -56,11 +56,11 @@ void TPCUnbinnedResidualReader::run(ProcessingContext& pc)
   assert(currEntry < mTreeIn->GetEntries()); // this should not happen
   mTreeIn->GetEntry(currEntry);
   LOG(info) << "Pushing " << mUnbinnedResid.size() << " unbinned residuals at entry " << currEntry;
-  pc.outputs().snapshot(Output{"GLO", "UNBINNEDRES", 0, Lifetime::Timeframe}, mUnbinnedResid);
-  pc.outputs().snapshot(Output{"GLO", "TRKREFS", 0, Lifetime::Timeframe}, mTrackDataCompact);
+  pc.outputs().snapshot(Output{"GLO", "UNBINNEDRES", 0}, mUnbinnedResid);
+  pc.outputs().snapshot(Output{"GLO", "TRKREFS", 0}, mTrackDataCompact);
   if (mTrackInput) {
     LOG(info) << "Pushing " << mTrackData.size() << " reference tracks for these residuals";
-    pc.outputs().snapshot(Output{"GLO", "TRKDATA", 0, Lifetime::Timeframe}, mTrackData);
+    pc.outputs().snapshot(Output{"GLO", "TRKDATA", 0}, mTrackData);
   }
 
   if (mTreeIn->GetReadEntry() + 1 >= mTreeIn->GetEntries()) {
diff --git a/Detectors/HMPID/workflow/src/ClustersReaderSpec.cxx b/Detectors/HMPID/workflow/src/ClustersReaderSpec.cxx
index e71cd42845dd4..9ac5074acb505 100644
--- a/Detectors/HMPID/workflow/src/ClustersReaderSpec.cxx
+++ b/Detectors/HMPID/workflow/src/ClustersReaderSpec.cxx
@@ -71,8 +71,8 @@ void ClusterReaderTask::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
 
-  pc.outputs().snapshot(Output{"HMP", "CLUSTERS", 0, Lifetime::Timeframe}, mClustersFromFile);
-  pc.outputs().snapshot(Output{"HMP", "INTRECORDS1", 0, Lifetime::Timeframe}, mClusterTriggersFromFile);
+  pc.outputs().snapshot(Output{"HMP", "CLUSTERS", 0}, mClustersFromFile);
+  pc.outputs().snapshot(Output{"HMP", "INTRECORDS1", 0}, mClusterTriggersFromFile);
   mClustersReceived += mClustersFromFile.size();
   LOG(info) << "[HMPID ClusterReader - run() ] clusters  = " << mClustersFromFile.size();
 
diff --git a/Detectors/HMPID/workflow/src/DataDecoderSpec.cxx b/Detectors/HMPID/workflow/src/DataDecoderSpec.cxx
index 3a3b5a6338d81..1a6509fc75654 100644
--- a/Detectors/HMPID/workflow/src/DataDecoderSpec.cxx
+++ b/Detectors/HMPID/workflow/src/DataDecoderSpec.cxx
@@ -87,8 +87,8 @@ void DataDecoderTask::run(framework::ProcessingContext& pc)
   //  decodeReadout(pc);
   // decodeRawFile(pc);
 
-  pc.outputs().snapshot(o2::framework::Output{"HMP", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, mDeco->mDigits);
-  pc.outputs().snapshot(o2::framework::Output{"HMP", "INTRECORDS", 0, o2::framework::Lifetime::Timeframe}, mDeco->mIntReco);
+  pc.outputs().snapshot(o2::framework::Output{"HMP", "DIGITS", 0}, mDeco->mDigits);
+  pc.outputs().snapshot(o2::framework::Output{"HMP", "INTRECORDS", 0}, mDeco->mIntReco);
 
   LOG(debug) << "Writing   Digitis=" << mDeco->mDigits.size() << "/" << mTotalDigits << " Frame=" << mTotalFrames << " IntRec " << mDeco->mIntReco;
   mExTimer.elapseMes("Decoding... Digits decoded = " + std::to_string(mTotalDigits) + " Frames received = " + std::to_string(mTotalFrames));
diff --git a/Detectors/HMPID/workflow/src/DataDecoderSpec2.cxx b/Detectors/HMPID/workflow/src/DataDecoderSpec2.cxx
index 61051ab48577e..92a966938786a 100644
--- a/Detectors/HMPID/workflow/src/DataDecoderSpec2.cxx
+++ b/Detectors/HMPID/workflow/src/DataDecoderSpec2.cxx
@@ -93,8 +93,8 @@ void DataDecoderTask2::run(framework::ProcessingContext& pc)
 
   // Output the Digits/Triggers vector
   orderTriggers();
-  pc.outputs().snapshot(o2::framework::Output{"HMP", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, mDeco->mDigits);
-  pc.outputs().snapshot(o2::framework::Output{"HMP", "INTRECORDS", 0, o2::framework::Lifetime::Timeframe}, mTriggers);
+  pc.outputs().snapshot(o2::framework::Output{"HMP", "DIGITS", 0}, mDeco->mDigits);
+  pc.outputs().snapshot(o2::framework::Output{"HMP", "INTRECORDS", 0}, mTriggers);
 
   mExTimer.elapseMes("Decoding... Digits decoded = " + std::to_string(mTotalDigits) + " Frames received = " + std::to_string(mTotalFrames));
   return;
diff --git a/Detectors/HMPID/workflow/src/DigitsReaderSpec.cxx b/Detectors/HMPID/workflow/src/DigitsReaderSpec.cxx
index ed83db476b01a..88f6df2bce2e7 100644
--- a/Detectors/HMPID/workflow/src/DigitsReaderSpec.cxx
+++ b/Detectors/HMPID/workflow/src/DigitsReaderSpec.cxx
@@ -115,8 +115,8 @@ void DigitReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
 
-  pc.outputs().snapshot(Output{"HMP", "DIGITS", 0, Lifetime::Timeframe}, mDigitsFromFile);
-  pc.outputs().snapshot(Output{"HMP", "INTRECORDS", 0, Lifetime::Timeframe}, mTriggersFromFile);
+  pc.outputs().snapshot(Output{"HMP", "DIGITS", 0}, mDigitsFromFile);
+  pc.outputs().snapshot(Output{"HMP", "INTRECORDS", 0}, mTriggersFromFile);
   mDigitsReceived += mDigitsFromFile.size();
   LOG(info) << "[HMPID DigitsReader - run() ] digits  = " << mDigitsFromFile.size();
 
diff --git a/Detectors/HMPID/workflow/src/DigitsToClustersSpec.cxx b/Detectors/HMPID/workflow/src/DigitsToClustersSpec.cxx
index 377bc0f7af45e..81c04a9875c51 100644
--- a/Detectors/HMPID/workflow/src/DigitsToClustersSpec.cxx
+++ b/Detectors/HMPID/workflow/src/DigitsToClustersSpec.cxx
@@ -116,8 +116,8 @@ void DigitsToClustersTask::run(framework::ProcessingContext& pc)
   mDigitsReceived += digits.size();
   mClustersReceived += clusters.size();
 
-  pc.outputs().snapshot(o2::framework::Output{"HMP", "CLUSTERS", 0, o2::framework::Lifetime::Timeframe}, clusters);
-  pc.outputs().snapshot(o2::framework::Output{"HMP", "INTRECORDS1", 0, o2::framework::Lifetime::Timeframe}, clusterTriggers);
+  pc.outputs().snapshot(o2::framework::Output{"HMP", "CLUSTERS", 0}, clusters);
+  pc.outputs().snapshot(o2::framework::Output{"HMP", "INTRECORDS1", 0}, clusterTriggers);
 
   mExTimer.elapseMes("Clusterization of Digits received = " + std::to_string(mDigitsReceived));
   mExTimer.elapseMes("Clusterization of Clusters received = " + std::to_string(mClustersReceived));
diff --git a/Detectors/HMPID/workflow/src/EntropyEncoderSpec.cxx b/Detectors/HMPID/workflow/src/EntropyEncoderSpec.cxx
index 15bd0987a0ccc..95723f42d0fd6 100644
--- a/Detectors/HMPID/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/HMPID/workflow/src/EntropyEncoderSpec.cxx
@@ -72,7 +72,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"HMP", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"HMP", "CTFDATA", 0});
 
   auto iosize = mCTFCoder.encode(buffer, triggers, digits);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
diff --git a/Detectors/HMPID/workflow/src/HMPMatchedReaderSpec.cxx b/Detectors/HMPID/workflow/src/HMPMatchedReaderSpec.cxx
index e466a8a4e00c4..3ddb71e282a96 100644
--- a/Detectors/HMPID/workflow/src/HMPMatchedReaderSpec.cxx
+++ b/Detectors/HMPID/workflow/src/HMPMatchedReaderSpec.cxx
@@ -62,9 +62,9 @@ void HMPMatchedReader::run(ProcessingContext& pc)
   mTree->GetEntry(currEntry);
   LOG(debug) << "Pushing " << mMatches.size() << " HMP matchings at entry " << currEntry;
 
-  pc.outputs().snapshot(Output{o2::header::gDataOriginHMP, "MATCHES", 0, Lifetime::Timeframe}, mMatches);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginHMP, "MATCHES", 0}, mMatches);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginHMP, "MCLABELS", 0, Lifetime::Timeframe}, mLabelHMP);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginHMP, "MCLABELS", 0}, mLabelHMP);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/ITSMFT/ITS/QC/TestDataReaderWorkflow/src/TestDataReader.cxx b/Detectors/ITSMFT/ITS/QC/TestDataReaderWorkflow/src/TestDataReader.cxx
index d6ef8c947ed49..964f342c58b15 100644
--- a/Detectors/ITSMFT/ITS/QC/TestDataReaderWorkflow/src/TestDataReader.cxx
+++ b/Detectors/ITSMFT/ITS/QC/TestDataReaderWorkflow/src/TestDataReader.cxx
@@ -139,14 +139,14 @@ void TestDataReader::run(ProcessingContext& pc)
   if (mDiffFolderName.size() == 0) {
     cout << "No New Run -- No Need to Reset" << endl;
     mResetCommand = 0;
-    pc.outputs().snapshot(Output{"ITS", "TEST", 0, Lifetime::Timeframe}, mResetCommand);
+    pc.outputs().snapshot(Output{"ITS", "TEST", 0}, mResetCommand);
   }
 
   // New folders found, send the reset signal and reload configuration
   if (mDiffFolderName.size() > 0) {
     cout << "New Run Started -- Reset All Histograms" << endl;
     mResetCommand = 1;
-    pc.outputs().snapshot(Output{"ITS", "TEST", 0, Lifetime::Timeframe}, mResetCommand);
+    pc.outputs().snapshot(Output{"ITS", "TEST", 0}, mResetCommand);
     for (int i = 0; i < sNError; i++) {
       mErrors[i] = 0;
     }
@@ -264,11 +264,11 @@ void TestDataReader::run(ProcessingContext& pc)
         mErrorsVecTest.push_back(mErrors);
         mFileDone = 1;
         mFileInfo = mFileDone + mFileRemain * 10;
-        pc.outputs().snapshot(Output{"ITS", "Run", 0, Lifetime::Timeframe}, mRunNumber);
-        pc.outputs().snapshot(Output{"ITS", "File", 0, Lifetime::Timeframe}, mFileID);
-        pc.outputs().snapshot(Output{"ITS", "Error", 0, Lifetime::Timeframe}, mErrorsVecTest[0]);
-        pc.outputs().snapshot(Output{"ITS", "Finish", 0, Lifetime::Timeframe}, mFileInfo);
-        pc.outputs().snapshot(Output{"ITS", "DIGITS", 0, Lifetime::Timeframe}, mMultiDigitsTest);
+        pc.outputs().snapshot(Output{"ITS", "Run", 0}, mRunNumber);
+        pc.outputs().snapshot(Output{"ITS", "File", 0}, mFileID);
+        pc.outputs().snapshot(Output{"ITS", "Error", 0}, mErrorsVecTest[0]);
+        pc.outputs().snapshot(Output{"ITS", "Finish", 0}, mFileInfo);
+        pc.outputs().snapshot(Output{"ITS", "DIGITS", 0}, mMultiDigitsTest);
         mNewFileInj = 0;
         mErrorsVecTest.clear();
         mDigitsTest.clear();
@@ -422,10 +422,10 @@ void TestDataReader::run(ProcessingContext& pc)
 
     cout << "RunIDS = " << mRunNumber << "   FileIDS = " << mFileID << endl;
 
-    pc.outputs().snapshot(Output{"ITS", "Run", 0, Lifetime::Timeframe}, mRunNumber);
-    pc.outputs().snapshot(Output{"ITS", "File", 0, Lifetime::Timeframe}, mFileID);
+    pc.outputs().snapshot(Output{"ITS", "Run", 0}, mRunNumber);
+    pc.outputs().snapshot(Output{"ITS", "File", 0}, mFileID);
 
-    pc.outputs().snapshot(Output{"ITS", "Error", 0, Lifetime::Timeframe}, mErrorsVec[j]);
+    pc.outputs().snapshot(Output{"ITS", "Error", 0}, mErrorsVec[j]);
     mIndexPushEx = mIndexPush + mNDigits[j];
     LOG(debug) << "IndexPushEx = " << mIndexPushEx << "  mDigits.size() " << mDigits.size();
     if (mIndexPushEx > mDigits.size() - 5) {
@@ -436,11 +436,11 @@ void TestDataReader::run(ProcessingContext& pc)
 
     mFileInfo = mFileDone + mFileRemain * 10;
 
-    pc.outputs().snapshot(Output{"ITS", "Finish", 0, Lifetime::Timeframe}, mFileInfo);
+    pc.outputs().snapshot(Output{"ITS", "Finish", 0}, mFileInfo);
 
     LOG(debug) << "mIndexPush = " << mIndexPush << "    Chip ID Pushing " << mDigits[mIndexPush].getChipIndex();
 
-    pc.outputs().snapshot(Output{"ITS", "DIGITS", 0, Lifetime::Timeframe}, mMultiDigits);
+    pc.outputs().snapshot(Output{"ITS", "DIGITS", 0}, mMultiDigits);
 
     mMultiDigits.clear();
     mIndexPush = mIndexPush + mNDigits[j];
@@ -453,7 +453,7 @@ void TestDataReader::run(ProcessingContext& pc)
   //            << "mIndexPush = " << mIndexPush << "     mDigits.size() = " << mDigits.size();
   // while (mIndexPush < mDigits.size()) {
   // LOG(debug) << "mDigits.size() = " << mDigits.size();
-  //   pc.outputs().snapshot(Output{"ITS", "DIGITS", 0, Lifetime::Timeframe}, mDigits[mIndexPush++]);
+  //   pc.outputs().snapshot(Output{"ITS", "DIGITS", 0}, mDigits[mIndexPush++]);
   //   if (mIndexPush % 100000 == 0)
   //     LOG(debug) << "mIndexPush = " << mIndexPush << "    Chip ID Pushing " << mDigits[mIndexPush].getChipIndex();
   // }
@@ -483,7 +483,7 @@ void TestDataReader::run(ProcessingContext& pc)
     j = 0;
     mNDigits.clear();
     mFileDone = 1;
-    pc.outputs().snapshot(Output{"TST", "Finish", 0, Lifetime::Timeframe}, mFileDone);
+    pc.outputs().snapshot(Output{"TST", "Finish", 0}, mFileDone);
     PercentDone = 0;
     mErrorsVec.clear();
   }
diff --git a/Detectors/ITSMFT/ITS/workflow/src/ClustererSpec.cxx b/Detectors/ITSMFT/ITS/workflow/src/ClustererSpec.cxx
index 89596e8357b74..71987d0480566 100644
--- a/Detectors/ITSMFT/ITS/workflow/src/ClustererSpec.cxx
+++ b/Detectors/ITSMFT/ITS/workflow/src/ClustererSpec.cxx
@@ -88,17 +88,17 @@ void ClustererDPL::run(ProcessingContext& pc)
     clusterLabels = std::make_unique<o2::dataformats::MCTruthContainer<o2::MCCompLabel>>();
   }
   mClusterer->process(mNThreads, reader, &clusCompVec, &clusPattVec, &clusROFVec, clusterLabels.get());
-  pc.outputs().snapshot(Output{orig, "COMPCLUSTERS", 0, Lifetime::Timeframe}, clusCompVec);
-  pc.outputs().snapshot(Output{orig, "CLUSTERSROF", 0, Lifetime::Timeframe}, clusROFVec);
-  pc.outputs().snapshot(Output{orig, "PATTERNS", 0, Lifetime::Timeframe}, clusPattVec);
+  pc.outputs().snapshot(Output{orig, "COMPCLUSTERS", 0}, clusCompVec);
+  pc.outputs().snapshot(Output{orig, "CLUSTERSROF", 0}, clusROFVec);
+  pc.outputs().snapshot(Output{orig, "PATTERNS", 0}, clusPattVec);
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{orig, "CLUSTERSMCTR", 0, Lifetime::Timeframe}, *clusterLabels.get()); // at the moment requires snapshot
+    pc.outputs().snapshot(Output{orig, "CLUSTERSMCTR", 0}, *clusterLabels.get()); // at the moment requires snapshot
     std::vector<o2::itsmft::MC2ROFRecord> clusterMC2ROframes(mc2rofs.size());
     for (int i = mc2rofs.size(); i--;) {
       clusterMC2ROframes[i] = mc2rofs[i]; // Simply, replicate it from digits ?
     }
-    pc.outputs().snapshot(Output{orig, "CLUSTERSMC2ROF", 0, Lifetime::Timeframe}, clusterMC2ROframes);
+    pc.outputs().snapshot(Output{orig, "CLUSTERSMC2ROF", 0}, clusterMC2ROframes);
   }
 
   // TODO: in principle, after masking "overflow" pixels the MC2ROFRecord maxROF supposed to change, nominally to minROF
diff --git a/Detectors/ITSMFT/ITS/workflow/src/CookedTrackerSpec.cxx b/Detectors/ITSMFT/ITS/workflow/src/CookedTrackerSpec.cxx
index e3f585c81cd97..e4c9d43541e1f 100644
--- a/Detectors/ITSMFT/ITS/workflow/src/CookedTrackerSpec.cxx
+++ b/Detectors/ITSMFT/ITS/workflow/src/CookedTrackerSpec.cxx
@@ -95,7 +95,7 @@ void CookedTrackerDPL::run(ProcessingContext& pc)
     physTriggers = pc.inputs().get<gsl::span<o2::itsmft::PhysTrigger>>("phystrig");
   }
 
-  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "ITSTrackROF", 0, Lifetime::Timeframe}, rofsinput.begin(), rofsinput.end());
+  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "ITSTrackROF", 0}, rofsinput.begin(), rofsinput.end());
 
   std::unique_ptr<const o2::dataformats::MCTruthContainer<o2::MCCompLabel>> labels;
   gsl::span<itsmft::MC2ROFRecord const> mc2rofs;
@@ -116,11 +116,11 @@ void CookedTrackerDPL::run(ProcessingContext& pc)
   o2::its::ROframe event(0, 7);
   mVertexerPtr->adoptTimeFrame(mTimeFrame);
 
-  auto& vertROFvec = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "VERTICESROF", 0, Lifetime::Timeframe});
-  auto& vertices = pc.outputs().make<std::vector<Vertex>>(Output{"ITS", "VERTICES", 0, Lifetime::Timeframe});
-  auto& tracks = pc.outputs().make<std::vector<o2::its::TrackITS>>(Output{"ITS", "TRACKS", 0, Lifetime::Timeframe});
-  auto& clusIdx = pc.outputs().make<std::vector<int>>(Output{"ITS", "TRACKCLSID", 0, Lifetime::Timeframe});
-  auto& irFrames = pc.outputs().make<std::vector<o2::dataformats::IRFrame>>(Output{"ITS", "IRFRAMES", 0, Lifetime::Timeframe});
+  auto& vertROFvec = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "VERTICESROF", 0});
+  auto& vertices = pc.outputs().make<std::vector<Vertex>>(Output{"ITS", "VERTICES", 0});
+  auto& tracks = pc.outputs().make<std::vector<o2::its::TrackITS>>(Output{"ITS", "TRACKS", 0});
+  auto& clusIdx = pc.outputs().make<std::vector<int>>(Output{"ITS", "TRACKCLSID", 0});
+  auto& irFrames = pc.outputs().make<std::vector<o2::dataformats::IRFrame>>(Output{"ITS", "IRFRAMES", 0});
 
   const auto& alpParams = o2::itsmft::DPLAlpideParam<o2::detectors::DetID::ITS>::Instance(); // RS: this should come from CCDB
   int nBCPerTF = mTracker.getContinuousMode() ? alpParams.roFrameLengthInBC : alpParams.roFrameLengthTrig;
@@ -201,8 +201,8 @@ void CookedTrackerDPL::run(ProcessingContext& pc)
   LOG(info) << "ITSCookedTracker pushed " << tracks.size() << " tracks and " << vertices.size() << " vertices";
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"ITS", "TRACKSMCTR", 0, Lifetime::Timeframe}, trackLabels);
-    pc.outputs().snapshot(Output{"ITS", "ITSTrackMC2ROF", 0, Lifetime::Timeframe}, mc2rofs);
+    pc.outputs().snapshot(Output{"ITS", "TRACKSMCTR", 0}, trackLabels);
+    pc.outputs().snapshot(Output{"ITS", "ITSTrackMC2ROF", 0}, mc2rofs);
   }
   mTimer.Stop();
 }
diff --git a/Detectors/ITSMFT/ITS/workflow/src/DCSGeneratorSpec.cxx b/Detectors/ITSMFT/ITS/workflow/src/DCSGeneratorSpec.cxx
index 48bbaed171a85..2ea0c833f3871 100644
--- a/Detectors/ITSMFT/ITS/workflow/src/DCSGeneratorSpec.cxx
+++ b/Detectors/ITSMFT/ITS/workflow/src/DCSGeneratorSpec.cxx
@@ -117,7 +117,7 @@ void ITSDCSDataGenerator::run(o2::framework::ProcessingContext& pc)
   auto timeNow = std::chrono::system_clock::now();
   timingInfo.creation = std::chrono::duration_cast<std::chrono::milliseconds>(timeNow.time_since_epoch()).count(); // in ms
 
-  pc.outputs().snapshot(Output{"ITS", mDataDescription, 0, Lifetime::Timeframe}, dpcoms);
+  pc.outputs().snapshot(Output{"ITS", mDataDescription, 0}, dpcoms);
   mTFs++;
 }
 } // namespace
diff --git a/Detectors/ITSMFT/ITS/workflow/src/TrackReaderSpec.cxx b/Detectors/ITSMFT/ITS/workflow/src/TrackReaderSpec.cxx
index 416ba5b3b07cc..8e72faae9fd37 100644
--- a/Detectors/ITSMFT/ITS/workflow/src/TrackReaderSpec.cxx
+++ b/Detectors/ITSMFT/ITS/workflow/src/TrackReaderSpec.cxx
@@ -44,14 +44,14 @@ void TrackReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mTracks.size() << " track in " << mROFRec.size() << " ROFs at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "ITSTrackROF", 0, Lifetime::Timeframe}, mROFRec);
-  pc.outputs().snapshot(Output{mOrigin, "TRACKS", 0, Lifetime::Timeframe}, mTracks);
-  pc.outputs().snapshot(Output{mOrigin, "TRACKCLSID", 0, Lifetime::Timeframe}, mClusInd);
-  pc.outputs().snapshot(Output{"ITS", "VERTICES", 0, Lifetime::Timeframe}, mVertices);
-  pc.outputs().snapshot(Output{"ITS", "VERTICESROF", 0, Lifetime::Timeframe}, mVerticesROFRec);
+  pc.outputs().snapshot(Output{mOrigin, "ITSTrackROF", 0}, mROFRec);
+  pc.outputs().snapshot(Output{mOrigin, "TRACKS", 0}, mTracks);
+  pc.outputs().snapshot(Output{mOrigin, "TRACKCLSID", 0}, mClusInd);
+  pc.outputs().snapshot(Output{"ITS", "VERTICES", 0}, mVertices);
+  pc.outputs().snapshot(Output{"ITS", "VERTICESROF", 0}, mVerticesROFRec);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{mOrigin, "TRACKSMCTR", 0, Lifetime::Timeframe}, mMCTruth);
-    pc.outputs().snapshot(Output{mOrigin, "VERTICESMCTR", 0, Lifetime::Timeframe}, mMCVertTruth);
+    pc.outputs().snapshot(Output{mOrigin, "TRACKSMCTR", 0}, mMCTruth);
+    pc.outputs().snapshot(Output{mOrigin, "VERTICESMCTR", 0}, mMCVertTruth);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx b/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx
index 35671f646bd04..331e42907530a 100644
--- a/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx
+++ b/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx
@@ -150,8 +150,8 @@ void TrackerDPL::run(ProcessingContext& pc)
   }
 
   auto rofsinput = pc.inputs().get<gsl::span<o2::itsmft::ROFRecord>>("ROframes");
-  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "ITSTrackROF", 0, Lifetime::Timeframe}, rofsinput.begin(), rofsinput.end());
-  auto& irFrames = pc.outputs().make<std::vector<o2::dataformats::IRFrame>>(Output{"ITS", "IRFRAMES", 0, Lifetime::Timeframe});
+  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "ITSTrackROF", 0}, rofsinput.begin(), rofsinput.end());
+  auto& irFrames = pc.outputs().make<std::vector<o2::dataformats::IRFrame>>(Output{"ITS", "IRFRAMES", 0});
   const auto& alpParams = o2::itsmft::DPLAlpideParam<o2::detectors::DetID::ITS>::Instance(); // RS: this should come from CCDB
 
   irFrames.reserve(rofs.size());
@@ -164,19 +164,19 @@ void TrackerDPL::run(ProcessingContext& pc)
   if (mIsMC) {
     labels = pc.inputs().get<const dataformats::MCTruthContainer<MCCompLabel>*>("itsmclabels").release();
     // get the array as read-only span, a snapshot is sent forward
-    pc.outputs().snapshot(Output{"ITS", "ITSTrackMC2ROF", 0, Lifetime::Timeframe}, pc.inputs().get<gsl::span<itsmft::MC2ROFRecord>>("ITSMC2ROframes"));
+    pc.outputs().snapshot(Output{"ITS", "ITSTrackMC2ROF", 0}, pc.inputs().get<gsl::span<itsmft::MC2ROFRecord>>("ITSMC2ROframes"));
     LOG(info) << labels->getIndexedSize() << " MC label objects , in " << mc2rofs.size() << " MC events";
   }
 
-  auto& allClusIdx = pc.outputs().make<std::vector<int>>(Output{"ITS", "TRACKCLSID", 0, Lifetime::Timeframe});
-  auto& allTracks = pc.outputs().make<std::vector<o2::its::TrackITS>>(Output{"ITS", "TRACKS", 0, Lifetime::Timeframe});
-  auto& vertROFvec = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "VERTICESROF", 0, Lifetime::Timeframe});
-  auto& vertices = pc.outputs().make<std::vector<Vertex>>(Output{"ITS", "VERTICES", 0, Lifetime::Timeframe});
+  auto& allClusIdx = pc.outputs().make<std::vector<int>>(Output{"ITS", "TRACKCLSID", 0});
+  auto& allTracks = pc.outputs().make<std::vector<o2::its::TrackITS>>(Output{"ITS", "TRACKS", 0});
+  auto& vertROFvec = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "VERTICESROF", 0});
+  auto& vertices = pc.outputs().make<std::vector<Vertex>>(Output{"ITS", "VERTICES", 0});
 
   // MC
   static pmr::vector<o2::MCCompLabel> dummyMCLabTracks, dummyMCLabVerts;
-  auto& allTrackLabels = mIsMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"ITS", "TRACKSMCTR", 0, Lifetime::Timeframe}) : dummyMCLabTracks;
-  auto& allVerticesLabels = mIsMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"ITS", "VERTICESMCTR", 0, Lifetime::Timeframe}) : dummyMCLabVerts;
+  auto& allTrackLabels = mIsMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"ITS", "TRACKSMCTR", 0}) : dummyMCLabTracks;
+  auto& allVerticesLabels = mIsMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"ITS", "VERTICESMCTR", 0}) : dummyMCLabVerts;
 
   std::uint32_t roFrame = 0;
 
diff --git a/Detectors/ITSMFT/ITS/workflow/src/VertexReaderSpec.cxx b/Detectors/ITSMFT/ITS/workflow/src/VertexReaderSpec.cxx
index bbf033a31abee..e92f08af23c0d 100644
--- a/Detectors/ITSMFT/ITS/workflow/src/VertexReaderSpec.cxx
+++ b/Detectors/ITSMFT/ITS/workflow/src/VertexReaderSpec.cxx
@@ -41,8 +41,8 @@ void VertexReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mVerticesPtr->size() << " vertices in " << mVerticesROFRecPtr->size()
             << " ROFs at entry " << ent;
-  pc.outputs().snapshot(Output{"ITS", "VERTICES", 0, Lifetime::Timeframe}, mVertices);
-  pc.outputs().snapshot(Output{"ITS", "VERTICESROF", 0, Lifetime::Timeframe}, mVerticesROFRec);
+  pc.outputs().snapshot(Output{"ITS", "VERTICES", 0}, mVertices);
+  pc.outputs().snapshot(Output{"ITS", "VERTICESROF", 0}, mVerticesROFRec);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/ITSMFT/MFT/workflow/src/ClustererSpec.cxx b/Detectors/ITSMFT/MFT/workflow/src/ClustererSpec.cxx
index f6279be58cfd9..bc8568112e447 100644
--- a/Detectors/ITSMFT/MFT/workflow/src/ClustererSpec.cxx
+++ b/Detectors/ITSMFT/MFT/workflow/src/ClustererSpec.cxx
@@ -89,17 +89,17 @@ void ClustererDPL::run(ProcessingContext& pc)
     clusterLabels = std::make_unique<o2::dataformats::MCTruthContainer<o2::MCCompLabel>>();
   }
   mClusterer->process(mNThreads, reader, &clusCompVec, &clusPattVec, &clusROFVec, clusterLabels.get());
-  pc.outputs().snapshot(Output{orig, "COMPCLUSTERS", 0, Lifetime::Timeframe}, clusCompVec);
-  pc.outputs().snapshot(Output{orig, "CLUSTERSROF", 0, Lifetime::Timeframe}, clusROFVec);
-  pc.outputs().snapshot(Output{orig, "PATTERNS", 0, Lifetime::Timeframe}, clusPattVec);
+  pc.outputs().snapshot(Output{orig, "COMPCLUSTERS", 0}, clusCompVec);
+  pc.outputs().snapshot(Output{orig, "CLUSTERSROF", 0}, clusROFVec);
+  pc.outputs().snapshot(Output{orig, "PATTERNS", 0}, clusPattVec);
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{orig, "CLUSTERSMCTR", 0, Lifetime::Timeframe}, *clusterLabels.get()); // at the moment requires snapshot
+    pc.outputs().snapshot(Output{orig, "CLUSTERSMCTR", 0}, *clusterLabels.get()); // at the moment requires snapshot
     std::vector<o2::itsmft::MC2ROFRecord> clusterMC2ROframes(mc2rofs.size());
     for (int i = mc2rofs.size(); i--;) {
       clusterMC2ROframes[i] = mc2rofs[i]; // Simply, replicate it from digits ?
     }
-    pc.outputs().snapshot(Output{orig, "CLUSTERSMC2ROF", 0, Lifetime::Timeframe}, clusterMC2ROframes);
+    pc.outputs().snapshot(Output{orig, "CLUSTERSMC2ROF", 0}, clusterMC2ROframes);
   }
 
   // TODO: in principle, after masking "overflow" pixels the MC2ROFRecord maxROF supposed to change, nominally to minROF
diff --git a/Detectors/ITSMFT/MFT/workflow/src/TrackReaderSpec.cxx b/Detectors/ITSMFT/MFT/workflow/src/TrackReaderSpec.cxx
index 8d9b889d4d2aa..1a2ae573af536 100644
--- a/Detectors/ITSMFT/MFT/workflow/src/TrackReaderSpec.cxx
+++ b/Detectors/ITSMFT/MFT/workflow/src/TrackReaderSpec.cxx
@@ -45,11 +45,11 @@ void TrackReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mTracks.size() << " track in " << mROFRec.size() << " ROFs at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "MFTTrackROF", 0, Lifetime::Timeframe}, mROFRec);
-  pc.outputs().snapshot(Output{mOrigin, "TRACKS", 0, Lifetime::Timeframe}, mTracks);
-  pc.outputs().snapshot(Output{mOrigin, "TRACKCLSID", 0, Lifetime::Timeframe}, mClusInd);
+  pc.outputs().snapshot(Output{mOrigin, "MFTTrackROF", 0}, mROFRec);
+  pc.outputs().snapshot(Output{mOrigin, "TRACKS", 0}, mTracks);
+  pc.outputs().snapshot(Output{mOrigin, "TRACKCLSID", 0}, mClusInd);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{mOrigin, "TRACKSMCTR", 0, Lifetime::Timeframe}, mMCTruth);
+    pc.outputs().snapshot(Output{mOrigin, "TRACKSMCTR", 0}, mMCTruth);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/ITSMFT/MFT/workflow/src/TrackerSpec.cxx b/Detectors/ITSMFT/MFT/workflow/src/TrackerSpec.cxx
index 7bf7d27a69e4c..5d22a6f2d2306 100644
--- a/Detectors/ITSMFT/MFT/workflow/src/TrackerSpec.cxx
+++ b/Detectors/ITSMFT/MFT/workflow/src/TrackerSpec.cxx
@@ -72,7 +72,7 @@ void TrackerDPL::run(ProcessingContext& pc)
   // the output vector however is created directly inside the message memory thus avoiding copy by
   // snapshot
   auto rofsinput = pc.inputs().get<const std::vector<o2::itsmft::ROFRecord>>("ROframes");
-  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"MFT", "MFTTrackROF", 0, Lifetime::Timeframe}, rofsinput.begin(), rofsinput.end());
+  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"MFT", "MFTTrackROF", 0}, rofsinput.begin(), rofsinput.end());
 
   ROFFilter filter = [](const o2::itsmft::ROFRecord& r) { return true; };
 
@@ -104,12 +104,12 @@ void TrackerDPL::run(ProcessingContext& pc)
     LOG(info) << labels->getIndexedSize() << " MC label objects , in " << mc2rofs.size() << " MC events";
   }
 
-  auto& allClusIdx = pc.outputs().make<std::vector<int>>(Output{"MFT", "TRACKCLSID", 0, Lifetime::Timeframe});
+  auto& allClusIdx = pc.outputs().make<std::vector<int>>(Output{"MFT", "TRACKCLSID", 0});
   std::vector<o2::MCCompLabel> trackLabels;
   std::vector<o2::MCCompLabel> allTrackLabels;
   std::vector<o2::mft::TrackLTF> tracks;
   std::vector<o2::mft::TrackLTFL> tracksL;
-  auto& allTracksMFT = pc.outputs().make<std::vector<o2::mft::TrackMFT>>(Output{"MFT", "TRACKS", 0, Lifetime::Timeframe});
+  auto& allTracksMFT = pc.outputs().make<std::vector<o2::mft::TrackMFT>>(Output{"MFT", "TRACKS", 0});
 
   std::uint32_t roFrameId = 0;
   int nROFs = rofs.size();
@@ -327,8 +327,8 @@ void TrackerDPL::run(ProcessingContext& pc)
   LOG(info) << "MFTTracker pushed " << allTracksMFT.size() << " tracks";
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"MFT", "TRACKSMCTR", 0, Lifetime::Timeframe}, allTrackLabels);
-    pc.outputs().snapshot(Output{"MFT", "TRACKSMC2ROF", 0, Lifetime::Timeframe}, mc2rofs);
+    pc.outputs().snapshot(Output{"MFT", "TRACKSMCTR", 0}, allTrackLabels);
+    pc.outputs().snapshot(Output{"MFT", "TRACKSMC2ROF", 0}, mc2rofs);
   }
 
   mTimer[SWTot].Stop();
diff --git a/Detectors/ITSMFT/common/workflow/src/ClusterReaderSpec.cxx b/Detectors/ITSMFT/common/workflow/src/ClusterReaderSpec.cxx
index ad7f26b1b4680..ea906056c7898 100644
--- a/Detectors/ITSMFT/common/workflow/src/ClusterReaderSpec.cxx
+++ b/Detectors/ITSMFT/common/workflow/src/ClusterReaderSpec.cxx
@@ -58,18 +58,18 @@ void ClusterReader::run(ProcessingContext& pc)
 
   // This is a very ugly way of providing DataDescription, which anyway does not need to contain detector name.
   // To be fixed once the names-definition class is ready
-  pc.outputs().snapshot(Output{mOrigin, "CLUSTERSROF", 0, Lifetime::Timeframe}, mClusROFRec);
-  pc.outputs().snapshot(Output{mOrigin, "COMPCLUSTERS", 0, Lifetime::Timeframe}, mClusterCompArray);
+  pc.outputs().snapshot(Output{mOrigin, "CLUSTERSROF", 0}, mClusROFRec);
+  pc.outputs().snapshot(Output{mOrigin, "COMPCLUSTERS", 0}, mClusterCompArray);
   if (mUsePatterns) {
-    pc.outputs().snapshot(Output{mOrigin, "PATTERNS", 0, Lifetime::Timeframe}, mPatternsArray);
+    pc.outputs().snapshot(Output{mOrigin, "PATTERNS", 0}, mPatternsArray);
   }
   if (mUseMC) {
-    pc.outputs().snapshot(Output{mOrigin, "CLUSTERSMCTR", 0, Lifetime::Timeframe}, mClusterMCTruth);
-    pc.outputs().snapshot(Output{mOrigin, "CLUSTERSMC2ROF", 0, Lifetime::Timeframe}, mClusMC2ROFs);
+    pc.outputs().snapshot(Output{mOrigin, "CLUSTERSMCTR", 0}, mClusterMCTruth);
+    pc.outputs().snapshot(Output{mOrigin, "CLUSTERSMC2ROF", 0}, mClusMC2ROFs);
   }
   if (mTriggerOut) {
     std::vector<o2::itsmft::PhysTrigger> dummyTrig;
-    pc.outputs().snapshot(Output{mOrigin, "PHYSTRIG", 0, Lifetime::Timeframe}, dummyTrig);
+    pc.outputs().snapshot(Output{mOrigin, "PHYSTRIG", 0}, dummyTrig);
   }
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/ITSMFT/common/workflow/src/DigitReaderSpec.cxx b/Detectors/ITSMFT/common/workflow/src/DigitReaderSpec.cxx
index fbdd63c73d782..ef9494c85bf4b 100644
--- a/Detectors/ITSMFT/common/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/ITSMFT/common/workflow/src/DigitReaderSpec.cxx
@@ -74,20 +74,20 @@ void DigitReader::run(ProcessingContext& pc)
 
   // This is a very ugly way of providing DataDescription, which anyway does not need to contain detector name.
   // To be fixed once the names-definition class is ready
-  pc.outputs().snapshot(Output{mOrigin, "DIGITSROF", 0, Lifetime::Timeframe}, mDigROFRec);
-  pc.outputs().snapshot(Output{mOrigin, "DIGITS", 0, Lifetime::Timeframe}, mDigits);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITSROF", 0}, mDigROFRec);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITS", 0}, mDigits);
   if (mUseCalib) {
-    pc.outputs().snapshot(Output{mOrigin, "GBTCALIB", 0, Lifetime::Timeframe}, mCalib);
+    pc.outputs().snapshot(Output{mOrigin, "GBTCALIB", 0}, mCalib);
   }
   if (mTriggerOut) {
     std::vector<o2::itsmft::PhysTrigger> dummyTrig;
-    pc.outputs().snapshot(Output{mOrigin, "PHYSTRIG", 0, Lifetime::Timeframe}, dummyTrig);
+    pc.outputs().snapshot(Output{mOrigin, "PHYSTRIG", 0}, dummyTrig);
   }
   if (mUseMC) {
-    auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{mOrigin, "DIGITSMCTR", 0, Lifetime::Timeframe});
+    auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{mOrigin, "DIGITSMCTR", 0});
     plabels->copyandflatten(sharedlabels);
     delete plabels;
-    pc.outputs().snapshot(Output{mOrigin, "DIGITSMC2ROF", 0, Lifetime::Timeframe}, mDigMC2ROFs);
+    pc.outputs().snapshot(Output{mOrigin, "DIGITSMC2ROF", 0}, mDigMC2ROFs);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/ITSMFT/common/workflow/src/EntropyEncoderSpec.cxx b/Detectors/ITSMFT/common/workflow/src/EntropyEncoderSpec.cxx
index ccb6f0539909e..4b35f6cc44e39 100644
--- a/Detectors/ITSMFT/common/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/ITSMFT/common/workflow/src/EntropyEncoderSpec.cxx
@@ -56,7 +56,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{mOrigin, "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{mOrigin, "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, rofs, compClusters, pspan, mPattIdConverter, mStrobeLength);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   if (mSelIR) {
diff --git a/Detectors/ITSMFT/common/workflow/src/STFDecoderSpec.cxx b/Detectors/ITSMFT/common/workflow/src/STFDecoderSpec.cxx
index 73132163d102c..a31ec0f1a4af2 100644
--- a/Detectors/ITSMFT/common/workflow/src/STFDecoderSpec.cxx
+++ b/Detectors/ITSMFT/common/workflow/src/STFDecoderSpec.cxx
@@ -139,7 +139,7 @@ void STFDecoder<Mapping>::run(ProcessingContext& pc)
   std::vector<Digit> digVec;
   std::vector<GBTCalibData> calVec;
   std::vector<ROFRecord> digROFVec;
-  auto& chipStatus = pc.outputs().make<std::vector<char>>(Output{orig, "CHIPSSTATUS", 0, Lifetime::Timeframe}, (size_t)Mapping::getNChips());
+  auto& chipStatus = pc.outputs().make<std::vector<char>>(Output{orig, "CHIPSSTATUS", 0}, (size_t)Mapping::getNChips());
 
   try {
     mDecoder->startNewTF(pc.inputs());
@@ -202,29 +202,29 @@ void STFDecoder<Mapping>::run(ProcessingContext& pc)
     }
   }
   if (mDoDigits) {
-    pc.outputs().snapshot(Output{orig, "DIGITS", 0, Lifetime::Timeframe}, digVec);
-    pc.outputs().snapshot(Output{orig, "DIGITSROF", 0, Lifetime::Timeframe}, digROFVec);
+    pc.outputs().snapshot(Output{orig, "DIGITS", 0}, digVec);
+    pc.outputs().snapshot(Output{orig, "DIGITSROF", 0}, digROFVec);
     mEstNDig = std::max(mEstNDig, size_t(digVec.size() * 1.2));
     mEstNROF = std::max(mEstNROF, size_t(digROFVec.size() * 1.2));
     if (mDoCalibData) {
-      pc.outputs().snapshot(Output{orig, "GBTCALIB", 0, Lifetime::Timeframe}, calVec);
+      pc.outputs().snapshot(Output{orig, "GBTCALIB", 0}, calVec);
       mEstNCalib = std::max(mEstNCalib, size_t(calVec.size() * 1.2));
     }
   }
 
   if (mDoClusters) { // we are not obliged to create vectors which are not requested, but other devices might not know the options of this one
-    pc.outputs().snapshot(Output{orig, "COMPCLUSTERS", 0, Lifetime::Timeframe}, clusCompVec);
-    pc.outputs().snapshot(Output{orig, "PATTERNS", 0, Lifetime::Timeframe}, clusPattVec);
-    pc.outputs().snapshot(Output{orig, "CLUSTERSROF", 0, Lifetime::Timeframe}, clusROFVec);
+    pc.outputs().snapshot(Output{orig, "COMPCLUSTERS", 0}, clusCompVec);
+    pc.outputs().snapshot(Output{orig, "PATTERNS", 0}, clusPattVec);
+    pc.outputs().snapshot(Output{orig, "CLUSTERSROF", 0}, clusROFVec);
     mEstNClus = std::max(mEstNClus, size_t(clusCompVec.size() * 1.2));
     mEstNClusPatt = std::max(mEstNClusPatt, size_t(clusPattVec.size() * 1.2));
     mEstNROF = std::max(mEstNROF, size_t(clusROFVec.size() * 1.2));
   }
-  auto& linkErrors = pc.outputs().make<std::vector<GBTLinkDecodingStat>>(Output{orig, "LinkErrors", 0, Lifetime::Timeframe});
-  auto& decErrors = pc.outputs().make<std::vector<ChipError>>(Output{orig, "ChipErrors", 0, Lifetime::Timeframe});
+  auto& linkErrors = pc.outputs().make<std::vector<GBTLinkDecodingStat>>(Output{orig, "LinkErrors", 0});
+  auto& decErrors = pc.outputs().make<std::vector<ChipError>>(Output{orig, "ChipErrors", 0});
   mDecoder->collectDecodingErrors(linkErrors, decErrors);
 
-  pc.outputs().snapshot(Output{orig, "PHYSTRIG", 0, Lifetime::Timeframe}, mDecoder->getExternalTriggers());
+  pc.outputs().snapshot(Output{orig, "PHYSTRIG", 0}, mDecoder->getExternalTriggers());
 
   if (mDumpOnError != int(GBTLink::RawDataDumps::DUMP_NONE)) {
     mDecoder->produceRawDataDumps(mDumpOnError, pc.services().get<o2::framework::TimingInfo>());
diff --git a/Detectors/MUON/MCH/IO/src/DigitReaderSpec.cxx b/Detectors/MUON/MCH/IO/src/DigitReaderSpec.cxx
index da4c37035c82b..541e6c5fa26c0 100644
--- a/Detectors/MUON/MCH/IO/src/DigitReaderSpec.cxx
+++ b/Detectors/MUON/MCH/IO/src/DigitReaderSpec.cxx
@@ -54,18 +54,18 @@ class DigitsReaderDeviceDPL
       mReader = std::make_unique<RootTreeReader>("o2sim", filename.c_str(), -1,
                                                  RootTreeReader::PublishingMode::Single,
                                                  RootTreeReader::BranchDefinition<std::vector<Digit>>{
-                                                   Output{header::gDataOriginMCH, mDescriptions[0], 0, Lifetime::Timeframe}, "MCHDigit"},
+                                                   Output{header::gDataOriginMCH, mDescriptions[0], 0}, "MCHDigit"},
                                                  RootTreeReader::BranchDefinition<std::vector<ROFRecord>>{
-                                                   Output{header::gDataOriginMCH, mDescriptions[1], 0, Lifetime::Timeframe}, "MCHROFRecords"},
+                                                   Output{header::gDataOriginMCH, mDescriptions[1], 0}, "MCHROFRecords"},
                                                  RootTreeReader::BranchDefinition<dataformats::MCTruthContainer<MCCompLabel>>{
-                                                   Output{header::gDataOriginMCH, mDescriptions[2], 0, Lifetime::Timeframe}, "MCHMCLabels"});
+                                                   Output{header::gDataOriginMCH, mDescriptions[2], 0}, "MCHMCLabels"});
     } else {
       mReader = std::make_unique<RootTreeReader>("o2sim", filename.c_str(), -1,
                                                  RootTreeReader::PublishingMode::Single,
                                                  RootTreeReader::BranchDefinition<std::vector<Digit>>{
-                                                   Output{header::gDataOriginMCH, mDescriptions[0], 0, Lifetime::Timeframe}, "MCHDigit"},
+                                                   Output{header::gDataOriginMCH, mDescriptions[0], 0}, "MCHDigit"},
                                                  RootTreeReader::BranchDefinition<std::vector<ROFRecord>>{
-                                                   Output{header::gDataOriginMCH, mDescriptions[1], 0, Lifetime::Timeframe}, "MCHROFRecords"});
+                                                   Output{header::gDataOriginMCH, mDescriptions[1], 0}, "MCHROFRecords"});
     }
   }
 
diff --git a/Detectors/MUON/MCH/Workflow/src/TrackAtVertexSpec.cxx b/Detectors/MUON/MCH/Workflow/src/TrackAtVertexSpec.cxx
index 080c36706a0eb..8b981d4ee4abd 100644
--- a/Detectors/MUON/MCH/Workflow/src/TrackAtVertexSpec.cxx
+++ b/Detectors/MUON/MCH/Workflow/src/TrackAtVertexSpec.cxx
@@ -139,7 +139,7 @@ class TrackAtVertexTask
     }
 
     // create the output message
-    auto msgOut = pc.outputs().make<char>(Output{"MCH", "TRACKSATVERTEX", 0, Lifetime::Timeframe},
+    auto msgOut = pc.outputs().make<char>(Output{"MCH", "TRACKSATVERTEX", 0},
                                           mTracksAtVtx.size() * sizeof(int) + nTracksTot * sizeof(TrackAtVtxStruct));
 
     // write the tracks
diff --git a/Detectors/MUON/MCH/Workflow/src/entropy-encoder-workflow.cxx b/Detectors/MUON/MCH/Workflow/src/entropy-encoder-workflow.cxx
index 50e085e238d55..058202dfb802b 100644
--- a/Detectors/MUON/MCH/Workflow/src/entropy-encoder-workflow.cxx
+++ b/Detectors/MUON/MCH/Workflow/src/entropy-encoder-workflow.cxx
@@ -72,7 +72,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"MCH", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"MCH", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, rofs, digits);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   mTimer.Stop();
diff --git a/Detectors/MUON/MID/Workflow/src/ClusterizerSpec.cxx b/Detectors/MUON/MID/Workflow/src/ClusterizerSpec.cxx
index 5657fd375f118..c544ce19fcdea 100644
--- a/Detectors/MUON/MID/Workflow/src/ClusterizerSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/ClusterizerSpec.cxx
@@ -99,13 +99,13 @@ class ClusterizerDeviceDPL
       // Clear the index correlations that will be used in the next cluster processing
       mCorrelation.clear();
 
-      pc.outputs().snapshot(of::Output{"MID", "CLUSTERSLABELS", 0, of::Lifetime::Timeframe}, mClusterLabeler.getContainer());
+      pc.outputs().snapshot(of::Output{"MID", "CLUSTERSLABELS", 0}, mClusterLabeler.getContainer());
       LOG(debug) << "Sent " << mClusterLabeler.getContainer().getIndexedSize() << " indexed clusters";
     }
 
-    pc.outputs().snapshot(of::Output{"MID", "CLUSTERS", 0, of::Lifetime::Timeframe}, mClusterizer.getClusters());
+    pc.outputs().snapshot(of::Output{"MID", "CLUSTERS", 0}, mClusterizer.getClusters());
     LOG(debug) << "Sent " << mClusterizer.getClusters().size() << " clusters";
-    pc.outputs().snapshot(of::Output{"MID", "CLUSTERSROF", 0, of::Lifetime::Timeframe}, mClusterizer.getROFRecords());
+    pc.outputs().snapshot(of::Output{"MID", "CLUSTERSROF", 0}, mClusterizer.getROFRecords());
     LOG(debug) << "Sent " << mClusterizer.getROFRecords().size() << " ROF";
 
     mTimer += std::chrono::high_resolution_clock::now() - tStart;
@@ -141,4 +141,4 @@ framework::DataProcessorSpec getClusterizerSpec(bool isMC, std::string_view inDa
     of::AlgorithmSpec{of::adaptFromTask<o2::mid::ClusterizerDeviceDPL>(isMC)}};
 }
 } // namespace mid
-} // namespace o2
\ No newline at end of file
+} // namespace o2
diff --git a/Detectors/MUON/MID/Workflow/src/ColumnDataSpecsUtils.cxx b/Detectors/MUON/MID/Workflow/src/ColumnDataSpecsUtils.cxx
index 98f9b8c08c654..e0d41cd8d91d2 100644
--- a/Detectors/MUON/MID/Workflow/src/ColumnDataSpecsUtils.cxx
+++ b/Detectors/MUON/MID/Workflow/src/ColumnDataSpecsUtils.cxx
@@ -127,7 +127,7 @@ std::vector<framework::Output> buildOutputs(std::vector<framework::OutputSpec> o
   std::vector<framework::Output> outputs;
   for (auto& outSpec : outputSpecs) {
     auto matcher = framework::DataSpecUtils::asConcreteDataMatcher(outSpec);
-    outputs.emplace_back(framework::Output{matcher.origin, matcher.description, matcher.subSpec, framework::Lifetime::Timeframe});
+    outputs.emplace_back(framework::Output{matcher.origin, matcher.description, matcher.subSpec});
   }
   return outputs;
 }
@@ -161,4 +161,4 @@ std::unique_ptr<const o2::dataformats::MCTruthContainer<MCLabel>> getLabels(fram
 
 } // namespace specs
 } // namespace mid
-} // namespace o2
\ No newline at end of file
+} // namespace o2
diff --git a/Detectors/MUON/MID/Workflow/src/DecodedDataAggregatorSpec.cxx b/Detectors/MUON/MID/Workflow/src/DecodedDataAggregatorSpec.cxx
index f03eb6715798a..192b4c52be9cc 100644
--- a/Detectors/MUON/MID/Workflow/src/DecodedDataAggregatorSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/DecodedDataAggregatorSpec.cxx
@@ -60,8 +60,8 @@ class DecodedDataAggregatorDeviceDPL
 
     for (o2::header::DataHeader::SubSpecificationType subSpec = 0; subSpec < 3; ++subSpec) {
       EventType evtType = static_cast<EventType>(subSpec);
-      pc.outputs().snapshot(of::Output{o2::header::gDataOriginMID, "DATA", subSpec, of::Lifetime::Timeframe}, mAggregator.getData(evtType));
-      pc.outputs().snapshot(of::Output{o2::header::gDataOriginMID, "DATAROF", subSpec, of::Lifetime::Timeframe}, mAggregator.getROFRecords(evtType));
+      pc.outputs().snapshot(of::Output{o2::header::gDataOriginMID, "DATA", subSpec}, mAggregator.getData(evtType));
+      pc.outputs().snapshot(of::Output{o2::header::gDataOriginMID, "DATAROF", subSpec}, mAggregator.getROFRecords(evtType));
     }
 
     mTimer += std::chrono::high_resolution_clock::now() - tStart;
diff --git a/Detectors/MUON/MID/Workflow/src/DigitReaderSpec.cxx b/Detectors/MUON/MID/Workflow/src/DigitReaderSpec.cxx
index 3445fad66642e..7f380d1afe13a 100644
--- a/Detectors/MUON/MID/Workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/DigitReaderSpec.cxx
@@ -55,19 +55,19 @@ class DigitsReaderDeviceDPL
       mReader = std::make_unique<RootTreeReader>("o2sim", filename.c_str(), -1,
                                                  RootTreeReader::PublishingMode::Single,
                                                  RootTreeReader::BranchDefinition<std::vector<ColumnData>>{
-                                                   Output{header::gDataOriginMID, mDescriptions[0], 0, Lifetime::Timeframe}, "MIDDigit"},
+                                                   Output{header::gDataOriginMID, mDescriptions[0], 0}, "MIDDigit"},
                                                  RootTreeReader::BranchDefinition<std::vector<ROFRecord>>{
-                                                   Output{header::gDataOriginMID, mDescriptions[1], 0, Lifetime::Timeframe}, "MIDROFRecords"},
+                                                   Output{header::gDataOriginMID, mDescriptions[1], 0}, "MIDROFRecords"},
                                                  RootTreeReader::BranchDefinition<dataformats::MCTruthContainer<MCLabel>>{
-                                                   Output{header::gDataOriginMID, mDescriptions[2], 0, Lifetime::Timeframe}, "MIDDigitMCLabels"},
+                                                   Output{header::gDataOriginMID, mDescriptions[2], 0}, "MIDDigitMCLabels"},
                                                  &mPublishDigits);
     } else {
       mReader = std::make_unique<RootTreeReader>("o2sim", filename.c_str(), -1,
                                                  RootTreeReader::PublishingMode::Single,
                                                  RootTreeReader::BranchDefinition<std::vector<ColumnData>>{
-                                                   Output{header::gDataOriginMID, mDescriptions[0], 0, Lifetime::Timeframe}, "MIDDigit"},
+                                                   Output{header::gDataOriginMID, mDescriptions[0], 0}, "MIDDigit"},
                                                  RootTreeReader::BranchDefinition<std::vector<ROFRecord>>{
-                                                   Output{header::gDataOriginMID, mDescriptions[1], 0, Lifetime::Timeframe}, "MIDROFRecords"},
+                                                   Output{header::gDataOriginMID, mDescriptions[1], 0}, "MIDROFRecords"},
                                                  &mPublishDigits);
     }
   }
diff --git a/Detectors/MUON/MID/Workflow/src/EntropyDecoderSpec.cxx b/Detectors/MUON/MID/Workflow/src/EntropyDecoderSpec.cxx
index e0030eb102c8d..5a8df6f8e81cb 100644
--- a/Detectors/MUON/MID/Workflow/src/EntropyDecoderSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/EntropyDecoderSpec.cxx
@@ -66,9 +66,9 @@ void EntropyDecoderSpec::run(ProcessingContext& pc)
   size_t insize = 0;
   for (uint32_t it = 0; it < NEvTypes; it++) {
     insize += cols[it].size() * sizeof(o2::mid::ColumnData);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginMID, "DATA", it, Lifetime::Timeframe}, cols[it]);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginMID, "DATA", it}, cols[it]);
     insize += rofs[it].size() * sizeof(o2::mid::ROFRecord);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginMID, "DATAROF", it, Lifetime::Timeframe}, rofs[it]);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginMID, "DATAROF", it}, rofs[it]);
   }
   iosize.rawIn = insize;
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
diff --git a/Detectors/MUON/MID/Workflow/src/EntropyEncoderSpec.cxx b/Detectors/MUON/MID/Workflow/src/EntropyEncoderSpec.cxx
index 44bf3dcc0ae6f..a472d6e28ff16 100644
--- a/Detectors/MUON/MID/Workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/EntropyEncoderSpec.cxx
@@ -83,7 +83,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   // build references for looping over the data in BC increasing direction
   tfData.buildReferences(mCTFCoder.getIRFramesSelector());
 
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{header::gDataOriginMID, "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{header::gDataOriginMID, "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, tfData);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   iosize.rawIn = insize;
diff --git a/Detectors/MUON/MID/Workflow/src/MaskMakerSpec.cxx b/Detectors/MUON/MID/Workflow/src/MaskMakerSpec.cxx
index 99acaa08ac5f0..28d2ff953ea23 100644
--- a/Detectors/MUON/MID/Workflow/src/MaskMakerSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/MaskMakerSpec.cxx
@@ -117,7 +117,7 @@ class MaskMakerDeviceDPL
     if (mCounterSinceReset >= mNReset) {
       for (size_t itype = 0; itype < 2; ++itype) {
         auto masks = o2::mid::makeMasks(mScalers[itype], mCounterSinceReset, mThreshold, mRefMasks);
-        pc.outputs().snapshot(of::Output{header::gDataOriginMID, "MASKS", static_cast<header::DataHeader::SubSpecificationType>(itype + 1), of::Lifetime::Timeframe}, masks);
+        pc.outputs().snapshot(of::Output{header::gDataOriginMID, "MASKS", static_cast<header::DataHeader::SubSpecificationType>(itype + 1)}, masks);
       }
       mCounterSinceReset = 0;
       for (auto& scaler : mScalers) {
@@ -160,4 +160,4 @@ framework::DataProcessorSpec getMaskMakerSpec(const FEEIdConfig& feeIdConfig, co
     of::Options{{"mid-mask-threshold", of::VariantType::Double, 0.9, {"Tolerated occupancy before producing a map"}}, {"mid-mask-reset", of::VariantType::Int, 100, {"Number of calibration events to be checked before resetting the scalers"}}}};
 }
 } // namespace mid
-} // namespace o2
\ No newline at end of file
+} // namespace o2
diff --git a/Detectors/MUON/MID/Workflow/src/RawGBTDecoderSpec.cxx b/Detectors/MUON/MID/Workflow/src/RawGBTDecoderSpec.cxx
index 6d23ce9cdab1f..ba15644e2b17a 100644
--- a/Detectors/MUON/MID/Workflow/src/RawGBTDecoderSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/RawGBTDecoderSpec.cxx
@@ -82,8 +82,8 @@ class RawGBTDecoderDeviceDPL
 
     mTimerAlgo += std::chrono::high_resolution_clock::now() - tAlgoStart;
 
-    pc.outputs().snapshot(o2::framework::Output{header::gDataOriginMID, "DECODED", dh->subSpecification, o2::framework::Lifetime::Timeframe}, data);
-    pc.outputs().snapshot(o2::framework::Output{header::gDataOriginMID, "DECODEDROF", dh->subSpecification, o2::framework::Lifetime::Timeframe}, rofRecords);
+    pc.outputs().snapshot(o2::framework::Output{header::gDataOriginMID, "DECODED", dh->subSpecification}, data);
+    pc.outputs().snapshot(o2::framework::Output{header::gDataOriginMID, "DECODEDROF", dh->subSpecification}, rofRecords);
 
     mTimer += std::chrono::high_resolution_clock::now() - tStart;
     mNROFs += rofRecords.size();
diff --git a/Detectors/MUON/MID/Workflow/src/TrackerSpec.cxx b/Detectors/MUON/MID/Workflow/src/TrackerSpec.cxx
index be1dd9124315c..e1044768e8ea6 100644
--- a/Detectors/MUON/MID/Workflow/src/TrackerSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/TrackerSpec.cxx
@@ -80,20 +80,20 @@ class TrackerDeviceDPL
     if (mIsMC) {
       std::unique_ptr<const o2::dataformats::MCTruthContainer<MCClusterLabel>> labels = pc.inputs().get<const o2::dataformats::MCTruthContainer<MCClusterLabel>*>("mid_clusterlabels");
       mTrackLabeler.process(mTracker->getClusters(), tracks, *labels);
-      pc.outputs().snapshot(of::Output{"MID", "TRACKLABELS", 0, of::Lifetime::Timeframe}, mTrackLabeler.getTracksLabels());
+      pc.outputs().snapshot(of::Output{"MID", "TRACKLABELS", 0}, mTrackLabeler.getTracksLabels());
       LOG(debug) << "Sent " << mTrackLabeler.getTracksLabels().size() << " indexed tracks.";
-      pc.outputs().snapshot(of::Output{"MID", "TRCLUSLABELS", 0, of::Lifetime::Timeframe}, mTrackLabeler.getTrackClustersLabels());
+      pc.outputs().snapshot(of::Output{"MID", "TRCLUSLABELS", 0}, mTrackLabeler.getTrackClustersLabels());
       LOG(debug) << "Sent " << mTrackLabeler.getTrackClustersLabels().getIndexedSize() << " indexed track clusters.";
     }
 
-    pc.outputs().snapshot(of::Output{"MID", "TRACKS", 0, of::Lifetime::Timeframe}, tracks);
+    pc.outputs().snapshot(of::Output{"MID", "TRACKS", 0}, tracks);
     LOG(debug) << "Sent " << tracks.size() << " tracks.";
-    pc.outputs().snapshot(of::Output{"MID", "TRACKCLUSTERS", 0, of::Lifetime::Timeframe}, mTracker->getClusters());
+    pc.outputs().snapshot(of::Output{"MID", "TRACKCLUSTERS", 0}, mTracker->getClusters());
     LOG(debug) << "Sent " << mTracker->getClusters().size() << " track clusters.";
 
-    pc.outputs().snapshot(of::Output{"MID", "TRACKROFS", 0, of::Lifetime::Timeframe}, mTracker->getTrackROFRecords());
+    pc.outputs().snapshot(of::Output{"MID", "TRACKROFS", 0}, mTracker->getTrackROFRecords());
     LOG(debug) << "Sent " << mTracker->getTrackROFRecords().size() << " ROFs.";
-    pc.outputs().snapshot(of::Output{"MID", "TRCLUSROFS", 0, of::Lifetime::Timeframe}, mTracker->getClusterROFRecords());
+    pc.outputs().snapshot(of::Output{"MID", "TRCLUSROFS", 0}, mTracker->getClusterROFRecords());
     LOG(debug) << "Sent " << mTracker->getClusterROFRecords().size() << " ROFs.";
 
     mTimer += std::chrono::high_resolution_clock::now() - tStart;
diff --git a/Detectors/MUON/MID/Workflow/src/ZeroSuppressionSpec.cxx b/Detectors/MUON/MID/Workflow/src/ZeroSuppressionSpec.cxx
index ccfe1641b87b6..5d89eee81c629 100644
--- a/Detectors/MUON/MID/Workflow/src/ZeroSuppressionSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/ZeroSuppressionSpec.cxx
@@ -90,7 +90,7 @@ class ZeroSuppressionDeviceDPL
     }
 
     if (mUseMC) {
-      pc.outputs().snapshot(of::Output{header::gDataOriginMID, "DATALABELS", 0, of::Lifetime::Timeframe}, outMCContainer);
+      pc.outputs().snapshot(of::Output{header::gDataOriginMID, "DATALABELS", 0}, outMCContainer);
     }
   }
 
@@ -113,4 +113,4 @@ framework::DataProcessorSpec getZeroSuppressionSpec(bool useMC, std::string_view
     of::AlgorithmSpec{of::adaptFromTask<o2::mid::ZeroSuppressionDeviceDPL>(useMC)}};
 }
 } // namespace mid
-} // namespace o2
\ No newline at end of file
+} // namespace o2
diff --git a/Detectors/PHOS/calib/src/PHOSBadMapCalibDevice.cxx b/Detectors/PHOS/calib/src/PHOSBadMapCalibDevice.cxx
index 226630a8b1d81..4f81300431934 100644
--- a/Detectors/PHOS/calib/src/PHOSBadMapCalibDevice.cxx
+++ b/Detectors/PHOS/calib/src/PHOSBadMapCalibDevice.cxx
@@ -177,7 +177,7 @@ void PHOSBadMapCalibDevice::sendOutput(DataAllocator& output)
 
   // Send change to QC
   LOG(info) << "[PHOSBadMapCalibDevice - run] Sending QC ";
-  output.snapshot(o2::framework::Output{"PHS", "BADMAPDIFF", 0, o2::framework::Lifetime::Timeframe}, mBadMapDiff);
+  output.snapshot(o2::framework::Output{"PHS", "BADMAPDIFF", 0}, mBadMapDiff);
 }
 
 bool PHOSBadMapCalibDevice::calculateBadMap()
diff --git a/Detectors/PHOS/calib/src/PHOSHGLGRatioCalibDevice.cxx b/Detectors/PHOS/calib/src/PHOSHGLGRatioCalibDevice.cxx
index 0a37589d3abe7..5c07c6fee8493 100644
--- a/Detectors/PHOS/calib/src/PHOSHGLGRatioCalibDevice.cxx
+++ b/Detectors/PHOS/calib/src/PHOSHGLGRatioCalibDevice.cxx
@@ -229,7 +229,7 @@ void PHOSHGLGRatioCalibDevice::sendOutput(DataAllocator& output)
   }
   // Anyway send change to QC
   LOG(info) << "[PHOSHGLGRatioCalibDevice - sendOutput] Sending QC ";
-  output.snapshot(o2::framework::Output{"PHS", "CALIBDIFF", 0, o2::framework::Lifetime::Timeframe}, mRatioDiff);
+  output.snapshot(o2::framework::Output{"PHS", "CALIBDIFF", 0}, mRatioDiff);
 }
 
 DataProcessorSpec o2::phos::getHGLGRatioCalibSpec(bool useCCDB, bool forceUpdate)
diff --git a/Detectors/PHOS/calib/src/PHOSPedestalCalibDevice.cxx b/Detectors/PHOS/calib/src/PHOSPedestalCalibDevice.cxx
index 96f680f710fb0..42289a910debb 100644
--- a/Detectors/PHOS/calib/src/PHOSPedestalCalibDevice.cxx
+++ b/Detectors/PHOS/calib/src/PHOSPedestalCalibDevice.cxx
@@ -135,7 +135,7 @@ void PHOSPedestalCalibDevice::sendOutput(DataAllocator& output)
   }
   // Anyway send change to QC
   LOG(info) << "[PHOSPedestalCalibDevice - run] Sending QC ";
-  output.snapshot(o2::framework::Output{"PHS", "CALIBDIFF", 0, o2::framework::Lifetime::Timeframe}, mPedDiff);
+  output.snapshot(o2::framework::Output{"PHS", "CALIBDIFF", 0}, mPedDiff);
 }
 
 void PHOSPedestalCalibDevice::calculatePedestals()
diff --git a/Detectors/PHOS/workflow/src/CellConverterSpec.cxx b/Detectors/PHOS/workflow/src/CellConverterSpec.cxx
index 330dd27c45251..370df48a8fc40 100644
--- a/Detectors/PHOS/workflow/src/CellConverterSpec.cxx
+++ b/Detectors/PHOS/workflow/src/CellConverterSpec.cxx
@@ -44,12 +44,12 @@ void CellConverterSpec::run(framework::ProcessingContext& ctx)
   auto digitsTR = ctx.inputs().get<std::vector<o2::phos::TriggerRecord>>("digitTriggerRecords");
   if (!digitsTR.size()) { // nothing to process
     mOutputCells.clear();
-    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLS", 0, o2::framework::Lifetime::Timeframe}, mOutputCells);
+    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLS", 0}, mOutputCells);
     mOutputCellTrigRecs.clear();
-    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", 0, o2::framework::Lifetime::Timeframe}, mOutputCellTrigRecs);
+    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", 0}, mOutputCellTrigRecs);
     if (mPropagateMC) {
       mOutputTruthCont.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLSMCTR", 0, o2::framework::Lifetime::Timeframe}, mOutputTruthCont);
+      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLSMCTR", 0}, mOutputTruthCont);
     }
     return;
   }
@@ -128,10 +128,10 @@ void CellConverterSpec::run(framework::ProcessingContext& ctx)
   }
   LOG(info) << "[PHOSCellConverter - run] Writing " << mOutputCells.size() << " cells, " << mOutputCellTrigRecs.size() << " Trig Records " << mOutputTruthCont.getNElements() << " PHOS labels ";
 
-  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLS", 0, o2::framework::Lifetime::Timeframe}, mOutputCells);
-  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", 0, o2::framework::Lifetime::Timeframe}, mOutputCellTrigRecs);
+  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLS", 0}, mOutputCells);
+  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", 0}, mOutputCellTrigRecs);
   if (mPropagateMC) {
-    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLSMCTR", 0, o2::framework::Lifetime::Timeframe}, mOutputTruthCont);
+    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLSMCTR", 0}, mOutputTruthCont);
   }
 }
 
diff --git a/Detectors/PHOS/workflow/src/CellReaderSpec.cxx b/Detectors/PHOS/workflow/src/CellReaderSpec.cxx
index 64f54c1baf189..c7d93fc20301f 100644
--- a/Detectors/PHOS/workflow/src/CellReaderSpec.cxx
+++ b/Detectors/PHOS/workflow/src/CellReaderSpec.cxx
@@ -44,10 +44,10 @@ void CellReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mCells.size() << " Cells in " << mTRs.size() << " TriggerRecords at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "CELLS", 0, Lifetime::Timeframe}, mCells);
-  pc.outputs().snapshot(Output{mOrigin, "CELLTRIGREC", 0, Lifetime::Timeframe}, mTRs);
+  pc.outputs().snapshot(Output{mOrigin, "CELLS", 0}, mCells);
+  pc.outputs().snapshot(Output{mOrigin, "CELLTRIGREC", 0}, mTRs);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{mOrigin, "CELLSMCTR", 0, Lifetime::Timeframe}, mMCTruth);
+    pc.outputs().snapshot(Output{mOrigin, "CELLSMCTR", 0}, mMCTruth);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/PHOS/workflow/src/ClusterizerSpec.cxx b/Detectors/PHOS/workflow/src/ClusterizerSpec.cxx
index 27bce0f8745f4..13aa21258125d 100644
--- a/Detectors/PHOS/workflow/src/ClusterizerSpec.cxx
+++ b/Detectors/PHOS/workflow/src/ClusterizerSpec.cxx
@@ -78,16 +78,16 @@ void ClusterizerSpec::run(framework::ProcessingContext& ctx)
     auto digitsTR = ctx.inputs().get<std::vector<o2::phos::TriggerRecord>>("digitTriggerRecords");
     if (!digitsTR.size()) { // nothing to process
       mOutputClusters.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERS", 0, o2::framework::Lifetime::Timeframe}, mOutputClusters);
+      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERS", 0}, mOutputClusters);
       if (mFullCluOutput) {
         mOutputCluElements.clear();
-        ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUELEMENTS", 0, o2::framework::Lifetime::Timeframe}, mOutputCluElements);
+        ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUELEMENTS", 0}, mOutputCluElements);
       }
       mOutputClusterTrigRecs.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERTRIGREC", 0, o2::framework::Lifetime::Timeframe}, mOutputClusterTrigRecs);
+      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERTRIGREC", 0}, mOutputClusterTrigRecs);
       if (mPropagateMC) {
         mOutputTruthCont.clear();
-        ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERTRUEMC", 0, o2::framework::Lifetime::Timeframe}, mOutputTruthCont);
+        ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERTRUEMC", 0}, mOutputTruthCont);
       }
       return;
     }
@@ -120,13 +120,13 @@ void ClusterizerSpec::run(framework::ProcessingContext& ctx)
   } else {
     LOG(debug) << "[PHOSClusterizer - run] Writing " << mOutputClusters.size() << " clusters and " << mOutputClusterTrigRecs.size() << " TR";
   }
-  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERS", 0, o2::framework::Lifetime::Timeframe}, mOutputClusters);
+  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERS", 0}, mOutputClusters);
   if (mFullCluOutput) {
-    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUELEMENTS", 0, o2::framework::Lifetime::Timeframe}, mOutputCluElements);
+    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUELEMENTS", 0}, mOutputCluElements);
   }
-  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERTRIGREC", 0, o2::framework::Lifetime::Timeframe}, mOutputClusterTrigRecs);
+  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERTRIGREC", 0}, mOutputClusterTrigRecs);
   if (mPropagateMC) {
-    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERTRUEMC", 0, o2::framework::Lifetime::Timeframe}, mOutputTruthCont);
+    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERTRUEMC", 0}, mOutputTruthCont);
   }
 }
 
diff --git a/Detectors/PHOS/workflow/src/DigitReaderSpec.cxx b/Detectors/PHOS/workflow/src/DigitReaderSpec.cxx
index 737856051af0c..70f5077b2f0c9 100644
--- a/Detectors/PHOS/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/PHOS/workflow/src/DigitReaderSpec.cxx
@@ -44,10 +44,10 @@ void DigitReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mDigits.size() << " Digits in " << mTRs.size() << " TriggerRecords at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "DIGITS", 0, Lifetime::Timeframe}, mDigits);
-  pc.outputs().snapshot(Output{mOrigin, "DIGITTRIGREC", 0, Lifetime::Timeframe}, mTRs);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITS", 0}, mDigits);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITTRIGREC", 0}, mTRs);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{mOrigin, "DIGITSMCTR", 0, Lifetime::Timeframe}, mMCTruth);
+    pc.outputs().snapshot(Output{mOrigin, "DIGITSMCTR", 0}, mMCTruth);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/PHOS/workflow/src/EntropyEncoderSpec.cxx b/Detectors/PHOS/workflow/src/EntropyEncoderSpec.cxx
index 226184c1179c6..a932a45f1bb53 100644
--- a/Detectors/PHOS/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/PHOS/workflow/src/EntropyEncoderSpec.cxx
@@ -54,7 +54,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"PHS", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"PHS", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, triggers, cells);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   if (mSelIR) {
diff --git a/Detectors/PHOS/workflow/src/EventBuilderSpec.cxx b/Detectors/PHOS/workflow/src/EventBuilderSpec.cxx
index d6562fdadfee6..9a1289889424e 100644
--- a/Detectors/PHOS/workflow/src/EventBuilderSpec.cxx
+++ b/Detectors/PHOS/workflow/src/EventBuilderSpec.cxx
@@ -178,8 +178,8 @@ void EventBuilderSpec::run(framework::ProcessingContext& ctx)
     }
   }
 
-  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginPHS, "CELLS", 0, framework::Lifetime::Timeframe}, outputCells);
-  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginPHS, "CELLTRIGREC", 0, framework::Lifetime::Timeframe}, outputTriggers);
+  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginPHS, "CELLS", 0}, outputCells);
+  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginPHS, "CELLTRIGREC", 0}, outputTriggers);
 }
 
 o2::framework::DataProcessorSpec o2::phos::getEventBuilderSpec()
diff --git a/Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx b/Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx
index ea0336f0ab19a..292e083d4bb48 100644
--- a/Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx
+++ b/Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx
@@ -108,11 +108,11 @@ void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
              contDeadBeef == maxWarn ? fmt::format(". {} such inputs in row received, stopping reporting", contDeadBeef) : "");
       }
       mOutputCells.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLS", mflpId, o2::framework::Lifetime::Timeframe}, mOutputCells);
+      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLS", mflpId}, mOutputCells);
       mOutputTriggerRecords.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", mflpId, o2::framework::Lifetime::Timeframe}, mOutputTriggerRecords);
+      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", mflpId}, mOutputTriggerRecords);
       mOutputHWErrors.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"PHS", "RAWHWERRORS", 0, o2::framework::Lifetime::Timeframe}, mOutputHWErrors);
+      ctx.outputs().snapshot(o2::framework::Output{"PHS", "RAWHWERRORS", 0}, mOutputHWErrors);
       if (mFillChi2) {
         mOutputFitChi.clear();
         ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLFITQA", 0, o2::framework::Lifetime::QA}, mOutputFitChi);
@@ -290,9 +290,9 @@ void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
   mLastSize = 1.1 * mOutputCells.size();
 
   LOG(debug) << "[PHOSRawToCellConverter - run] Writing " << mOutputCells.size() << " cells ...";
-  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLS", mflpId, o2::framework::Lifetime::Timeframe}, mOutputCells);
-  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", mflpId, o2::framework::Lifetime::Timeframe}, mOutputTriggerRecords);
-  ctx.outputs().snapshot(o2::framework::Output{"PHS", "RAWHWERRORS", 0, o2::framework::Lifetime::Timeframe}, mOutputHWErrors);
+  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLS", mflpId}, mOutputCells);
+  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", mflpId}, mOutputTriggerRecords);
+  ctx.outputs().snapshot(o2::framework::Output{"PHS", "RAWHWERRORS", 0}, mOutputHWErrors);
   if (mFillChi2) {
     ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLFITQA", 0, o2::framework::Lifetime::QA}, mOutputFitChi);
   }
diff --git a/Detectors/PHOS/workflow/src/ReaderSpec.cxx b/Detectors/PHOS/workflow/src/ReaderSpec.cxx
index b68046b47c156..9a1fb23a553df 100644
--- a/Detectors/PHOS/workflow/src/ReaderSpec.cxx
+++ b/Detectors/PHOS/workflow/src/ReaderSpec.cxx
@@ -58,7 +58,6 @@ DataProcessorSpec getDigitsReaderSpec(bool propagateMC)
       processAttributes->terminateOnEod = ic.options().get<bool>("terminate-on-eod");
       processAttributes->finished = false;
       processAttributes->datatype = "PHOSDigit";
-      constexpr auto persistency = Lifetime::Timeframe;
       o2::header::DataHeader::SubSpecificationType subSpec = 0;
       if (propagateMC) {
         processAttributes->reader = std::make_shared<RootTreeReader>(treename.c_str(), // tree name
@@ -66,19 +65,19 @@ DataProcessorSpec getDigitsReaderSpec(bool propagateMC)
                                                                      nofEvents,        // number of entries to publish
                                                                      publishingMode,
                                                                      RootTreeReader::BranchDefinition<std::vector<o2::phos::Digit>>{
-                                                                       Output{"PHS", "DIGITS", subSpec, persistency}, "PHOSDigit"},
+                                                                       Output{"PHS", "DIGITS", subSpec}, "PHOSDigit"},
                                                                      RootTreeReader::BranchDefinition<std::vector<o2::phos::TriggerRecord>>{
-                                                                       Output{"PHS", "DIGITTRIGREC", subSpec, persistency}, "PHOSDigitTrigRecords"},
-                                                                     Output{"PHS", "DIGITSMCTR", subSpec, persistency}, "PHOSDigitMCTruth"); // name of mc label branch
+                                                                       Output{"PHS", "DIGITTRIGREC", subSpec}, "PHOSDigitTrigRecords"},
+                                                                     Output{"PHS", "DIGITSMCTR", subSpec}, "PHOSDigitMCTruth"); // name of mc label branch
       } else {
         processAttributes->reader = std::make_shared<RootTreeReader>(treename.c_str(), // tree name
                                                                      filename.c_str(), // input file name
                                                                      nofEvents,        // number of entries to publish
                                                                      publishingMode,
                                                                      RootTreeReader::BranchDefinition<std::vector<o2::phos::Digit>>{
-                                                                       Output{"PHS", "DIGITS", subSpec, persistency}, "PHOSDigit"},
+                                                                       Output{"PHS", "DIGITS", subSpec}, "PHOSDigit"},
                                                                      RootTreeReader::BranchDefinition<std::vector<o2::phos::TriggerRecord>>{
-                                                                       Output{"PHS", "DIGITTRIGREC", subSpec, persistency}, "PHOSDigitTrigRecords"});
+                                                                       Output{"PHS", "DIGITTRIGREC", subSpec}, "PHOSDigitTrigRecords"});
       }
     }
 
@@ -156,7 +155,6 @@ DataProcessorSpec getCellReaderSpec(bool propagateMC)
       processAttributes->terminateOnEod = ic.options().get<bool>("terminate-on-eod");
       processAttributes->finished = false;
       processAttributes->datatype = "PHOSCell";
-      constexpr auto persistency = Lifetime::Timeframe;
       o2::header::DataHeader::SubSpecificationType subSpec = 0;
       if (propagateMC) {
         processAttributes->reader = std::make_shared<RootTreeReader>(treename.c_str(), // tree name
@@ -164,10 +162,10 @@ DataProcessorSpec getCellReaderSpec(bool propagateMC)
                                                                      nofEvents,        // number of entries to publish
                                                                      publishingMode,
                                                                      RootTreeReader::BranchDefinition<std::vector<o2::phos::Cell>>{
-                                                                       Output{"PHS", "CELLS", subSpec, persistency}, "PHOSCell"},
+                                                                       Output{"PHS", "CELLS", subSpec}, "PHOSCell"},
                                                                      RootTreeReader::BranchDefinition<std::vector<o2::phos::TriggerRecord>>{
-                                                                       Output{"PHS", "CELLTRIGREC", subSpec, persistency}, "PHOSCellTrigRec"},
-                                                                     Output{"PHS", "CELLSMCTR", subSpec, persistency},
+                                                                       Output{"PHS", "CELLTRIGREC", subSpec}, "PHOSCellTrigRec"},
+                                                                     Output{"PHS", "CELLSMCTR", subSpec},
                                                                      "PHOSCellTrueMC"); // name of mc label branch
       } else {
         processAttributes->reader = std::make_shared<RootTreeReader>(treename.c_str(), // tree name
@@ -175,9 +173,9 @@ DataProcessorSpec getCellReaderSpec(bool propagateMC)
                                                                      nofEvents,        // number of entries to publish
                                                                      publishingMode,
                                                                      RootTreeReader::BranchDefinition<std::vector<o2::phos::Cell>>{
-                                                                       Output{"PHS", "CELLS", subSpec, persistency}, "PHOSCell"},
+                                                                       Output{"PHS", "CELLS", subSpec}, "PHOSCell"},
                                                                      RootTreeReader::BranchDefinition<std::vector<o2::phos::TriggerRecord>>{
-                                                                       Output{"PHS", "CELLTRIGREC", subSpec, persistency}, "PHOSCellTrigRec"});
+                                                                       Output{"PHS", "CELLTRIGREC", subSpec}, "PHOSCellTrigRec"});
       }
     }
 
diff --git a/Detectors/PHOS/workflow/src/StandaloneAODProducerSpec.cxx b/Detectors/PHOS/workflow/src/StandaloneAODProducerSpec.cxx
index 736a569cacd34..454be7a5fcb83 100644
--- a/Detectors/PHOS/workflow/src/StandaloneAODProducerSpec.cxx
+++ b/Detectors/PHOS/workflow/src/StandaloneAODProducerSpec.cxx
@@ -136,8 +136,8 @@ void StandaloneAODProducerSpec::run(ProcessingContext& pc)
   } // end of event loop
   // std::cout << "Finished cell loop" << std::endl;
 
-  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0, Lifetime::Timeframe}, tfNumber);
-  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0, Lifetime::Timeframe}, "");
+  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0}, tfNumber);
+  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0}, "");
 
   mTimer.Stop();
 }
diff --git a/Detectors/TOF/calibration/testWorkflow/TOFCalibCollectorSpec.h b/Detectors/TOF/calibration/testWorkflow/TOFCalibCollectorSpec.h
index f42c5676d55e5..4beef9541eb55 100644
--- a/Detectors/TOF/calibration/testWorkflow/TOFCalibCollectorSpec.h
+++ b/Detectors/TOF/calibration/testWorkflow/TOFCalibCollectorSpec.h
@@ -113,8 +113,8 @@ class TOFCalibCollectorDevice : public o2::framework::Task
       auto entries = collectedInfo.size();
       // this means that we are ready to send the output
       auto entriesPerChannel = mCollector->getEntriesPerChannel();
-      output.snapshot(Output{o2::header::gDataOriginTOF, "COLLECTEDINFO", 0, Lifetime::Timeframe}, collectedInfo);
-      output.snapshot(Output{o2::header::gDataOriginTOF, "ENTRIESCH", 0, Lifetime::Timeframe}, entriesPerChannel);
+      output.snapshot(Output{o2::header::gDataOriginTOF, "COLLECTEDINFO", 0}, collectedInfo);
+      output.snapshot(Output{o2::header::gDataOriginTOF, "ENTRIESCH", 0}, entriesPerChannel);
       mCollector->initOutput(); // reset the output for the next round
     }
   }
diff --git a/Detectors/TOF/workflow/src/CompressedDecodingTask.cxx b/Detectors/TOF/workflow/src/CompressedDecodingTask.cxx
index ebc61f05e85c0..fc0e2b9b2c30d 100644
--- a/Detectors/TOF/workflow/src/CompressedDecodingTask.cxx
+++ b/Detectors/TOF/workflow/src/CompressedDecodingTask.cxx
@@ -109,18 +109,18 @@ void CompressedDecodingTask::postData(ProcessingContext& pc)
   // LOG(info) << "TOF: N tof window decoded = " << n_tof_window << "(orbits = " << n_orbits << ") with " << digit_size << " digits";
 
   // add digits in the output snapshot
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITS", 0, Lifetime::Timeframe}, *alldigits);
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "READOUTWINDOW", 0, Lifetime::Timeframe}, *row);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITS", 0}, *alldigits);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "READOUTWINDOW", 0}, *row);
 
   std::vector<uint8_t>& patterns = mDecoder.getPatterns();
 
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "PATTERNS", 0, Lifetime::Timeframe}, patterns);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "PATTERNS", 0}, patterns);
 
   std::vector<uint64_t>& errors = mDecoder.getErrors();
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "ERRORS", 0, Lifetime::Timeframe}, errors);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "ERRORS", 0}, errors);
 
   DigitHeader& digitH = mDecoder.getDigitHeader();
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITHEADER", 0, Lifetime::Timeframe}, digitH);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITHEADER", 0}, digitH);
 
   auto diagnosticFrequency = mDecoder.getDiagnosticFrequency();
   diagnosticFrequency.setTimeStamp(mCreationTime / 1000);
@@ -130,7 +130,7 @@ void CompressedDecodingTask::postData(ProcessingContext& pc)
   diagnosticFrequency.setTFIDInfo(tfinfo);
 
   //diagnosticFrequency.print();
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIAFREQ", 0, Lifetime::Timeframe}, diagnosticFrequency);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIAFREQ", 0}, diagnosticFrequency);
 
   mDecoder.clear();
 
diff --git a/Detectors/TOF/workflow/src/EntropyDecoderSpec.cxx b/Detectors/TOF/workflow/src/EntropyDecoderSpec.cxx
index b2c735af87241..400914c64021f 100644
--- a/Detectors/TOF/workflow/src/EntropyDecoderSpec.cxx
+++ b/Detectors/TOF/workflow/src/EntropyDecoderSpec.cxx
@@ -81,7 +81,7 @@ void EntropyDecoderSpec::run(ProcessingContext& pc)
   o2::dataformats::TFIDInfo tfinfo;
   o2::base::TFIDInfoHelper::fillTFIDInfo(pc, tfinfo);
   diagnostic.setTFIDInfo(tfinfo);
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIAFREQ", 0, Lifetime::Timeframe}, diagnostic);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIAFREQ", 0}, diagnostic);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   mTimer.Stop();
   LOG(info) << "Decoded " << digits.size() << " digits in " << row.size() << " ROF, (" << iosize.asString() << ") in " << mTimer.CpuTime() - cput << " s";
diff --git a/Detectors/TOF/workflow/src/EntropyEncoderSpec.cxx b/Detectors/TOF/workflow/src/EntropyEncoderSpec.cxx
index 8f215a0572961..3fc47955f53c0 100644
--- a/Detectors/TOF/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/TOF/workflow/src/EntropyEncoderSpec.cxx
@@ -55,7 +55,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{o2::header::gDataOriginTOF, "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{o2::header::gDataOriginTOF, "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, rofs, compDigits, pspan);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   if (mSelIR) {
diff --git a/Detectors/TOF/workflow/src/TOFClusterizerSpec.cxx b/Detectors/TOF/workflow/src/TOFClusterizerSpec.cxx
index 4d3e0589c186d..48b1508692c24 100644
--- a/Detectors/TOF/workflow/src/TOFClusterizerSpec.cxx
+++ b/Detectors/TOF/workflow/src/TOFClusterizerSpec.cxx
@@ -224,25 +224,25 @@ class TOFDPLClustererTask
     }
 
     // send clusters
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERS", 0, Lifetime::Timeframe}, mClustersArray);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERSMULT", 0, Lifetime::Timeframe}, mMultPerLongBC);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERS", 0}, mClustersArray);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERSMULT", 0}, mMultPerLongBC);
     // send labels
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERSMCTR", 0, Lifetime::Timeframe}, mClsLabels);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERSMCTR", 0}, mClsLabels);
     }
 
     if (mIsCalib) {
       std::vector<CalibInfoCluster>* clusterCalInfo = mClusterer.getInfoFromCluster();
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOCALCLUS", 0, Lifetime::Timeframe}, *clusterCalInfo);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOCALCLUS", 0}, *clusterCalInfo);
     }
 
     if (mIsCosmic) {
       std::vector<CosmicInfo>* cosmicInfo = mCosmicProcessor.getCosmicInfo();
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOCOSMICS", 0, Lifetime::Timeframe}, *cosmicInfo);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOCOSMICS", 0}, *cosmicInfo);
       std::vector<CalibInfoTrackCl>* cosmicTrack = mCosmicProcessor.getCosmicTrack();
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOTRACKCOS", 0, Lifetime::Timeframe}, *cosmicTrack);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOTRACKCOS", 0}, *cosmicTrack);
       std::vector<int>* cosmicTrackSize = mCosmicProcessor.getCosmicTrackSize();
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOTRACKSIZE", 0, Lifetime::Timeframe}, *cosmicTrackSize);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOTRACKSIZE", 0}, *cosmicTrackSize);
     }
 
     mTimer.Stop();
diff --git a/Detectors/TOF/workflowIO/src/CalibClusReaderSpec.cxx b/Detectors/TOF/workflowIO/src/CalibClusReaderSpec.cxx
index 9be38858250ca..116f93a06c208 100644
--- a/Detectors/TOF/workflowIO/src/CalibClusReaderSpec.cxx
+++ b/Detectors/TOF/workflowIO/src/CalibClusReaderSpec.cxx
@@ -39,13 +39,13 @@ void CalibClusReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(debug) << "Pushing " << mPclusInfos->size() << " TOF clusters calib info at entry " << ent;
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOCALCLUS", 0, Lifetime::Timeframe}, mClusInfos);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOCALCLUS", 0}, mClusInfos);
 
   if (mIsCosmics) {
     LOG(debug) << "Pushing " << mPcosmicInfo->size() << " TOF cosmics info at entry " << ent;
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOCOSMICS", 0, Lifetime::Timeframe}, mCosmicInfo);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOTRACKCOS", 0, Lifetime::Timeframe}, mCosmicTrack);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOTRACKSIZE", 0, Lifetime::Timeframe}, mCosmicTrackSize);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOCOSMICS", 0}, mCosmicInfo);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOTRACKCOS", 0}, mCosmicTrack);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOTRACKSIZE", 0}, mCosmicTrackSize);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/TOF/workflowIO/src/CalibInfoReaderSpec.cxx b/Detectors/TOF/workflowIO/src/CalibInfoReaderSpec.cxx
index d81905fb38888..1854be2f1b88f 100644
--- a/Detectors/TOF/workflowIO/src/CalibInfoReaderSpec.cxx
+++ b/Detectors/TOF/workflowIO/src/CalibInfoReaderSpec.cxx
@@ -90,9 +90,9 @@ void CalibInfoReader::run(ProcessingContext& pc)
       LOG(debug) << "Current entry " << mCurrentEntry;
       LOG(debug) << "Send " << mVect.size() << " calib infos";
 
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, mTOFTPC ? ddCalib_tpc : ddCalib, 0, Lifetime::Timeframe}, mVect);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, mTOFTPC ? ddCalib_tpc : ddCalib, 0}, mVect);
 
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, ddDia, 0, Lifetime::Timeframe}, mDia);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, ddDia, 0}, mDia);
       usleep(100);
     }
     mGlobalEntry++;
diff --git a/Detectors/TOF/workflowIO/src/ClusterReaderSpec.cxx b/Detectors/TOF/workflowIO/src/ClusterReaderSpec.cxx
index 70f10b109a70e..e2979a8fc0dbf 100644
--- a/Detectors/TOF/workflowIO/src/ClusterReaderSpec.cxx
+++ b/Detectors/TOF/workflowIO/src/ClusterReaderSpec.cxx
@@ -44,10 +44,10 @@ void ClusterReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(debug) << "Pushing " << mClustersPtr->size() << " TOF clusters at entry " << ent;
 
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERS", 0, Lifetime::Timeframe}, mClusters);
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERSMULT", 0, Lifetime::Timeframe}, mClustersMult);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERS", 0}, mClusters);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERSMULT", 0}, mClustersMult);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERSMCTR", 0, Lifetime::Timeframe}, mLabels);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERSMCTR", 0}, mLabels);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/TOF/workflowIO/src/DigitReaderSpec.cxx b/Detectors/TOF/workflowIO/src/DigitReaderSpec.cxx
index f9fd57cf6b58f..30ddea9d5b9e7 100644
--- a/Detectors/TOF/workflowIO/src/DigitReaderSpec.cxx
+++ b/Detectors/TOF/workflowIO/src/DigitReaderSpec.cxx
@@ -81,18 +81,18 @@ void DigitReader::run(ProcessingContext& pc)
     mDiagnostic = mFiller.getDiagnosticFrequency();
 
     // add digits loaded in the output snapshot
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITS", 0, Lifetime::Timeframe}, mDigits);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "READOUTWINDOW", 0, Lifetime::Timeframe}, mRow);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "PATTERNS", 0, Lifetime::Timeframe}, mPatterns);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIAFREQ", 0, Lifetime::Timeframe}, mDiagnostic);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITS", 0}, mDigits);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "READOUTWINDOW", 0}, mRow);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "PATTERNS", 0}, mPatterns);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIAFREQ", 0}, mDiagnostic);
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITSMCTR", 0, Lifetime::Timeframe}, mLabels);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITSMCTR", 0}, mLabels);
     }
 
     static o2::parameters::GRPObject::ROMode roMode = o2::parameters::GRPObject::CONTINUOUS;
 
     LOG(debug) << "TOF: Sending ROMode= " << roMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "ROMode", 0, Lifetime::Timeframe}, roMode);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "ROMode", 0}, roMode);
   } else {
     LOG(error) << "Cannot read the TOF digits !";
     return;
diff --git a/Detectors/TOF/workflowIO/src/TOFMatchedReaderSpec.cxx b/Detectors/TOF/workflowIO/src/TOFMatchedReaderSpec.cxx
index 43264eb70cb58..48ccb83ca3cc3 100644
--- a/Detectors/TOF/workflowIO/src/TOFMatchedReaderSpec.cxx
+++ b/Detectors/TOF/workflowIO/src/TOFMatchedReaderSpec.cxx
@@ -74,12 +74,12 @@ void TOFMatchedReader::run(ProcessingContext& pc)
   LOG(debug) << "Pushing " << mMatches.size() << " TOF matchings at entry " << currEntry;
 
   uint32_t tpcMatchSS = o2::globaltracking::getSubSpec(mSubSpecStrict && (!mMode) ? o2::globaltracking::MatchingType::Strict : o2::globaltracking::MatchingType::Standard);
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, ddMatchInfo[mMode], tpcMatchSS, Lifetime::Timeframe}, mMatches);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, ddMatchInfo[mMode], tpcMatchSS}, mMatches);
   if (mReadTracks && (!mMode)) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "TOFTRACKS_TPC", tpcMatchSS, Lifetime::Timeframe}, mTracks);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "TOFTRACKS_TPC", tpcMatchSS}, mTracks);
   }
   if (mUseMC) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, ddMCMatchTOF[mMode], tpcMatchSS, Lifetime::Timeframe}, mLabelTOF);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, ddMCMatchTOF[mMode], tpcMatchSS}, mLabelTOF);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFLPIDCSpec.h b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFLPIDCSpec.h
index 6316824966856..02857398f0e07 100644
--- a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFLPIDCSpec.h
+++ b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFLPIDCSpec.h
@@ -195,14 +195,14 @@ class TPCFLPIDCDevice : public o2::framework::Task
 
     fill1DIDCs(cru);
     LOGP(debug, "Sending 1D-IDCs to EPNs of size {} and weights of size {}", mOneDIDCs.first.size(), mOneDIDCs.second.size());
-    output.snapshot(Output{gDataOriginTPC, getDataDescription1DIDCEPN(), subSpec, Lifetime::Timeframe}, mOneDIDCs.first);
-    output.snapshot(Output{gDataOriginTPC, getDataDescription1DIDCEPNWeights(), subSpec, Lifetime::Timeframe}, mOneDIDCs.second);
+    output.snapshot(Output{gDataOriginTPC, getDataDescription1DIDCEPN(), subSpec}, mOneDIDCs.first);
+    output.snapshot(Output{gDataOriginTPC, getDataDescription1DIDCEPNWeights(), subSpec}, mOneDIDCs.second);
   }
 
   void sendOutput(DataAllocator& output, const uint32_t cru)
   {
     const header::DataHeader::SubSpecificationType subSpec{cru << 7};
-    output.adoptContainer(Output{gDataOriginTPC, getDataDescriptionIDCGroup(CRU(cru).side()), subSpec, Lifetime::Timeframe}, std::move(mIDCs[cru]));
+    output.adoptContainer(Output{gDataOriginTPC, getDataDescriptionIDCGroup(CRU(cru).side()), subSpec}, std::move(mIDCs[cru]));
   }
 
   void fill1DIDCs(const uint32_t cru)
diff --git a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformEPNSpec.h b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformEPNSpec.h
index 7fd4c923feb19..e52234b3bb9ef 100644
--- a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformEPNSpec.h
+++ b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformEPNSpec.h
@@ -120,7 +120,7 @@ class TPCFourierTransformEPNSpec : public o2::framework::Task
 
   void sendOutput(DataAllocator& output, const Side side)
   {
-    output.snapshot(Output{gDataOriginTPC, getDataDescription(), header::DataHeader::SubSpecificationType{side}, Lifetime::Timeframe}, mIDCFourierTransform.getFourierCoefficients().getFourierCoefficients());
+    output.snapshot(Output{gDataOriginTPC, getDataDescription(), header::DataHeader::SubSpecificationType{side}}, mIDCFourierTransform.getFourierCoefficients().getFourierCoefficients());
   }
 };
 
diff --git a/Detectors/TPC/workflow/include/TPCWorkflow/TPCIntegrateIDCSpec.h b/Detectors/TPC/workflow/include/TPCWorkflow/TPCIntegrateIDCSpec.h
index 99d64220a2238..3ce0553f8dd7f 100644
--- a/Detectors/TPC/workflow/include/TPCWorkflow/TPCIntegrateIDCSpec.h
+++ b/Detectors/TPC/workflow/include/TPCWorkflow/TPCIntegrateIDCSpec.h
@@ -92,14 +92,14 @@ class TPCIntegrateIDCDevice : public o2::framework::Task
     for (const auto& idcs : mIDCs[sector].get()) {
       const header::DataHeader::SubSpecificationType subSpec{cru << 7};
       if (mIDCFormat == IDCFormat::Sim) {
-        output.snapshot(Output{gDataOriginTPC, getDataDescription(mIDCFormat), subSpec, Lifetime::Timeframe}, idcs);
+        output.snapshot(Output{gDataOriginTPC, getDataDescription(mIDCFormat), subSpec}, idcs);
       } else {
         // TODO
         // convert to format from thorsten here
         // send.......
         // DUMMY FOR NOW
         // const TPCCRUHeader cruheader{cru, mIntegrationIntervalsPerTF};
-        output.snapshot(Output{gDataOriginTPC, getDataDescription(mIDCFormat), subSpec, Lifetime::Timeframe}, idcs);
+        output.snapshot(Output{gDataOriginTPC, getDataDescription(mIDCFormat), subSpec}, idcs);
       }
       ++cru;
     }
diff --git a/Detectors/TPC/workflow/readers/include/TPCReaderWorkflow/PublisherSpec.h b/Detectors/TPC/workflow/readers/include/TPCReaderWorkflow/PublisherSpec.h
index 531bf635a1e5b..d73f80adf8b4b 100644
--- a/Detectors/TPC/workflow/readers/include/TPCReaderWorkflow/PublisherSpec.h
+++ b/Detectors/TPC/workflow/readers/include/TPCReaderWorkflow/PublisherSpec.h
@@ -62,22 +62,21 @@ framework::DataProcessorSpec getPublisherSpec(PublisherConf const& config, bool
 
   // a creator callback for the actual reader instance
   auto creator = [dto, mco, propagateMC](const char* treename, const char* filename, int nofEvents, Reader::PublishingMode publishingMode, o2::header::DataHeader::SubSpecificationType subSpec, const char* branchname, const char* mcbranchname, Reader::SpecialPublishHook* publishhook = nullptr) {
-    constexpr auto persistency = o2::framework::Lifetime::Timeframe;
     if (propagateMC) {
       return std::make_shared<Reader>(treename,
                                       filename,
                                       nofEvents,
                                       publishingMode,
-                                      Output{mco.origin, mco.description, subSpec, persistency},
+                                      Output{mco.origin, mco.description, subSpec},
                                       mcbranchname,
-                                      Reader::BranchDefinition<T>{Output{dto.origin, dto.description, subSpec, persistency}, branchname},
+                                      Reader::BranchDefinition<T>{Output{dto.origin, dto.description, subSpec}, branchname},
                                       publishhook);
     } else {
       return std::make_shared<Reader>(treename,
                                       filename,
                                       nofEvents,
                                       publishingMode,
-                                      Reader::BranchDefinition<T>{Output{dto.origin, dto.description, subSpec, persistency}, branchname},
+                                      Reader::BranchDefinition<T>{Output{dto.origin, dto.description, subSpec}, branchname},
                                       publishhook);
     }
   };
diff --git a/Detectors/TPC/workflow/readers/src/PublisherSpec.cxx b/Detectors/TPC/workflow/readers/src/PublisherSpec.cxx
index e02a78b3dca3a..7abd744440517 100644
--- a/Detectors/TPC/workflow/readers/src/PublisherSpec.cxx
+++ b/Detectors/TPC/workflow/readers/src/PublisherSpec.cxx
@@ -204,9 +204,9 @@ DataProcessorSpec createPublisherSpec(PublisherConf const& config, bool propagat
         header.sectorBits = 0;
         header.activeSectors = processAttributes->activeSectors;
         for (auto const& subSpec : processAttributes->zeroLengthOutputs) {
-          pc.outputs().make<char>({dto.origin, dto.description, subSpec, Lifetime::Timeframe, {header}});
+          pc.outputs().make<char>({dto.origin, dto.description, subSpec, {header}});
           if (pc.outputs().isAllowed({mco.origin, mco.description, subSpec})) {
-            pc.outputs().make<char>({mco.origin, mco.description, subSpec, Lifetime::Timeframe, {header}});
+            pc.outputs().make<char>({mco.origin, mco.description, subSpec, {header}});
           }
         }
       }
diff --git a/Detectors/TPC/workflow/readers/src/TrackReaderSpec.cxx b/Detectors/TPC/workflow/readers/src/TrackReaderSpec.cxx
index 66575dd843458..bfe1644bd8b5d 100644
--- a/Detectors/TPC/workflow/readers/src/TrackReaderSpec.cxx
+++ b/Detectors/TPC/workflow/readers/src/TrackReaderSpec.cxx
@@ -68,10 +68,10 @@ void TrackReader::run(ProcessingContext& pc)
     }
   }
 
-  pc.outputs().snapshot(Output{"TPC", "TRACKS", 0, Lifetime::Timeframe}, mTracksOut);
-  pc.outputs().snapshot(Output{"TPC", "CLUSREFS", 0, Lifetime::Timeframe}, mCluRefVecOut);
+  pc.outputs().snapshot(Output{"TPC", "TRACKS", 0}, mTracksOut);
+  pc.outputs().snapshot(Output{"TPC", "CLUSREFS", 0}, mCluRefVecOut);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"TPC", "TRACKSMCLBL", 0, Lifetime::Timeframe}, mMCTruthOut);
+    pc.outputs().snapshot(Output{"TPC", "TRACKSMCLBL", 0}, mMCTruthOut);
   }
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/TPC/workflow/readers/src/TriggerReaderSpec.cxx b/Detectors/TPC/workflow/readers/src/TriggerReaderSpec.cxx
index 2c363f4f7ed93..5dd3c16b3b1a8 100644
--- a/Detectors/TPC/workflow/readers/src/TriggerReaderSpec.cxx
+++ b/Detectors/TPC/workflow/readers/src/TriggerReaderSpec.cxx
@@ -36,7 +36,7 @@ void TriggerReader::run(ProcessingContext& pc)
   auto ent = mTree->GetReadEntry() + 1;
   mTree->GetEntry(ent);
 
-  pc.outputs().snapshot(Output{"TPC", "TRIGGERWORDS", 0, Lifetime::Timeframe}, *mTrig);
+  pc.outputs().snapshot(Output{"TPC", "TRIGGERWORDS", 0}, *mTrig);
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Detectors/TPC/workflow/src/ChunkedDigitPublisher.cxx b/Detectors/TPC/workflow/src/ChunkedDigitPublisher.cxx
index cf11e3c7b39ce..adf0cba944c03 100644
--- a/Detectors/TPC/workflow/src/ChunkedDigitPublisher.cxx
+++ b/Detectors/TPC/workflow/src/ChunkedDigitPublisher.cxx
@@ -111,8 +111,7 @@ auto makePublishBuffer(framework::ProcessingContext& pc, int sector, uint64_t ac
 
   o2::tpc::TPCSectorHeader header{sector};
   header.activeSectors = activeSectors;
-  return &pc.outputs().make<T>(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(sector), Lifetime::Timeframe,
-                                      header});
+  return &pc.outputs().make<T>(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(sector), header});
 }
 
 template <>
@@ -138,7 +137,7 @@ void publishBuffer<MCTruthContainer>(framework::ProcessingContext& pc, int secto
   LabelType* sharedlabels;
 #pragma omp critical
   sharedlabels = &pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(
-    Output{"TPC", "DIGITSMCTR", static_cast<SubSpecificationType>(sector), Lifetime::Timeframe, header});
+    Output{"TPC", "DIGITSMCTR", static_cast<SubSpecificationType>(sector), header});
 
   accum->flatten_to(*sharedlabels);
   delete accum;
diff --git a/Detectors/TPC/workflow/src/ClusterDecoderRawSpec.cxx b/Detectors/TPC/workflow/src/ClusterDecoderRawSpec.cxx
index bb20bf86d531a..9e2e70e8dde46 100644
--- a/Detectors/TPC/workflow/src/ClusterDecoderRawSpec.cxx
+++ b/Detectors/TPC/workflow/src/ClusterDecoderRawSpec.cxx
@@ -88,7 +88,7 @@ DataProcessorSpec getClusterDecoderRawSpec(bool sendMC)
           o2::header::Stack actual{*sectorHeaderMC};
           std::swap(mcHeaderStack, actual);
           if (sectorHeaderMC->sector() < 0) {
-            pc.outputs().snapshot(Output{gDataOriginTPC, DataDescription("CLNATIVEMCLBL"), fanSpec, Lifetime::Timeframe, std::move(mcHeaderStack)}, fanSpec);
+            pc.outputs().snapshot(Output{gDataOriginTPC, DataDescription("CLNATIVEMCLBL"), fanSpec, std::move(mcHeaderStack)}, fanSpec);
           }
         }
       }
@@ -97,7 +97,7 @@ DataProcessorSpec getClusterDecoderRawSpec(bool sendMC)
         o2::header::Stack actual{*sectorHeader};
         std::swap(rawHeaderStack, actual);
         if (sectorHeader->sector() < 0) {
-          pc.outputs().snapshot(Output{gDataOriginTPC, DataDescription("CLUSTERNATIVE"), fanSpec, Lifetime::Timeframe, std::move(rawHeaderStack)}, fanSpec);
+          pc.outputs().snapshot(Output{gDataOriginTPC, DataDescription("CLUSTERNATIVE"), fanSpec, std::move(rawHeaderStack)}, fanSpec);
           return;
         }
       }
@@ -167,7 +167,7 @@ DataProcessorSpec getClusterDecoderRawSpec(bool sendMC)
       // containers are created for clusters and MC labels per (sector,globalPadRow) address
       char* outputBuffer = nullptr;
       auto outputAllocator = [&pc, &fanSpec, &outputBuffer, &rawHeaderStack](size_t size) -> char* {
-        outputBuffer = pc.outputs().newChunk(Output{gDataOriginTPC, DataDescription("CLUSTERNATIVE"), fanSpec, Lifetime::Timeframe, std::move(rawHeaderStack)}, size).data();
+        outputBuffer = pc.outputs().newChunk(Output{gDataOriginTPC, DataDescription("CLUSTERNATIVE"), fanSpec, std::move(rawHeaderStack)}, size).data();
         return outputBuffer;
       };
       MCLabelContainer mcout;
@@ -188,7 +188,7 @@ DataProcessorSpec getClusterDecoderRawSpec(bool sendMC)
         // serialize the complete list of MC label containers
         ConstMCLabelContainer labelsFlat;
         mcout.flatten_to(labelsFlat);
-        pc.outputs().snapshot(Output{gDataOriginTPC, DataDescription("CLNATIVEMCLBL"), fanSpec, Lifetime::Timeframe, std::move(mcHeaderStack)}, labelsFlat);
+        pc.outputs().snapshot(Output{gDataOriginTPC, DataDescription("CLNATIVEMCLBL"), fanSpec, std::move(mcHeaderStack)}, labelsFlat);
       }
     };
 
diff --git a/Detectors/TPC/workflow/src/ClustererSpec.cxx b/Detectors/TPC/workflow/src/ClustererSpec.cxx
index 71043db3935be..310551a67753e 100644
--- a/Detectors/TPC/workflow/src/ClustererSpec.cxx
+++ b/Detectors/TPC/workflow/src/ClustererSpec.cxx
@@ -81,9 +81,9 @@ DataProcessorSpec getClustererSpec(bool sendMC)
         // forward the control information
         // FIXME define and use flags in TPCSectorHeader
         o2::tpc::TPCSectorHeader header{sector};
-        pc.outputs().snapshot(Output{gDataOriginTPC, "CLUSTERHW", fanSpec, Lifetime::Timeframe, {header}}, fanSpec);
+        pc.outputs().snapshot(Output{gDataOriginTPC, "CLUSTERHW", fanSpec, {header}}, fanSpec);
         if (DataRefUtils::isValid(mclabelref)) {
-          pc.outputs().snapshot(Output{gDataOriginTPC, "CLUSTERHWMCLBL", fanSpec, Lifetime::Timeframe, {header}}, fanSpec);
+          pc.outputs().snapshot(Output{gDataOriginTPC, "CLUSTERHWMCLBL", fanSpec, {header}}, fanSpec);
         }
         return;
       }
@@ -131,12 +131,12 @@ DataProcessorSpec getClustererSpec(bool sendMC)
       }
       // FIXME: that should be a case for pmr, want to send the content of the vector as a binary
       // block by using move semantics
-      auto outputPages = pc.outputs().make<ClusterHardwareContainer8kb>(Output{gDataOriginTPC, "CLUSTERHW", fanSpec, Lifetime::Timeframe, {*sectorHeader}}, clusterArray.size());
+      auto outputPages = pc.outputs().make<ClusterHardwareContainer8kb>(Output{gDataOriginTPC, "CLUSTERHW", fanSpec, {*sectorHeader}}, clusterArray.size());
       std::copy(clusterArray.begin(), clusterArray.end(), outputPages.begin());
       if (DataRefUtils::isValid(mclabelref)) {
         ConstMCLabelContainer mcflat;
         mctruthArray.flatten_to(mcflat);
-        pc.outputs().snapshot(Output{gDataOriginTPC, "CLUSTERHWMCLBL", fanSpec, Lifetime::Timeframe, {*sectorHeader}}, mcflat);
+        pc.outputs().snapshot(Output{gDataOriginTPC, "CLUSTERHWMCLBL", fanSpec, {*sectorHeader}}, mcflat);
       }
     };
 
diff --git a/Detectors/TPC/workflow/src/EntropyEncoderSpec.cxx b/Detectors/TPC/workflow/src/EntropyEncoderSpec.cxx
index 7e03748da0bc4..75fedb4a0effe 100644
--- a/Detectors/TPC/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/TPC/workflow/src/EntropyEncoderSpec.cxx
@@ -133,7 +133,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   auto triggers = pc.inputs().get<gsl::span<o2::tpc::TriggerInfoDLBZS>>("trigger");
   auto cput = mTimer.CpuTime();
   mTimer.Start(false);
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"TPC", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"TPC", "CTFDATA", 0});
   std::vector<bool> rejectHits, rejectTracks, rejectTrackHits, rejectTrackHitsReduced;
   CompressedClusters clustersFiltered = clusters;
   std::vector<std::pair<std::vector<unsigned int>, std::vector<unsigned short>>> tmpBuffer(std::max<int>(mNThreads, 1));
diff --git a/Detectors/TPC/workflow/src/KryptonClustererSpec.cxx b/Detectors/TPC/workflow/src/KryptonClustererSpec.cxx
index d0119f719193c..e946488073bf9 100644
--- a/Detectors/TPC/workflow/src/KryptonClustererSpec.cxx
+++ b/Detectors/TPC/workflow/src/KryptonClustererSpec.cxx
@@ -75,7 +75,7 @@ class KrBoxClusterFinderDevice : public o2::framework::Task
   {
     o2::tpc::TPCSectorHeader header{sector};
     header.activeSectors = (0x1 << sector);
-    output.snapshot(Output{gDataOriginTPC, "KRCLUSTERS", static_cast<SubSpecificationType>(sector), Lifetime::Timeframe, header}, clusters);
+    output.snapshot(Output{gDataOriginTPC, "KRCLUSTERS", static_cast<SubSpecificationType>(sector), header}, clusters);
   }
 };
 
diff --git a/Detectors/TPC/workflow/src/KryptonRawFilterSpec.cxx b/Detectors/TPC/workflow/src/KryptonRawFilterSpec.cxx
index 36dfa5edc65f7..d968876eefccb 100644
--- a/Detectors/TPC/workflow/src/KryptonRawFilterSpec.cxx
+++ b/Detectors/TPC/workflow/src/KryptonRawFilterSpec.cxx
@@ -223,7 +223,7 @@ class KrRawFilterDevice : public o2::framework::Task
   {
     o2::tpc::TPCSectorHeader header{sector};
     header.activeSectors = (0x1 << sector);
-    output.snapshot(Output{gDataOriginTPC, "FILTERDIG", static_cast<SubSpecificationType>(sector), Lifetime::Timeframe, header}, digits);
+    output.snapshot(Output{gDataOriginTPC, "FILTERDIG", static_cast<SubSpecificationType>(sector), header}, digits);
   }
 };
 
diff --git a/Detectors/TPC/workflow/src/LinkZSToDigitsSpec.cxx b/Detectors/TPC/workflow/src/LinkZSToDigitsSpec.cxx
index c7758d9f2bb4e..c42a93f7a7f04 100644
--- a/Detectors/TPC/workflow/src/LinkZSToDigitsSpec.cxx
+++ b/Detectors/TPC/workflow/src/LinkZSToDigitsSpec.cxx
@@ -110,7 +110,7 @@ o2::framework::DataProcessorSpec getLinkZSToDigitsSpec(int channel, const std::s
         header.activeSectors = processAttributes->activeSectors;
         // digit for now are transported per sector, not per lane
         // pc.outputs().snapshot(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(channel), Lifetime::Timeframe, header},
-        pc.outputs().snapshot(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(sector), Lifetime::Timeframe, header},
+        pc.outputs().snapshot(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(sector), header},
                               const_cast<std::vector<o2::tpc::Digit>&>(digits));
       };
 
diff --git a/Detectors/TPC/workflow/src/MIPTrackFilterSpec.cxx b/Detectors/TPC/workflow/src/MIPTrackFilterSpec.cxx
index d4291552ec151..2e60ba0bc72f6 100644
--- a/Detectors/TPC/workflow/src/MIPTrackFilterSpec.cxx
+++ b/Detectors/TPC/workflow/src/MIPTrackFilterSpec.cxx
@@ -141,7 +141,7 @@ void MIPTrackFilterDevice::run(ProcessingContext& pc)
   mMIPTracks.clear();
 }
 
-void MIPTrackFilterDevice::sendOutput(DataAllocator& output) { output.snapshot(Output{header::gDataOriginTPC, "MIPS", 0, Lifetime::Timeframe}, mMIPTracks); }
+void MIPTrackFilterDevice::sendOutput(DataAllocator& output) { output.snapshot(Output{header::gDataOriginTPC, "MIPS", 0}, mMIPTracks); }
 
 void MIPTrackFilterDevice::endOfStream(EndOfStreamContext& eos)
 {
diff --git a/Detectors/TPC/workflow/src/RawToDigitsSpec.cxx b/Detectors/TPC/workflow/src/RawToDigitsSpec.cxx
index 5c465be53b8c8..658a60fb75fbf 100644
--- a/Detectors/TPC/workflow/src/RawToDigitsSpec.cxx
+++ b/Detectors/TPC/workflow/src/RawToDigitsSpec.cxx
@@ -226,10 +226,10 @@ class TPCDigitDumpDevice : public o2::framework::Task
       o2::tpc::TPCSectorHeader header{isector};
       header.activeSectors = mActiveSectors;
       // digit for now are transported per sector, not per lane
-      output.snapshot(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(isector), Lifetime::Timeframe, header},
+      output.snapshot(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(isector), header},
                       mDigitDump.getDigits(isector));
       if (mSendCEdigits) {
-        output.snapshot(Output{"TPC", "CEDIGITS", static_cast<SubSpecificationType>(isector), Lifetime::Timeframe, header},
+        output.snapshot(Output{"TPC", "CEDIGITS", static_cast<SubSpecificationType>(isector), header},
                         ceDigits[isector]);
       }
     }
diff --git a/Detectors/TPC/workflow/src/SACProcessorSpec.cxx b/Detectors/TPC/workflow/src/SACProcessorSpec.cxx
index 226bd8a410dc9..e69533a0bb6d3 100644
--- a/Detectors/TPC/workflow/src/SACProcessorSpec.cxx
+++ b/Detectors/TPC/workflow/src/SACProcessorSpec.cxx
@@ -129,8 +129,8 @@ class SACProcessorDevice : public Task
 
   void sendData(DataAllocator& output)
   {
-    output.snapshot(Output{"TPC", "REFTIMESAC", 0, Lifetime::Timeframe}, mDecoder.getDecodedData().referenceTime);
-    output.snapshot(Output{"TPC", "DECODEDSAC", 0, Lifetime::Timeframe}, mDecoder.getDecodedData().getGoodData());
+    output.snapshot(Output{"TPC", "REFTIMESAC", 0}, mDecoder.getDecodedData().referenceTime);
+    output.snapshot(Output{"TPC", "DECODEDSAC", 0}, mDecoder.getDecodedData().getGoodData());
     mDecoder.clearDecodedData();
   }
 
diff --git a/Detectors/TPC/workflow/src/ZSSpec.cxx b/Detectors/TPC/workflow/src/ZSSpec.cxx
index c73d2ae406ebd..00c78ba463507 100644
--- a/Detectors/TPC/workflow/src/ZSSpec.cxx
+++ b/Detectors/TPC/workflow/src/ZSSpec.cxx
@@ -133,8 +133,8 @@ DataProcessorSpec getZSEncoderSpec(std::vector<int> const& tpcSectors, bool outR
       }
       o2::tpc::TPCSectorHeader sh{0};
       gsl::span<ZeroSuppressedContainer8kb> outp(&page[0], offset);
-      pc.outputs().snapshot(Output{gDataOriginTPC, "TPCZS", 0, Lifetime::Timeframe, sh}, outp);
-      pc.outputs().snapshot(Output{gDataOriginTPC, "ZSSIZES", 0, Lifetime::Timeframe, sh}, sizes);
+      pc.outputs().snapshot(Output{gDataOriginTPC, "TPCZS", 0, sh}, outp);
+      pc.outputs().snapshot(Output{gDataOriginTPC, "ZSSIZES", 0, sh}, sizes);
 
       if (outRaw) {
         // ===| set up raw writer |===================================================
@@ -301,7 +301,7 @@ DataProcessorSpec getZStoDigitsSpec(std::vector<int> const& tpcSectors)
       for (int i = 0; i < NSectors; i++) {
         LOG(info) << "digits in sector " << i << " : " << outDigits[i].size();
         o2::tpc::TPCSectorHeader sh{i};
-        pc.outputs().snapshot(Output{gDataOriginTPC, "DIGITS", (unsigned int)i, Lifetime::Timeframe, sh}, outDigits[i]);
+        pc.outputs().snapshot(Output{gDataOriginTPC, "DIGITS", (unsigned int)i, sh}, outDigits[i]);
       }
     };
     return processingFct;
diff --git a/Detectors/TPC/workflow/test/test_ft_EPN_Aggregator.cxx b/Detectors/TPC/workflow/test/test_ft_EPN_Aggregator.cxx
index ed4be20a22569..e8255f98eff6c 100644
--- a/Detectors/TPC/workflow/test/test_ft_EPN_Aggregator.cxx
+++ b/Detectors/TPC/workflow/test/test_ft_EPN_Aggregator.cxx
@@ -243,7 +243,7 @@ DataProcessorSpec generateIDCsCRU(int lane, const unsigned int maxTFs, const std
               }
             }
           }
-          ctx.outputs().adoptContainer(Output{gDataOriginTPC, TPCIntegrateIDCDevice::getDataDescription(TPCIntegrateIDCDevice::IDCFormat::Sim), o2::header::DataHeader::SubSpecificationType{icru << 7}, Lifetime::Timeframe}, std::move(idcs));
+          ctx.outputs().adoptContainer(Output{gDataOriginTPC, TPCIntegrateIDCDevice::getDataDescription(TPCIntegrateIDCDevice::IDCFormat::Sim), o2::header::DataHeader::SubSpecificationType{icru << 7}}, std::move(idcs));
         }
 
         if (delay) {
diff --git a/Detectors/TRD/reconstruction/src/EventRecord.cxx b/Detectors/TRD/reconstruction/src/EventRecord.cxx
index 147b052a8ca3f..0e1a134e45b8f 100644
--- a/Detectors/TRD/reconstruction/src/EventRecord.cxx
+++ b/Detectors/TRD/reconstruction/src/EventRecord.cxx
@@ -77,16 +77,16 @@ void EventRecordContainer::sendData(o2::framework::ProcessingContext& pc, bool g
     counters.push_back(event.getCounters());
   }
 
-  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "DIGITS", 0, o2::framework::Lifetime::Timeframe}, digits);
-  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "TRACKLETS", 0, o2::framework::Lifetime::Timeframe}, tracklets);
-  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "TRKTRGRD", 0, o2::framework::Lifetime::Timeframe}, triggers);
+  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "DIGITS", 0}, digits);
+  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "TRACKLETS", 0}, tracklets);
+  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "TRKTRGRD", 0}, triggers);
   if (generatestats) {
     accumulateStats();
     o2::base::TFIDInfoHelper::fillTFIDInfo(pc, mTFStats.mTFIDInfo);
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "RAWSTATS", 0, o2::framework::Lifetime::Timeframe}, mTFStats);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "RAWSTATS", 0}, mTFStats);
   }
   if (sendLinkStats) {
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "LINKSTATS", 0, o2::framework::Lifetime::Timeframe}, counters);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "LINKSTATS", 0}, counters);
   }
 
   std::chrono::duration<double, std::micro> dataReadTime = std::chrono::high_resolution_clock::now() - dataReadStart;
diff --git a/Detectors/TRD/workflow/include/TRDWorkflow/KrClustererSpec.h b/Detectors/TRD/workflow/include/TRDWorkflow/KrClustererSpec.h
index 26be31d51c084..8f50c1af91183 100644
--- a/Detectors/TRD/workflow/include/TRDWorkflow/KrClustererSpec.h
+++ b/Detectors/TRD/workflow/include/TRDWorkflow/KrClustererSpec.h
@@ -67,8 +67,8 @@ void TRDKrClustererDevice::run(ProcessingContext& pc)
   LOGP(info, "Found {} Kr clusters in {} input trigger records. Timing: CPU: {}, Real: {}",
        mKrClFinder.getKrClusters().size(), triggerRecords.size(), timer.CpuTime(), timer.RealTime());
 
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "KRCLUSTER", 0, Lifetime::Timeframe}, mKrClFinder.getKrClusters());
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGKRCLS", 0, Lifetime::Timeframe}, mKrClFinder.getKrTrigRecs());
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "KRCLUSTER", 0}, mKrClFinder.getKrClusters());
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGKRCLS", 0}, mKrClFinder.getKrTrigRecs());
 }
 
 void TRDKrClustererDevice::endOfStream(EndOfStreamContext& ec)
diff --git a/Detectors/TRD/workflow/include/TRDWorkflow/TRDGlobalTrackingQCSpec.h b/Detectors/TRD/workflow/include/TRDWorkflow/TRDGlobalTrackingQCSpec.h
index 375b36c54af17..a3c5b28aa8b63 100644
--- a/Detectors/TRD/workflow/include/TRDWorkflow/TRDGlobalTrackingQCSpec.h
+++ b/Detectors/TRD/workflow/include/TRDWorkflow/TRDGlobalTrackingQCSpec.h
@@ -63,7 +63,7 @@ class TRDGlobalTrackingQC : public Task
     mQC.reset();
     mQC.setInput(recoData);
     mQC.run();
-    pc.outputs().snapshot(Output{"TRD", "TRACKINGQC", 0, Lifetime::Timeframe}, mQC.getTrackQC());
+    pc.outputs().snapshot(Output{"TRD", "TRACKINGQC", 0}, mQC.getTrackQC());
   }
   void endOfStream(framework::EndOfStreamContext& ec) final {}
   void finaliseCCDB(framework::ConcreteDataMatcher& matcher, void* obj) final
diff --git a/Detectors/TRD/workflow/include/TRDWorkflow/TRDPulseHeightSpec.h b/Detectors/TRD/workflow/include/TRDWorkflow/TRDPulseHeightSpec.h
index 6ba185ee28ccb..a1e35f3a3482e 100644
--- a/Detectors/TRD/workflow/include/TRDWorkflow/TRDPulseHeightSpec.h
+++ b/Detectors/TRD/workflow/include/TRDWorkflow/TRDPulseHeightSpec.h
@@ -52,7 +52,7 @@ class PuseHeightDevice : public o2::framework::Task
     }
     if (mRunStopRequested) {
       std::vector<PHData> mPHValues{}; // the calibration expects data at every TF, so inject dummy
-      pc.outputs().snapshot(Output{"TRD", "PULSEHEIGHT", 0, Lifetime::Timeframe}, mPHValues);
+      pc.outputs().snapshot(Output{"TRD", "PULSEHEIGHT", 0}, mPHValues);
       return;
     }
     RecoContainer recoData;
@@ -61,7 +61,7 @@ class PuseHeightDevice : public o2::framework::Task
     mPulseHeight->setInput(recoData, &digits);
     mPulseHeight->reset();
     mPulseHeight->process();
-    pc.outputs().snapshot(Output{"TRD", "PULSEHEIGHT", 0, Lifetime::Timeframe}, mPulseHeight->getPHData());
+    pc.outputs().snapshot(Output{"TRD", "PULSEHEIGHT", 0}, mPulseHeight->getPHData());
     if (pc.transitionState() == TransitionHandlingState::Requested) {
       LOG(info) << "Run stop requested, finalizing";
       mRunStopRequested = true;
diff --git a/Detectors/TRD/workflow/io/src/TRDCalibReaderSpec.cxx b/Detectors/TRD/workflow/io/src/TRDCalibReaderSpec.cxx
index d3a565b4c9804..cb76bb8bbad69 100644
--- a/Detectors/TRD/workflow/io/src/TRDCalibReaderSpec.cxx
+++ b/Detectors/TRD/workflow/io/src/TRDCalibReaderSpec.cxx
@@ -53,7 +53,7 @@ void TRDCalibReader::run(ProcessingContext& pc)
   assert(currEntry < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(currEntry);
   LOG(info) << "Pushing angular residual histograms filled with " << mAngResids.getNEntries() << " entries at tree entry " << currEntry;
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "ANGRESHISTS", 0, Lifetime::Timeframe}, mAngResids);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "ANGRESHISTS", 0}, mAngResids);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/TRD/workflow/io/src/TRDDigitReaderSpec.cxx b/Detectors/TRD/workflow/io/src/TRDDigitReaderSpec.cxx
index 4ca6fd46e4492..92d001bc5120a 100644
--- a/Detectors/TRD/workflow/io/src/TRDDigitReaderSpec.cxx
+++ b/Detectors/TRD/workflow/io/src/TRDDigitReaderSpec.cxx
@@ -58,13 +58,13 @@ void TRDDigitReaderSpec::run(ProcessingContext& pc)
   assert(currEntry < mTreeDigits->GetEntries()); // this should not happen
   mTreeDigits->GetEntry(currEntry);
   LOGP(info, "Pushing {} digits for tree entry {}", mDigits.size(), currEntry);
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "DIGITS", mSubSpec, Lifetime::Timeframe}, mDigits);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "DIGITS", mSubSpec}, mDigits);
   if (mUseTriggerRecords) {
     LOGP(info, "Pushing {} trigger records for tree entry {}", mTriggerRecords.size(), currEntry);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRKTRGRD", mSubSpec, Lifetime::Timeframe}, mTriggerRecords);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRKTRGRD", mSubSpec}, mTriggerRecords);
   }
   if (mUseMC) {
-    auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{o2::header::gDataOriginTRD, "LABELS", 0, Lifetime::Timeframe});
+    auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{o2::header::gDataOriginTRD, "LABELS", 0});
     mLabels->copyandflatten(sharedlabels);
   }
   if (mTreeDigits->GetReadEntry() + 1 >= mTreeDigits->GetEntries()) {
diff --git a/Detectors/TRD/workflow/io/src/TRDPHReaderSpec.cxx b/Detectors/TRD/workflow/io/src/TRDPHReaderSpec.cxx
index 235c58c2ad302..7f873beb2ece1 100644
--- a/Detectors/TRD/workflow/io/src/TRDPHReaderSpec.cxx
+++ b/Detectors/TRD/workflow/io/src/TRDPHReaderSpec.cxx
@@ -54,7 +54,7 @@ void TRDPHReader::run(ProcessingContext& pc)
   mTree->GetEntry(currEntry);
 
   LOG(info) << "Pushing vector of PH values filled with " << mPHValues.size() << " entries at tree entry " << currEntry;
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "PULSEHEIGHT", 0, Lifetime::Timeframe}, mPHValues);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "PULSEHEIGHT", 0}, mPHValues);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/TRD/workflow/io/src/TRDTrackReaderSpec.cxx b/Detectors/TRD/workflow/io/src/TRDTrackReaderSpec.cxx
index 20a7b5d5ac234..cd9702a3d2385 100644
--- a/Detectors/TRD/workflow/io/src/TRDTrackReaderSpec.cxx
+++ b/Detectors/TRD/workflow/io/src/TRDTrackReaderSpec.cxx
@@ -50,18 +50,18 @@ void TRDTrackReader::run(ProcessingContext& pc)
 
   if (mMode == Mode::TPCTRD) {
     uint32_t ss = o2::globaltracking::getSubSpec(mSubSpecStrict ? o2::globaltracking::MatchingType::Strict : o2::globaltracking::MatchingType::Standard);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MATCH_TPC", ss, Lifetime::Timeframe}, mTracks);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGREC_TPC", ss, Lifetime::Timeframe}, mTrigRec);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MATCH_TPC", ss}, mTracks);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGREC_TPC", ss}, mTrigRec);
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_TPC", ss, Lifetime::Timeframe}, mLabelsMatch);
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_TPC_TRD", ss, Lifetime::Timeframe}, mLabelsTrd);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_TPC", ss}, mLabelsMatch);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_TPC_TRD", ss}, mLabelsTrd);
     }
   } else if (mMode == Mode::ITSTPCTRD) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MATCH_ITSTPC", 0, Lifetime::Timeframe}, mTracks);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGREC_ITSTPC", 0, Lifetime::Timeframe}, mTrigRec);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MATCH_ITSTPC", 0}, mTracks);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGREC_ITSTPC", 0}, mTrigRec);
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_ITSTPC", 0, Lifetime::Timeframe}, mLabelsMatch);
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_ITSTPC_TRD", 0, Lifetime::Timeframe}, mLabelsTrd);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_ITSTPC", 0}, mLabelsMatch);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_ITSTPC_TRD", 0}, mLabelsTrd);
     }
   }
 
diff --git a/Detectors/TRD/workflow/io/src/TRDTrackletReaderSpec.cxx b/Detectors/TRD/workflow/io/src/TRDTrackletReaderSpec.cxx
index c588da696e321..864cc3d665af5 100644
--- a/Detectors/TRD/workflow/io/src/TRDTrackletReaderSpec.cxx
+++ b/Detectors/TRD/workflow/io/src/TRDTrackletReaderSpec.cxx
@@ -72,20 +72,20 @@ void TRDTrackletReader::run(ProcessingContext& pc)
   mTreeTrklt->GetEntry(currEntry);
   LOG(info) << "Pushing " << mTriggerRecords.size() << " TRD trigger records at entry " << currEntry;
   LOG(info) << "Pushing " << mTracklets.size() << " uncalibrated TRD tracklets for these trigger records";
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRACKLETS", 0, Lifetime::Timeframe}, mTracklets);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRACKLETS", 0}, mTracklets);
   if (mUseTrackletTransform) {
     assert(mTreeTrklt->GetEntries() == mTreeCTrklt->GetEntries());
     mTreeCTrklt->GetEntry(currEntry);
     LOG(info) << "Pushing " << mTrackletsCal.size() << " calibrated TRD tracklets for these trigger records";
     LOG(info) << "Pushing " << mTrigRecMask.size() << " flags for the given TRD trigger records";
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "CTRACKLETS", 0, Lifetime::Timeframe}, mTrackletsCal);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRIGRECMASK", 0, Lifetime::Timeframe}, mTrigRecMask);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "CTRACKLETS", 0}, mTrackletsCal);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRIGRECMASK", 0}, mTrigRecMask);
   }
 
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRKTRGRD", 0, Lifetime::Timeframe}, mTriggerRecords);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRKTRGRD", 0}, mTriggerRecords);
   if (mUseMC) {
     LOG(info) << "Pushing " << mLabels.getNElements() << " TRD tracklet labels";
-    pc.outputs().snapshot(Output{"TRD", "TRKLABELS", 0, Lifetime::Timeframe}, mLabels);
+    pc.outputs().snapshot(Output{"TRD", "TRKLABELS", 0}, mLabels);
   }
 
   if (mTreeTrklt->GetReadEntry() + 1 >= mTreeTrklt->GetEntries()) {
diff --git a/Detectors/TRD/workflow/src/EntropyEncoderSpec.cxx b/Detectors/TRD/workflow/src/EntropyEncoderSpec.cxx
index 07243d2b316cb..d345dd74141ed 100644
--- a/Detectors/TRD/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/TRD/workflow/src/EntropyEncoderSpec.cxx
@@ -76,7 +76,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"TRD", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"TRD", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, triggers, tracklets, digits);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   if (mSelIR) {
diff --git a/Detectors/TRD/workflow/src/TRDDigitizerSpec.cxx b/Detectors/TRD/workflow/src/TRDDigitizerSpec.cxx
index c0a4d85acd1bc..1cd145a6c2e36 100644
--- a/Detectors/TRD/workflow/src/TRDDigitizerSpec.cxx
+++ b/Detectors/TRD/workflow/src/TRDDigitizerSpec.cxx
@@ -177,17 +177,17 @@ class TRDDPLDigitizerTask : public o2::base::BaseDPLDigitizer
     LOGF(info, "TRD digitization timing: Cpu: %.3e Real: %.3e s", timer.CpuTime(), timer.RealTime());
 
     LOG(info) << "TRD: Sending " << digitsAccum.size() << " digits";
-    pc.outputs().snapshot(Output{"TRD", "DIGITS", 1, Lifetime::Timeframe}, digitsAccum);
+    pc.outputs().snapshot(Output{"TRD", "DIGITS", 1}, digitsAccum);
     if (mctruth) {
       LOG(info) << "TRD: Sending " << labelsAccum.getNElements() << " labels";
       // we are flattening the labels and write to managed shared memory container for further communication
-      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{"TRD", "LABELS", 0, Lifetime::Timeframe});
+      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{"TRD", "LABELS", 0});
       labelsAccum.flatten_to(sharedlabels);
     }
     LOG(info) << "TRD: Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{"TRD", "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{"TRD", "ROMode", 0}, mROMode);
     LOG(info) << "TRD: Sending trigger records";
-    pc.outputs().snapshot(Output{"TRD", "TRKTRGRD", 1, Lifetime::Timeframe}, triggers);
+    pc.outputs().snapshot(Output{"TRD", "TRKTRGRD", 1}, triggers);
     // we should be only called once; tell DPL that this process is ready to exit
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
     finished = true;
diff --git a/Detectors/TRD/workflow/src/TRDGlobalTrackingSpec.cxx b/Detectors/TRD/workflow/src/TRDGlobalTrackingSpec.cxx
index 1be328b4bcbe9..abe3a6f45ec70 100644
--- a/Detectors/TRD/workflow/src/TRDGlobalTrackingSpec.cxx
+++ b/Detectors/TRD/workflow/src/TRDGlobalTrackingSpec.cxx
@@ -496,19 +496,19 @@ void TRDGlobalTracking::run(ProcessingContext& pc)
 
   uint32_t ss = o2::globaltracking::getSubSpec(mStrict ? o2::globaltracking::MatchingType::Strict : o2::globaltracking::MatchingType::Standard);
   if (GTrackID::includesSource(GTrackID::Source::ITSTPC, mTrkMask)) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MATCH_ITSTPC", 0, Lifetime::Timeframe}, tracksOutITSTPC);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGREC_ITSTPC", 0, Lifetime::Timeframe}, trackTrigRecITSTPC);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MATCH_ITSTPC", 0}, tracksOutITSTPC);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGREC_ITSTPC", 0}, trackTrigRecITSTPC);
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_ITSTPC", 0, Lifetime::Timeframe}, matchLabelsITSTPC);
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_ITSTPC_TRD", 0, Lifetime::Timeframe}, trdLabelsITSTPC);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_ITSTPC", 0}, matchLabelsITSTPC);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_ITSTPC_TRD", 0}, trdLabelsITSTPC);
     }
   }
   if (GTrackID::includesSource(GTrackID::Source::TPC, mTrkMask)) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MATCH_TPC", ss, Lifetime::Timeframe}, tracksOutTPC);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGREC_TPC", ss, Lifetime::Timeframe}, trackTrigRecTPC);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MATCH_TPC", ss}, tracksOutTPC);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGREC_TPC", ss}, trackTrigRecTPC);
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_TPC", ss, Lifetime::Timeframe}, matchLabelsTPC);
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_TPC_TRD", ss, Lifetime::Timeframe}, trdLabelsTPC);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_TPC", ss}, matchLabelsTPC);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_TPC_TRD", ss}, trdLabelsTPC);
     }
   }
 
diff --git a/Detectors/TRD/workflow/src/TRDTrackletTransformerSpec.cxx b/Detectors/TRD/workflow/src/TRDTrackletTransformerSpec.cxx
index bb6d552cf3fee..0ac920c6e49c6 100644
--- a/Detectors/TRD/workflow/src/TRDTrackletTransformerSpec.cxx
+++ b/Detectors/TRD/workflow/src/TRDTrackletTransformerSpec.cxx
@@ -113,8 +113,8 @@ void TRDTrackletTransformerSpec::run(o2::framework::ProcessingContext& pc)
 
   LOGF(info, "Found %lu tracklets in %lu trigger records. Applied filter for ITS IR frames: %i. Transformed %i tracklets.", tracklets.size(), trigRecs.size(), mTrigRecFilterActive, nTrackletsTransformed);
 
-  pc.outputs().snapshot(Output{"TRD", "CTRACKLETS", 0, Lifetime::Timeframe}, calibratedTracklets);
-  pc.outputs().snapshot(Output{"TRD", "TRIGRECMASK", 0, Lifetime::Timeframe}, trigRecBitfield);
+  pc.outputs().snapshot(Output{"TRD", "CTRACKLETS", 0}, calibratedTracklets);
+  pc.outputs().snapshot(Output{"TRD", "TRIGRECMASK", 0}, trigRecBitfield);
 }
 
 void TRDTrackletTransformerSpec::updateTimeDependentParams(ProcessingContext& pc)
diff --git a/Detectors/TRD/workflow/src/TRDTrapSimulatorSpec.cxx b/Detectors/TRD/workflow/src/TRDTrapSimulatorSpec.cxx
index 4c6ac24a0dc43..20d0b719b9bf7 100644
--- a/Detectors/TRD/workflow/src/TRDTrapSimulatorSpec.cxx
+++ b/Detectors/TRD/workflow/src/TRDTrapSimulatorSpec.cxx
@@ -328,11 +328,11 @@ void TRDDPLTrapSimulatorTask::run(o2::framework::ProcessingContext& pc)
   LOG(info) << "Digit Sorting took: " << std::chrono::duration_cast<std::chrono::milliseconds>(sortTime).count() << "ms";
   LOG(info) << "Processing time for parallel region: " << std::chrono::duration_cast<std::chrono::milliseconds>(parallelTime).count() << "ms";
 
-  pc.outputs().snapshot(Output{"TRD", "TRACKLETS", 0, Lifetime::Timeframe}, tracklets);
-  pc.outputs().snapshot(Output{"TRD", "TRKTRGRD", 0, Lifetime::Timeframe}, triggerRecords);
-  pc.outputs().snapshot(Output{"TRD", "DIGITS", 0, Lifetime::Timeframe}, digitsOut);
+  pc.outputs().snapshot(Output{"TRD", "TRACKLETS", 0}, tracklets);
+  pc.outputs().snapshot(Output{"TRD", "TRKTRGRD", 0}, triggerRecords);
+  pc.outputs().snapshot(Output{"TRD", "DIGITS", 0}, digitsOut);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"TRD", "TRKLABELS", 0, Lifetime::Timeframe}, lblTracklets);
+    pc.outputs().snapshot(Output{"TRD", "TRKLABELS", 0}, lblTracklets);
   }
 
   LOG(debug) << "TRD Trap Simulator Device exiting";
diff --git a/Detectors/TRD/workflow/src/TrackBasedCalibSpec.cxx b/Detectors/TRD/workflow/src/TrackBasedCalibSpec.cxx
index 6011e194685ad..4418b9af61300 100644
--- a/Detectors/TRD/workflow/src/TrackBasedCalibSpec.cxx
+++ b/Detectors/TRD/workflow/src/TrackBasedCalibSpec.cxx
@@ -75,12 +75,12 @@ void TRDTrackBasedCalibDevice::run(ProcessingContext& pc)
 
   if (mDoVdExBCalib) {
     mCalibrator.calculateAngResHistos();
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "ANGRESHISTS", 0, Lifetime::Timeframe}, mCalibrator.getAngResHistos());
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "ANGRESHISTS", 0}, mCalibrator.getAngResHistos());
   }
 
   if (mDoGainCalib) {
     mCalibrator.calculateGainCalibObjs();
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "GAINCALIBHISTS", 0, Lifetime::Timeframe}, mCalibrator.getGainCalibHistos());
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "GAINCALIBHISTS", 0}, mCalibrator.getGainCalibHistos());
   }
 
   mCalibrator.reset();
diff --git a/Detectors/Upgrades/ITS3/workflow/src/ClustererSpec.cxx b/Detectors/Upgrades/ITS3/workflow/src/ClustererSpec.cxx
index e6c9b5bc4b910..5512964eec950 100644
--- a/Detectors/Upgrades/ITS3/workflow/src/ClustererSpec.cxx
+++ b/Detectors/Upgrades/ITS3/workflow/src/ClustererSpec.cxx
@@ -109,17 +109,17 @@ void ClustererDPL::run(ProcessingContext& pc)
     clusterLabels = std::make_unique<o2::dataformats::MCTruthContainer<o2::MCCompLabel>>();
   }
   mClusterer->process(mNThreads, reader, &clusCompVec, mPatterns ? &clusPattVec : nullptr, &clusROFVec, clusterLabels.get());
-  pc.outputs().snapshot(Output{orig, "COMPCLUSTERS", 0, Lifetime::Timeframe}, clusCompVec);
-  pc.outputs().snapshot(Output{orig, "CLUSTERSROF", 0, Lifetime::Timeframe}, clusROFVec);
-  pc.outputs().snapshot(Output{orig, "PATTERNS", 0, Lifetime::Timeframe}, clusPattVec);
+  pc.outputs().snapshot(Output{orig, "COMPCLUSTERS", 0}, clusCompVec);
+  pc.outputs().snapshot(Output{orig, "CLUSTERSROF", 0}, clusROFVec);
+  pc.outputs().snapshot(Output{orig, "PATTERNS", 0}, clusPattVec);
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{orig, "CLUSTERSMCTR", 0, Lifetime::Timeframe}, *clusterLabels.get()); // at the moment requires snapshot
+    pc.outputs().snapshot(Output{orig, "CLUSTERSMCTR", 0}, *clusterLabels.get()); // at the moment requires snapshot
     std::vector<o2::itsmft::MC2ROFRecord> clusterMC2ROframes(mc2rofs.size());
     for (int i = mc2rofs.size(); i--;) {
       clusterMC2ROframes[i] = mc2rofs[i]; // Simply, replicate it from digits ?
     }
-    pc.outputs().snapshot(Output{orig, "CLUSTERSMC2ROF", 0, Lifetime::Timeframe}, clusterMC2ROframes);
+    pc.outputs().snapshot(Output{orig, "CLUSTERSMC2ROF", 0}, clusterMC2ROframes);
   }
 
   // TODO: in principle, after masking "overflow" pixels the MC2ROFRecord maxROF supposed to change, nominally to minROF
diff --git a/Detectors/Upgrades/ITS3/workflow/src/DigitReaderSpec.cxx b/Detectors/Upgrades/ITS3/workflow/src/DigitReaderSpec.cxx
index a679beed2c530..c2380be77f956 100644
--- a/Detectors/Upgrades/ITS3/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/Upgrades/ITS3/workflow/src/DigitReaderSpec.cxx
@@ -71,17 +71,17 @@ void DigitReader::run(ProcessingContext& pc)
 
   // This is a very ugly way of providing DataDescription, which anyway does not need to contain detector name.
   // To be fixed once the names-definition class is ready
-  pc.outputs().snapshot(Output{mOrigin, "DIGITSROF", 0, Lifetime::Timeframe}, mDigROFRec);
-  pc.outputs().snapshot(Output{mOrigin, "DIGITS", 0, Lifetime::Timeframe}, mDigits);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITSROF", 0}, mDigROFRec);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITS", 0}, mDigits);
   if (mUseCalib) {
-    pc.outputs().snapshot(Output{mOrigin, "GBTCALIB", 0, Lifetime::Timeframe}, mCalib);
+    pc.outputs().snapshot(Output{mOrigin, "GBTCALIB", 0}, mCalib);
   }
 
   if (mUseMC) {
-    auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{mOrigin, "DIGITSMCTR", 0, Lifetime::Timeframe});
+    auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{mOrigin, "DIGITSMCTR", 0});
     plabels->copyandflatten(sharedlabels);
     delete plabels;
-    pc.outputs().snapshot(Output{mOrigin, "DIGITSMC2ROF", 0, Lifetime::Timeframe}, mDigMC2ROFs);
+    pc.outputs().snapshot(Output{mOrigin, "DIGITSMC2ROF", 0}, mDigMC2ROFs);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/Upgrades/ITS3/workflow/src/TrackReaderSpec.cxx b/Detectors/Upgrades/ITS3/workflow/src/TrackReaderSpec.cxx
index 3d048bf981ea3..54d8951b5a0d0 100644
--- a/Detectors/Upgrades/ITS3/workflow/src/TrackReaderSpec.cxx
+++ b/Detectors/Upgrades/ITS3/workflow/src/TrackReaderSpec.cxx
@@ -44,14 +44,14 @@ void TrackReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mTracks.size() << " track in " << mROFRec.size() << " ROFs at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "IT3TrackROF", 0, Lifetime::Timeframe}, mROFRec);
-  pc.outputs().snapshot(Output{mOrigin, "TRACKS", 0, Lifetime::Timeframe}, mTracks);
-  pc.outputs().snapshot(Output{mOrigin, "TRACKCLSID", 0, Lifetime::Timeframe}, mClusInd);
-  pc.outputs().snapshot(Output{"IT3", "VERTICES", 0, Lifetime::Timeframe}, mVertices);
-  pc.outputs().snapshot(Output{"IT3", "VERTICESROF", 0, Lifetime::Timeframe}, mVerticesROFRec);
+  pc.outputs().snapshot(Output{mOrigin, "IT3TrackROF", 0}, mROFRec);
+  pc.outputs().snapshot(Output{mOrigin, "TRACKS", 0}, mTracks);
+  pc.outputs().snapshot(Output{mOrigin, "TRACKCLSID", 0}, mClusInd);
+  pc.outputs().snapshot(Output{"IT3", "VERTICES", 0}, mVertices);
+  pc.outputs().snapshot(Output{"IT3", "VERTICESROF", 0}, mVerticesROFRec);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{mOrigin, "TRACKSMCTR", 0, Lifetime::Timeframe}, mMCTruth);
-    pc.outputs().snapshot(Output{mOrigin, "VERTICESMCTR", 0, Lifetime::Timeframe}, mMCVertTruth);
+    pc.outputs().snapshot(Output{mOrigin, "TRACKSMCTR", 0}, mMCTruth);
+    pc.outputs().snapshot(Output{mOrigin, "VERTICESMCTR", 0}, mMCVertTruth);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx b/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx
index e2078046d1966..e4240401928b6 100644
--- a/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx
+++ b/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx
@@ -171,9 +171,9 @@ void TrackerDPL::run(ProcessingContext& pc)
   // the output vector however is created directly inside the message memory thus avoiding copy by
   // snapshot
   auto rofsinput = pc.inputs().get<gsl::span<o2::itsmft::ROFRecord>>("ROframes");
-  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"IT3", "IT3TrackROF", 0, Lifetime::Timeframe}, rofsinput.begin(), rofsinput.end());
+  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"IT3", "IT3TrackROF", 0}, rofsinput.begin(), rofsinput.end());
 
-  auto& irFrames = pc.outputs().make<std::vector<o2::dataformats::IRFrame>>(Output{"IT3", "IRFRAMES", 0, Lifetime::Timeframe});
+  auto& irFrames = pc.outputs().make<std::vector<o2::dataformats::IRFrame>>(Output{"IT3", "IRFRAMES", 0});
 
   const auto& alpParams = o2::itsmft::DPLAlpideParam<o2::detectors::DetID::ITS>::Instance(); // RS: this should come from CCDB
   int nBCPerTF = alpParams.roFrameLengthInBC;
@@ -189,15 +189,15 @@ void TrackerDPL::run(ProcessingContext& pc)
     LOG(info) << labels->getIndexedSize() << " MC label objects , in " << mc2rofs.size() << " MC events";
   }
 
-  auto& allClusIdx = pc.outputs().make<std::vector<int>>(Output{"IT3", "TRACKCLSID", 0, Lifetime::Timeframe});
+  auto& allClusIdx = pc.outputs().make<std::vector<int>>(Output{"IT3", "TRACKCLSID", 0});
   std::vector<o2::MCCompLabel> trackLabels;
   std::vector<MCCompLabel> verticesLabels;
-  auto& allTracks = pc.outputs().make<std::vector<o2::its::TrackITS>>(Output{"IT3", "TRACKS", 0, Lifetime::Timeframe});
+  auto& allTracks = pc.outputs().make<std::vector<o2::its::TrackITS>>(Output{"IT3", "TRACKS", 0});
   std::vector<o2::MCCompLabel> allTrackLabels;
   std::vector<o2::MCCompLabel> allVerticesLabels;
 
-  auto& vertROFvec = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"IT3", "VERTICESROF", 0, Lifetime::Timeframe});
-  auto& vertices = pc.outputs().make<std::vector<Vertex>>(Output{"IT3", "VERTICES", 0, Lifetime::Timeframe});
+  auto& vertROFvec = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"IT3", "VERTICESROF", 0});
+  auto& vertices = pc.outputs().make<std::vector<Vertex>>(Output{"IT3", "VERTICES", 0});
 
   TimeFrame* timeFrame = mChainITS->GetITSTimeframe();
   timeFrame->resizeVectors(mNLayers);
@@ -314,9 +314,9 @@ void TrackerDPL::run(ProcessingContext& pc)
       LOGP(info, "ITS3Tracker pushed {} track labels", allTrackLabels.size());
       LOGP(info, "ITS3Tracker pushed {} vertex labels", allVerticesLabels.size());
 
-      pc.outputs().snapshot(Output{"IT3", "TRACKSMCTR", 0, Lifetime::Timeframe}, allTrackLabels);
-      pc.outputs().snapshot(Output{"IT3", "VERTICESMCTR", 0, Lifetime::Timeframe}, allVerticesLabels);
-      pc.outputs().snapshot(Output{"IT3", "IT3TrackMC2ROF", 0, Lifetime::Timeframe}, mc2rofs);
+      pc.outputs().snapshot(Output{"IT3", "TRACKSMCTR", 0}, allTrackLabels);
+      pc.outputs().snapshot(Output{"IT3", "VERTICESMCTR", 0}, allVerticesLabels);
+      pc.outputs().snapshot(Output{"IT3", "IT3TrackMC2ROF", 0}, mc2rofs);
     }
   }
   mTimer.Stop();
diff --git a/Detectors/Upgrades/ITS3/workflow/src/VertexReaderSpec.cxx b/Detectors/Upgrades/ITS3/workflow/src/VertexReaderSpec.cxx
index 015aa1a78977b..25fd3e530161a 100644
--- a/Detectors/Upgrades/ITS3/workflow/src/VertexReaderSpec.cxx
+++ b/Detectors/Upgrades/ITS3/workflow/src/VertexReaderSpec.cxx
@@ -41,8 +41,8 @@ void VertexReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mVerticesPtr->size() << " vertices in " << mVerticesROFRecPtr->size()
             << " ROFs at entry " << ent;
-  pc.outputs().snapshot(Output{"IT3", "VERTICES", 0, Lifetime::Timeframe}, mVertices);
-  pc.outputs().snapshot(Output{"IT3", "VERTICESROF", 0, Lifetime::Timeframe}, mVerticesROFRec);
+  pc.outputs().snapshot(Output{"IT3", "VERTICES", 0}, mVertices);
+  pc.outputs().snapshot(Output{"IT3", "VERTICESROF", 0}, mVerticesROFRec);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/Vertexing/src/SVertexer.cxx b/Detectors/Vertexing/src/SVertexer.cxx
index de167af74526e..120d49d44b4e5 100644
--- a/Detectors/Vertexing/src/SVertexer.cxx
+++ b/Detectors/Vertexing/src/SVertexer.cxx
@@ -117,15 +117,15 @@ void SVertexer::produceOutput(o2::framework::ProcessingContext& pc)
   std::sort(nbodySortID.begin(), nbodySortID.end(), [](const vid& a, const vid& b) { return a.vtxID < b.vtxID; });
 
   // dpl output
-  auto& v0sIdx = pc.outputs().make<std::vector<V0Index>>(o2f::Output{"GLO", "V0S_IDX", 0, o2f::Lifetime::Timeframe});
-  auto& cascsIdx = pc.outputs().make<std::vector<CascadeIndex>>(o2f::Output{"GLO", "CASCS_IDX", 0, o2f::Lifetime::Timeframe});
-  auto& body3Idx = pc.outputs().make<std::vector<Decay3BodyIndex>>(o2f::Output{"GLO", "DECAYS3BODY_IDX", 0, o2f::Lifetime::Timeframe});
-  auto& fullv0s = pc.outputs().make<std::vector<V0>>(o2f::Output{"GLO", "V0S", 0, o2f::Lifetime::Timeframe});
-  auto& fullcascs = pc.outputs().make<std::vector<Cascade>>(o2f::Output{"GLO", "CASCS", 0, o2f::Lifetime::Timeframe});
-  auto& full3body = pc.outputs().make<std::vector<Decay3Body>>(o2f::Output{"GLO", "DECAYS3BODY", 0, o2f::Lifetime::Timeframe});
-  auto& v0Refs = pc.outputs().make<std::vector<RRef>>(o2f::Output{"GLO", "PVTX_V0REFS", 0, o2f::Lifetime::Timeframe});
-  auto& cascRefs = pc.outputs().make<std::vector<RRef>>(o2f::Output{"GLO", "PVTX_CASCREFS", 0, o2f::Lifetime::Timeframe});
-  auto& vtx3bodyRefs = pc.outputs().make<std::vector<RRef>>(o2f::Output{"GLO", "PVTX_3BODYREFS", 0, o2f::Lifetime::Timeframe});
+  auto& v0sIdx = pc.outputs().make<std::vector<V0Index>>(o2f::Output{"GLO", "V0S_IDX", 0});
+  auto& cascsIdx = pc.outputs().make<std::vector<CascadeIndex>>(o2f::Output{"GLO", "CASCS_IDX", 0});
+  auto& body3Idx = pc.outputs().make<std::vector<Decay3BodyIndex>>(o2f::Output{"GLO", "DECAYS3BODY_IDX", 0});
+  auto& fullv0s = pc.outputs().make<std::vector<V0>>(o2f::Output{"GLO", "V0S", 0});
+  auto& fullcascs = pc.outputs().make<std::vector<Cascade>>(o2f::Output{"GLO", "CASCS", 0});
+  auto& full3body = pc.outputs().make<std::vector<Decay3Body>>(o2f::Output{"GLO", "DECAYS3BODY", 0});
+  auto& v0Refs = pc.outputs().make<std::vector<RRef>>(o2f::Output{"GLO", "PVTX_V0REFS", 0});
+  auto& cascRefs = pc.outputs().make<std::vector<RRef>>(o2f::Output{"GLO", "PVTX_CASCREFS", 0});
+  auto& vtx3bodyRefs = pc.outputs().make<std::vector<RRef>>(o2f::Output{"GLO", "PVTX_3BODYREFS", 0});
 
   // sorted V0s
   v0sIdx.reserve(mNV0s);
@@ -214,8 +214,8 @@ void SVertexer::produceOutput(o2::framework::ProcessingContext& pc)
       }
     }
 
-    auto& strTracksOut = pc.outputs().make<std::vector<o2::dataformats::StrangeTrack>>(o2f::Output{"GLO", "STRANGETRACKS", 0, o2f::Lifetime::Timeframe});
-    auto& strClustOut = pc.outputs().make<std::vector<o2::strangeness_tracking::ClusAttachments>>(o2f::Output{"GLO", "CLUSUPDATES", 0, o2f::Lifetime::Timeframe});
+    auto& strTracksOut = pc.outputs().make<std::vector<o2::dataformats::StrangeTrack>>(o2f::Output{"GLO", "STRANGETRACKS", 0});
+    auto& strClustOut = pc.outputs().make<std::vector<o2::strangeness_tracking::ClusAttachments>>(o2f::Output{"GLO", "CLUSUPDATES", 0});
     o2::pmr::vector<o2::MCCompLabel> mcLabsOut;
     strTracksOut.resize(mNStrangeTracks);
     strClustOut.resize(mNStrangeTracks);
@@ -239,7 +239,7 @@ void SVertexer::produceOutput(o2::framework::ProcessingContext& pc)
     }
 
     if (mStrTracker->getMCTruthOn()) {
-      auto& strTrMCLableOut = pc.outputs().make<std::vector<o2::MCCompLabel>>(o2f::Output{"GLO", "STRANGETRACKS_MC", 0, o2f::Lifetime::Timeframe});
+      auto& strTrMCLableOut = pc.outputs().make<std::vector<o2::MCCompLabel>>(o2f::Output{"GLO", "STRANGETRACKS_MC", 0});
       strTrMCLableOut.swap(mcLabsOut);
     }
   }
diff --git a/Detectors/ZDC/calib/src/InterCalibEPNSpec.cxx b/Detectors/ZDC/calib/src/InterCalibEPNSpec.cxx
index 80b23c696b5a4..a1a83aada50f0 100644
--- a/Detectors/ZDC/calib/src/InterCalibEPNSpec.cxx
+++ b/Detectors/ZDC/calib/src/InterCalibEPNSpec.cxx
@@ -101,16 +101,16 @@ void InterCalibEPNSpec::run(ProcessingContext& pc)
   mWorker.process(bcrec, energy, tdc, info);
 
   // Send intermediate calibration data and histograms
-  o2::framework::Output output("ZDC", "INTERCALIBDATA", 0, Lifetime::Timeframe);
+  o2::framework::Output output("ZDC", "INTERCALIBDATA", 0);
   pc.outputs().snapshot(output, mWorker.mData);
   for (int ih = 0; ih < (2 * InterCalibData::NH); ih++) {
     if (mWorker.mH[ih] != nullptr) {
-      o2::framework::Output output("ZDC", "INTER_1DH", ih, Lifetime::Timeframe);
+      o2::framework::Output output("ZDC", "INTER_1DH", ih);
       pc.outputs().snapshot(output, mWorker.mH[ih]->getBase());
     }
   }
   for (int ih = 0; ih < InterCalibData::NH; ih++) {
-    o2::framework::Output output("ZDC", "INTER_2DH", ih, Lifetime::Timeframe);
+    o2::framework::Output output("ZDC", "INTER_2DH", ih);
     pc.outputs().snapshot(output, mWorker.mC[ih]->getBase());
   }
 }
diff --git a/Detectors/ZDC/calib/src/TDCCalibEPNSpec.cxx b/Detectors/ZDC/calib/src/TDCCalibEPNSpec.cxx
index 8b90ee3b0af36..6125baaa197f8 100644
--- a/Detectors/ZDC/calib/src/TDCCalibEPNSpec.cxx
+++ b/Detectors/ZDC/calib/src/TDCCalibEPNSpec.cxx
@@ -114,10 +114,10 @@ void TDCCalibEPNSpec::run(ProcessingContext& pc)
     if (mVerbosity > DbgMedium && mModTF > 0) {
       LOG(info) << "Send intermediate calibration data mProcessed=" << mProcessed << " >= mModTF=" << mModTF;
     }
-    o2::framework::Output output("ZDC", "TDCCALIBDATA", 0, Lifetime::Timeframe);
+    o2::framework::Output output("ZDC", "TDCCALIBDATA", 0);
     pc.outputs().snapshot(output, mWorker.mData);
     for (int ih = 0; ih < TDCCalibData::NTDC; ih++) {
-      o2::framework::Output output("ZDC", "TDC_1DH", ih, Lifetime::Timeframe);
+      o2::framework::Output output("ZDC", "TDC_1DH", ih);
       pc.outputs().snapshot(output, mWorker.mTDC[ih]->getBase());
     }
     mWorker.clear();
diff --git a/Detectors/ZDC/calib/src/WaveformCalibEPNSpec.cxx b/Detectors/ZDC/calib/src/WaveformCalibEPNSpec.cxx
index 62ee862905524..72f606cc4f6ed 100644
--- a/Detectors/ZDC/calib/src/WaveformCalibEPNSpec.cxx
+++ b/Detectors/ZDC/calib/src/WaveformCalibEPNSpec.cxx
@@ -107,7 +107,7 @@ void WaveformCalibEPNSpec::run(ProcessingContext& pc)
   mWorker.process(bcrec, energy, tdc, info, wave);
 
   // Send intermediate calibration data
-  o2::framework::Output output("ZDC", "WAVECALIBDATA", 0, Lifetime::Timeframe);
+  o2::framework::Output output("ZDC", "WAVECALIBDATA", 0);
   pc.outputs().snapshot(output, mWorker.mData);
 }
 
diff --git a/Detectors/ZDC/raw/include/ZDCRaw/RawReaderZDC.h b/Detectors/ZDC/raw/include/ZDCRaw/RawReaderZDC.h
index b27f7e34353c0..70f7ce30647f5 100644
--- a/Detectors/ZDC/raw/include/ZDCRaw/RawReaderZDC.h
+++ b/Detectors/ZDC/raw/include/ZDCRaw/RawReaderZDC.h
@@ -95,9 +95,9 @@ class RawReaderZDC
 
   void makeSnapshot(o2::framework::ProcessingContext& pc)
   {
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginZDC, "DIGITSBC", 0, o2::framework::Lifetime::Timeframe}, mDigitsBC);
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginZDC, "DIGITSCH", 0, o2::framework::Lifetime::Timeframe}, mDigitsCh);
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginZDC, "DIGITSPD", 0, o2::framework::Lifetime::Timeframe}, mOrbitData);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginZDC, "DIGITSBC", 0}, mDigitsBC);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginZDC, "DIGITSCH", 0}, mDigitsCh);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginZDC, "DIGITSPD", 0}, mOrbitData);
   }
 };
 } // namespace zdc
diff --git a/Detectors/ZDC/workflow/src/DigitReaderSpec.cxx b/Detectors/ZDC/workflow/src/DigitReaderSpec.cxx
index 59b0e34d16ecd..e952111e0c6c3 100644
--- a/Detectors/ZDC/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/ZDC/workflow/src/DigitReaderSpec.cxx
@@ -69,11 +69,11 @@ void DigitReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "ZDCDigitReader pushed " << zdcOrbitData.size() << " orbits with " << zdcBCData.size() << " bcs and " << zdcChData.size() << " digits";
-  pc.outputs().snapshot(Output{"ZDC", "DIGITSPD", 0, Lifetime::Timeframe}, zdcOrbitData);
-  pc.outputs().snapshot(Output{"ZDC", "DIGITSBC", 0, Lifetime::Timeframe}, zdcBCData);
-  pc.outputs().snapshot(Output{"ZDC", "DIGITSCH", 0, Lifetime::Timeframe}, zdcChData);
+  pc.outputs().snapshot(Output{"ZDC", "DIGITSPD", 0}, zdcOrbitData);
+  pc.outputs().snapshot(Output{"ZDC", "DIGITSBC", 0}, zdcBCData);
+  pc.outputs().snapshot(Output{"ZDC", "DIGITSCH", 0}, zdcChData);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"ZDC", "DIGITSLBL", 0, Lifetime::Timeframe}, labels);
+    pc.outputs().snapshot(Output{"ZDC", "DIGITSLBL", 0}, labels);
   }
   uint64_t nextEntry = mTree->GetReadEntry() + 1;
   if (nextEntry >= mTree->GetEntries() || (mLastEntry >= 0 && nextEntry > mLastEntry)) {
diff --git a/Detectors/ZDC/workflow/src/DigitRecoSpec.cxx b/Detectors/ZDC/workflow/src/DigitRecoSpec.cxx
index d504d33483081..e9b63c5b49d5b 100644
--- a/Detectors/ZDC/workflow/src/DigitRecoSpec.cxx
+++ b/Detectors/ZDC/workflow/src/DigitRecoSpec.cxx
@@ -264,11 +264,11 @@ void DigitRecoSpec::run(ProcessingContext& pc)
   }
   // TODO: rate information for all channels
   // TODO: summary of reconstruction to be collected by DQM?
-  pc.outputs().snapshot(Output{"ZDC", "BCREC", 0, Lifetime::Timeframe}, recEvent.mRecBC);
-  pc.outputs().snapshot(Output{"ZDC", "ENERGY", 0, Lifetime::Timeframe}, recEvent.mEnergy);
-  pc.outputs().snapshot(Output{"ZDC", "TDCDATA", 0, Lifetime::Timeframe}, recEvent.mTDCData);
-  pc.outputs().snapshot(Output{"ZDC", "INFO", 0, Lifetime::Timeframe}, recEvent.mInfo);
-  pc.outputs().snapshot(Output{"ZDC", "WAVE", 0, Lifetime::Timeframe}, recEvent.mWaveform);
+  pc.outputs().snapshot(Output{"ZDC", "BCREC", 0}, recEvent.mRecBC);
+  pc.outputs().snapshot(Output{"ZDC", "ENERGY", 0}, recEvent.mEnergy);
+  pc.outputs().snapshot(Output{"ZDC", "TDCDATA", 0}, recEvent.mTDCData);
+  pc.outputs().snapshot(Output{"ZDC", "INFO", 0}, recEvent.mInfo);
+  pc.outputs().snapshot(Output{"ZDC", "WAVE", 0}, recEvent.mWaveform);
   mTimer.Stop();
 }
 
diff --git a/Detectors/ZDC/workflow/src/EntropyEncoderSpec.cxx b/Detectors/ZDC/workflow/src/EntropyEncoderSpec.cxx
index 7c30a0f34d151..abbd821fcb749 100644
--- a/Detectors/ZDC/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/ZDC/workflow/src/EntropyEncoderSpec.cxx
@@ -58,7 +58,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
 
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"ZDC", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"ZDC", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, bcdata, chans, peds);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   if (mSelIR) {
diff --git a/Detectors/ZDC/workflow/src/RecEventReaderSpec.cxx b/Detectors/ZDC/workflow/src/RecEventReaderSpec.cxx
index acb0cef3e29be..18c620e427569 100644
--- a/Detectors/ZDC/workflow/src/RecEventReaderSpec.cxx
+++ b/Detectors/ZDC/workflow/src/RecEventReaderSpec.cxx
@@ -49,10 +49,10 @@ void RecEventReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
 
   LOG(info) << "ZDC RecEventReader pushes " << mBCRecData->size() << " events with " << mBCRecData->size() << " energy, " << mZDCTDCData->size() << " TDC and " << mZDCInfo->size() << " info records at entry " << ent;
-  pc.outputs().snapshot(Output{"ZDC", "BCREC", 0, Lifetime::Timeframe}, *mBCRecData);
-  pc.outputs().snapshot(Output{"ZDC", "ENERGY", 0, Lifetime::Timeframe}, *mZDCEnergy);
-  pc.outputs().snapshot(Output{"ZDC", "TDCDATA", 0, Lifetime::Timeframe}, *mZDCTDCData);
-  pc.outputs().snapshot(Output{"ZDC", "INFO", 0, Lifetime::Timeframe}, *mZDCInfo);
+  pc.outputs().snapshot(Output{"ZDC", "BCREC", 0}, *mBCRecData);
+  pc.outputs().snapshot(Output{"ZDC", "ENERGY", 0}, *mZDCEnergy);
+  pc.outputs().snapshot(Output{"ZDC", "TDCDATA", 0}, *mZDCTDCData);
+  pc.outputs().snapshot(Output{"ZDC", "INFO", 0}, *mZDCInfo);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/ZDC/workflow/src/RecoReaderSpec.cxx b/Detectors/ZDC/workflow/src/RecoReaderSpec.cxx
index 20b9510243be4..ebea51c932c5c 100644
--- a/Detectors/ZDC/workflow/src/RecoReaderSpec.cxx
+++ b/Detectors/ZDC/workflow/src/RecoReaderSpec.cxx
@@ -64,10 +64,10 @@ void RecoReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "ZDCRecoReader pushed " << RecBC.size() << " b.c. " << Energy.size() << " Energies " << TDCData.size() << " TDCs " << Info.size() << " Infos";
-  pc.outputs().snapshot(Output{"ZDC", "BCREC", 0, Lifetime::Timeframe}, RecBC);
-  pc.outputs().snapshot(Output{"ZDC", "ENERGY", 0, Lifetime::Timeframe}, Energy);
-  pc.outputs().snapshot(Output{"ZDC", "TDCDATA", 0, Lifetime::Timeframe}, TDCData);
-  pc.outputs().snapshot(Output{"ZDC", "INFO", 0, Lifetime::Timeframe}, Info);
+  pc.outputs().snapshot(Output{"ZDC", "BCREC", 0}, RecBC);
+  pc.outputs().snapshot(Output{"ZDC", "ENERGY", 0}, Energy);
+  pc.outputs().snapshot(Output{"ZDC", "TDCDATA", 0}, TDCData);
+  pc.outputs().snapshot(Output{"ZDC", "INFO", 0}, Info);
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/GPU/Workflow/src/GPUWorkflowITS.cxx b/GPU/Workflow/src/GPUWorkflowITS.cxx
index e82f2348255d7..1785ddd05f39c 100644
--- a/GPU/Workflow/src/GPUWorkflowITS.cxx
+++ b/GPU/Workflow/src/GPUWorkflowITS.cxx
@@ -134,8 +134,8 @@ int GPURecoWorkflowSpec::runITSTracking(o2::framework::ProcessingContext& pc)
 
   auto rofsinput = pc.inputs().get<gsl::span<o2::itsmft::ROFRecord>>("ROframes");
 
-  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "ITSTrackROF", 0, Lifetime::Timeframe}, rofsinput.begin(), rofsinput.end());
-  auto& irFrames = pc.outputs().make<std::vector<o2::dataformats::IRFrame>>(Output{"ITS", "IRFRAMES", 0, Lifetime::Timeframe});
+  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "ITSTrackROF", 0}, rofsinput.begin(), rofsinput.end());
+  auto& irFrames = pc.outputs().make<std::vector<o2::dataformats::IRFrame>>(Output{"ITS", "IRFRAMES", 0});
   irFrames.reserve(rofs.size());
 
   const auto& alpParams = o2::itsmft::DPLAlpideParam<o2::detectors::DetID::ITS>::Instance(); // RS: this should come from CCDB
@@ -148,19 +148,19 @@ int GPURecoWorkflowSpec::runITSTracking(o2::framework::ProcessingContext& pc)
   if (mSpecConfig.processMC) {
     labels = pc.inputs().get<const dataformats::MCTruthContainer<MCCompLabel>*>("itsmclabels").release();
     // get the array as read-only span, a snapshot is sent forward
-    pc.outputs().snapshot(Output{"ITS", "ITSTrackMC2ROF", 0, Lifetime::Timeframe}, pc.inputs().get<gsl::span<itsmft::MC2ROFRecord>>("ITSMC2ROframes"));
+    pc.outputs().snapshot(Output{"ITS", "ITSTrackMC2ROF", 0}, pc.inputs().get<gsl::span<itsmft::MC2ROFRecord>>("ITSMC2ROframes"));
     LOG(info) << labels->getIndexedSize() << " MC label objects , in " << mc2rofs.size() << " MC events";
   }
 
-  auto& allClusIdx = pc.outputs().make<std::vector<int>>(Output{"ITS", "TRACKCLSID", 0, Lifetime::Timeframe});
-  auto& allTracks = pc.outputs().make<std::vector<o2::its::TrackITS>>(Output{"ITS", "TRACKS", 0, Lifetime::Timeframe});
-  auto& vertROFvec = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "VERTICESROF", 0, Lifetime::Timeframe});
-  auto& vertices = pc.outputs().make<std::vector<Vertex>>(Output{"ITS", "VERTICES", 0, Lifetime::Timeframe});
+  auto& allClusIdx = pc.outputs().make<std::vector<int>>(Output{"ITS", "TRACKCLSID", 0});
+  auto& allTracks = pc.outputs().make<std::vector<o2::its::TrackITS>>(Output{"ITS", "TRACKS", 0});
+  auto& vertROFvec = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "VERTICESROF", 0});
+  auto& vertices = pc.outputs().make<std::vector<Vertex>>(Output{"ITS", "VERTICES", 0});
 
   // MC
   static pmr::vector<o2::MCCompLabel> dummyMCLabTracks, dummyMCLabVerts;
-  auto& allTrackLabels = mSpecConfig.processMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"ITS", "TRACKSMCTR", 0, Lifetime::Timeframe}) : dummyMCLabTracks;
-  auto& allVerticesLabels = mSpecConfig.processMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"ITS", "VERTICESMCTR", 0, Lifetime::Timeframe}) : dummyMCLabVerts;
+  auto& allTrackLabels = mSpecConfig.processMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"ITS", "TRACKSMCTR", 0}) : dummyMCLabTracks;
+  auto& allVerticesLabels = mSpecConfig.processMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"ITS", "VERTICESMCTR", 0}) : dummyMCLabVerts;
 
   std::uint32_t roFrame = 0;
 
diff --git a/GPU/Workflow/src/GPUWorkflowPipeline.cxx b/GPU/Workflow/src/GPUWorkflowPipeline.cxx
index c50f8f67cbd32..0352151b342c6 100644
--- a/GPU/Workflow/src/GPUWorkflowPipeline.cxx
+++ b/GPU/Workflow/src/GPUWorkflowPipeline.cxx
@@ -177,7 +177,7 @@ int GPURecoWorkflowSpec::handlePipeline(ProcessingContext& pc, GPUTrackingInOutP
     ptrs.tpcZS = &tpcZS;
   }
   if (mSpecConfig.enableDoublePipeline == 2) {
-    auto prepareBuffer = pc.outputs().make<DataAllocator::UninitializedVector<char>>(Output{gDataOriginGPU, "PIPELINEPREPARE", 0, Lifetime::Timeframe}, 0u);
+    auto prepareBuffer = pc.outputs().make<DataAllocator::UninitializedVector<char>>(Output{gDataOriginGPU, "PIPELINEPREPARE", 0}, 0u);
 
     size_t ptrsTotal = 0;
     const void* firstPtr = nullptr;
diff --git a/GPU/Workflow/src/GPUWorkflowSpec.cxx b/GPU/Workflow/src/GPUWorkflowSpec.cxx
index 40b6c13495a66..fb482df5b11c0 100644
--- a/GPU/Workflow/src/GPUWorkflowSpec.cxx
+++ b/GPU/Workflow/src/GPUWorkflowSpec.cxx
@@ -734,7 +734,7 @@ void GPURecoWorkflowSpec::run(ProcessingContext& pc)
   };
 
   setOutputAllocator("COMPCLUSTERSFLAT", mSpecConfig.outputCompClustersFlat, outputRegions.compressedClusters, std::make_tuple(gDataOriginTPC, (DataDescription) "COMPCLUSTERSFLAT", 0));
-  setOutputAllocator("CLUSTERNATIVE", mClusterOutputIds.size() > 0, outputRegions.clustersNative, std::make_tuple(gDataOriginTPC, mSpecConfig.sendClustersPerSector ? (DataDescription) "CLUSTERNATIVETMP" : (DataDescription) "CLUSTERNATIVE", NSectors, Lifetime::Timeframe, clusterOutputSectorHeader), sizeof(o2::tpc::ClusterCountIndex));
+  setOutputAllocator("CLUSTERNATIVE", mClusterOutputIds.size() > 0, outputRegions.clustersNative, std::make_tuple(gDataOriginTPC, mSpecConfig.sendClustersPerSector ? (DataDescription) "CLUSTERNATIVETMP" : (DataDescription) "CLUSTERNATIVE", NSectors, clusterOutputSectorHeader), sizeof(o2::tpc::ClusterCountIndex));
   setOutputAllocator("CLSHAREDMAP", mSpecConfig.outputSharedClusterMap, outputRegions.sharedClusterMap, std::make_tuple(gDataOriginTPC, (DataDescription) "CLSHAREDMAP", 0));
   setOutputAllocator("TRACKS", mSpecConfig.outputTracks, outputRegions.tpcTracksO2, std::make_tuple(gDataOriginTPC, (DataDescription) "TRACKS", 0));
   setOutputAllocator("CLUSREFS", mSpecConfig.outputTracks, outputRegions.tpcTracksO2ClusRefs, std::make_tuple(gDataOriginTPC, (DataDescription) "CLUSREFS", 0));
@@ -909,7 +909,7 @@ void GPURecoWorkflowSpec::run(ProcessingContext& pc)
         if (mTPCSectorMask & (1ul << i)) {
           DataHeader::SubSpecificationType subspec = i;
           clusterOutputSectorHeader.sectorBits = (1ul << i);
-          char* buffer = pc.outputs().make<char>({gDataOriginTPC, "CLUSTERNATIVE", subspec, Lifetime::Timeframe, {clusterOutputSectorHeader}}, accessIndex.nClustersSector[i] * sizeof(*accessIndex.clustersLinear) + sizeof(o2::tpc::ClusterCountIndex)).data();
+          char* buffer = pc.outputs().make<char>({gDataOriginTPC, "CLUSTERNATIVE", subspec, {clusterOutputSectorHeader}}, accessIndex.nClustersSector[i] * sizeof(*accessIndex.clustersLinear) + sizeof(o2::tpc::ClusterCountIndex)).data();
           o2::tpc::ClusterCountIndex* outIndex = reinterpret_cast<o2::tpc::ClusterCountIndex*>(buffer);
           memset(outIndex, 0, sizeof(*outIndex));
           for (int j = 0; j < o2::tpc::constants::MAXGLOBALPADROW; j++) {
@@ -926,7 +926,7 @@ void GPURecoWorkflowSpec::run(ProcessingContext& pc)
             }
             ConstMCLabelContainer contflat;
             cont.flatten_to(contflat);
-            pc.outputs().snapshot({gDataOriginTPC, "CLNATIVEMCLBL", subspec, Lifetime::Timeframe, {clusterOutputSectorHeader}}, contflat);
+            pc.outputs().snapshot({gDataOriginTPC, "CLNATIVEMCLBL", subspec, {clusterOutputSectorHeader}}, contflat);
           }
         }
       }
@@ -937,7 +937,7 @@ void GPURecoWorkflowSpec::run(ProcessingContext& pc)
       static_assert(sizeof(o2::tpc::ClusterCountIndex) == sizeof(accessIndex.nClusters));
       memcpy(outIndex, &accessIndex.nClusters[0][0], sizeof(o2::tpc::ClusterCountIndex));
       if (mSpecConfig.processMC && mSpecConfig.caClusterer && accessIndex.clustersMCTruth) {
-        pc.outputs().snapshot({gDataOriginTPC, "CLNATIVEMCLBL", subspec, Lifetime::Timeframe, {clusterOutputSectorHeader}}, clustersMCBuffer.first);
+        pc.outputs().snapshot({gDataOriginTPC, "CLNATIVEMCLBL", subspec, {clusterOutputSectorHeader}}, clustersMCBuffer.first);
       }
     }
   }
@@ -952,17 +952,17 @@ void GPURecoWorkflowSpec::run(ProcessingContext& pc)
     if (sendQAOutput) {
       mQA->postprocessExternal(copy1, copy2, copy3, copy4, out, mQATaskMask ? mQATaskMask : -1);
     }
-    pc.outputs().snapshot({gDataOriginTPC, "TRACKINGQA", 0, Lifetime::Timeframe}, out);
+    pc.outputs().snapshot({gDataOriginTPC, "TRACKINGQA", 0}, out);
     if (sendQAOutput) {
       mQA->cleanup();
     }
   }
   if (mSpecConfig.outputErrorQA) {
-    pc.outputs().snapshot({gDataOriginGPU, "ERRORQA", 0, Lifetime::Timeframe}, mErrorQA);
+    pc.outputs().snapshot({gDataOriginGPU, "ERRORQA", 0}, mErrorQA);
     mErrorQA.clear(); // FIXME: This is a race condition once we run multi-threaded!
   }
   if (mSpecConfig.tpcTriggerHandling && !(mSpecConfig.zsOnTheFly || mSpecConfig.zsDecoder)) {
-    pc.outputs().make<DataAllocator::UninitializedVector<outputDataType>>(Output{gDataOriginTPC, "TRIGGERWORDS", 0, Lifetime::Timeframe}, 0u);
+    pc.outputs().make<DataAllocator::UninitializedVector<outputDataType>>(Output{gDataOriginTPC, "TRIGGERWORDS", 0}, 0u);
   }
   mTimer->Stop();
   LOG(info) << "GPU Reconstruction time for this TF " << mTimer->CpuTime() - cput << " s (cpu), " << mTimer->RealTime() - realt << " s (wall)";
diff --git a/Steer/DigitizerWorkflow/src/CPVDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/CPVDigitizerSpec.cxx
index 57fb11249b312..fc64106e39d79 100644
--- a/Steer/DigitizerWorkflow/src/CPVDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/CPVDigitizerSpec.cxx
@@ -171,15 +171,15 @@ void DigitizerSpec::run(framework::ProcessingContext& pc)
   }
   LOG(debug) << "Have " << mLabels.getNElements() << " CPV labels ";
   // here we have all digits and we can send them to consumer (aka snapshot it onto output)
-  pc.outputs().snapshot(Output{"CPV", "DIGITS", 0, Lifetime::Timeframe}, mDigitsOut);
-  pc.outputs().snapshot(Output{"CPV", "DIGITTRIGREC", 0, Lifetime::Timeframe}, triggers);
+  pc.outputs().snapshot(Output{"CPV", "DIGITS", 0}, mDigitsOut);
+  pc.outputs().snapshot(Output{"CPV", "DIGITTRIGREC", 0}, triggers);
   if (pc.outputs().isAllowed({"CPV", "DIGITSMCTR", 0})) {
-    pc.outputs().snapshot(Output{"CPV", "DIGITSMCTR", 0, Lifetime::Timeframe}, mLabels);
+    pc.outputs().snapshot(Output{"CPV", "DIGITSMCTR", 0}, mLabels);
   }
   // CPV is always a triggered detector
   const o2::parameters::GRPObject::ROMode roMode = o2::parameters::GRPObject::PRESENT;
   LOG(debug) << "CPV: Sending ROMode= " << roMode << " to GRPUpdater";
-  pc.outputs().snapshot(Output{"CPV", "ROMode", 0, Lifetime::Timeframe}, roMode);
+  pc.outputs().snapshot(Output{"CPV", "ROMode", 0}, roMode);
 
   timer.Stop();
   LOG(info) << "Digitization took " << timer.CpuTime() << "s";
diff --git a/Steer/DigitizerWorkflow/src/CTPDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/CTPDigitizerSpec.cxx
index 412e6d6129a76..a72b9e26581eb 100644
--- a/Steer/DigitizerWorkflow/src/CTPDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/CTPDigitizerSpec.cxx
@@ -79,9 +79,9 @@ class CTPDPLDigitizerTask : public o2::base::BaseDPLDigitizer
     auto digits = mDigitizer.process(ginputs);
     // send out to next stage
     LOG(info) << "CTP DIGITS being sent.";
-    pc.outputs().snapshot(Output{"CTP", "DIGITS", 0, Lifetime::Timeframe}, digits);
+    pc.outputs().snapshot(Output{"CTP", "DIGITS", 0}, digits);
     LOG(info) << "CTP PRESENT being sent.";
-    pc.outputs().snapshot(Output{"CTP", "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{"CTP", "ROMode", 0}, mROMode);
     timer.Stop();
     LOG(info) << "CTP Digitization took " << timer.CpuTime() << "s";
   }
diff --git a/Steer/DigitizerWorkflow/src/FDDDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/FDDDigitizerSpec.cxx
index 99150a3cb83fb..901c7e709d265 100644
--- a/Steer/DigitizerWorkflow/src/FDDDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/FDDDigitizerSpec.cxx
@@ -120,17 +120,17 @@ class FDDDPLDigitizerTask : public o2::base::BaseDPLDigitizer
     mDigitizer.flush(mDigitsBC, mDigitsCh, mDigitsTrig, labels);
 
     // send out to next stage
-    pc.outputs().snapshot(Output{"FDD", "DIGITSBC", 0, Lifetime::Timeframe}, mDigitsBC);
-    pc.outputs().snapshot(Output{"FDD", "DIGITSCH", 0, Lifetime::Timeframe}, mDigitsCh);
-    pc.outputs().snapshot(Output{"FDD", "TRIGGERINPUT", 0, Lifetime::Timeframe}, mDigitsTrig);
+    pc.outputs().snapshot(Output{"FDD", "DIGITSBC", 0}, mDigitsBC);
+    pc.outputs().snapshot(Output{"FDD", "DIGITSCH", 0}, mDigitsCh);
+    pc.outputs().snapshot(Output{"FDD", "TRIGGERINPUT", 0}, mDigitsTrig);
     if (pc.outputs().isAllowed({"FDD", "DIGITLBL", 0})) {
-      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::fdd::MCLabel>>(Output{"FDD", "DIGITLBL", 0, Lifetime::Timeframe});
+      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::fdd::MCLabel>>(Output{"FDD", "DIGITLBL", 0});
       labels.flatten_to(sharedlabels);
       labels.clear_andfreememory();
     }
 
     LOG(info) << "FDD: Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{"FDD", "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{"FDD", "ROMode", 0}, mROMode);
 
     // we should be only called once; tell DPL that this process is ready to exit
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Steer/DigitizerWorkflow/src/FT0DigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/FT0DigitizerSpec.cxx
index 4eda984358634..9098240d17627 100644
--- a/Steer/DigitizerWorkflow/src/FT0DigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/FT0DigitizerSpec.cxx
@@ -138,14 +138,14 @@ class FT0DPLDigitizerTask : public o2::base::BaseDPLDigitizer
     mDigitizer.flush_all(mDigitsBC, mDigitsCh, mDigitsTrig, labels);
 
     // send out to next stage
-    pc.outputs().snapshot(Output{"FT0", "DIGITSBC", 0, Lifetime::Timeframe}, mDigitsBC);
-    pc.outputs().snapshot(Output{"FT0", "DIGITSCH", 0, Lifetime::Timeframe}, mDigitsCh);
-    pc.outputs().snapshot(Output{"FT0", "TRIGGERINPUT", 0, Lifetime::Timeframe}, mDigitsTrig);
+    pc.outputs().snapshot(Output{"FT0", "DIGITSBC", 0}, mDigitsBC);
+    pc.outputs().snapshot(Output{"FT0", "DIGITSCH", 0}, mDigitsCh);
+    pc.outputs().snapshot(Output{"FT0", "TRIGGERINPUT", 0}, mDigitsTrig);
     if (pc.outputs().isAllowed({"FT0", "DIGITSMCTR", 0})) {
-      pc.outputs().snapshot(Output{"FT0", "DIGITSMCTR", 0, Lifetime::Timeframe}, labels);
+      pc.outputs().snapshot(Output{"FT0", "DIGITSMCTR", 0}, labels);
     }
     LOG(info) << "FT0: Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{"FT0", "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{"FT0", "ROMode", 0}, mROMode);
 
     timer.Stop();
     LOG(info) << "Digitization took " << timer.CpuTime() << "s";
diff --git a/Steer/DigitizerWorkflow/src/FV0DigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/FV0DigitizerSpec.cxx
index 151175e0c0758..bb5174dfcbb2c 100644
--- a/Steer/DigitizerWorkflow/src/FV0DigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/FV0DigitizerSpec.cxx
@@ -102,14 +102,14 @@ class FV0DPLDigitizerTask : public o2::base::BaseDPLDigitizer
     LOG(info) << "FV0: Sending " << mDigitsBC.size() << " digitsBC and " << mDigitsCh.size() << " digitsCh.";
 
     // send out to next stage
-    pc.outputs().snapshot(Output{"FV0", "DIGITSBC", 0, Lifetime::Timeframe}, mDigitsBC);
-    pc.outputs().snapshot(Output{"FV0", "DIGITSCH", 0, Lifetime::Timeframe}, mDigitsCh);
-    pc.outputs().snapshot(Output{"FV0", "TRIGGERINPUT", 0, Lifetime::Timeframe}, mDigitsTrig);
+    pc.outputs().snapshot(Output{"FV0", "DIGITSBC", 0}, mDigitsBC);
+    pc.outputs().snapshot(Output{"FV0", "DIGITSCH", 0}, mDigitsCh);
+    pc.outputs().snapshot(Output{"FV0", "TRIGGERINPUT", 0}, mDigitsTrig);
     if (pc.outputs().isAllowed({"FV0", "DIGITLBL", 0})) {
-      pc.outputs().snapshot(Output{"FV0", "DIGITLBL", 0, Lifetime::Timeframe}, mLabels);
+      pc.outputs().snapshot(Output{"FV0", "DIGITLBL", 0}, mLabels);
     }
     LOG(info) << "FV0: Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{"FV0", "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{"FV0", "ROMode", 0}, mROMode);
 
     // we should be only called once; tell DPL that this process is ready to exit
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Steer/DigitizerWorkflow/src/HMPIDDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/HMPIDDigitizerSpec.cxx
index 22ab3e5d7ef84..0adb9eb0a90df 100644
--- a/Steer/DigitizerWorkflow/src/HMPIDDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/HMPIDDigitizerSpec.cxx
@@ -132,13 +132,13 @@ class HMPIDDPLDigitizerTask : public o2::base::BaseDPLDigitizer
     flushDigitsAndLabels();
 
     // send out to next stage
-    pc.outputs().snapshot(Output{"HMP", "DIGITS", 0, Lifetime::Timeframe}, digitsAccum);
-    pc.outputs().snapshot(Output{"HMP", "INTRECORDS", 0, Lifetime::Timeframe}, mIntRecord);
+    pc.outputs().snapshot(Output{"HMP", "DIGITS", 0}, digitsAccum);
+    pc.outputs().snapshot(Output{"HMP", "INTRECORDS", 0}, mIntRecord);
     if (pc.outputs().isAllowed({"HMP", "DIGITLBL", 0})) {
-      pc.outputs().snapshot(Output{"HMP", "DIGITLBL", 0, Lifetime::Timeframe}, labelAccum);
+      pc.outputs().snapshot(Output{"HMP", "DIGITLBL", 0}, labelAccum);
     }
     LOG(info) << "HMP: Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{"HMP", "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{"HMP", "ROMode", 0}, mROMode);
 
     // we should be only called once; tell DPL that this process is ready to exit
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Steer/DigitizerWorkflow/src/ITS3DigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/ITS3DigitizerSpec.cxx
index 9f03bd6733729..21f808f1e1ed2 100644
--- a/Steer/DigitizerWorkflow/src/ITS3DigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/ITS3DigitizerSpec.cxx
@@ -143,7 +143,7 @@ class ITS3DPLDigitizerTask : BaseDPLDigitizer
     mDigitizer.setMCLabels(&mLabels);
 
     // digits are directly put into DPL owned resource
-    auto& digitsAccum = pc.outputs().make<std::vector<itsmft::Digit>>(Output{mOrigin, "DIGITS", 0, Lifetime::Timeframe});
+    auto& digitsAccum = pc.outputs().make<std::vector<itsmft::Digit>>(Output{mOrigin, "DIGITS", 0});
 
     auto accumulate = [this, &digitsAccum]() {
       // accumulate result of single event processing, called after processing every event supplied
@@ -216,17 +216,17 @@ class ITS3DPLDigitizerTask : BaseDPLDigitizer
 
     // here we have all digits and labels and we can send them to consumer (aka snapshot it onto output)
 
-    pc.outputs().snapshot(Output{mOrigin, "DIGITSROF", 0, Lifetime::Timeframe}, mROFRecordsAccum);
+    pc.outputs().snapshot(Output{mOrigin, "DIGITSROF", 0}, mROFRecordsAccum);
     if (mWithMCTruth) {
-      pc.outputs().snapshot(Output{mOrigin, "DIGITSMC2ROF", 0, Lifetime::Timeframe}, mMC2ROFRecordsAccum);
-      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{mOrigin, "DIGITSMCTR", 0, Lifetime::Timeframe});
+      pc.outputs().snapshot(Output{mOrigin, "DIGITSMC2ROF", 0}, mMC2ROFRecordsAccum);
+      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{mOrigin, "DIGITSMCTR", 0});
       mLabelsAccum.flatten_to(sharedlabels);
       // free space of existing label containers
       mLabels.clear_andfreememory();
       mLabelsAccum.clear_andfreememory();
     }
     LOG(info) << mID.getName() << ": Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{mOrigin, "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{mOrigin, "ROMode", 0}, mROMode);
 
     timer.Stop();
     LOG(info) << "Digitization took " << timer.CpuTime() << "s";
diff --git a/Steer/DigitizerWorkflow/src/ITSMFTDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/ITSMFTDigitizerSpec.cxx
index e1291fe0cb0d8..2a8c28bb4dbbd 100644
--- a/Steer/DigitizerWorkflow/src/ITSMFTDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/ITSMFTDigitizerSpec.cxx
@@ -82,7 +82,7 @@ class ITSMFTDPLDigitizerTask : BaseDPLDigitizer
     mDigitizer.setMCLabels(&mLabels);
 
     // digits are directly put into DPL owned resource
-    auto& digitsAccum = pc.outputs().make<std::vector<itsmft::Digit>>(Output{mOrigin, "DIGITS", 0, Lifetime::Timeframe});
+    auto& digitsAccum = pc.outputs().make<std::vector<itsmft::Digit>>(Output{mOrigin, "DIGITS", 0});
 
     auto accumulate = [this, &digitsAccum]() {
       // accumulate result of single event processing, called after processing every event supplied
@@ -160,17 +160,17 @@ class ITSMFTDPLDigitizerTask : BaseDPLDigitizer
 
     // here we have all digits and labels and we can send them to consumer (aka snapshot it onto output)
 
-    pc.outputs().snapshot(Output{mOrigin, "DIGITSROF", 0, Lifetime::Timeframe}, mROFRecordsAccum);
+    pc.outputs().snapshot(Output{mOrigin, "DIGITSROF", 0}, mROFRecordsAccum);
     if (mWithMCTruth) {
-      pc.outputs().snapshot(Output{mOrigin, "DIGITSMC2ROF", 0, Lifetime::Timeframe}, mMC2ROFRecordsAccum);
-      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{mOrigin, "DIGITSMCTR", 0, Lifetime::Timeframe});
+      pc.outputs().snapshot(Output{mOrigin, "DIGITSMC2ROF", 0}, mMC2ROFRecordsAccum);
+      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{mOrigin, "DIGITSMCTR", 0});
       mLabelsAccum.flatten_to(sharedlabels);
       // free space of existing label containers
       mLabels.clear_andfreememory();
       mLabelsAccum.clear_andfreememory();
     }
     LOG(info) << mID.getName() << ": Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{mOrigin, "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{mOrigin, "ROMode", 0}, mROMode);
 
     timer.Stop();
     LOG(info) << "Digitization took " << timer.CpuTime() << "s";
diff --git a/Steer/DigitizerWorkflow/src/MCHDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/MCHDigitizerSpec.cxx
index fc92020f25b21..187c24f962c1f 100644
--- a/Steer/DigitizerWorkflow/src/MCHDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/MCHDigitizerSpec.cxx
@@ -111,12 +111,12 @@ class MCHDPLDigitizerTask : public o2::base::BaseDPLDigitizer
     dataformats::MCLabelContainer labels{};
     auto nPileup = mDigitizer->digitize(rofs, digits, labels);
 
-    pc.outputs().snapshot(Output{"MCH", "DIGITS", 0, Lifetime::Timeframe}, digits);
-    pc.outputs().snapshot(Output{"MCH", "DIGITROFS", 0, Lifetime::Timeframe}, rofs);
+    pc.outputs().snapshot(Output{"MCH", "DIGITS", 0}, digits);
+    pc.outputs().snapshot(Output{"MCH", "DIGITROFS", 0}, rofs);
     if (pc.outputs().isAllowed({"MCH", "DIGITSLABELS", 0})) {
-      pc.outputs().snapshot(Output{"MCH", "DIGITSLABELS", 0, Lifetime::Timeframe}, labels);
+      pc.outputs().snapshot(Output{"MCH", "DIGITSLABELS", 0}, labels);
     }
-    pc.outputs().snapshot(Output{"MCH", "ROMode", 0, Lifetime::Timeframe},
+    pc.outputs().snapshot(Output{"MCH", "ROMode", 0},
                           DigitizerParam::Instance().continuous ? o2::parameters::GRPObject::CONTINUOUS : o2::parameters::GRPObject::TRIGGERING);
 
     // we should be only called once; tell DPL that this process is ready to exit
diff --git a/Steer/DigitizerWorkflow/src/MCTruthReaderSpec.h b/Steer/DigitizerWorkflow/src/MCTruthReaderSpec.h
index e3359af01b1e5..8204f4e437cc3 100644
--- a/Steer/DigitizerWorkflow/src/MCTruthReaderSpec.h
+++ b/Steer/DigitizerWorkflow/src/MCTruthReaderSpec.h
@@ -61,7 +61,7 @@ class MCTruthReaderTask : public o2::framework::Task
       br->GetEntry(0);
 
       // publish the labels in a const shared memory container
-      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{"TST", "LABELS2", 0, Lifetime::Timeframe});
+      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{"TST", "LABELS2", 0});
       iocontainer->copyandflatten(sharedlabels);
 
     } else {
@@ -74,7 +74,7 @@ class MCTruthReaderTask : public o2::framework::Task
       LOG(info) << "MCCONTAINER CHECK" << mccontainer->getNElements();
 
       // publish the original labels
-      pc.outputs().snapshot(Output{"TST", "LABELS2", 0, Lifetime::Timeframe}, *mccontainer);
+      pc.outputs().snapshot(Output{"TST", "LABELS2", 0}, *mccontainer);
     }
     // we should be only called once; tell DPL that this process is ready to exit
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Steer/DigitizerWorkflow/src/MCTruthSourceSpec.cxx b/Steer/DigitizerWorkflow/src/MCTruthSourceSpec.cxx
index 1b0f325cba9d6..6888b118fb06c 100644
--- a/Steer/DigitizerWorkflow/src/MCTruthSourceSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/MCTruthSourceSpec.cxx
@@ -54,12 +54,12 @@ class MCTruthSourceTask : public o2::framework::Task
     if (mNew) {
       LOG(info) << "New serialization";
       // we need to flatten it and write to managed shared memory container
-      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{"TST", "LABELS", 0, Lifetime::Timeframe});
+      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{"TST", "LABELS", 0});
       container.flatten_to(sharedlabels);
       sleep(1);
     } else {
       LOG(info) << "Old serialization";
-      pc.outputs().snapshot({"TST", "LABELS", 0, Lifetime::Timeframe}, container);
+      pc.outputs().snapshot({"TST", "LABELS", 0}, container);
       sleep(1);
     }
 
diff --git a/Steer/DigitizerWorkflow/src/MCTruthWriterSpec.cxx b/Steer/DigitizerWorkflow/src/MCTruthWriterSpec.cxx
index 9ce78d97418c3..78fb8988a2419 100644
--- a/Steer/DigitizerWorkflow/src/MCTruthWriterSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/MCTruthWriterSpec.cxx
@@ -84,7 +84,7 @@ class MCTruthWriterTask : public o2::framework::Task
     }
     if (mIO) {
       // this triggers the reader process
-      pc.outputs().snapshot({"TST", "TRIGGERREAD", 0, Lifetime::Timeframe}, labelfilename);
+      pc.outputs().snapshot({"TST", "TRIGGERREAD", 0}, labelfilename);
     }
 
     // we should be only called once; tell DPL that this process is ready to exit
diff --git a/Steer/DigitizerWorkflow/src/MIDDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/MIDDigitizerSpec.cxx
index c0a3be730c411..1dbd392fb9f73 100644
--- a/Steer/DigitizerWorkflow/src/MIDDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/MIDDigitizerSpec.cxx
@@ -117,13 +117,13 @@ class MIDDPLDigitizerTask : public o2::base::BaseDPLDigitizer
     mDigitsMerger.process(digitsAccum, labelsAccum, rofRecords);
 
     LOG(debug) << "MID: Sending " << digitsAccum.size() << " digits.";
-    pc.outputs().snapshot(Output{"MID", "DIGITS", 0, Lifetime::Timeframe}, mDigitsMerger.getColumnData());
-    pc.outputs().snapshot(Output{"MID", "DIGITSROF", 0, Lifetime::Timeframe}, mDigitsMerger.getROFRecords());
+    pc.outputs().snapshot(Output{"MID", "DIGITS", 0}, mDigitsMerger.getColumnData());
+    pc.outputs().snapshot(Output{"MID", "DIGITSROF", 0}, mDigitsMerger.getROFRecords());
     if (pc.outputs().isAllowed({"MID", "DIGITLABELS", 0})) {
-      pc.outputs().snapshot(Output{"MID", "DIGITLABELS", 0, Lifetime::Timeframe}, mDigitsMerger.getMCContainer());
+      pc.outputs().snapshot(Output{"MID", "DIGITLABELS", 0}, mDigitsMerger.getMCContainer());
     }
     LOG(debug) << "MID: Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{"MID", "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{"MID", "ROMode", 0}, mROMode);
 
     // we should be only called once; tell DPL that this process is ready to exit
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Steer/DigitizerWorkflow/src/PHOSDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/PHOSDigitizerSpec.cxx
index db20e811941c2..0b7b968d22ff6 100644
--- a/Steer/DigitizerWorkflow/src/PHOSDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/PHOSDigitizerSpec.cxx
@@ -160,16 +160,16 @@ void DigitizerSpec::run(framework::ProcessingContext& pc)
   }
   LOG(debug) << "Have " << mLabels.getNElements() << " PHOS labels ";
   // here we have all digits and we can send them to consumer (aka snapshot it onto output)
-  pc.outputs().snapshot(Output{"PHS", "DIGITS", 0, Lifetime::Timeframe}, mDigitsOut);
-  pc.outputs().snapshot(Output{"PHS", "DIGITTRIGREC", 0, Lifetime::Timeframe}, triggers);
+  pc.outputs().snapshot(Output{"PHS", "DIGITS", 0}, mDigitsOut);
+  pc.outputs().snapshot(Output{"PHS", "DIGITTRIGREC", 0}, triggers);
   if (pc.outputs().isAllowed({"PHS", "DIGITSMCTR", 0})) {
-    pc.outputs().snapshot(Output{"PHS", "DIGITSMCTR", 0, Lifetime::Timeframe}, mLabels);
+    pc.outputs().snapshot(Output{"PHS", "DIGITSMCTR", 0}, mLabels);
   }
 
   // PHOS is always a triggering detector
   const o2::parameters::GRPObject::ROMode roMode = o2::parameters::GRPObject::TRIGGERING;
   LOG(debug) << "PHOS: Sending ROMode= " << roMode << " to GRPUpdater";
-  pc.outputs().snapshot(Output{"PHS", "ROMode", 0, Lifetime::Timeframe}, roMode);
+  pc.outputs().snapshot(Output{"PHS", "ROMode", 0}, roMode);
 
   timer.Stop();
   LOG(info) << "Digitization took " << timer.CpuTime() << "s";
diff --git a/Steer/DigitizerWorkflow/src/TOFDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/TOFDigitizerSpec.cxx
index c74e5ca6327fc..e512659686c86 100644
--- a/Steer/DigitizerWorkflow/src/TOFDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/TOFDigitizerSpec.cxx
@@ -259,21 +259,21 @@ class TOFDPLDigitizerTask : public o2::base::BaseDPLDigitizer
     LOG(info) << "Post " << digitsVector->size() << " digits in " << readoutwindow->size() << " RO windows";
 
     // here we have all digits and we can send them to consumer (aka snapshot it onto output)
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITS", 0, Lifetime::Timeframe}, *digitsVector);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITS", 0}, *digitsVector);
     if (pc.outputs().isAllowed({o2::header::gDataOriginTOF, "DIGITSMCTR", 0})) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITSMCTR", 0, Lifetime::Timeframe}, *mcLabVecOfVec);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITSMCTR", 0}, *mcLabVecOfVec);
     }
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "READOUTWINDOW", 0, Lifetime::Timeframe}, *readoutwindow);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "READOUTWINDOW", 0}, *readoutwindow);
 
     // send empty pattern from digitizer (it may change in future)
     std::vector<uint8_t>& patterns = mDigitizer->getPatterns();
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "PATTERNS", 0, Lifetime::Timeframe}, patterns);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "PATTERNS", 0}, patterns);
 
     DigitHeader& digitH = mDigitizer->getDigitHeader();
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITHEADER", 0, Lifetime::Timeframe}, digitH);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITHEADER", 0}, digitH);
 
     LOG(info) << "TOF: Sending ROMode= " << roMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "ROMode", 0, Lifetime::Timeframe}, roMode);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "ROMode", 0}, roMode);
 
     timer.Stop();
     LOG(info) << "Digitization took " << timer.CpuTime() << "s";
diff --git a/Steer/DigitizerWorkflow/src/TPCDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/TPCDigitizerSpec.cxx
index 59f8589a56449..3f24bb1a9ee12 100644
--- a/Steer/DigitizerWorkflow/src/TPCDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/TPCDigitizerSpec.cxx
@@ -282,7 +282,7 @@ class TPCDPLDigitizerTask : public BaseDPLDigitizer
       auto roMode = isContinuous ? o2::parameters::GRPObject::CONTINUOUS : o2::parameters::GRPObject::PRESENT;
       LOG(info) << "TPC: Sending ROMode= " << (mDigitizer.isContinuousReadout() ? "Continuous" : "Triggered")
                 << " to GRPUpdater from channel " << dh->subSpecification;
-      pc.outputs().snapshot(Output{"TPC", "ROMode", 0, Lifetime::Timeframe}, roMode);
+      pc.outputs().snapshot(Output{"TPC", "ROMode", 0}, roMode);
     }
     mWriteGRP = false;
 
@@ -309,7 +309,7 @@ class TPCDPLDigitizerTask : public BaseDPLDigitizer
         return ContainerType(nullptr);
       } else {
         // default case
-        return &pc.outputs().make<std::vector<o2::tpc::Digit>>(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(dh->subSpecification), Lifetime::Timeframe, header});
+        return &pc.outputs().make<std::vector<o2::tpc::Digit>>(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(dh->subSpecification), header});
       }
     };
     // lambda that snapshots the common mode vector to be sent out; prepares and attaches header with sector information
@@ -318,8 +318,7 @@ class TPCDPLDigitizerTask : public BaseDPLDigitizer
       header.activeSectors = activeSectors;
       if (!mInternalWriter) {
         // note that snapshoting only works with non-const references (to be fixed?)
-        pc.outputs().snapshot(Output{"TPC", "COMMONMODE", static_cast<SubSpecificationType>(dh->subSpecification), Lifetime::Timeframe,
-                                     header},
+        pc.outputs().snapshot(Output{"TPC", "COMMONMODE", static_cast<SubSpecificationType>(dh->subSpecification), header},
                               const_cast<std::vector<o2::tpc::CommonMode>&>(commonMode));
       }
     };
@@ -329,7 +328,7 @@ class TPCDPLDigitizerTask : public BaseDPLDigitizer
       header.activeSectors = activeSectors;
       if (mWithMCTruth) {
         if (!mInternalWriter) {
-          auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{"TPC", "DIGITSMCTR", static_cast<SubSpecificationType>(dh->subSpecification), Lifetime::Timeframe, header});
+          auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{"TPC", "DIGITSMCTR", static_cast<SubSpecificationType>(dh->subSpecification), header});
           labels.flatten_to(sharedlabels);
         }
       }
@@ -340,8 +339,7 @@ class TPCDPLDigitizerTask : public BaseDPLDigitizer
       header.activeSectors = activeSectors;
       if (!mInternalWriter) {
         LOG(info) << "TPC: Send TRIGGERS for sector " << sector << " channel " << dh->subSpecification << " | size " << events.size();
-        pc.outputs().snapshot(Output{"TPC", "DIGTRIGGERS", static_cast<SubSpecificationType>(dh->subSpecification), Lifetime::Timeframe,
-                                     header},
+        pc.outputs().snapshot(Output{"TPC", "DIGTRIGGERS", static_cast<SubSpecificationType>(dh->subSpecification), header},
                               const_cast<std::vector<DigiGroupRef>&>(events));
       }
     };
diff --git a/Steer/DigitizerWorkflow/src/ZDCDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/ZDCDigitizerSpec.cxx
index d616df2fadb35..cd07f6389c43f 100644
--- a/Steer/DigitizerWorkflow/src/ZDCDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/ZDCDigitizerSpec.cxx
@@ -147,15 +147,15 @@ class ZDCDPLDigitizerTask : public o2::base::BaseDPLDigitizer
     mDigitizer.Finalize(mDigitsBC, mOrbitData);
 
     // send out to next stage
-    pc.outputs().snapshot(Output{"ZDC", "DIGITSBC", 0, Lifetime::Timeframe}, mDigitsBC);
-    pc.outputs().snapshot(Output{"ZDC", "DIGITSCH", 0, Lifetime::Timeframe}, mDigitsCh);
-    pc.outputs().snapshot(Output{"ZDC", "DIGITSPD", 0, Lifetime::Timeframe}, mOrbitData);
+    pc.outputs().snapshot(Output{"ZDC", "DIGITSBC", 0}, mDigitsBC);
+    pc.outputs().snapshot(Output{"ZDC", "DIGITSCH", 0}, mDigitsCh);
+    pc.outputs().snapshot(Output{"ZDC", "DIGITSPD", 0}, mOrbitData);
     if (pc.outputs().isAllowed({"ZDC", "DIGITSLBL", 0})) {
-      pc.outputs().snapshot(Output{"ZDC", "DIGITSLBL", 0, Lifetime::Timeframe}, mLabels);
+      pc.outputs().snapshot(Output{"ZDC", "DIGITSLBL", 0}, mLabels);
     }
 
     LOG(info) << "ZDC: Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{"ZDC", "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{"ZDC", "ROMode", 0}, mROMode);
 
     // we should be only called once; tell DPL that this process is ready to exit
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Utilities/DataSampling/src/DataSamplingPolicy.cxx b/Utilities/DataSampling/src/DataSamplingPolicy.cxx
index de7aa5d8f6865..44a7a63efcf51 100644
--- a/Utilities/DataSampling/src/DataSamplingPolicy.cxx
+++ b/Utilities/DataSampling/src/DataSamplingPolicy.cxx
@@ -128,7 +128,7 @@ Output DataSamplingPolicy::prepareOutput(const ConcreteDataMatcher& input, Lifet
   auto result = mPaths.find(input);
   if (result != mPaths.end()) {
     auto dataType = DataSpecUtils::asConcreteDataTypeMatcher(result->second);
-    return Output{dataType.origin, dataType.description, input.subSpec, lifetime};
+    return Output{dataType.origin, dataType.description, input.subSpec};
   } else {
     return Output{header::gDataOriginInvalid, header::gDataDescriptionInvalid};
   }
diff --git a/Utilities/DataSampling/src/Dispatcher.cxx b/Utilities/DataSampling/src/Dispatcher.cxx
index b32218ea43e58..de78993ff083e 100644
--- a/Utilities/DataSampling/src/Dispatcher.cxx
+++ b/Utilities/DataSampling/src/Dispatcher.cxx
@@ -114,7 +114,6 @@ void Dispatcher::run(ProcessingContext& ctx)
               routeAsConcreteDataType.origin,
               routeAsConcreteDataType.description,
               partInputHeader->subSpecification,
-              part.spec->lifetime,
               std::move(headerStack)};
             send(ctx.outputs(), part, output);
           }
diff --git a/run/dpl_eventgen.cxx b/run/dpl_eventgen.cxx
index 9700ac32c6938..3090944031ad3 100644
--- a/run/dpl_eventgen.cxx
+++ b/run/dpl_eventgen.cxx
@@ -69,8 +69,8 @@ struct GeneratorTask {
     for (auto i = 0; i < std::min((int)aggregate, nEvents - eventCounter); ++i) {
       mctracks.clear();
       genservice.generateEvent_MCTracks(mctracks, mcheader);
-      pc.outputs().snapshot(Output{"MC", "MCHEADER", 0, Lifetime::Timeframe}, mcheader);
-      pc.outputs().snapshot(Output{"MC", "MCTRACKS", 0, Lifetime::Timeframe}, mctracks);
+      pc.outputs().snapshot(Output{"MC", "MCHEADER", 0}, mcheader);
+      pc.outputs().snapshot(Output{"MC", "MCTRACKS", 0}, mctracks);
       ++eventCounter;
     }
     // report number of TFs injected for the rate limiter to work
diff --git a/run/o2sim_hepmc_publisher.cxx b/run/o2sim_hepmc_publisher.cxx
index 76fbddc1993f3..7172248ca68ac 100644
--- a/run/o2sim_hepmc_publisher.cxx
+++ b/run/o2sim_hepmc_publisher.cxx
@@ -135,8 +135,8 @@ struct O2simHepmcPublisher {
       }
 
       // add to the message
-      pc.outputs().snapshot(Output{"MC", "MCHEADER", 0, Lifetime::Timeframe}, mcHeader);
-      pc.outputs().snapshot(Output{"MC", "MCTRACKS", 0, Lifetime::Timeframe}, mcTracks);
+      pc.outputs().snapshot(Output{"MC", "MCHEADER", 0}, mcHeader);
+      pc.outputs().snapshot(Output{"MC", "MCTRACKS", 0}, mcTracks);
       mcTracks.clear();
       ++eventCounter;
     }
diff --git a/run/o2sim_kine_publisher.cxx b/run/o2sim_kine_publisher.cxx
index 35fd48af199e0..83f82fd69cca3 100644
--- a/run/o2sim_kine_publisher.cxx
+++ b/run/o2sim_kine_publisher.cxx
@@ -44,8 +44,8 @@ struct O2simKinePublisher {
     for (auto i = 0; i < std::min((int)aggregate, nEvents - eventCounter); ++i) {
       auto mcevent = mcKinReader->getMCEventHeader(0, eventCounter);
       auto mctracks = mcKinReader->getTracks(0, eventCounter);
-      pc.outputs().snapshot(Output{"MC", "MCHEADER", 0, Lifetime::Timeframe}, mcevent);
-      pc.outputs().snapshot(Output{"MC", "MCTRACKS", 0, Lifetime::Timeframe}, mctracks);
+      pc.outputs().snapshot(Output{"MC", "MCHEADER", 0}, mcevent);
+      pc.outputs().snapshot(Output{"MC", "MCTRACKS", 0}, mctracks);
       ++eventCounter;
     }
     // report number of TFs injected for the rate limiter to work
diff --git a/run/o2sim_mctracks_to_aod.cxx b/run/o2sim_mctracks_to_aod.cxx
index e5460e346c3f9..ca64ce0705f1a 100644
--- a/run/o2sim_mctracks_to_aod.cxx
+++ b/run/o2sim_mctracks_to_aod.cxx
@@ -171,8 +171,8 @@ struct MctracksToAod {
       }
     }
     ++timeframe;
-    pc.outputs().snapshot(Output{"TFF", "TFFilename", 0, Lifetime::Timeframe}, "");
-    pc.outputs().snapshot(Output{"TFN", "TFNumber", 0, Lifetime::Timeframe}, timeframe);
+    pc.outputs().snapshot(Output{"TFF", "TFFilename", 0}, "");
+    pc.outputs().snapshot(Output{"TFN", "TFNumber", 0}, timeframe);
   }
 };
 

From 5654c48597898d436a8504946c6b497832594a48 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 5 Dec 2023 11:55:54 +0100
Subject: [PATCH 150/726] Remove Lifetime specification from Output

The .lifetime field in Output is to be removed. This includes
cases in which the value is not the default.
---
 .../src/GlobalFwdMatchingAssessmentSpec.cxx               | 2 +-
 Detectors/ITSMFT/MFT/workflow/src/MFTAssessmentSpec.cxx   | 2 +-
 Detectors/PHOS/calib/src/PHOSL1phaseCalibDevice.cxx       | 2 +-
 Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx    | 4 ++--
 .../workflow/include/TPCWorkflow/TPCDistributeSACSpec.h   | 2 +-
 Detectors/TPC/workflow/src/OccupancyFilterSpec.cxx        | 2 +-
 Detectors/TPC/workflow/src/TPCVDriftTglCalibSpec.cxx      | 4 ++--
 Detectors/ZDC/calib/src/BaselineCalibEPNSpec.cxx          | 2 +-
 Detectors/ZDC/calib/src/NoiseCalibEPNSpec.cxx             | 8 ++++----
 9 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/Detectors/GlobalTrackingWorkflow/src/GlobalFwdMatchingAssessmentSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/GlobalFwdMatchingAssessmentSpec.cxx
index e4c045d2a7d93..e896c097cd03d 100644
--- a/Detectors/GlobalTrackingWorkflow/src/GlobalFwdMatchingAssessmentSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/GlobalFwdMatchingAssessmentSpec.cxx
@@ -109,7 +109,7 @@ void GlobalFwdAssessmentSpec::sendOutput(DataAllocator& output)
   TObjArray objar;
   mGloFwdAssessment->getHistos(objar);
 
-  output.snapshot(Output{"GLO", "FWDASSESSMENT", 0, Lifetime::Sporadic}, objar);
+  output.snapshot(Output{"GLO", "FWDASSESSMENT", 0}, objar);
 
   TFile* f = new TFile(Form("GlobalForwardAssessment.root"), "RECREATE");
   objar.Write();
diff --git a/Detectors/ITSMFT/MFT/workflow/src/MFTAssessmentSpec.cxx b/Detectors/ITSMFT/MFT/workflow/src/MFTAssessmentSpec.cxx
index 9778120a75389..d3e4c33457ae0 100644
--- a/Detectors/ITSMFT/MFT/workflow/src/MFTAssessmentSpec.cxx
+++ b/Detectors/ITSMFT/MFT/workflow/src/MFTAssessmentSpec.cxx
@@ -90,7 +90,7 @@ void MFTAssessmentSpec::sendOutput(DataAllocator& output)
   TObjArray objar;
   mMFTAssessment->getHistos(objar);
 
-  output.snapshot(Output{"MFT", "MFTASSESSMENT", 0, Lifetime::Sporadic}, objar);
+  output.snapshot(Output{"MFT", "MFTASSESSMENT", 0}, objar);
 
   TFile* f = new TFile(Form("MFTAssessment.root"), "RECREATE");
   objar.Write();
diff --git a/Detectors/PHOS/calib/src/PHOSL1phaseCalibDevice.cxx b/Detectors/PHOS/calib/src/PHOSL1phaseCalibDevice.cxx
index 1ed04baa77892..9ce61d0af38cb 100644
--- a/Detectors/PHOS/calib/src/PHOSL1phaseCalibDevice.cxx
+++ b/Detectors/PHOS/calib/src/PHOSL1phaseCalibDevice.cxx
@@ -71,7 +71,7 @@ void PHOSL1phaseCalibDevice::endOfStream(o2::framework::EndOfStreamContext& ec)
   ec.outputs().snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, "PHOS_L1phase", 0}, info);
   // Send summary to QC
   LOG(info) << "Sending histos to QC ";
-  ec.outputs().snapshot(o2::framework::Output{"PHS", "L1PHASEHISTO", 0, o2::framework::Lifetime::Sporadic}, mCalibrator->getQcHistos());
+  ec.outputs().snapshot(o2::framework::Output{"PHS", "L1PHASEHISTO", 0}, mCalibrator->getQcHistos());
 }
 
 o2::framework::DataProcessorSpec o2::phos::getPHOSL1phaseCalibDeviceSpec()
diff --git a/Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx b/Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx
index 292e083d4bb48..bc650dfaa54b8 100644
--- a/Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx
+++ b/Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx
@@ -115,7 +115,7 @@ void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
       ctx.outputs().snapshot(o2::framework::Output{"PHS", "RAWHWERRORS", 0}, mOutputHWErrors);
       if (mFillChi2) {
         mOutputFitChi.clear();
-        ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLFITQA", 0, o2::framework::Lifetime::QA}, mOutputFitChi);
+        ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLFITQA", 0}, mOutputFitChi);
       }
       return; // empty TF, nothing to process
     }
@@ -294,7 +294,7 @@ void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
   ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", mflpId}, mOutputTriggerRecords);
   ctx.outputs().snapshot(o2::framework::Output{"PHS", "RAWHWERRORS", 0}, mOutputHWErrors);
   if (mFillChi2) {
-    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLFITQA", 0, o2::framework::Lifetime::QA}, mOutputFitChi);
+    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLFITQA", 0}, mOutputFitChi);
   }
 }
 
diff --git a/Detectors/TPC/workflow/include/TPCWorkflow/TPCDistributeSACSpec.h b/Detectors/TPC/workflow/include/TPCWorkflow/TPCDistributeSACSpec.h
index f7c207ba3a27b..ee05a58b5dcbe 100644
--- a/Detectors/TPC/workflow/include/TPCWorkflow/TPCDistributeSACSpec.h
+++ b/Detectors/TPC/workflow/include/TPCWorkflow/TPCDistributeSACSpec.h
@@ -102,7 +102,7 @@ class TPCDistributeSACSpec : public o2::framework::Task
   void sendOutput(o2::framework::ProcessingContext& pc)
   {
     LOGP(info, "Sending SACs on lane: {} for {} TFs", mCurrentOutLane, mProcessedTFs);
-    pc.outputs().snapshot(Output{gDataOriginTPC, getDataDescriptionSACCCDB(), 0, Lifetime::Sporadic}, mCCDBTimeStamp);
+    pc.outputs().snapshot(Output{gDataOriginTPC, getDataDescriptionSACCCDB(), 0}, mCCDBTimeStamp);
     for (unsigned int i = 0; i < o2::tpc::GEMSTACKS; ++i) {
       pc.outputs().snapshot(Output{gDataOriginTPC, mDataDescrOut[mCurrentOutLane], header::DataHeader::SubSpecificationType{i}}, mSACs[i]);
     }
diff --git a/Detectors/TPC/workflow/src/OccupancyFilterSpec.cxx b/Detectors/TPC/workflow/src/OccupancyFilterSpec.cxx
index 9891d80a39503..5ef2bdc7adc51 100644
--- a/Detectors/TPC/workflow/src/OccupancyFilterSpec.cxx
+++ b/Detectors/TPC/workflow/src/OccupancyFilterSpec.cxx
@@ -106,7 +106,7 @@ class OccupancyFilterDevice : public o2::framework::Task
   {
     o2::tpc::TPCSectorHeader header{sector};
     header.activeSectors = (0x1 << sector);
-    output.snapshot(Output{gDataOriginTPC, "FILTERDIG", static_cast<SubSpecificationType>(sector), Lifetime::Sporadic, header}, digits);
+    output.snapshot(Output{gDataOriginTPC, "FILTERDIG", static_cast<SubSpecificationType>(sector), header}, digits);
   }
 };
 
diff --git a/Detectors/TPC/workflow/src/TPCVDriftTglCalibSpec.cxx b/Detectors/TPC/workflow/src/TPCVDriftTglCalibSpec.cxx
index d48dde325c614..ea50a32c6a906 100644
--- a/Detectors/TPC/workflow/src/TPCVDriftTglCalibSpec.cxx
+++ b/Detectors/TPC/workflow/src/TPCVDriftTglCalibSpec.cxx
@@ -104,8 +104,8 @@ void TPCVDriftTglCalibSpec::sendOutput(DataAllocator& output)
     auto image = o2::ccdb::CcdbApi::createObjectImage(&payloadVec[i], &w);
     LOG(info) << "Sending object " << w.getPath() << "/" << w.getFileName() << " of size " << image->size()
               << " bytes, valid for " << w.getStartValidityTimestamp() << " : " << w.getEndValidityTimestamp();
-    output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBPayload, "TPCVDTGL", i, Lifetime::Sporadic}, *image.get()); // vector<char>
-    output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, "TPCVDTGL", i, Lifetime::Sporadic}, w);            // root-serialized
+    output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBPayload, "TPCVDTGL", i}, *image.get()); // vector<char>
+    output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, "TPCVDTGL", i}, w);            // root-serialized
   }
   if (payloadVec.size()) {
     mCalibrator->initOutput(); // reset the outputs once they are already sent
diff --git a/Detectors/ZDC/calib/src/BaselineCalibEPNSpec.cxx b/Detectors/ZDC/calib/src/BaselineCalibEPNSpec.cxx
index 422221157f877..328f824918cd4 100644
--- a/Detectors/ZDC/calib/src/BaselineCalibEPNSpec.cxx
+++ b/Detectors/ZDC/calib/src/BaselineCalibEPNSpec.cxx
@@ -127,7 +127,7 @@ void BaselineCalibEPNSpec::run(ProcessingContext& pc)
     }
     // Send intermediate calibration data
     auto& summary = mWorker.mData.getSummary();
-    o2::framework::Output outputData("ZDC", "BASECALIBDATA", 0, Lifetime::Sporadic);
+    o2::framework::Output outputData("ZDC", "BASECALIBDATA", 0);
     pc.outputs().snapshot(outputData, summary);
     if (pc.transitionState() == TransitionHandlingState::Requested) {
       // End of processing for this run
diff --git a/Detectors/ZDC/calib/src/NoiseCalibEPNSpec.cxx b/Detectors/ZDC/calib/src/NoiseCalibEPNSpec.cxx
index 32fe2ee2eb1fb..186cec68517ea 100644
--- a/Detectors/ZDC/calib/src/NoiseCalibEPNSpec.cxx
+++ b/Detectors/ZDC/calib/src/NoiseCalibEPNSpec.cxx
@@ -113,19 +113,19 @@ void NoiseCalibEPNSpec::run(ProcessingContext& pc)
     if (mVerbosity > DbgMedium && mModTF > 0) {
       LOG(info) << "Send intermediate calibration data mProcessed=" << mProcessed << " >= mModTF=" << mModTF;
     }
-    o2::framework::Output outputData("ZDC", "NOISECALIBDATA", 0, Lifetime::Sporadic);
+    o2::framework::Output outputData("ZDC", "NOISECALIBDATA", 0);
     pc.outputs().snapshot(outputData, summary);
     for (int ih = 0; ih < NChannels; ih++) {
       {
-        o2::framework::Output output("ZDC", "NOISE_1DH", ih, Lifetime::Sporadic);
+        o2::framework::Output output("ZDC", "NOISE_1DH", ih);
         pc.outputs().snapshot(output, mWorker.mH[0][ih]->getBase());
       }
       {
-        o2::framework::Output output("ZDC", "NOISE_1DH_S", ih, Lifetime::Sporadic);
+        o2::framework::Output output("ZDC", "NOISE_1DH_S", ih);
         pc.outputs().snapshot(output, mWorker.mH[1][ih]->getBase());
       }
       {
-        o2::framework::Output output("ZDC", "NOISE_1DH_D", ih, Lifetime::Sporadic);
+        o2::framework::Output output("ZDC", "NOISE_1DH_D", ih);
         pc.outputs().snapshot(output, mWorker.mH[2][ih]->getBase());
       }
     }

From 5d49fa20979e4532ebc7522c441fbf58acc1c224 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 5 Dec 2023 11:53:34 +0100
Subject: [PATCH 151/726] DPL: drop lifetime member from Output

This is apparently not really used for anything and in general not passed
correctly. Lifetime is a static property of the data flow, so it does not
make sense to have it on a per message basis.
---
 Framework/CCDBSupport/src/CCDBHelpers.cxx     |  4 ++--
 Framework/Core/include/Framework/Output.h     | 18 ++++-------------
 Framework/Core/src/CommonServices.cxx         |  2 +-
 Framework/Core/src/DataAllocator.cxx          |  4 ++--
 Framework/Core/test/test_DataAllocator.cxx    | 20 +++++++++----------
 Framework/Core/test/test_ParallelPipeline.cxx |  8 ++++----
 Framework/Core/test/test_ProcessorOptions.cxx |  2 +-
 .../Core/test/test_StaggeringWorkflow.cxx     |  6 +++---
 Framework/Core/test/test_TableBuilder.cxx     |  2 +-
 .../Utils/include/DPLUtils/RootTreeReader.h   |  8 ++++----
 Framework/Utils/src/Utils.cxx                 |  2 +-
 Framework/Utils/test/test_RootTreeReader.cxx  |  5 ++---
 .../test/test_RootTreeWriterWorkflow.cxx      |  2 +-
 13 files changed, 36 insertions(+), 47 deletions(-)

diff --git a/Framework/CCDBSupport/src/CCDBHelpers.cxx b/Framework/CCDBSupport/src/CCDBHelpers.cxx
index 0e067dc2f2d52..e45e1c6996a98 100644
--- a/Framework/CCDBSupport/src/CCDBHelpers.cxx
+++ b/Framework/CCDBSupport/src/CCDBHelpers.cxx
@@ -185,7 +185,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
     LOGP(debug, "Fetching object for route {}", DataSpecUtils::describe(route.matcher));
     objCnt++;
     auto concrete = DataSpecUtils::asConcreteDataMatcher(route.matcher);
-    Output output{concrete.origin, concrete.description, concrete.subSpec, route.matcher.lifetime};
+    Output output{concrete.origin, concrete.description, concrete.subSpec};
     auto&& v = allocator.makeVector<char>(output);
     std::map<std::string, std::string> metadata;
     std::map<std::string, std::string> headers;
@@ -343,7 +343,7 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
             checkValidity = true; // never skip check if the cache is empty
           }
           LOG(debug) << "checkValidity = " << checkValidity << " for TF " << timingInfo.timeslice;
-          Output output{"CTP", "OrbitReset", 0, Lifetime::Condition};
+          Output output{"CTP", "OrbitReset", 0};
           Long64_t newOrbitResetTime = orbitResetTime;
           auto&& v = allocator.makeVector<char>(output);
           const auto& api = helper->getAPI(path);
diff --git a/Framework/Core/include/Framework/Output.h b/Framework/Core/include/Framework/Output.h
index b7d26e8dfd7ee..a2abf5f7600a9 100644
--- a/Framework/Core/include/Framework/Output.h
+++ b/Framework/Core/include/Framework/Output.h
@@ -28,7 +28,6 @@ struct Output {
   header::DataOrigin origin;
   header::DataDescription description;
   header::DataHeader::SubSpecificationType subSpec = 0;
-  enum Lifetime lifetime = Lifetime::Timeframe;
   header::Stack metaHeader = {};
 
   Output(header::DataOrigin o, header::DataDescription d) : origin(o), description(d) {}
@@ -38,14 +37,8 @@ struct Output {
   {
   }
 
-  Output(header::DataOrigin o, header::DataDescription d, header::DataHeader::SubSpecificationType s, Lifetime l)
-    : origin(o), description(d), subSpec(s), lifetime(l)
-  {
-  }
-
-  Output(header::DataOrigin o, header::DataDescription d, header::DataHeader::SubSpecificationType s, Lifetime l,
-         header::Stack&& stack)
-    : origin(o), description(d), subSpec(s), lifetime(l), metaHeader(std::move(stack))
+  Output(header::DataOrigin o, header::DataDescription d, header::DataHeader::SubSpecificationType s, header::Stack&& stack)
+    : origin(o), description(d), subSpec(s), metaHeader(std::move(stack))
   {
   }
 
@@ -60,7 +53,6 @@ struct Output {
     : origin(rhs.origin),
       description(rhs.description),
       subSpec(rhs.subSpec),
-      lifetime(rhs.lifetime),
       metaHeader(std::move(rhs.metaHeader))
   {
   }
@@ -72,16 +64,14 @@ struct Output {
     origin = rhs.origin;
     description = rhs.description;
     subSpec = rhs.subSpec;
-    lifetime = rhs.lifetime;
     metaHeader = std::move(rhs.metaHeader);
     return *this;
   }
 
   bool operator==(const Output& that) const
   {
-    return origin == that.origin && description == that.description && subSpec == that.subSpec &&
-           lifetime == that.lifetime;
-  };
+    return origin == that.origin && description == that.description && subSpec == that.subSpec;
+  }
 };
 
 } // namespace o2
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 7d502216bc509..05b8fba47e532 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -425,7 +425,7 @@ o2::framework::ServiceSpec CommonServices::ccdbSupportSpec()
           if (concrete.subSpec == 0) {
             continue;
           }
-          auto& stfDist = pc.outputs().make<o2::header::STFHeader>(Output{concrete.origin, concrete.description, concrete.subSpec, output.matcher.lifetime});
+          auto& stfDist = pc.outputs().make<o2::header::STFHeader>(Output{concrete.origin, concrete.description, concrete.subSpec});
           stfDist.id = timingInfo.timeslice;
           stfDist.firstOrbit = timingInfo.firstTForbit;
           stfDist.runNumber = timingInfo.runNumber;
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index 296e094bffe26..1230d2e942f6e 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -294,7 +294,7 @@ Output DataAllocator::getOutputByBind(OutputRef&& ref)
     if (allowedOutputRoutes[ri].matcher.binding.value == ref.label) {
       auto spec = allowedOutputRoutes[ri].matcher;
       auto dataType = DataSpecUtils::asConcreteDataTypeMatcher(spec);
-      return Output{dataType.origin, dataType.description, ref.subSpec, spec.lifetime, std::move(ref.headerStack)};
+      return Output{dataType.origin, dataType.description, ref.subSpec, std::move(ref.headerStack)};
     }
   }
   std::string availableRoutes;
@@ -342,7 +342,7 @@ void DataAllocator::cookDeadBeef(const Output& spec)
   // We get the output route from the original spec, but we send it
   // using the binding of the deadbeef subSpecification.
   RouteIndex routeIndex = matchDataHeader(spec, timingInfo.timeslice);
-  auto deadBeefOutput = Output{spec.origin, spec.description, 0xdeadbeef, Lifetime::Timeframe};
+  auto deadBeefOutput = Output{spec.origin, spec.description, 0xdeadbeef};
   auto headerMessage = headerMessageFromOutput(deadBeefOutput, routeIndex, header::gSerializationMethodNone, 0);
 
   addPartToContext(proxy.createOutputMessage(routeIndex, 0), deadBeefOutput, header::gSerializationMethodNone);
diff --git a/Framework/Core/test/test_DataAllocator.cxx b/Framework/Core/test/test_DataAllocator.cxx
index d880da72d7cb4..894fb5f38368a 100644
--- a/Framework/Core/test/test_DataAllocator.cxx
+++ b/Framework/Core/test/test_DataAllocator.cxx
@@ -45,7 +45,7 @@ using namespace o2::framework;
 // this function is only used to do the static checks for API return types
 void doTypeChecks()
 {
-  const Output output{"TST", "DUMMY", 0, Lifetime::Timeframe};
+  const Output output{"TST", "DUMMY", 0};
   // we require references to objects owned by allocator context
   static_assert(std::is_lvalue_reference<decltype(std::declval<DataAllocator>().make<int>(output))>::value);
   static_assert(std::is_lvalue_reference<decltype(std::declval<DataAllocator>().make<std::string>(output, "test"))>::value);
@@ -83,24 +83,24 @@ DataProcessorSpec getSourceSpec()
     // picked by the framework is no serialization
     test::MetaHeader meta1{42};
     test::MetaHeader meta2{23};
-    pc.outputs().snapshot(Output{"TST", "MESSAGEABLE", 0, Lifetime::Timeframe, {meta1, meta2}}, a);
-    pc.outputs().snapshot(Output{"TST", "MSGBLEROOTSRLZ", 0, Lifetime::Timeframe},
+    pc.outputs().snapshot(Output{"TST", "MESSAGEABLE", 0, {meta1, meta2}}, a);
+    pc.outputs().snapshot(Output{"TST", "MSGBLEROOTSRLZ", 0},
                           o2::framework::ROOTSerialized<decltype(a)>(a));
     // class Polymorphic is not messageable, so the serialization type is deduced
     // from the fact that the type has a dictionary and can be ROOT-serialized.
-    pc.outputs().snapshot(Output{"TST", "ROOTNONTOBJECT", 0, Lifetime::Timeframe}, b);
+    pc.outputs().snapshot(Output{"TST", "ROOTNONTOBJECT", 0}, b);
     // vector of ROOT serializable class
-    pc.outputs().snapshot(Output{"TST", "ROOTVECTOR", 0, Lifetime::Timeframe}, c);
+    pc.outputs().snapshot(Output{"TST", "ROOTVECTOR", 0}, c);
     // deque of simple types
-    pc.outputs().snapshot(Output{"TST", "DEQUE", 0, Lifetime::Timeframe}, testDequePayload);
+    pc.outputs().snapshot(Output{"TST", "DEQUE", 0}, testDequePayload);
     // likewise, passed anonymously with char type and class name
     o2::framework::ROOTSerialized<char, const char> d(*((char*)&c), "vector<o2::test::Polymorphic>");
-    pc.outputs().snapshot(Output{"TST", "ROOTSERLZDVEC", 0, Lifetime::Timeframe}, d);
+    pc.outputs().snapshot(Output{"TST", "ROOTSERLZDVEC", 0}, d);
     // vector of ROOT serializable class wrapped with TClass info as hint
     auto* cl = TClass::GetClass(typeid(decltype(c)));
     ASSERT_ERROR(cl != nullptr);
     o2::framework::ROOTSerialized<char, TClass> e(*((char*)&c), cl);
-    pc.outputs().snapshot(Output{"TST", "ROOTSERLZDVEC2", 0, Lifetime::Timeframe}, e);
+    pc.outputs().snapshot(Output{"TST", "ROOTSERLZDVEC2", 0}, e);
     // test the 'make' methods
     pc.outputs().make<o2::test::TriviallyCopyable>(OutputRef{"makesingle", 0}) = a;
     auto& multi = pc.outputs().make<o2::test::TriviallyCopyable>(OutputRef{"makespan", 0}, 3);
@@ -111,7 +111,7 @@ DataProcessorSpec getSourceSpec()
     // test the adopt method
     auto freefct = [](void* data, void* hint) {}; // simply ignore the cleanup for the test
     static std::string teststring = "adoptchunk";
-    pc.outputs().adoptChunk(Output{"TST", "ADOPTCHUNK", 0, Lifetime::Timeframe}, teststring.data(), teststring.length(), freefct, nullptr);
+    pc.outputs().adoptChunk(Output{"TST", "ADOPTCHUNK", 0}, teststring.data(), teststring.length(), freefct, nullptr);
     // test resizable data chunk, initial size 0 and grow
     auto& growchunk = pc.outputs().newChunk(OutputRef{"growchunk", 0}, 0);
     growchunk.resize(sizeof(o2::test::TriviallyCopyable));
@@ -307,7 +307,7 @@ DataProcessorSpec getSinkSpec()
     ASSERT_ERROR((object12[0] == o2::test::TriviallyCopyable{42, 23, 0xdead}));
     ASSERT_ERROR((object12[1] == o2::test::TriviallyCopyable{10, 20, 0xacdc}));
     // forward the read-only span on a different route
-    pc.outputs().snapshot(Output{"TST", "MSGABLVECTORCPY", 0, Lifetime::Timeframe}, object12);
+    pc.outputs().snapshot(Output{"TST", "MSGABLVECTORCPY", 0}, object12);
 
     LOG(info) << "extracting TNamed object from input13";
     auto object13 = pc.inputs().get<TNamed*>("input13");
diff --git a/Framework/Core/test/test_ParallelPipeline.cxx b/Framework/Core/test/test_ParallelPipeline.cxx
index e13869873fca1..47106855a8f74 100644
--- a/Framework/Core/test/test_ParallelPipeline.cxx
+++ b/Framework/Core/test/test_ParallelPipeline.cxx
@@ -64,7 +64,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
                     << *input.spec << ": " << *((int*)input.payload);
          auto const* dataheader = DataRefUtils::getHeader<o2::header::DataHeader*>(input);
          //auto& data = ctx.outputs().make<int>(OutputRef{"output", dataheader->subSpecification});
-         auto& data = ctx.outputs().make<int>(Output{"TST", "PREPROC", dataheader->subSpecification, Lifetime::Timeframe});
+         auto& data = ctx.outputs().make<int>(Output{"TST", "PREPROC", dataheader->subSpecification});
          ASSERT_ERROR(ctx.inputs().get<int>(input.spec->binding.c_str()) == parallelContext.index1D());
          data = parallelContext.index1D();
        }
@@ -85,10 +85,10 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
          // TODO: there is a bug in the API for using OutputRef, returns an rvalue which can not be bound to
          // lvalue reference
          //auto& data = ctx.outputs().make<int>(OutputRef{"output", dataheader->subSpecification});
-         auto& data = ctx.outputs().make<int>(Output{"TST", "DATA", dataheader->subSpecification, Lifetime::Timeframe});
+         auto& data = ctx.outputs().make<int>(Output{"TST", "DATA", dataheader->subSpecification});
          data = ctx.inputs().get<int>(input.spec->binding.c_str());
          //auto& meta = ctx.outputs().make<int>(OutputRef{"metadt", dataheader->subSpecification});
-         auto& meta = ctx.outputs().make<int>(Output{"TST", "META", dataheader->subSpecification, Lifetime::Timeframe});
+         auto& meta = ctx.outputs().make<int>(Output{"TST", "META", dataheader->subSpecification});
          meta = dataheader->subSpecification;
        }
      }}},
@@ -145,7 +145,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
           if (multiplicities[pipeline] == 0) {
             continue;
           }
-          ctx.outputs().make<int>(Output{"TST", "TRIGGER", subspecs[index], Lifetime::Timeframe}) = pipeline;
+          ctx.outputs().make<int>(Output{"TST", "TRIGGER", subspecs[index]}) = pipeline;
           multiplicities[pipeline++]--;
           if (pipeline >= nPipelines) {
             pipeline = 0;
diff --git a/Framework/Core/test/test_ProcessorOptions.cxx b/Framework/Core/test/test_ProcessorOptions.cxx
index 8de90e0e23441..1751176eb639e 100644
--- a/Framework/Core/test/test_ProcessorOptions.cxx
+++ b/Framework/Core/test/test_ProcessorOptions.cxx
@@ -45,7 +45,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const&)
             }
             // there is nothing to do, simply stop the workflow but we have to send at least one message
             // to make sure that the callback of the consumer is called
-            ctx.outputs().make<int>(Output{"TST", "TEST", 0, Lifetime::Timeframe}) = 42;
+            ctx.outputs().make<int>(Output{"TST", "TEST", 0}) = 42;
             ctx.services().get<ControlService>().endOfStream();
             *isReady = true;
           };
diff --git a/Framework/Core/test/test_StaggeringWorkflow.cxx b/Framework/Core/test/test_StaggeringWorkflow.cxx
index 0e98c4324bc76..edc07d598465b 100644
--- a/Framework/Core/test/test_StaggeringWorkflow.cxx
+++ b/Framework/Core/test/test_StaggeringWorkflow.cxx
@@ -85,9 +85,9 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
       // since the snapshot copy is ready for sending it is scheduled but held back
       // because of the CompletionPolicy trigger matcher. This message will be
       // sent together with the second message.
-      outputs.snapshot(Output{"PROD", "CHANNEL", subspec, Lifetime::Timeframe}, subspec);
+      outputs.snapshot(Output{"PROD", "CHANNEL", subspec}, subspec);
       device.waitFor(100);
-      outputs.snapshot(Output{"PROD", "TRIGGER", subspec, Lifetime::Timeframe}, subspec);
+      outputs.snapshot(Output{"PROD", "TRIGGER", subspec}, subspec);
       device.waitFor(100);
     }
     control.endOfStream();
@@ -106,7 +106,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
       LOG(info) << "processing " << input.spec->binding << " " << data;
       // check if the channel binding starts with 'trigger'
       if (input.spec->binding.find("trigger") == 0) {
-        pc.outputs().make<MyDataType>(Output{"PROC", "CHANNEL", data, Lifetime::Timeframe}) = data;
+        pc.outputs().make<MyDataType>(Output{"PROC", "CHANNEL", data}) = data;
       }
       nActiveInputs++;
     }
diff --git a/Framework/Core/test/test_TableBuilder.cxx b/Framework/Core/test/test_TableBuilder.cxx
index a7b4739054d03..26a0f5a930b73 100644
--- a/Framework/Core/test/test_TableBuilder.cxx
+++ b/Framework/Core/test/test_TableBuilder.cxx
@@ -240,7 +240,7 @@ TEST_CASE("TestSoAIntegration")
 
 TEST_CASE("TestDataAllocatorReturnType")
 {
-  const Output output{"TST", "DUMMY", 0, Lifetime::Timeframe};
+  const Output output{"TST", "DUMMY", 0};
 }
 
 TEST_CASE("TestPodInjestion")
diff --git a/Framework/Utils/include/DPLUtils/RootTreeReader.h b/Framework/Utils/include/DPLUtils/RootTreeReader.h
index 42a8c9d01d0d0..109d79f9fad78 100644
--- a/Framework/Utils/include/DPLUtils/RootTreeReader.h
+++ b/Framework/Utils/include/DPLUtils/RootTreeReader.h
@@ -51,11 +51,11 @@ struct DefaultKey {
   enum Lifetime lifetime = Lifetime::Timeframe;
 
   DefaultKey(const Output& desc)
-    : origin(desc.origin), description(desc.description), subSpec(desc.subSpec), lifetime(desc.lifetime)
+    : origin(desc.origin), description(desc.description), subSpec(desc.subSpec)
   {
   }
 
-  operator Output() const { return Output{origin, description, subSpec, lifetime}; }
+  operator Output() const { return Output{origin, description, subSpec}; }
 };
 } // namespace rtr
 
@@ -302,7 +302,7 @@ class GenericRootTreeReader
       }
 
       auto snapshot = [&context, &stackcreator](const KeyType& key, const auto& object) {
-        context.outputs().snapshot(Output{key.origin, key.description, key.subSpec, key.lifetime, std::move(stackcreator())}, object);
+        context.outputs().snapshot(Output{key.origin, key.description, key.subSpec, std::move(stackcreator())}, object);
       };
 
       char* data = nullptr;
@@ -310,7 +310,7 @@ class GenericRootTreeReader
       mBranch->GetEntry(entry);
 
       // execute hook if it was registered; if this return true do not proceed further
-      if (mPublishHook != nullptr && (*mPublishHook).hook(mName, context, Output{mKey.origin, mKey.description, mKey.subSpec, mKey.lifetime, std::move(stackcreator())}, data)) {
+      if (mPublishHook != nullptr && (*mPublishHook).hook(mName, context, Output{mKey.origin, mKey.description, mKey.subSpec, std::move(stackcreator())}, data)) {
 
       }
       // try to figureout when we need to do something special
diff --git a/Framework/Utils/src/Utils.cxx b/Framework/Utils/src/Utils.cxx
index 3a52c46e074af..1dc0882c2f55d 100644
--- a/Framework/Utils/src/Utils.cxx
+++ b/Framework/Utils/src/Utils.cxx
@@ -28,7 +28,7 @@ namespace workflows
 Output getOutput(const o2f::OutputSpec outputSpec)
 {
   auto concrete = DataSpecUtils::asConcreteDataMatcher(outputSpec);
-  return Output{concrete.origin, concrete.description, concrete.subSpec, outputSpec.lifetime};
+  return Output{concrete.origin, concrete.description, concrete.subSpec};
 }
 
 // This method can convert a vector of OutputSpec into a vector of Output.
diff --git a/Framework/Utils/test/test_RootTreeReader.cxx b/Framework/Utils/test/test_RootTreeReader.cxx
index 0390a29f807b1..bf652fca6f97a 100644
--- a/Framework/Utils/test/test_RootTreeReader.cxx
+++ b/Framework/Utils/test/test_RootTreeReader.cxx
@@ -67,11 +67,10 @@ DataProcessorSpec getSourceSpec()
       testFile->Close();
     }
 
-    constexpr auto persistency = Lifetime::Transient;
     auto reader = std::make_shared<RootTreeReader>("testtree",       // tree name
                                                    fileName.c_str(), // input file name
-                                                   RootTreeReader::BranchDefinition<std::vector<o2::test::TriviallyCopyable>>{Output{"TST", "ARRAYOFMSGBL", 0, persistency}, "msgblarray"},
-                                                   Output{"TST", "ARRAYOFDATA", 0, persistency},
+                                                   RootTreeReader::BranchDefinition<std::vector<o2::test::TriviallyCopyable>>{Output{"TST", "ARRAYOFMSGBL", 0}, "msgblarray"},
+                                                   Output{"TST", "ARRAYOFDATA", 0},
                                                    "dataarray",
                                                    RootTreeReader::PublishingMode::Single);
 
diff --git a/Framework/Utils/test/test_RootTreeWriterWorkflow.cxx b/Framework/Utils/test/test_RootTreeWriterWorkflow.cxx
index 3213c1775c758..9a9b849d8d9d2 100644
--- a/Framework/Utils/test/test_RootTreeWriterWorkflow.cxx
+++ b/Framework/Utils/test/test_RootTreeWriterWorkflow.cxx
@@ -143,7 +143,7 @@ DataProcessorSpec getSourceSpec()
       o2::test::Polymorphic a(*counter);
       pc.outputs().snapshot(OutputRef{"output", 0}, a);
       pc.outputs().snapshot(OutputRef{"output", 1}, a);
-      int& metadata = pc.outputs().make<int>(Output{"TST", "METADATA", 0, Lifetime::Timeframe});
+      int& metadata = pc.outputs().make<int>(Output{"TST", "METADATA", 0});
       metadata = *counter;
       *counter = *counter + 1;
       if (*counter >= sTreeSize) {

From 148366b7096871e052a7585f3ec321974807c84c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 5 Dec 2023 15:25:58 +0100
Subject: [PATCH 152/726] Remove default Lifetime::Timeframe when instanciating
 an Output

This lifetime field will be removed soon, and this is unambiguous, because
it's in any case the default.
---
 .../AOD/src/AODMcProducerWorkflowSpec.cxx     |  8 +--
 Detectors/AOD/src/AODProducerWorkflowSpec.cxx |  8 +--
 .../src/CPVBadMapCalibDevice.cxx              |  2 +-
 .../src/CPVGainCalibDevice.cxx                |  2 +-
 .../src/CPVPedestalCalibDevice.cxx            |  2 +-
 .../CPV/workflow/src/ClusterReaderSpec.cxx    |  6 +--
 .../CPV/workflow/src/ClusterizerSpec.cxx      | 16 +++---
 .../CPV/workflow/src/DigitReaderSpec.cxx      |  6 +--
 .../CPV/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../workflow/src/RawToDigitConverterSpec.cxx  | 18 +++----
 .../CTP/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 Detectors/CTP/workflow/src/RawDecoderSpec.cxx |  8 +--
 .../CTP/workflowIO/src/DigitReaderSpec.cxx    |  4 +-
 .../testWorkflow/src/DCSDataReplaySpec.cxx    |  2 +-
 .../src/DCSRandomDataGeneratorSpec.cxx        |  2 +-
 .../include/EMCALWorkflow/PublisherSpec.h     | 11 ++--
 .../workflow/src/AnalysisClusterSpec.cxx      |  2 +-
 .../EMCAL/workflow/src/CellConverterSpec.cxx  |  6 +--
 .../workflow/src/CellRecalibratorSpec.cxx     | 10 ++--
 .../EMCAL/workflow/src/ClusterizerSpec.cxx    |  8 +--
 .../EMCAL/workflow/src/EMCALDigitizerSpec.cxx | 10 ++--
 .../EMCAL/workflow/src/EntropyEncoderSpec.cxx |  2 +-
 .../workflow/src/RawToCellConverterSpec.cxx   |  6 +--
 .../src/StandaloneAODProducerSpec.cxx         |  4 +-
 .../src/emc-channel-data-producer.cxx         |  6 +--
 .../include/FDDWorkflow/RawReaderFDD.h        |  4 +-
 .../FIT/FDD/workflow/src/DigitReaderSpec.cxx  |  8 +--
 .../FDD/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../FDD/workflow/src/RecPointReaderSpec.cxx   |  4 +-
 .../FDD/workflow/src/ReconstructorSpec.cxx    |  4 +-
 .../FT0CalibSlewingCollectorSpec.h            |  4 +-
 .../FT0TimeSpectraProcessor-Workflow.cxx      |  2 +-
 .../FIT/FT0/workflow/src/DigitReaderSpec.cxx  |  8 +--
 .../FT0/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../FT0/workflow/src/RecPointReaderSpec.cxx   |  4 +-
 .../FT0/workflow/src/ReconstructionSpec.cxx   |  4 +-
 .../FIT/FV0/workflow/src/DigitReaderSpec.cxx  |  8 +--
 .../FV0/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../FV0/workflow/src/RecPointReaderSpec.cxx   |  4 +-
 .../FV0/workflow/src/ReconstructionSpec.cxx   |  4 +-
 .../include/FITWorkflow/RawReaderFIT.h        | 14 ++---
 .../FOCAL/workflow/src/RawDecoderSpec.cxx     | 10 ++--
 .../Filtering/src/FilteredTFReaderSpec.cxx    | 14 ++---
 .../helpers/src/NoInpDummyOutSpec.cxx         |  2 +-
 .../qc/src/ITSTPCMatchingQCSpec.cxx           |  2 +-
 .../readers/src/GlobalFwdTrackReaderSpec.cxx  |  4 +-
 .../readers/src/IRFrameReaderSpec.cxx         |  2 +-
 .../readers/src/MatchedMFTMCHReaderSpec.cxx   |  2 +-
 .../readers/src/PrimaryVertexReaderSpec.cxx   |  8 +--
 .../readers/src/SecondaryVertexReaderSpec.cxx | 18 +++----
 .../src/StrangenessTrackingReaderSpec.cxx     |  6 +--
 .../readers/src/TrackCosmicsReaderSpec.cxx    |  4 +-
 .../readers/src/TrackTPCITSReaderSpec.cxx     | 10 ++--
 .../src/CosmicsMatchingSpec.cxx               |  4 +-
 .../src/GlobalFwdMatchingSpec.cxx             | 12 ++---
 .../src/HMPMatcherSpec.cxx                    |  4 +-
 .../src/PrimaryVertexingSpec.cxx              |  8 +--
 .../src/StrangenessTrackingSpec.cxx           |  6 +--
 .../src/TOFMatcherSpec.cxx                    | 54 +++++++++----------
 .../src/TPCITSMatchingSpec.cxx                | 12 ++---
 .../src/VertexTrackMatcherSpec.cxx            |  4 +-
 .../study/src/TPCDataFilter.cxx               | 10 ++--
 .../src/TPCInterpolationSpec.cxx              | 12 ++---
 .../src/TPCUnbinnedResidualReaderSpec.cxx     |  6 +--
 .../HMPID/workflow/src/ClustersReaderSpec.cxx |  4 +-
 .../HMPID/workflow/src/DataDecoderSpec.cxx    |  4 +-
 .../HMPID/workflow/src/DataDecoderSpec2.cxx   |  4 +-
 .../HMPID/workflow/src/DigitsReaderSpec.cxx   |  4 +-
 .../workflow/src/DigitsToClustersSpec.cxx     |  4 +-
 .../HMPID/workflow/src/EntropyEncoderSpec.cxx |  2 +-
 .../workflow/src/HMPMatchedReaderSpec.cxx     |  4 +-
 .../src/TestDataReader.cxx                    | 28 +++++-----
 .../ITSMFT/ITS/workflow/src/ClustererSpec.cxx | 10 ++--
 .../ITS/workflow/src/CookedTrackerSpec.cxx    | 16 +++---
 .../ITS/workflow/src/DCSGeneratorSpec.cxx     |  2 +-
 .../ITS/workflow/src/TrackReaderSpec.cxx      | 14 ++---
 .../ITSMFT/ITS/workflow/src/TrackerSpec.cxx   | 18 +++----
 .../ITS/workflow/src/VertexReaderSpec.cxx     |  4 +-
 .../ITSMFT/MFT/workflow/src/ClustererSpec.cxx | 10 ++--
 .../MFT/workflow/src/TrackReaderSpec.cxx      |  8 +--
 .../ITSMFT/MFT/workflow/src/TrackerSpec.cxx   | 10 ++--
 .../common/workflow/src/ClusterReaderSpec.cxx | 12 ++---
 .../common/workflow/src/DigitReaderSpec.cxx   | 12 ++---
 .../workflow/src/EntropyEncoderSpec.cxx       |  2 +-
 .../common/workflow/src/STFDecoderSpec.cxx    | 20 +++----
 Detectors/MUON/MCH/IO/src/DigitReaderSpec.cxx | 10 ++--
 .../MCH/Workflow/src/TrackAtVertexSpec.cxx    |  2 +-
 .../Workflow/src/entropy-encoder-workflow.cxx |  2 +-
 .../MUON/MID/Workflow/src/ClusterizerSpec.cxx |  8 +--
 .../MID/Workflow/src/ColumnDataSpecsUtils.cxx |  4 +-
 .../src/DecodedDataAggregatorSpec.cxx         |  4 +-
 .../MUON/MID/Workflow/src/DigitReaderSpec.cxx | 10 ++--
 .../MID/Workflow/src/EntropyDecoderSpec.cxx   |  4 +-
 .../MID/Workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../MUON/MID/Workflow/src/MaskMakerSpec.cxx   |  4 +-
 .../MID/Workflow/src/RawGBTDecoderSpec.cxx    |  4 +-
 .../MUON/MID/Workflow/src/TrackerSpec.cxx     | 12 ++---
 .../MID/Workflow/src/ZeroSuppressionSpec.cxx  |  4 +-
 .../PHOS/calib/src/PHOSBadMapCalibDevice.cxx  |  2 +-
 .../calib/src/PHOSHGLGRatioCalibDevice.cxx    |  2 +-
 .../calib/src/PHOSPedestalCalibDevice.cxx     |  2 +-
 .../PHOS/workflow/src/CellConverterSpec.cxx   | 12 ++---
 .../PHOS/workflow/src/CellReaderSpec.cxx      |  6 +--
 .../PHOS/workflow/src/ClusterizerSpec.cxx     | 16 +++---
 .../PHOS/workflow/src/DigitReaderSpec.cxx     |  6 +--
 .../PHOS/workflow/src/EntropyEncoderSpec.cxx  |  2 +-
 .../PHOS/workflow/src/EventBuilderSpec.cxx    |  4 +-
 .../workflow/src/RawToCellConverterSpec.cxx   | 12 ++---
 Detectors/PHOS/workflow/src/ReaderSpec.cxx    | 22 ++++----
 .../src/StandaloneAODProducerSpec.cxx         |  4 +-
 .../testWorkflow/TOFCalibCollectorSpec.h      |  4 +-
 .../workflow/src/CompressedDecodingTask.cxx   | 12 ++---
 .../TOF/workflow/src/EntropyDecoderSpec.cxx   |  2 +-
 .../TOF/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../TOF/workflow/src/TOFClusterizerSpec.cxx   | 14 ++---
 .../workflowIO/src/CalibClusReaderSpec.cxx    |  8 +--
 .../workflowIO/src/CalibInfoReaderSpec.cxx    |  4 +-
 .../TOF/workflowIO/src/ClusterReaderSpec.cxx  |  6 +--
 .../TOF/workflowIO/src/DigitReaderSpec.cxx    | 12 ++---
 .../workflowIO/src/TOFMatchedReaderSpec.cxx   |  6 +--
 .../include/TPCWorkflow/TPCFLPIDCSpec.h       |  6 +--
 .../TPCWorkflow/TPCFourierTransformEPNSpec.h  |  2 +-
 .../include/TPCWorkflow/TPCIntegrateIDCSpec.h |  4 +-
 .../include/TPCReaderWorkflow/PublisherSpec.h |  7 ++-
 .../workflow/readers/src/PublisherSpec.cxx    |  4 +-
 .../workflow/readers/src/TrackReaderSpec.cxx  |  6 +--
 .../readers/src/TriggerReaderSpec.cxx         |  2 +-
 .../workflow/src/ChunkedDigitPublisher.cxx    |  5 +-
 .../workflow/src/ClusterDecoderRawSpec.cxx    |  8 +--
 Detectors/TPC/workflow/src/ClustererSpec.cxx  |  8 +--
 .../TPC/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../TPC/workflow/src/KryptonClustererSpec.cxx |  2 +-
 .../TPC/workflow/src/KryptonRawFilterSpec.cxx |  2 +-
 .../TPC/workflow/src/LinkZSToDigitsSpec.cxx   |  2 +-
 .../TPC/workflow/src/MIPTrackFilterSpec.cxx   |  2 +-
 .../TPC/workflow/src/RawToDigitsSpec.cxx      |  4 +-
 .../TPC/workflow/src/SACProcessorSpec.cxx     |  4 +-
 Detectors/TPC/workflow/src/ZSSpec.cxx         |  6 +--
 .../workflow/test/test_ft_EPN_Aggregator.cxx  |  2 +-
 .../TRD/reconstruction/src/EventRecord.cxx    | 10 ++--
 .../include/TRDWorkflow/KrClustererSpec.h     |  4 +-
 .../TRDWorkflow/TRDGlobalTrackingQCSpec.h     |  2 +-
 .../include/TRDWorkflow/TRDPulseHeightSpec.h  |  4 +-
 .../workflow/io/src/TRDCalibReaderSpec.cxx    |  2 +-
 .../workflow/io/src/TRDDigitReaderSpec.cxx    |  6 +--
 .../TRD/workflow/io/src/TRDPHReaderSpec.cxx   |  2 +-
 .../workflow/io/src/TRDTrackReaderSpec.cxx    | 16 +++---
 .../workflow/io/src/TRDTrackletReaderSpec.cxx | 10 ++--
 .../TRD/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../TRD/workflow/src/TRDDigitizerSpec.cxx     |  8 +--
 .../workflow/src/TRDGlobalTrackingSpec.cxx    | 16 +++---
 .../src/TRDTrackletTransformerSpec.cxx        |  4 +-
 .../TRD/workflow/src/TRDTrapSimulatorSpec.cxx |  8 +--
 .../TRD/workflow/src/TrackBasedCalibSpec.cxx  |  4 +-
 .../ITS3/workflow/src/ClustererSpec.cxx       | 10 ++--
 .../ITS3/workflow/src/DigitReaderSpec.cxx     | 10 ++--
 .../ITS3/workflow/src/TrackReaderSpec.cxx     | 14 ++---
 .../ITS3/workflow/src/TrackerSpec.cxx         | 18 +++----
 .../ITS3/workflow/src/VertexReaderSpec.cxx    |  4 +-
 Detectors/Vertexing/src/SVertexer.cxx         | 24 ++++-----
 Detectors/ZDC/calib/src/InterCalibEPNSpec.cxx |  6 +--
 Detectors/ZDC/calib/src/TDCCalibEPNSpec.cxx   |  4 +-
 .../ZDC/calib/src/WaveformCalibEPNSpec.cxx    |  2 +-
 .../ZDC/raw/include/ZDCRaw/RawReaderZDC.h     |  6 +--
 .../ZDC/workflow/src/DigitReaderSpec.cxx      |  8 +--
 Detectors/ZDC/workflow/src/DigitRecoSpec.cxx  | 10 ++--
 .../ZDC/workflow/src/EntropyEncoderSpec.cxx   |  2 +-
 .../ZDC/workflow/src/RecEventReaderSpec.cxx   |  8 +--
 Detectors/ZDC/workflow/src/RecoReaderSpec.cxx |  8 +--
 Framework/Core/include/Framework/Output.h     |  5 ++
 GPU/Workflow/src/GPUWorkflowITS.cxx           | 18 +++----
 GPU/Workflow/src/GPUWorkflowPipeline.cxx      |  2 +-
 GPU/Workflow/src/GPUWorkflowSpec.cxx          | 14 ++---
 .../src/CPVDigitizerSpec.cxx                  |  8 +--
 .../src/CTPDigitizerSpec.cxx                  |  4 +-
 .../src/FDDDigitizerSpec.cxx                  | 10 ++--
 .../src/FT0DigitizerSpec.cxx                  | 10 ++--
 .../src/FV0DigitizerSpec.cxx                  | 10 ++--
 .../src/HMPIDDigitizerSpec.cxx                |  8 +--
 .../src/ITS3DigitizerSpec.cxx                 | 10 ++--
 .../src/ITSMFTDigitizerSpec.cxx               | 10 ++--
 .../src/MCHDigitizerSpec.cxx                  |  8 +--
 .../DigitizerWorkflow/src/MCTruthReaderSpec.h |  4 +-
 .../src/MCTruthSourceSpec.cxx                 |  4 +-
 .../src/MCTruthWriterSpec.cxx                 |  2 +-
 .../src/MIDDigitizerSpec.cxx                  |  8 +--
 .../src/PHOSDigitizerSpec.cxx                 |  8 +--
 .../src/TOFDigitizerSpec.cxx                  | 12 ++---
 .../src/TPCDigitizerSpec.cxx                  | 12 ++---
 .../src/ZDCDigitizerSpec.cxx                  | 10 ++--
 .../DataSampling/src/DataSamplingPolicy.cxx   |  2 +-
 Utilities/DataSampling/src/Dispatcher.cxx     |  1 -
 run/dpl_eventgen.cxx                          |  4 +-
 run/o2sim_hepmc_publisher.cxx                 |  4 +-
 run/o2sim_kine_publisher.cxx                  |  4 +-
 run/o2sim_mctracks_to_aod.cxx                 |  4 +-
 196 files changed, 685 insertions(+), 688 deletions(-)

diff --git a/Detectors/AOD/src/AODMcProducerWorkflowSpec.cxx b/Detectors/AOD/src/AODMcProducerWorkflowSpec.cxx
index 2e60b70c9855a..4c1623188e55c 100644
--- a/Detectors/AOD/src/AODMcProducerWorkflowSpec.cxx
+++ b/Detectors/AOD/src/AODMcProducerWorkflowSpec.cxx
@@ -296,13 +296,13 @@ void AODMcProducerWorkflowDPL::run(ProcessingContext& pc)
     TString ROOTVersion = ROOT_RELEASE;
     mMetaDataKeys = {"DataType", "Run", "O2Version", "ROOTVersion", "RecoPassName", "AnchorProduction", "AnchorPassName", "LPMProductionTag"};
     mMetaDataVals = {dataType, "3", O2Version, ROOTVersion, mRecoPass, mAnchorProd, mAnchorPass, mLPMProdTag};
-    pc.outputs().snapshot(Output{"AMD", "AODMetadataKeys", 0, Lifetime::Timeframe}, mMetaDataKeys);
-    pc.outputs().snapshot(Output{"AMD", "AODMetadataVals", 0, Lifetime::Timeframe}, mMetaDataVals);
+    pc.outputs().snapshot(Output{"AMD", "AODMetadataKeys", 0}, mMetaDataKeys);
+    pc.outputs().snapshot(Output{"AMD", "AODMetadataVals", 0}, mMetaDataVals);
     mIsMDSent = true;
   }
 
-  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0, Lifetime::Timeframe}, tfNumber);
-  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0, Lifetime::Timeframe}, "");
+  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0}, tfNumber);
+  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0}, "");
 
   pc.services().get<ControlService>().endOfStream();
   pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Detectors/AOD/src/AODProducerWorkflowSpec.cxx b/Detectors/AOD/src/AODProducerWorkflowSpec.cxx
index 323d4b9f7f41a..e97c90f14f64f 100644
--- a/Detectors/AOD/src/AODProducerWorkflowSpec.cxx
+++ b/Detectors/AOD/src/AODProducerWorkflowSpec.cxx
@@ -2241,11 +2241,11 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc)
   TString ROOTVersion = ROOT_RELEASE;
   mMetaDataKeys = {"DataType", "Run", "O2Version", "ROOTVersion", "RecoPassName", "AnchorProduction", "AnchorPassName", "LPMProductionTag"};
   mMetaDataVals = {dataType, "3", O2Version, ROOTVersion, mRecoPass, mAnchorProd, mAnchorPass, mLPMProdTag};
-  pc.outputs().snapshot(Output{"AMD", "AODMetadataKeys", 0, Lifetime::Timeframe}, mMetaDataKeys);
-  pc.outputs().snapshot(Output{"AMD", "AODMetadataVals", 0, Lifetime::Timeframe}, mMetaDataVals);
+  pc.outputs().snapshot(Output{"AMD", "AODMetadataKeys", 0}, mMetaDataKeys);
+  pc.outputs().snapshot(Output{"AMD", "AODMetadataVals", 0}, mMetaDataVals);
 
-  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0, Lifetime::Timeframe}, tfNumber);
-  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0, Lifetime::Timeframe}, "");
+  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0}, tfNumber);
+  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0}, "");
 
   mTimer.Stop();
 }
diff --git a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVBadMapCalibDevice.cxx b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVBadMapCalibDevice.cxx
index b1c61d25da9c8..f2a08e280f954 100644
--- a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVBadMapCalibDevice.cxx
+++ b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVBadMapCalibDevice.cxx
@@ -174,7 +174,7 @@ void CPVBadMapCalibDevice::sendOutput(DataAllocator& output)
     output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, "CPV_BadChanMap", subSpec}, info);
   }
 
-  output.snapshot(o2::framework::Output{"CPV", "BADMAPCHANGE", 0, o2::framework::Lifetime::Timeframe}, mMapDiff);
+  output.snapshot(o2::framework::Output{"CPV", "BADMAPCHANGE", 0}, mMapDiff);
 }
 
 bool CPVBadMapCalibDevice::differFromCurrent()
diff --git a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVGainCalibDevice.cxx b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVGainCalibDevice.cxx
index b5e98f00d021f..d7f187e3a88cc 100644
--- a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVGainCalibDevice.cxx
+++ b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVGainCalibDevice.cxx
@@ -143,7 +143,7 @@ void CPVGainCalibDevice::sendOutput(DataAllocator& output)
     fout.Close();
   }
   // Anyway send change to QC
-  output.snapshot(o2::framework::Output{"CPV", "GAINDIFF", 0, o2::framework::Lifetime::Timeframe}, mGainRatio);
+  output.snapshot(o2::framework::Output{"CPV", "GAINDIFF", 0}, mGainRatio);
 }
 
 void CPVGainCalibDevice::calculateGains()
diff --git a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVPedestalCalibDevice.cxx b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVPedestalCalibDevice.cxx
index 125e8e1bc8c40..5dd414ab7aeb8 100644
--- a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVPedestalCalibDevice.cxx
+++ b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVPedestalCalibDevice.cxx
@@ -118,7 +118,7 @@ void CPVPedestalCalibDevice::sendOutput(DataAllocator& output)
   }
   // Anyway send change to QC
   LOG(info) << "[CPVPedestalCalibDevice - run] Writing ";
-  output.snapshot(o2::framework::Output{"CPV", "PEDDIFF", 0, o2::framework::Lifetime::Timeframe}, mPedDiff);
+  output.snapshot(o2::framework::Output{"CPV", "PEDDIFF", 0}, mPedDiff);
 
   // Write pedestal distributions to calculate bad map
   std::string filename = mPath + "CPVPedestals.root";
diff --git a/Detectors/CPV/workflow/src/ClusterReaderSpec.cxx b/Detectors/CPV/workflow/src/ClusterReaderSpec.cxx
index f9d82eb23bf68..f9d0817325c36 100644
--- a/Detectors/CPV/workflow/src/ClusterReaderSpec.cxx
+++ b/Detectors/CPV/workflow/src/ClusterReaderSpec.cxx
@@ -44,10 +44,10 @@ void ClusterReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mClusters.size() << " Clusters in " << mTRs.size() << " TriggerRecords at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "CLUSTERS", 0, Lifetime::Timeframe}, mClusters);
-  pc.outputs().snapshot(Output{mOrigin, "CLUSTERTRIGRECS", 0, Lifetime::Timeframe}, mTRs);
+  pc.outputs().snapshot(Output{mOrigin, "CLUSTERS", 0}, mClusters);
+  pc.outputs().snapshot(Output{mOrigin, "CLUSTERTRIGRECS", 0}, mTRs);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{mOrigin, "CLUSTERTRUEMC", 0, Lifetime::Timeframe}, mMCTruth);
+    pc.outputs().snapshot(Output{mOrigin, "CLUSTERTRUEMC", 0}, mMCTruth);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/CPV/workflow/src/ClusterizerSpec.cxx b/Detectors/CPV/workflow/src/ClusterizerSpec.cxx
index 87d0b86a98f5c..e004c3cec8949 100644
--- a/Detectors/CPV/workflow/src/ClusterizerSpec.cxx
+++ b/Detectors/CPV/workflow/src/ClusterizerSpec.cxx
@@ -49,14 +49,14 @@ void ClusterizerSpec::run(framework::ProcessingContext& ctx)
   if (!digits.size()) { // nothing to process
     LOG(info) << "ClusterizerSpec::run() : no digits; moving on";
     mOutputClusters.clear();
-    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERS", 0, o2::framework::Lifetime::Timeframe}, mOutputClusters);
+    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERS", 0}, mOutputClusters);
     mOutputClusterTrigRecs.clear();
-    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRIGRECS", 0, o2::framework::Lifetime::Timeframe}, mOutputClusterTrigRecs);
+    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRIGRECS", 0}, mOutputClusterTrigRecs);
     mCalibDigits.clear();
-    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CALIBDIGITS", 0, o2::framework::Lifetime::Timeframe}, mCalibDigits);
+    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CALIBDIGITS", 0}, mCalibDigits);
     if (mPropagateMC) {
       mOutputTruthCont.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRUEMC", 0, o2::framework::Lifetime::Timeframe}, mOutputTruthCont);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRUEMC", 0}, mOutputTruthCont);
     }
     return;
   }
@@ -73,12 +73,12 @@ void ClusterizerSpec::run(framework::ProcessingContext& ctx)
 
   LOG(debug) << "CPVClusterizer::run() : Received " << digitsTR.size() << " TR, calling clusterizer ...";
 
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERS", 0, o2::framework::Lifetime::Timeframe}, mOutputClusters);
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRIGRECS", 0, o2::framework::Lifetime::Timeframe}, mOutputClusterTrigRecs);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERS", 0}, mOutputClusters);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRIGRECS", 0}, mOutputClusterTrigRecs);
   if (mPropagateMC) {
-    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRUEMC", 0, o2::framework::Lifetime::Timeframe}, mOutputTruthCont);
+    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRUEMC", 0}, mOutputTruthCont);
   }
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CALIBDIGITS", 0, o2::framework::Lifetime::Timeframe}, mCalibDigits);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CALIBDIGITS", 0}, mCalibDigits);
   LOG(info) << "Finished, wrote  " << mOutputClusters.size() << " clusters, " << mOutputClusterTrigRecs.size() << "TR and " << mOutputTruthCont.getIndexedSize() << " Labels";
 }
 o2::framework::DataProcessorSpec o2::cpv::reco_workflow::getClusterizerSpec(bool propagateMC)
diff --git a/Detectors/CPV/workflow/src/DigitReaderSpec.cxx b/Detectors/CPV/workflow/src/DigitReaderSpec.cxx
index ba74cb88b10d2..20fe497eb5d0c 100644
--- a/Detectors/CPV/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/CPV/workflow/src/DigitReaderSpec.cxx
@@ -44,10 +44,10 @@ void DigitReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mDigits.size() << " Digits in " << mTRs.size() << " TriggerRecords at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "DIGITS", 0, Lifetime::Timeframe}, mDigits);
-  pc.outputs().snapshot(Output{mOrigin, "DIGITTRIGREC", 0, Lifetime::Timeframe}, mTRs);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITS", 0}, mDigits);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITTRIGREC", 0}, mTRs);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{mOrigin, "DIGITSMCTR", 0, Lifetime::Timeframe}, mMCTruth);
+    pc.outputs().snapshot(Output{mOrigin, "DIGITSMCTR", 0}, mMCTruth);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/CPV/workflow/src/EntropyEncoderSpec.cxx b/Detectors/CPV/workflow/src/EntropyEncoderSpec.cxx
index 97f69fe928342..31ed720e66335 100644
--- a/Detectors/CPV/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/CPV/workflow/src/EntropyEncoderSpec.cxx
@@ -54,7 +54,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"CPV", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"CPV", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, triggers, clusters);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   if (mSelIR) {
diff --git a/Detectors/CPV/workflow/src/RawToDigitConverterSpec.cxx b/Detectors/CPV/workflow/src/RawToDigitConverterSpec.cxx
index fcdc75c38c67f..766902a2fdc95 100644
--- a/Detectors/CPV/workflow/src/RawToDigitConverterSpec.cxx
+++ b/Detectors/CPV/workflow/src/RawToDigitConverterSpec.cxx
@@ -136,11 +136,11 @@ void RawToDigitConverterSpec::run(framework::ProcessingContext& ctx)
              contDeadBeef == maxWarn ? fmt::format(". {} such inputs in row received, stopping reporting", contDeadBeef) : "");
       }
       mOutputDigits.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, mOutputDigits);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0}, mOutputDigits);
       mOutputTriggerRecords.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0, o2::framework::Lifetime::Timeframe}, mOutputTriggerRecords);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0}, mOutputTriggerRecords);
       mOutputHWErrors.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0, o2::framework::Lifetime::Timeframe}, mOutputHWErrors);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0}, mOutputHWErrors);
       return; // empty TF, nothing to process
     }
   }
@@ -272,11 +272,11 @@ void RawToDigitConverterSpec::run(framework::ProcessingContext& ctx)
     if (skipTF) {
       // Send no digits
       mOutputDigits.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, mOutputDigits);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0}, mOutputDigits);
       mOutputTriggerRecords.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0, o2::framework::Lifetime::Timeframe}, mOutputTriggerRecords);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0}, mOutputTriggerRecords);
       // Send errors
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0, o2::framework::Lifetime::Timeframe}, mOutputHWErrors);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0}, mOutputHWErrors);
       return;
     }
   }
@@ -305,9 +305,9 @@ void RawToDigitConverterSpec::run(framework::ProcessingContext& ctx)
   digitBuffer.clear();
 
   LOG(info) << "[CPVRawToDigitConverter - run] Sending " << mOutputDigits.size() << " digits in " << mOutputTriggerRecords.size() << "trigger records.";
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, mOutputDigits);
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0, o2::framework::Lifetime::Timeframe}, mOutputTriggerRecords);
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0, o2::framework::Lifetime::Timeframe}, mOutputHWErrors);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0}, mOutputDigits);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0}, mOutputTriggerRecords);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0}, mOutputHWErrors);
 }
 //_____________________________________________________________________________
 o2::framework::DataProcessorSpec o2::cpv::reco_workflow::getRawToDigitConverterSpec(bool askDISTSTF, bool isPedestal, bool useBadChannelMap, bool useGainCalibration)
diff --git a/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx b/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx
index 7ad4f821c12a9..ca16d10087fa6 100644
--- a/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx
@@ -64,7 +64,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
       lumi = lumiPrev;
     }
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"CTP", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"CTP", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, digits, lumi);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   mTimer.Stop();
diff --git a/Detectors/CTP/workflow/src/RawDecoderSpec.cxx b/Detectors/CTP/workflow/src/RawDecoderSpec.cxx
index 674c2f4208d8f..5f0cb3b759c48 100644
--- a/Detectors/CTP/workflow/src/RawDecoderSpec.cxx
+++ b/Detectors/CTP/workflow/src/RawDecoderSpec.cxx
@@ -80,10 +80,10 @@ void RawDecoderSpec::run(framework::ProcessingContext& ctx)
   auto& inputs = ctx.inputs();
   auto dummyOutput = [&ctx, this]() {
     if (this->mDoDigits) {
-      ctx.outputs().snapshot(o2::framework::Output{"CTP", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, this->mOutputDigits);
+      ctx.outputs().snapshot(o2::framework::Output{"CTP", "DIGITS", 0}, this->mOutputDigits);
     }
     if (this->mDoLumi) {
-      ctx.outputs().snapshot(o2::framework::Output{"CTP", "LUMI", 0, o2::framework::Lifetime::Timeframe}, this->mOutputLumiInfo);
+      ctx.outputs().snapshot(o2::framework::Output{"CTP", "LUMI", 0}, this->mOutputLumiInfo);
     }
   };
   // if we see requested data type input with 0xDEADBEEF subspec and 0 payload this means that the "delayed message"
@@ -117,7 +117,7 @@ void RawDecoderSpec::run(framework::ProcessingContext& ctx)
   }
   if (mDoDigits) {
     LOG(info) << "[CTPRawToDigitConverter - run] Writing " << mOutputDigits.size() << " digits. IR rejected:" << mDecoder.getIRRejected() << " TCR rejected:" << mDecoder.getTCRRejected();
-    ctx.outputs().snapshot(o2::framework::Output{"CTP", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, mOutputDigits);
+    ctx.outputs().snapshot(o2::framework::Output{"CTP", "DIGITS", 0}, mOutputDigits);
   }
   if (mDoLumi) {
     uint32_t tfCountsT = 0;
@@ -157,7 +157,7 @@ void RawDecoderSpec::run(framework::ProcessingContext& ctx)
       mOutputLumiInfo.printInputs();
       LOGP(info, "Orbit {}: {}/{} counts inp1/inp2 in {}/{} HBFs -> lumi_inp1 = {:.3e}+-{:.3e} lumi_inp2 = {:.3e}+-{:.3e}", mOutputLumiInfo.orbit, mCountsT, mCountsV, mNHBIntegratedT, mNHBIntegratedV, mOutputLumiInfo.getLumi(), mOutputLumiInfo.getLumiError(), mOutputLumiInfo.getLumiFV0(), mOutputLumiInfo.getLumiFV0Error());
     }
-    ctx.outputs().snapshot(o2::framework::Output{"CTP", "LUMI", 0, o2::framework::Lifetime::Timeframe}, mOutputLumiInfo);
+    ctx.outputs().snapshot(o2::framework::Output{"CTP", "LUMI", 0}, mOutputLumiInfo);
   }
 }
 o2::framework::DataProcessorSpec o2::ctp::reco_workflow::getRawDecoderSpec(bool askDISTSTF, bool digits, bool lumi)
diff --git a/Detectors/CTP/workflowIO/src/DigitReaderSpec.cxx b/Detectors/CTP/workflowIO/src/DigitReaderSpec.cxx
index 59f608bef2efb..ecf1c2e19b660 100644
--- a/Detectors/CTP/workflowIO/src/DigitReaderSpec.cxx
+++ b/Detectors/CTP/workflowIO/src/DigitReaderSpec.cxx
@@ -76,8 +76,8 @@ void DigitReader::run(ProcessingContext& pc)
 
   mTree->GetEntry(ent);
   LOG(info) << "DigitReader pushes " << mDigits.size() << " digits at entry " << ent;
-  pc.outputs().snapshot(Output{"CTP", "DIGITS", 0, Lifetime::Timeframe}, mDigits);
-  pc.outputs().snapshot(Output{"CTP", "LUMI", 0, Lifetime::Timeframe}, mLumi);
+  pc.outputs().snapshot(Output{"CTP", "DIGITS", 0}, mDigits);
+  pc.outputs().snapshot(Output{"CTP", "LUMI", 0}, mLumi);
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Detectors/DCS/testWorkflow/src/DCSDataReplaySpec.cxx b/Detectors/DCS/testWorkflow/src/DCSDataReplaySpec.cxx
index 05daf522f372a..783f6ae76e707 100644
--- a/Detectors/DCS/testWorkflow/src/DCSDataReplaySpec.cxx
+++ b/Detectors/DCS/testWorkflow/src/DCSDataReplaySpec.cxx
@@ -88,7 +88,7 @@ void DCSDataReplayer::run(o2::framework::ProcessingContext& pc)
 
   LOG(info)
     << "***************** TF " << tfid << " has generated " << dpcoms.size() << " DPs";
-  pc.outputs().snapshot(Output{"DCS", mDataDescription, 0, Lifetime::Timeframe}, dpcoms);
+  pc.outputs().snapshot(Output{"DCS", mDataDescription, 0}, dpcoms);
   mTFs++;
 }
 } // namespace
diff --git a/Detectors/DCS/testWorkflow/src/DCSRandomDataGeneratorSpec.cxx b/Detectors/DCS/testWorkflow/src/DCSRandomDataGeneratorSpec.cxx
index a82a5b4164e2a..899a4e110aea8 100644
--- a/Detectors/DCS/testWorkflow/src/DCSRandomDataGeneratorSpec.cxx
+++ b/Detectors/DCS/testWorkflow/src/DCSRandomDataGeneratorSpec.cxx
@@ -153,7 +153,7 @@ void DCSRandomDataGenerator::run(o2::framework::ProcessingContext& pc)
   auto timeNow = std::chrono::system_clock::now();
   timingInfo.creation = std::chrono::duration_cast<std::chrono::milliseconds>(timeNow.time_since_epoch()).count(); // in ms
 
-  pc.outputs().snapshot(Output{"DCS", mDataDescription, 0, Lifetime::Timeframe}, dpcoms);
+  pc.outputs().snapshot(Output{"DCS", mDataDescription, 0}, dpcoms);
   mTFs++;
 }
 } // namespace
diff --git a/Detectors/EMCAL/workflow/include/EMCALWorkflow/PublisherSpec.h b/Detectors/EMCAL/workflow/include/EMCALWorkflow/PublisherSpec.h
index f583653f1c3cb..468e66c43a0ab 100644
--- a/Detectors/EMCAL/workflow/include/EMCALWorkflow/PublisherSpec.h
+++ b/Detectors/EMCAL/workflow/include/EMCALWorkflow/PublisherSpec.h
@@ -60,23 +60,22 @@ framework::DataProcessorSpec getPublisherSpec(PublisherConf const& config, uint3
 
   // a creator callback for the actual reader instance
   auto creator = [dto, tro, mco, subspec, propagateMC](const char* treename, const char* filename, int nofEvents, Reader::PublishingMode publishingMode, const char* branchname, const char* triggerbranchname, const char* mcbranchname) {
-    constexpr auto persistency = o2::framework::Lifetime::Timeframe;
     if (propagateMC) {
       return std::make_shared<Reader>(treename,
                                       filename,
                                       nofEvents,
                                       publishingMode,
-                                      Output{mco.origin, mco.description, subspec, persistency},
+                                      Output{mco.origin, mco.description, subspec},
                                       mcbranchname,
-                                      Reader::BranchDefinition<T>{Output{dto.origin, dto.description, subspec, persistency}, branchname},
-                                      Reader::BranchDefinition<TriggerInputType>{Output{tro.origin, tro.description, subspec, persistency}, triggerbranchname});
+                                      Reader::BranchDefinition<T>{Output{dto.origin, dto.description, subspec}, branchname},
+                                      Reader::BranchDefinition<TriggerInputType>{Output{tro.origin, tro.description, subspec}, triggerbranchname});
     } else {
       return std::make_shared<Reader>(treename,
                                       filename,
                                       nofEvents,
                                       publishingMode,
-                                      Reader::BranchDefinition<T>{Output{dto.origin, dto.description, subspec, persistency}, branchname},
-                                      Reader::BranchDefinition<TriggerInputType>{Output{tro.origin, tro.description, subspec, persistency}, triggerbranchname});
+                                      Reader::BranchDefinition<T>{Output{dto.origin, dto.description, subspec}, branchname},
+                                      Reader::BranchDefinition<TriggerInputType>{Output{tro.origin, tro.description, subspec}, triggerbranchname});
     }
   };
 
diff --git a/Detectors/EMCAL/workflow/src/AnalysisClusterSpec.cxx b/Detectors/EMCAL/workflow/src/AnalysisClusterSpec.cxx
index 2c98a100ac17a..7b81fec681c14 100644
--- a/Detectors/EMCAL/workflow/src/AnalysisClusterSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/AnalysisClusterSpec.cxx
@@ -148,7 +148,7 @@ void AnalysisClusterSpec<InputType>::run(framework::ProcessingContext& ctx)
   }
 
   LOG(debug) << "[EMCALClusterizer - run] Writing " << mOutputAnaClusters->size() << " clusters ...";
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "ANALYSISCLUSTERS", 0, o2::framework::Lifetime::Timeframe}, *mOutputAnaClusters);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "ANALYSISCLUSTERS", 0}, *mOutputAnaClusters);
 }
 
 o2::framework::DataProcessorSpec o2::emcal::reco_workflow::getAnalysisClusterSpec(bool useDigits)
diff --git a/Detectors/EMCAL/workflow/src/CellConverterSpec.cxx b/Detectors/EMCAL/workflow/src/CellConverterSpec.cxx
index 489546a86eeaf..5fa7353e907e2 100644
--- a/Detectors/EMCAL/workflow/src/CellConverterSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/CellConverterSpec.cxx
@@ -161,10 +161,10 @@ void CellConverterSpec::run(framework::ProcessingContext& ctx)
     ncellsTrigger = 0;
   }
   LOG(debug) << "[EMCALCellConverter - run] Writing " << mOutputCells.size() << " cells ...";
-  ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLS", mSubspecificationOut, o2::framework::Lifetime::Timeframe}, mOutputCells);
-  ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLSTRGR", mSubspecificationOut, o2::framework::Lifetime::Timeframe}, mOutputTriggers);
+  ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLS", mSubspecificationOut}, mOutputCells);
+  ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLSTRGR", mSubspecificationOut}, mOutputTriggers);
   if (mPropagateMC) {
-    ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLSMCTR", mSubspecificationOut, o2::framework::Lifetime::Timeframe}, mOutputLabels);
+    ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLSMCTR", mSubspecificationOut}, mOutputLabels);
   }
 }
 
diff --git a/Detectors/EMCAL/workflow/src/CellRecalibratorSpec.cxx b/Detectors/EMCAL/workflow/src/CellRecalibratorSpec.cxx
index 493c205223161..3f19b49cd73bd 100644
--- a/Detectors/EMCAL/workflow/src/CellRecalibratorSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/CellRecalibratorSpec.cxx
@@ -113,15 +113,15 @@ void CellRecalibratorSpec::run(framework::ProcessingContext& ctx)
   }
 
   // send recalibrated objects
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLS", mOutputSubspec, o2::framework::Lifetime::Timeframe}, outputcells);
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLSTRGR", mOutputSubspec, o2::framework::Lifetime::Timeframe}, outputtriggers);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLS", mOutputSubspec}, outputcells);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLSTRGR", mOutputSubspec}, outputtriggers);
   if (outputMCLabels.has_value()) {
     LOG(info) << "Timeframe: " << inputMCLabels->getIndexedSize() << " label entries read, " << outputMCLabels->getIndexedSize() << " label entries kept";
-    ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLSMCTR", mOutputSubspec, o2::framework::Lifetime::Timeframe}, outputMCLabels.value());
+    ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLSMCTR", mOutputSubspec}, outputMCLabels.value());
   }
   if (mLEDsettings == LEDEventSettings::REDIRECT) {
-    ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLS", 10, o2::framework::Lifetime::Timeframe}, ledcells);
-    ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLSTRGR", 10, o2::framework::Lifetime::Timeframe}, ledtriggers);
+    ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLS", 10}, ledcells);
+    ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLSTRGR", 10}, ledtriggers);
   }
 }
 
diff --git a/Detectors/EMCAL/workflow/src/ClusterizerSpec.cxx b/Detectors/EMCAL/workflow/src/ClusterizerSpec.cxx
index eaa00d6d3d0b6..f938d02ce7e3f 100644
--- a/Detectors/EMCAL/workflow/src/ClusterizerSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/ClusterizerSpec.cxx
@@ -107,11 +107,11 @@ void ClusterizerSpec<InputType>::run(framework::ProcessingContext& ctx)
     currentStartIndices = mOutputCellDigitIndices->size();
   }
   LOG(debug) << "[EMCALClusterizer - run] Writing " << mOutputClusters->size() << " clusters ...";
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CLUSTERS", 0, o2::framework::Lifetime::Timeframe}, *mOutputClusters);
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "INDICES", 0, o2::framework::Lifetime::Timeframe}, *mOutputCellDigitIndices);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CLUSTERS", 0}, *mOutputClusters);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "INDICES", 0}, *mOutputCellDigitIndices);
 
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CLUSTERSTRGR", 0, o2::framework::Lifetime::Timeframe}, *mOutputTriggerRecord);
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "INDICESTRGR", 0, o2::framework::Lifetime::Timeframe}, *mOutputTriggerRecordIndices);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CLUSTERSTRGR", 0}, *mOutputTriggerRecord);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "INDICESTRGR", 0}, *mOutputTriggerRecordIndices);
   mTimer.Stop();
 }
 
diff --git a/Detectors/EMCAL/workflow/src/EMCALDigitizerSpec.cxx b/Detectors/EMCAL/workflow/src/EMCALDigitizerSpec.cxx
index 052e65af00bda..becf3c6cdcfc3 100644
--- a/Detectors/EMCAL/workflow/src/EMCALDigitizerSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/EMCALDigitizerSpec.cxx
@@ -140,15 +140,15 @@ void DigitizerSpec::run(framework::ProcessingContext& ctx)
   mDigitizer.finish();
 
   // here we have all digits and we can send them to consumer (aka snapshot it onto output)
-  ctx.outputs().snapshot(Output{"EMC", "DIGITS", 0, Lifetime::Timeframe}, mDigitizer.getDigits());
-  ctx.outputs().snapshot(Output{"EMC", "TRGRDIG", 0, Lifetime::Timeframe}, mDigitizer.getTriggerRecords());
+  ctx.outputs().snapshot(Output{"EMC", "DIGITS", 0}, mDigitizer.getDigits());
+  ctx.outputs().snapshot(Output{"EMC", "TRGRDIG", 0}, mDigitizer.getTriggerRecords());
   if (ctx.outputs().isAllowed({"EMC", "DIGITSMCTR", 0})) {
-    ctx.outputs().snapshot(Output{"EMC", "DIGITSMCTR", 0, Lifetime::Timeframe}, mDigitizer.getMCLabels());
+    ctx.outputs().snapshot(Output{"EMC", "DIGITSMCTR", 0}, mDigitizer.getMCLabels());
   }
   // EMCAL is always a triggering detector
   const o2::parameters::GRPObject::ROMode roMode = o2::parameters::GRPObject::TRIGGERING;
   LOG(info) << "EMCAL: Sending ROMode= " << roMode << " to GRPUpdater";
-  ctx.outputs().snapshot(Output{"EMC", "ROMode", 0, Lifetime::Timeframe}, roMode);
+  ctx.outputs().snapshot(Output{"EMC", "ROMode", 0}, roMode);
   // Create CTP digits
   std::vector<o2::ctp::CTPInputDigit> triggerinputs;
   for (auto& trg : mDigitizer.getTriggerRecords()) {
@@ -161,7 +161,7 @@ void DigitizerSpec::run(framework::ProcessingContext& ctx)
     nextdigit.inputsMask.set(0);
     triggerinputs.push_back(nextdigit);
   }
-  ctx.outputs().snapshot(Output{"EMC", "TRIGGERINPUT", 0, Lifetime::Timeframe}, triggerinputs);
+  ctx.outputs().snapshot(Output{"EMC", "TRIGGERINPUT", 0}, triggerinputs);
 
   timer.Stop();
   LOG(info) << "Digitization took " << timer.CpuTime() << "s";
diff --git a/Detectors/EMCAL/workflow/src/EntropyEncoderSpec.cxx b/Detectors/EMCAL/workflow/src/EntropyEncoderSpec.cxx
index da34fcfd8c534..773c4c65fc9fe 100644
--- a/Detectors/EMCAL/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/EntropyEncoderSpec.cxx
@@ -52,7 +52,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   auto triggers = pc.inputs().get<gsl::span<TriggerRecord>>("triggers");
   auto cells = pc.inputs().get<gsl::span<Cell>>("cells");
 
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"EMC", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"EMC", "CTFDATA", 0});
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
diff --git a/Detectors/EMCAL/workflow/src/RawToCellConverterSpec.cxx b/Detectors/EMCAL/workflow/src/RawToCellConverterSpec.cxx
index 178f4c4418b40..ac09b842d9ab8 100644
--- a/Detectors/EMCAL/workflow/src/RawToCellConverterSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/RawToCellConverterSpec.cxx
@@ -722,11 +722,11 @@ void RawToCellConverterSpec::handleMinorPageError(const RawReaderMemory::MinorEr
 void RawToCellConverterSpec::sendData(framework::ProcessingContext& ctx, const std::vector<o2::emcal::Cell>& cells, const std::vector<o2::emcal::TriggerRecord>& triggers, const std::vector<ErrorTypeFEE>& decodingErrors) const
 {
   constexpr auto originEMC = o2::header::gDataOriginEMC;
-  ctx.outputs().snapshot(framework::Output{originEMC, "CELLS", mSubspecification, framework::Lifetime::Timeframe}, cells);
-  ctx.outputs().snapshot(framework::Output{originEMC, "CELLSTRGR", mSubspecification, framework::Lifetime::Timeframe}, triggers);
+  ctx.outputs().snapshot(framework::Output{originEMC, "CELLS", mSubspecification}, cells);
+  ctx.outputs().snapshot(framework::Output{originEMC, "CELLSTRGR", mSubspecification}, triggers);
   if (mCreateRawDataErrors) {
     LOG(debug) << "Sending " << decodingErrors.size() << " decoding errors";
-    ctx.outputs().snapshot(framework::Output{originEMC, "DECODERERR", mSubspecification, framework::Lifetime::Timeframe}, decodingErrors);
+    ctx.outputs().snapshot(framework::Output{originEMC, "DECODERERR", mSubspecification}, decodingErrors);
   }
 }
 
diff --git a/Detectors/EMCAL/workflow/src/StandaloneAODProducerSpec.cxx b/Detectors/EMCAL/workflow/src/StandaloneAODProducerSpec.cxx
index 152e9dd54119d..73987ce6d1c1b 100644
--- a/Detectors/EMCAL/workflow/src/StandaloneAODProducerSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/StandaloneAODProducerSpec.cxx
@@ -136,8 +136,8 @@ void StandaloneAODProducerSpec::run(ProcessingContext& pc)
   } // end of event loop
   // std::cout << "Finished cell loop" << std::endl;
 
-  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0, Lifetime::Timeframe}, tfNumber);
-  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0, Lifetime::Timeframe}, "");
+  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0}, tfNumber);
+  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0}, "");
 
   mTimer.Stop();
 }
diff --git a/Detectors/EMCAL/workflow/src/emc-channel-data-producer.cxx b/Detectors/EMCAL/workflow/src/emc-channel-data-producer.cxx
index f878a933b2d8d..75e2cdbd9fce3 100644
--- a/Detectors/EMCAL/workflow/src/emc-channel-data-producer.cxx
+++ b/Detectors/EMCAL/workflow/src/emc-channel-data-producer.cxx
@@ -160,7 +160,7 @@ DataProcessorSpec generateData(const std::string nameRootFile, const std::string
         o2::pmr::vector<o2::emcal::TriggerRecord> TriggerOutput;
         TriggerOutput.emplace_back(0, 0, 0, CellOutput.size());
 
-        ctx.outputs().adoptContainer(Output{o2::header::gDataOriginEMC, "CELLS", 0, Lifetime::Timeframe}, std::move(CellOutput));
-        ctx.outputs().adoptContainer(Output{o2::header::gDataOriginEMC, "CELLSTRGR", 0, Lifetime::Timeframe}, std::move(TriggerOutput));
+        ctx.outputs().adoptContainer(Output{o2::header::gDataOriginEMC, "CELLS", 0}, std::move(CellOutput));
+        ctx.outputs().adoptContainer(Output{o2::header::gDataOriginEMC, "CELLSTRGR", 0}, std::move(TriggerOutput));
       }}};
-}
\ No newline at end of file
+}
diff --git a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawReaderFDD.h b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawReaderFDD.h
index e9612ae8c8bda..2d92c907c1d79 100644
--- a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawReaderFDD.h
+++ b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawReaderFDD.h
@@ -65,8 +65,8 @@ class RawReaderFDD : public RawReaderFDDBaseNorm
   }
   void makeSnapshot(o2::framework::ProcessingContext& pc)
   {
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFDD, "DIGITSBC", 0, o2::framework::Lifetime::Timeframe}, mVecDigits);
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFDD, "DIGITSCH", 0, o2::framework::Lifetime::Timeframe}, mVecChannelData);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFDD, "DIGITSBC", 0}, mVecDigits);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFDD, "DIGITSCH", 0}, mVecChannelData);
   }
   bool mDumpData;
   std::vector<Digit> mVecDigits;
diff --git a/Detectors/FIT/FDD/workflow/src/DigitReaderSpec.cxx b/Detectors/FIT/FDD/workflow/src/DigitReaderSpec.cxx
index 441c679a327e3..628a2160c6d0c 100644
--- a/Detectors/FIT/FDD/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/FIT/FDD/workflow/src/DigitReaderSpec.cxx
@@ -81,18 +81,18 @@ void DigitReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
 
   LOG(info) << "FDD DigitReader pushes " << digitsBC->size() << " digits";
-  pc.outputs().snapshot(Output{mOrigin, "DIGITSBC", 0, Lifetime::Timeframe}, *digitsBC);
-  pc.outputs().snapshot(Output{mOrigin, "DIGITSCH", 0, Lifetime::Timeframe}, *digitsCh);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITSBC", 0}, *digitsBC);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITSCH", 0}, *digitsCh);
 
   if (mUseMC) {
     // TODO: To be replaced with sending ConstMCTruthContainer as soon as reco workflow supports it
-    pc.outputs().snapshot(Output{mOrigin, "TRIGGERINPUT", 0, Lifetime::Timeframe}, *digitsTrig);
+    pc.outputs().snapshot(Output{mOrigin, "TRIGGERINPUT", 0}, *digitsTrig);
 
     std::vector<char> flatbuffer;
     mcTruthRootBuffer->copyandflatten(flatbuffer);
     o2::dataformats::MCTruthContainer<o2::fdd::MCLabel> mcTruth;
     mcTruth.restore_from(flatbuffer.data(), flatbuffer.size());
-    pc.outputs().snapshot(Output{mOrigin, "DIGITLBL", 0, Lifetime::Timeframe}, mcTruth);
+    pc.outputs().snapshot(Output{mOrigin, "DIGITLBL", 0}, mcTruth);
   }
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/FIT/FDD/workflow/src/EntropyEncoderSpec.cxx b/Detectors/FIT/FDD/workflow/src/EntropyEncoderSpec.cxx
index b79c033771743..abb2518e5ae0b 100644
--- a/Detectors/FIT/FDD/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/FIT/FDD/workflow/src/EntropyEncoderSpec.cxx
@@ -54,7 +54,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"FDD", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"FDD", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, digits, channels);
   if (mSelIR) {
     mCTFCoder.getIRFramesSelector().clear();
diff --git a/Detectors/FIT/FDD/workflow/src/RecPointReaderSpec.cxx b/Detectors/FIT/FDD/workflow/src/RecPointReaderSpec.cxx
index 3da258a672611..9b612c31d28e6 100644
--- a/Detectors/FIT/FDD/workflow/src/RecPointReaderSpec.cxx
+++ b/Detectors/FIT/FDD/workflow/src/RecPointReaderSpec.cxx
@@ -51,8 +51,8 @@ void RecPointReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
 
   LOG(info) << "FDD RecPointReader pushes " << mRecPoints->size() << " recpoints with " << mChannelData->size() << " channels at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, *mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, *mChannelData);
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, *mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, *mChannelData);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/FIT/FDD/workflow/src/ReconstructorSpec.cxx b/Detectors/FIT/FDD/workflow/src/ReconstructorSpec.cxx
index 8d76bc4ee45b5..b7a0b9876a2ee 100644
--- a/Detectors/FIT/FDD/workflow/src/ReconstructorSpec.cxx
+++ b/Detectors/FIT/FDD/workflow/src/ReconstructorSpec.cxx
@@ -54,8 +54,8 @@ void FDDReconstructorDPL::run(ProcessingContext& pc)
   }
   // do we ignore MC in this task?
   LOG(debug) << "FDD reconstruction pushes " << mRecPoints.size() << " RecPoints";
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, mRecChData);
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, mRecChData);
 }
 
 DataProcessorSpec getFDDReconstructorSpec(bool useMC)
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/FT0CalibSlewingCollectorSpec.h b/Detectors/FIT/FT0/calibration/testWorkflow/FT0CalibSlewingCollectorSpec.h
index e01fed6f20588..8a1a071a0a043 100644
--- a/Detectors/FIT/FT0/calibration/testWorkflow/FT0CalibSlewingCollectorSpec.h
+++ b/Detectors/FIT/FT0/calibration/testWorkflow/FT0CalibSlewingCollectorSpec.h
@@ -88,8 +88,8 @@ class FT0CalibCollectorDevice : public o2::framework::Task
       auto entries = collectedInfo.size();
       // this means that we are ready to send the output
       auto entriesPerChannel = mCollector->getEntriesPerChannel();
-      output.snapshot(Output{o2::header::gDataOriginFT0, "COLLECTEDINFO", 0, Lifetime::Timeframe}, collectedInfo);
-      output.snapshot(Output{o2::header::gDataOriginFT0, "ENTRIESCH", 0, Lifetime::Timeframe}, entriesPerChannel);
+      output.snapshot(Output{o2::header::gDataOriginFT0, "COLLECTEDINFO", 0}, collectedInfo);
+      output.snapshot(Output{o2::header::gDataOriginFT0, "ENTRIESCH", 0}, entriesPerChannel);
       mCollector->initOutput(); // reset the output for the next round
     }
   }
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/FT0TimeSpectraProcessor-Workflow.cxx b/Detectors/FIT/FT0/calibration/testWorkflow/FT0TimeSpectraProcessor-Workflow.cxx
index 541f5a9827a4a..f1dd64a250a3b 100644
--- a/Detectors/FIT/FT0/calibration/testWorkflow/FT0TimeSpectraProcessor-Workflow.cxx
+++ b/Detectors/FIT/FT0/calibration/testWorkflow/FT0TimeSpectraProcessor-Workflow.cxx
@@ -101,7 +101,7 @@ class FT0TimeSpectraProcessor final : public o2::framework::Task
       }
     }
 
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "TIME_SPECTRA", 0, o2::framework::Lifetime::Timeframe}, timeSpectraInfoObject.getBase());
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "TIME_SPECTRA", 0}, timeSpectraInfoObject.getBase());
   }
 };
 
diff --git a/Detectors/FIT/FT0/workflow/src/DigitReaderSpec.cxx b/Detectors/FIT/FT0/workflow/src/DigitReaderSpec.cxx
index e4f11d3f5bd1e..09586d778ac15 100644
--- a/Detectors/FIT/FT0/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/FIT/FT0/workflow/src/DigitReaderSpec.cxx
@@ -64,13 +64,13 @@ void DigitReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(debug) << "FT0DigitReader pushed " << channels.size() << " channels in " << digits.size() << " digits";
-  pc.outputs().snapshot(Output{"FT0", "DIGITSBC", 0, Lifetime::Timeframe}, digits);
-  pc.outputs().snapshot(Output{"FT0", "DIGITSCH", 0, Lifetime::Timeframe}, channels);
+  pc.outputs().snapshot(Output{"FT0", "DIGITSBC", 0}, digits);
+  pc.outputs().snapshot(Output{"FT0", "DIGITSCH", 0}, channels);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"FT0", "DIGITSMCTR", 0, Lifetime::Timeframe}, labels);
+    pc.outputs().snapshot(Output{"FT0", "DIGITSMCTR", 0}, labels);
   }
   if (mUseTrgInput) {
-    pc.outputs().snapshot(Output{"FT0", "TRIGGERINPUT", 0, Lifetime::Timeframe}, trgInput);
+    pc.outputs().snapshot(Output{"FT0", "TRIGGERINPUT", 0}, trgInput);
   }
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/FIT/FT0/workflow/src/EntropyEncoderSpec.cxx b/Detectors/FIT/FT0/workflow/src/EntropyEncoderSpec.cxx
index edf86b3d7da0d..81bdc2e729bb4 100644
--- a/Detectors/FIT/FT0/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/FIT/FT0/workflow/src/EntropyEncoderSpec.cxx
@@ -55,7 +55,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
 
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"FT0", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"FT0", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, digits, channels);
   if (mSelIR) {
     mCTFCoder.getIRFramesSelector().clear();
diff --git a/Detectors/FIT/FT0/workflow/src/RecPointReaderSpec.cxx b/Detectors/FIT/FT0/workflow/src/RecPointReaderSpec.cxx
index 69f21abc369c6..ba5ae4aa1356c 100644
--- a/Detectors/FIT/FT0/workflow/src/RecPointReaderSpec.cxx
+++ b/Detectors/FIT/FT0/workflow/src/RecPointReaderSpec.cxx
@@ -49,8 +49,8 @@ void RecPointReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
 
   LOG(debug) << "FT0 RecPointReader pushes " << mRecPoints->size() << " recpoints with " << mChannelData->size() << " channels at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, *mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, *mChannelData);
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, *mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, *mChannelData);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/FIT/FT0/workflow/src/ReconstructionSpec.cxx b/Detectors/FIT/FT0/workflow/src/ReconstructionSpec.cxx
index da5281d7e2ba1..40bc96ebca58e 100644
--- a/Detectors/FIT/FT0/workflow/src/ReconstructionSpec.cxx
+++ b/Detectors/FIT/FT0/workflow/src/ReconstructionSpec.cxx
@@ -74,8 +74,8 @@ void ReconstructionDPL::run(ProcessingContext& pc)
   mReco.processTF(digits, channels, mRecPoints, mRecChData);
   // do we ignore MC in this task?
   LOG(debug) << "FT0 reconstruction pushes " << mRecPoints.size() << " RecPoints";
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, mRecChData);
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, mRecChData);
 
   mTimer.Stop();
 }
diff --git a/Detectors/FIT/FV0/workflow/src/DigitReaderSpec.cxx b/Detectors/FIT/FV0/workflow/src/DigitReaderSpec.cxx
index 08a6c4cbb2e24..a49bda2cec18b 100644
--- a/Detectors/FIT/FV0/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/FIT/FV0/workflow/src/DigitReaderSpec.cxx
@@ -65,13 +65,13 @@ void DigitReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(debug) << "FV0DigitReader pushed " << channels.size() << " channels in " << digits.size() << " digits";
-  pc.outputs().snapshot(Output{"FV0", "DIGITSBC", 0, Lifetime::Timeframe}, digits);
-  pc.outputs().snapshot(Output{"FV0", "DIGITSCH", 0, Lifetime::Timeframe}, channels);
+  pc.outputs().snapshot(Output{"FV0", "DIGITSBC", 0}, digits);
+  pc.outputs().snapshot(Output{"FV0", "DIGITSCH", 0}, channels);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"FV0", "DIGITSMCTR", 0, Lifetime::Timeframe}, labels);
+    pc.outputs().snapshot(Output{"FV0", "DIGITSMCTR", 0}, labels);
   }
   if (mUseTrgInput) {
-    pc.outputs().snapshot(Output{"FV0", "TRIGGERINPUT", 0, Lifetime::Timeframe}, trgInput);
+    pc.outputs().snapshot(Output{"FV0", "TRIGGERINPUT", 0}, trgInput);
   }
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/FIT/FV0/workflow/src/EntropyEncoderSpec.cxx b/Detectors/FIT/FV0/workflow/src/EntropyEncoderSpec.cxx
index a115712a0c980..a25c16a5d697c 100644
--- a/Detectors/FIT/FV0/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/FIT/FV0/workflow/src/EntropyEncoderSpec.cxx
@@ -55,7 +55,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
 
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"FV0", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"FV0", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, digits, channels);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   if (mSelIR) {
diff --git a/Detectors/FIT/FV0/workflow/src/RecPointReaderSpec.cxx b/Detectors/FIT/FV0/workflow/src/RecPointReaderSpec.cxx
index 163beacf0c140..5997cac500ee6 100644
--- a/Detectors/FIT/FV0/workflow/src/RecPointReaderSpec.cxx
+++ b/Detectors/FIT/FV0/workflow/src/RecPointReaderSpec.cxx
@@ -49,8 +49,8 @@ void RecPointReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
 
   LOG(debug) << "FV0 RecPointReader pushes " << mRecPoints->size() << " recpoints with " << mChannelData->size() << " channels at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, *mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, *mChannelData);
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, *mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, *mChannelData);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/FIT/FV0/workflow/src/ReconstructionSpec.cxx b/Detectors/FIT/FV0/workflow/src/ReconstructionSpec.cxx
index d48b96fd4e626..520ac4dbaa563 100644
--- a/Detectors/FIT/FV0/workflow/src/ReconstructionSpec.cxx
+++ b/Detectors/FIT/FV0/workflow/src/ReconstructionSpec.cxx
@@ -68,8 +68,8 @@ void ReconstructionDPL::run(ProcessingContext& pc)
   }
 
   LOG(debug) << "FV0 reconstruction pushes " << mRecPoints.size() << " RecPoints";
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, mRecChData);
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, mRecChData);
 
   mTimer.Stop();
 }
diff --git a/Detectors/FIT/workflow/include/FITWorkflow/RawReaderFIT.h b/Detectors/FIT/workflow/include/FITWorkflow/RawReaderFIT.h
index 5c2cfce0a620c..cfe41e4bb474f 100644
--- a/Detectors/FIT/workflow/include/FITWorkflow/RawReaderFIT.h
+++ b/Detectors/FIT/workflow/include/FITWorkflow/RawReaderFIT.h
@@ -165,28 +165,28 @@ class RawReaderFIT : public RawReaderType
   }
   void makeSnapshot(o2::framework::ProcessingContext& pc) const
   {
-    pc.outputs().snapshot(o2::framework::Output{mDataOrigin, Digit_t::sChannelNameDPL, 0, o2::framework::Lifetime::Timeframe}, mVecDigit);
+    pc.outputs().snapshot(o2::framework::Output{mDataOrigin, Digit_t::sChannelNameDPL, 0}, mVecDigit);
     if constexpr (sSubDigitExists) {
       std::apply([&](const auto&... subDigit) {
-        ((pc.outputs().snapshot(o2::framework::Output{mDataOrigin, (std::decay<decltype(subDigit)>::type::value_type::sChannelNameDPL), 0, o2::framework::Lifetime::Timeframe}, subDigit)), ...);
+        ((pc.outputs().snapshot(o2::framework::Output{mDataOrigin, (std::decay<decltype(subDigit)>::type::value_type::sChannelNameDPL), 0}, subDigit)), ...);
       },
                  mVecSubDigit);
     }
     if constexpr (sSingleSubDigitExists) {
       std::apply([&](const auto&... singleSubDigit) {
-        ((pc.outputs().snapshot(o2::framework::Output{mDataOrigin, (std::decay<decltype(singleSubDigit)>::type::value_type::sChannelNameDPL), 0, o2::framework::Lifetime::Timeframe}, singleSubDigit)), ...);
+        ((pc.outputs().snapshot(o2::framework::Output{mDataOrigin, (std::decay<decltype(singleSubDigit)>::type::value_type::sChannelNameDPL), 0}, singleSubDigit)), ...);
       },
                  mVecSingleSubDigit);
     }
     if constexpr (sUseTrgInput) {
-      pc.outputs().snapshot(o2::framework::Output{mDataOrigin, DetTrigInput_t::sChannelNameDPL, 0, o2::framework::Lifetime::Timeframe}, mVecTrgInput);
+      pc.outputs().snapshot(o2::framework::Output{mDataOrigin, DetTrigInput_t::sChannelNameDPL, 0}, mVecTrgInput);
     }
-    pc.outputs().snapshot(o2::framework::Output{mDataOrigin, "RawDataMetric", 0, o2::framework::Lifetime::Timeframe}, mVecRawDataMetric);
+    pc.outputs().snapshot(o2::framework::Output{mDataOrigin, "RawDataMetric", 0}, mVecRawDataMetric);
   }
   template <typename VecDigitType>
   auto& getRefVec(o2::framework::ProcessingContext& pc)
   {
-    auto& refVec = pc.outputs().make<VecDigitType>(o2::framework::Output{mDataOrigin, VecDigitType::value_type::sChannelNameDPL, 0, o2::framework::Lifetime::Timeframe});
+    auto& refVec = pc.outputs().make<VecDigitType>(o2::framework::Output{mDataOrigin, VecDigitType::value_type::sChannelNameDPL, 0});
     return refVec;
   }
   void enableEmptyTFprotection()
@@ -204,4 +204,4 @@ class RawReaderFIT : public RawReaderType
 } // namespace fit
 } // namespace o2
 
-#endif
\ No newline at end of file
+#endif
diff --git a/Detectors/FOCAL/workflow/src/RawDecoderSpec.cxx b/Detectors/FOCAL/workflow/src/RawDecoderSpec.cxx
index d6004561f852d..c7260ae3c9815 100644
--- a/Detectors/FOCAL/workflow/src/RawDecoderSpec.cxx
+++ b/Detectors/FOCAL/workflow/src/RawDecoderSpec.cxx
@@ -274,10 +274,10 @@ void RawDecoderSpec::endOfStream(o2::framework::EndOfStreamContext& ec)
 
 void RawDecoderSpec::sendOutput(framework::ProcessingContext& ctx)
 {
-  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginFOC, "PADLAYERS", mOutputSubspec, framework::Lifetime::Timeframe}, mOutputPadLayers);
-  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginFOC, "PIXELHITS", mOutputSubspec, framework::Lifetime::Timeframe}, mOutputPixelHits);
-  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginFOC, "PIXELCHIPS", mOutputSubspec, framework::Lifetime::Timeframe}, mOutputPixelChips);
-  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginFOC, "TRIGGERS", mOutputSubspec, framework::Lifetime::Timeframe}, mOutputTriggerRecords);
+  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginFOC, "PADLAYERS", mOutputSubspec}, mOutputPadLayers);
+  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginFOC, "PIXELHITS", mOutputSubspec}, mOutputPixelHits);
+  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginFOC, "PIXELCHIPS", mOutputSubspec}, mOutputPixelChips);
+  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginFOC, "TRIGGERS", mOutputSubspec}, mOutputTriggerRecords);
 }
 
 void RawDecoderSpec::resetContainers()
@@ -689,4 +689,4 @@ o2::framework::DataProcessorSpec o2::focal::reco_workflow::getRawDecoderSpec(boo
                                             {"displayInconsistent", o2::framework::VariantType::Bool, false, {"Display information about inconsistent timeframes"}},
                                             {"pixeltype", o2::framework::VariantType::String, "OB", {"Pixel mapping type"}},
                                             {"pixelmapping", o2::framework::VariantType::String, "default", {"File with pixel mapping"}}}};
-}
\ No newline at end of file
+}
diff --git a/Detectors/Filtering/src/FilteredTFReaderSpec.cxx b/Detectors/Filtering/src/FilteredTFReaderSpec.cxx
index 3165208aef246..22fe1370040db 100644
--- a/Detectors/Filtering/src/FilteredTFReaderSpec.cxx
+++ b/Detectors/Filtering/src/FilteredTFReaderSpec.cxx
@@ -45,15 +45,15 @@ void FilteredTFReader::run(ProcessingContext& pc)
 
   LOG(info) << "Pushing filtered TF: " << mFiltTF.header.asString();
   // ITS
-  pc.outputs().snapshot(Output{"ITS", "ITSTrackROF", 0, Lifetime::Timeframe}, mFiltTF.ITSTrackROFs);
-  pc.outputs().snapshot(Output{"ITS", "TRACKS", 0, Lifetime::Timeframe}, mFiltTF.ITSTracks);
-  pc.outputs().snapshot(Output{"ITS", "TRACKCLSID", 0, Lifetime::Timeframe}, mFiltTF.ITSClusterIndices);
+  pc.outputs().snapshot(Output{"ITS", "ITSTrackROF", 0}, mFiltTF.ITSTrackROFs);
+  pc.outputs().snapshot(Output{"ITS", "TRACKS", 0}, mFiltTF.ITSTracks);
+  pc.outputs().snapshot(Output{"ITS", "TRACKCLSID", 0}, mFiltTF.ITSClusterIndices);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"ITS", "TRACKSMCTR", 0, Lifetime::Timeframe}, mFiltTF.ITSTrackMCTruth);
+    pc.outputs().snapshot(Output{"ITS", "TRACKSMCTR", 0}, mFiltTF.ITSTrackMCTruth);
   }
-  pc.outputs().snapshot(Output{"ITS", "CLUSTERSROF", 0, Lifetime::Timeframe}, mFiltTF.ITSClusterROFs);
-  pc.outputs().snapshot(Output{"ITS", "COMPCLUSTERS", 0, Lifetime::Timeframe}, mFiltTF.ITSClusters);
-  pc.outputs().snapshot(Output{"ITS", "PATTERNS", 0, Lifetime::Timeframe}, mFiltTF.ITSClusterPatterns);
+  pc.outputs().snapshot(Output{"ITS", "CLUSTERSROF", 0}, mFiltTF.ITSClusterROFs);
+  pc.outputs().snapshot(Output{"ITS", "COMPCLUSTERS", 0}, mFiltTF.ITSClusters);
+  pc.outputs().snapshot(Output{"ITS", "PATTERNS", 0}, mFiltTF.ITSClusterPatterns);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/GlobalTrackingWorkflow/helpers/src/NoInpDummyOutSpec.cxx b/Detectors/GlobalTrackingWorkflow/helpers/src/NoInpDummyOutSpec.cxx
index 34f445ed003da..17ad5f09027e1 100644
--- a/Detectors/GlobalTrackingWorkflow/helpers/src/NoInpDummyOutSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/helpers/src/NoInpDummyOutSpec.cxx
@@ -47,7 +47,7 @@ void NoInpDummyOut::run(ProcessingContext& pc)
 {
   static int counter = 0;
   // send just once dummy output to trigger the ccdb-fetcher
-  pc.outputs().make<std::vector<char>>(Output{"GLO", "DUMMY_OUT", 0, Lifetime::Timeframe});
+  pc.outputs().make<std::vector<char>>(Output{"GLO", "DUMMY_OUT", 0});
   if (mLoops >= 0 && ++counter >= mLoops) {
     pc.services().get<o2::framework::ControlService>().endOfStream();
     pc.services().get<o2::framework::ControlService>().readyToQuit(framework::QuitRequest::Me);
diff --git a/Detectors/GlobalTrackingWorkflow/qc/src/ITSTPCMatchingQCSpec.cxx b/Detectors/GlobalTrackingWorkflow/qc/src/ITSTPCMatchingQCSpec.cxx
index 3653fd8bcc90b..c8a402a48d659 100644
--- a/Detectors/GlobalTrackingWorkflow/qc/src/ITSTPCMatchingQCSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/qc/src/ITSTPCMatchingQCSpec.cxx
@@ -71,7 +71,7 @@ void ITSTPCMatchingQCDevice::sendOutput(DataAllocator& output)
 
   TObjArray objar;
   mMatchITSTPCQC->getHistos(objar);
-  output.snapshot(Output{"GLO", "ITSTPCMATCHQC", 0, Lifetime::Sporadic}, objar);
+  output.snapshot(Output{"GLO", "ITSTPCMATCHQC", 0}, objar);
 
   TFile* f = new TFile(Form("outITSTPCmatchingQC.root"), "RECREATE");
   objar.Write("ObjArray", TObject::kSingleKey);
diff --git a/Detectors/GlobalTrackingWorkflow/readers/src/GlobalFwdTrackReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/readers/src/GlobalFwdTrackReaderSpec.cxx
index 1fdc7b49616c3..11fa58333f89b 100644
--- a/Detectors/GlobalTrackingWorkflow/readers/src/GlobalFwdTrackReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/readers/src/GlobalFwdTrackReaderSpec.cxx
@@ -65,9 +65,9 @@ void GlobalFwdTrackReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mTracks.size() << " Global Forward tracks at entry " << ent;
 
-  pc.outputs().snapshot(Output{"GLO", "GLFWD", 0, Lifetime::Timeframe}, mTracks);
+  pc.outputs().snapshot(Output{"GLO", "GLFWD", 0}, mTracks);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"GLO", "GLFWD_MC", 0, Lifetime::Timeframe}, mLabels);
+    pc.outputs().snapshot(Output{"GLO", "GLFWD_MC", 0}, mLabels);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/GlobalTrackingWorkflow/readers/src/IRFrameReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/readers/src/IRFrameReaderSpec.cxx
index 012542e448d19..c1810a1deb743 100644
--- a/Detectors/GlobalTrackingWorkflow/readers/src/IRFrameReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/readers/src/IRFrameReaderSpec.cxx
@@ -63,7 +63,7 @@ void IRFrameReaderSpec::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(debug) << "Pushing " << mIRF.size() << " IR-frames in at entry " << ent;
-  pc.outputs().snapshot(Output{mDataOrigin, "IRFRAMES", mSubSpec, Lifetime::Timeframe}, mIRF);
+  pc.outputs().snapshot(Output{mDataOrigin, "IRFRAMES", mSubSpec}, mIRF);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/GlobalTrackingWorkflow/readers/src/MatchedMFTMCHReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/readers/src/MatchedMFTMCHReaderSpec.cxx
index 08c5c5871db84..5f02beebd1746 100644
--- a/Detectors/GlobalTrackingWorkflow/readers/src/MatchedMFTMCHReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/readers/src/MatchedMFTMCHReaderSpec.cxx
@@ -65,7 +65,7 @@ void MatchMFTMCHReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mTracks.size() << " MFTMCH matches at entry " << ent;
 
-  pc.outputs().snapshot(Output{"GLO", "MTC_MFTMCH", 0, Lifetime::Timeframe}, mTracks);
+  pc.outputs().snapshot(Output{"GLO", "MTC_MFTMCH", 0}, mTracks);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/GlobalTrackingWorkflow/readers/src/PrimaryVertexReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/readers/src/PrimaryVertexReaderSpec.cxx
index 06bdf8ff95a45..6e1aba8b2e1f3 100644
--- a/Detectors/GlobalTrackingWorkflow/readers/src/PrimaryVertexReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/readers/src/PrimaryVertexReaderSpec.cxx
@@ -84,12 +84,12 @@ void PrimaryVertexReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mVerticesPtr->size() << " vertices at entry " << ent;
 
-  pc.outputs().snapshot(Output{"GLO", "PVTX", 0, Lifetime::Timeframe}, mVertices);
-  pc.outputs().snapshot(Output{"GLO", "PVTX_TRMTC", 0, Lifetime::Timeframe}, mPV2MatchIdx);
-  pc.outputs().snapshot(Output{"GLO", "PVTX_TRMTCREFS", 0, Lifetime::Timeframe}, mPV2MatchIdxRef);
+  pc.outputs().snapshot(Output{"GLO", "PVTX", 0}, mVertices);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_TRMTC", 0}, mPV2MatchIdx);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_TRMTCREFS", 0}, mPV2MatchIdxRef);
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"GLO", "PVTX_MCTR", 0, Lifetime::Timeframe}, mLabels);
+    pc.outputs().snapshot(Output{"GLO", "PVTX_MCTR", 0}, mLabels);
   }
 
   if (mVerbose) {
diff --git a/Detectors/GlobalTrackingWorkflow/readers/src/SecondaryVertexReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/readers/src/SecondaryVertexReaderSpec.cxx
index e440a075d2558..4dd6324c83675 100644
--- a/Detectors/GlobalTrackingWorkflow/readers/src/SecondaryVertexReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/readers/src/SecondaryVertexReaderSpec.cxx
@@ -93,15 +93,15 @@ void SecondaryVertexReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mV0s.size() << " V0s and " << mCascs.size() << " cascades at entry " << ent;
 
-  pc.outputs().snapshot(Output{"GLO", "V0S_IDX", 0, Lifetime::Timeframe}, mV0sIdx);
-  pc.outputs().snapshot(Output{"GLO", "V0S", 0, Lifetime::Timeframe}, mV0s);
-  pc.outputs().snapshot(Output{"GLO", "PVTX_V0REFS", 0, Lifetime::Timeframe}, mPV2V0Ref);
-  pc.outputs().snapshot(Output{"GLO", "CASCS_IDX", 0, Lifetime::Timeframe}, mCascsIdx);
-  pc.outputs().snapshot(Output{"GLO", "CASCS", 0, Lifetime::Timeframe}, mCascs);
-  pc.outputs().snapshot(Output{"GLO", "PVTX_CASCREFS", 0, Lifetime::Timeframe}, mPV2CascRef);
-  pc.outputs().snapshot(Output{"GLO", "DECAYS3BODY_IDX", 0, Lifetime::Timeframe}, m3BodysIdx);
-  pc.outputs().snapshot(Output{"GLO", "DECAYS3BODY", 0, Lifetime::Timeframe}, m3Bodys);
-  pc.outputs().snapshot(Output{"GLO", "PVTX_3BODYREFS", 0, Lifetime::Timeframe}, mPV23BodyRef);
+  pc.outputs().snapshot(Output{"GLO", "V0S_IDX", 0}, mV0sIdx);
+  pc.outputs().snapshot(Output{"GLO", "V0S", 0}, mV0s);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_V0REFS", 0}, mPV2V0Ref);
+  pc.outputs().snapshot(Output{"GLO", "CASCS_IDX", 0}, mCascsIdx);
+  pc.outputs().snapshot(Output{"GLO", "CASCS", 0}, mCascs);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_CASCREFS", 0}, mPV2CascRef);
+  pc.outputs().snapshot(Output{"GLO", "DECAYS3BODY_IDX", 0}, m3BodysIdx);
+  pc.outputs().snapshot(Output{"GLO", "DECAYS3BODY", 0}, m3Bodys);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_3BODYREFS", 0}, mPV23BodyRef);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/GlobalTrackingWorkflow/readers/src/StrangenessTrackingReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/readers/src/StrangenessTrackingReaderSpec.cxx
index 23eff571105e6..8c7f87a720925 100644
--- a/Detectors/GlobalTrackingWorkflow/readers/src/StrangenessTrackingReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/readers/src/StrangenessTrackingReaderSpec.cxx
@@ -76,14 +76,14 @@ void StrangenessTrackingReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mStrangeTrack.size() << " strange tracks at entry " << ent;
-  pc.outputs().snapshot(Output{"GLO", "STRANGETRACKS", 0, Lifetime::Timeframe}, mStrangeTrack);
+  pc.outputs().snapshot(Output{"GLO", "STRANGETRACKS", 0}, mStrangeTrack);
 
   if (mUseMC) {
     LOG(info) << "Pushing " << mStrangeTrackMC.size() << " strange tracks MC labels at entry " << ent;
-    pc.outputs().snapshot(Output{"GLO", "STRANGETRACKS_MC", 0, Lifetime::Timeframe}, mStrangeTrackMC);
+    pc.outputs().snapshot(Output{"GLO", "STRANGETRACKS_MC", 0}, mStrangeTrackMC);
   }
 
-  // pc.outputs().snapshot(Output{"GLO", "PVTX_V0REFS", 0, Lifetime::Timeframe}, mPV2V0Ref);
+  // pc.outputs().snapshot(Output{"GLO", "PVTX_V0REFS", 0}, mPV2V0Ref);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/GlobalTrackingWorkflow/readers/src/TrackCosmicsReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/readers/src/TrackCosmicsReaderSpec.cxx
index 90db5d08acc58..7e3cdffd84a6d 100644
--- a/Detectors/GlobalTrackingWorkflow/readers/src/TrackCosmicsReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/readers/src/TrackCosmicsReaderSpec.cxx
@@ -41,9 +41,9 @@ void TrackCosmicsReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mTracks.size() << " Cosmic Tracks at entry " << ent;
 
-  pc.outputs().snapshot(Output{"GLO", "COSMICTRC", 0, Lifetime::Timeframe}, mTracks);
+  pc.outputs().snapshot(Output{"GLO", "COSMICTRC", 0}, mTracks);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"GLO", "COSMICTRC_MC", 0, Lifetime::Timeframe}, mLabels);
+    pc.outputs().snapshot(Output{"GLO", "COSMICTRC_MC", 0}, mLabels);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/GlobalTrackingWorkflow/readers/src/TrackTPCITSReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/readers/src/TrackTPCITSReaderSpec.cxx
index 8ae55e478603e..c7fd0d543ecf6 100644
--- a/Detectors/GlobalTrackingWorkflow/readers/src/TrackTPCITSReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/readers/src/TrackTPCITSReaderSpec.cxx
@@ -68,12 +68,12 @@ void TrackTPCITSReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mTracks.size() << " TPC-ITS matches at entry " << ent;
 
-  pc.outputs().snapshot(Output{"GLO", "TPCITS", 0, Lifetime::Timeframe}, mTracks);
-  pc.outputs().snapshot(Output{"GLO", "TPCITSAB_REFS", 0, Lifetime::Timeframe}, mABTrkClusRefs);
-  pc.outputs().snapshot(Output{"GLO", "TPCITSAB_CLID", 0, Lifetime::Timeframe}, mABTrkClIDs);
+  pc.outputs().snapshot(Output{"GLO", "TPCITS", 0}, mTracks);
+  pc.outputs().snapshot(Output{"GLO", "TPCITSAB_REFS", 0}, mABTrkClusRefs);
+  pc.outputs().snapshot(Output{"GLO", "TPCITSAB_CLID", 0}, mABTrkClIDs);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"GLO", "TPCITS_MC", 0, Lifetime::Timeframe}, mLabels);
-    pc.outputs().snapshot(Output{"GLO", "TPCITSAB_MC", 0, Lifetime::Timeframe}, mLabelsAB);
+    pc.outputs().snapshot(Output{"GLO", "TPCITS_MC", 0}, mLabels);
+    pc.outputs().snapshot(Output{"GLO", "TPCITSAB_MC", 0}, mLabelsAB);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/GlobalTrackingWorkflow/src/CosmicsMatchingSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/CosmicsMatchingSpec.cxx
index d8242fc2ccd79..a4c9d302019a5 100644
--- a/Detectors/GlobalTrackingWorkflow/src/CosmicsMatchingSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/CosmicsMatchingSpec.cxx
@@ -99,9 +99,9 @@ void CosmicsMatchingSpec::run(ProcessingContext& pc)
   updateTimeDependentParams(pc); // Make sure this is called after recoData.collectData, which may load some conditions
 
   mMatching.process(recoData);
-  pc.outputs().snapshot(Output{"GLO", "COSMICTRC", 0, Lifetime::Timeframe}, mMatching.getCosmicTracks());
+  pc.outputs().snapshot(Output{"GLO", "COSMICTRC", 0}, mMatching.getCosmicTracks());
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"GLO", "COSMICTRC_MC", 0, Lifetime::Timeframe}, mMatching.getCosmicTracksLbl());
+    pc.outputs().snapshot(Output{"GLO", "COSMICTRC_MC", 0}, mMatching.getCosmicTracksLbl());
   }
   mTimer.Stop();
 }
diff --git a/Detectors/GlobalTrackingWorkflow/src/GlobalFwdMatchingSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/GlobalFwdMatchingSpec.cxx
index 582f2be3c5f84..03dc823c62c42 100644
--- a/Detectors/GlobalTrackingWorkflow/src/GlobalFwdMatchingSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/GlobalFwdMatchingSpec.cxx
@@ -92,18 +92,18 @@ void GlobalFwdMatchingDPL::run(ProcessingContext& pc)
   const auto& matchingParam = GlobalFwdMatchingParam::Instance();
 
   if (matchingParam.saveMode == kSaveTrainingData) {
-    pc.outputs().snapshot(Output{"GLO", "GLFWDMFT", 0, Lifetime::Timeframe}, mMatching.getMFTMatchingPlaneParams());
-    pc.outputs().snapshot(Output{"GLO", "GLFWDMCH", 0, Lifetime::Timeframe}, mMatching.getMCHMatchingPlaneParams());
-    pc.outputs().snapshot(Output{"GLO", "GLFWDINF", 0, Lifetime::Timeframe}, mMatching.getMFTMCHMatchInfo());
+    pc.outputs().snapshot(Output{"GLO", "GLFWDMFT", 0}, mMatching.getMFTMatchingPlaneParams());
+    pc.outputs().snapshot(Output{"GLO", "GLFWDMCH", 0}, mMatching.getMCHMatchingPlaneParams());
+    pc.outputs().snapshot(Output{"GLO", "GLFWDINF", 0}, mMatching.getMFTMCHMatchInfo());
   } else {
-    pc.outputs().snapshot(Output{"GLO", "GLFWD", 0, Lifetime::Timeframe}, mMatching.getMatchedFwdTracks());
+    pc.outputs().snapshot(Output{"GLO", "GLFWD", 0}, mMatching.getMatchedFwdTracks());
   }
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"GLO", "GLFWD_MC", 0, Lifetime::Timeframe}, mMatching.getMatchLabels());
+    pc.outputs().snapshot(Output{"GLO", "GLFWD_MC", 0}, mMatching.getMatchLabels());
   }
   if (mMatchRootOutput) {
-    pc.outputs().snapshot(Output{"GLO", "MTC_MFTMCH", 0, Lifetime::Timeframe}, mMatching.getMFTMCHMatchInfo());
+    pc.outputs().snapshot(Output{"GLO", "MTC_MFTMCH", 0}, mMatching.getMFTMCHMatchInfo());
   }
   mTimer.Stop();
 }
diff --git a/Detectors/GlobalTrackingWorkflow/src/HMPMatcherSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/HMPMatcherSpec.cxx
index f8666b5561610..b0562fd4f3a1e 100644
--- a/Detectors/GlobalTrackingWorkflow/src/HMPMatcherSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/HMPMatcherSpec.cxx
@@ -128,9 +128,9 @@ void HMPMatcherSpec::run(ProcessingContext& pc)
 
   mMatcher.run(recoData);
 
-  pc.outputs().snapshot(Output{o2::header::gDataOriginHMP, "MATCHES", 0, Lifetime::Timeframe}, mMatcher.getMatchedTrackVector(o2::globaltracking::MatchHMP::trackType::CONSTR));
+  pc.outputs().snapshot(Output{o2::header::gDataOriginHMP, "MATCHES", 0}, mMatcher.getMatchedTrackVector(o2::globaltracking::MatchHMP::trackType::CONSTR));
   if (mUseMC) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginHMP, "MCLABELS", 0, Lifetime::Timeframe}, mMatcher.getMatchedHMPLabelsVector(o2::globaltracking::MatchHMP::trackType::CONSTR));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginHMP, "MCLABELS", 0}, mMatcher.getMatchedHMPLabelsVector(o2::globaltracking::MatchHMP::trackType::CONSTR));
   }
 
   mTimer.Stop();
diff --git a/Detectors/GlobalTrackingWorkflow/src/PrimaryVertexingSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/PrimaryVertexingSpec.cxx
index e70122e7a01b5..ff45790a95921 100644
--- a/Detectors/GlobalTrackingWorkflow/src/PrimaryVertexingSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/PrimaryVertexingSpec.cxx
@@ -152,12 +152,12 @@ void PrimaryVertexingSpec::run(ProcessingContext& pc)
     mVertexer.process(tracks, gids, ft0Data, vertices, vertexTrackIDs, v2tRefs, tracksMCInfo, lblVtx);
   }
 
-  pc.outputs().snapshot(Output{"GLO", "PVTX", 0, Lifetime::Timeframe}, vertices);
-  pc.outputs().snapshot(Output{"GLO", "PVTX_CONTIDREFS", 0, Lifetime::Timeframe}, v2tRefs);
-  pc.outputs().snapshot(Output{"GLO", "PVTX_CONTID", 0, Lifetime::Timeframe}, vertexTrackIDs);
+  pc.outputs().snapshot(Output{"GLO", "PVTX", 0}, vertices);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_CONTIDREFS"}, v2tRefs);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_CONTID", 0}, vertexTrackIDs);
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"GLO", "PVTX_MCTR", 0, Lifetime::Timeframe}, lblVtx);
+    pc.outputs().snapshot(Output{"GLO", "PVTX_MCTR", 0}, lblVtx);
   }
 
   mTimer.Stop();
diff --git a/Detectors/GlobalTrackingWorkflow/src/StrangenessTrackingSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/StrangenessTrackingSpec.cxx
index 46136ac6e2397..37b35df06ea72 100644
--- a/Detectors/GlobalTrackingWorkflow/src/StrangenessTrackingSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/StrangenessTrackingSpec.cxx
@@ -75,11 +75,11 @@ void StrangenessTrackerSpec::run(framework::ProcessingContext& pc)
   mTracker.loadData(recoData);
   mTracker.prepareITStracks();
   mTracker.process();
-  pc.outputs().snapshot(Output{"GLO", "STRANGETRACKS", 0, Lifetime::Timeframe}, mTracker.getStrangeTrackVec());
-  pc.outputs().snapshot(Output{"GLO", "CLUSUPDATES", 0, Lifetime::Timeframe}, mTracker.getClusAttachments());
+  pc.outputs().snapshot(Output{"GLO", "STRANGETRACKS", 0}, mTracker.getStrangeTrackVec());
+  pc.outputs().snapshot(Output{"GLO", "CLUSUPDATES", 0}, mTracker.getClusAttachments());
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"GLO", "STRANGETRACKS_MC", 0, Lifetime::Timeframe}, mTracker.getStrangeTrackLabels());
+    pc.outputs().snapshot(Output{"GLO", "STRANGETRACKS_MC", 0}, mTracker.getStrangeTrackLabels());
   }
 
   mTimer.Stop();
diff --git a/Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx
index 134ffbb75f341..ca4d3e2fc5dc8 100644
--- a/Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx
@@ -171,8 +171,8 @@ void TOFMatcherSpec::run(ProcessingContext& pc)
   static pmr::vector<o2::MCCompLabel> dummyMCLab;
 
   if (isTPCused) {
-    auto& mtcInfo = pc.outputs().make<std::vector<o2::dataformats::MatchInfoTOF>>(Output{o2::header::gDataOriginTOF, "MTC_TPC", ss, Lifetime::Timeframe});
-    auto& mclabels = mUseMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{o2::header::gDataOriginTOF, "MCMTC_TPC", ss, Lifetime::Timeframe}) : dummyMCLab;
+    auto& mtcInfo = pc.outputs().make<std::vector<o2::dataformats::MatchInfoTOF>>(Output{o2::header::gDataOriginTOF, "MTC_TPC", ss});
+    auto& mclabels = mUseMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{o2::header::gDataOriginTOF, "MCMTC_TPC", ss}) : dummyMCLab;
     auto& tracksTPCTOF = pc.outputs().make<std::vector<o2::dataformats::TrackTPCTOF>>(OutputRef{"tpctofTracks", ss});
     auto nmatch = mMatcher.getMatchedTrackVector(o2::dataformats::MatchInfoTOFReco::TrackType::TPC).size();
     LOG(debug) << (mDoTPCRefit ? "Refitting " : "Shifting Z for ") << nmatch << " matched TPC tracks with TOF time info";
@@ -180,48 +180,48 @@ void TOFMatcherSpec::run(ProcessingContext& pc)
   }
 
   if (isITSTPCused) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MTC_ITSTPC", 0, Lifetime::Timeframe}, mMatcher.getMatchedTrackVector(o2::dataformats::MatchInfoTOFReco::TrackType::ITSTPC));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MTC_ITSTPC", 0}, mMatcher.getMatchedTrackVector(o2::dataformats::MatchInfoTOFReco::TrackType::ITSTPC));
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MCMTC_ITSTPC", 0, Lifetime::Timeframe}, mMatcher.getMatchedTOFLabelsVector(o2::dataformats::MatchInfoTOFReco::TrackType::ITSTPC));
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MCMTC_ITSTPC", 0}, mMatcher.getMatchedTOFLabelsVector(o2::dataformats::MatchInfoTOFReco::TrackType::ITSTPC));
     }
   }
 
   if (isTPCTRDused) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MTC_TPCTRD", ss, Lifetime::Timeframe}, mMatcher.getMatchedTrackVector(o2::dataformats::MatchInfoTOFReco::TrackType::TPCTRD));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MTC_TPCTRD", ss}, mMatcher.getMatchedTrackVector(o2::dataformats::MatchInfoTOFReco::TrackType::TPCTRD));
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MCMTC_TPCTRD", ss, Lifetime::Timeframe}, mMatcher.getMatchedTOFLabelsVector(o2::dataformats::MatchInfoTOFReco::TrackType::TPCTRD));
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MCMTC_TPCTRD", ss}, mMatcher.getMatchedTOFLabelsVector(o2::dataformats::MatchInfoTOFReco::TrackType::TPCTRD));
     }
   }
 
   if (isITSTPCTRDused) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MTC_ITSTPCTRD", 0, Lifetime::Timeframe}, mMatcher.getMatchedTrackVector(o2::dataformats::MatchInfoTOFReco::TrackType::ITSTPCTRD));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MTC_ITSTPCTRD", 0}, mMatcher.getMatchedTrackVector(o2::dataformats::MatchInfoTOFReco::TrackType::ITSTPCTRD));
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MCMTC_ITSTPCTRD", 0, Lifetime::Timeframe}, mMatcher.getMatchedTOFLabelsVector(o2::dataformats::MatchInfoTOFReco::TrackType::ITSTPCTRD));
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MCMTC_ITSTPCTRD", 0}, mMatcher.getMatchedTOFLabelsVector(o2::dataformats::MatchInfoTOFReco::TrackType::ITSTPCTRD));
     }
   }
 
   // TODO: TRD-matched tracks
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CALIBDATA", 0, Lifetime::Timeframe}, mMatcher.getCalibVector());
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CALIBDATA", 0}, mMatcher.getCalibVector());
 
   if (mPushMatchable) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_0", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(0));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_1", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(1));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_2", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(2));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_3", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(3));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_4", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(4));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_5", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(5));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_6", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(6));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_7", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(7));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_8", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(8));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_9", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(9));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_10", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(10));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_11", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(11));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_12", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(12));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_13", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(13));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_14", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(14));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_15", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(15));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_16", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(16));
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_17", 0, Lifetime::Timeframe}, mMatcher.getMatchedTracksPair(17));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_0", 0}, mMatcher.getMatchedTracksPair(0));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_1", 0}, mMatcher.getMatchedTracksPair(1));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_2", 0}, mMatcher.getMatchedTracksPair(2));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_3", 0}, mMatcher.getMatchedTracksPair(3));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_4", 0}, mMatcher.getMatchedTracksPair(4));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_5", 0}, mMatcher.getMatchedTracksPair(5));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_6", 0}, mMatcher.getMatchedTracksPair(6));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_7", 0}, mMatcher.getMatchedTracksPair(7));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_8", 0}, mMatcher.getMatchedTracksPair(8));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_9", 0}, mMatcher.getMatchedTracksPair(9));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_10", 0}, mMatcher.getMatchedTracksPair(10));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_11", 0}, mMatcher.getMatchedTracksPair(11));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_12", 0}, mMatcher.getMatchedTracksPair(12));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_13", 0}, mMatcher.getMatchedTracksPair(13));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_14", 0}, mMatcher.getMatchedTracksPair(14));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_15", 0}, mMatcher.getMatchedTracksPair(15));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_16", 0}, mMatcher.getMatchedTracksPair(16));
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_17", 0}, mMatcher.getMatchedTracksPair(17));
   }
 
   mTimer.Stop();
diff --git a/Detectors/GlobalTrackingWorkflow/src/TPCITSMatchingSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/TPCITSMatchingSpec.cxx
index 1a061ee9707f5..aebcd96c4faa7 100644
--- a/Detectors/GlobalTrackingWorkflow/src/TPCITSMatchingSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/TPCITSMatchingSpec.cxx
@@ -103,12 +103,12 @@ void TPCITSMatchingDPL::run(ProcessingContext& pc)
   static pmr::vector<o2::MCCompLabel> dummyMCLab, dummyMCLabAB;
   static pmr::vector<o2::dataformats::Triplet<float, float, float>> dummyCalib;
 
-  auto& matchedTracks = pc.outputs().make<std::vector<o2::dataformats::TrackTPCITS>>(Output{"GLO", "TPCITS", 0, Lifetime::Timeframe});
-  auto& ABTrackletRefs = pc.outputs().make<std::vector<o2::itsmft::TrkClusRef>>(Output{"GLO", "TPCITSAB_REFS", 0, Lifetime::Timeframe});
-  auto& ABTrackletClusterIDs = pc.outputs().make<std::vector<int>>(Output{"GLO", "TPCITSAB_CLID", 0, Lifetime::Timeframe});
-  auto& matchLabels = mUseMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"GLO", "TPCITS_MC", 0, Lifetime::Timeframe}) : dummyMCLab;
-  auto& ABTrackletLabels = mUseMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"GLO", "TPCITSAB_MC", 0, Lifetime::Timeframe}) : dummyMCLabAB;
-  auto& calib = mCalibMode ? pc.outputs().make<std::vector<o2::dataformats::Triplet<float, float, float>>>(Output{"GLO", "TPCITS_VDTGL", 0, Lifetime::Timeframe}) : dummyCalib;
+  auto& matchedTracks = pc.outputs().make<std::vector<o2::dataformats::TrackTPCITS>>(Output{"GLO", "TPCITS", 0});
+  auto& ABTrackletRefs = pc.outputs().make<std::vector<o2::itsmft::TrkClusRef>>(Output{"GLO", "TPCITSAB_REFS", 0});
+  auto& ABTrackletClusterIDs = pc.outputs().make<std::vector<int>>(Output{"GLO", "TPCITSAB_CLID", 0});
+  auto& matchLabels = mUseMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"GLO", "TPCITS_MC", 0}) : dummyMCLab;
+  auto& ABTrackletLabels = mUseMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"GLO", "TPCITSAB_MC", 0}) : dummyMCLabAB;
+  auto& calib = mCalibMode ? pc.outputs().make<std::vector<o2::dataformats::Triplet<float, float, float>>>(Output{"GLO", "TPCITS_VDTGL", 0}) : dummyCalib;
 
   mMatching.run(recoData, matchedTracks, ABTrackletRefs, ABTrackletClusterIDs, matchLabels, ABTrackletLabels, calib);
 
diff --git a/Detectors/GlobalTrackingWorkflow/src/VertexTrackMatcherSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/VertexTrackMatcherSpec.cxx
index 281fd06dea7eb..e5d2afb1ee700 100644
--- a/Detectors/GlobalTrackingWorkflow/src/VertexTrackMatcherSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/VertexTrackMatcherSpec.cxx
@@ -78,8 +78,8 @@ void VertexTrackMatcherSpec::run(ProcessingContext& pc)
 
   mMatcher.process(recoData, trackIndex, vtxRefs);
 
-  pc.outputs().snapshot(Output{"GLO", "PVTX_TRMTC", 0, Lifetime::Timeframe}, trackIndex);
-  pc.outputs().snapshot(Output{"GLO", "PVTX_TRMTCREFS", 0, Lifetime::Timeframe}, vtxRefs);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_TRMTC", 0}, trackIndex);
+  pc.outputs().snapshot(Output{"GLO", "PVTX_TRMTCREFS", 0}, vtxRefs);
 
   mTimer.Stop();
   LOG(info) << "Made " << trackIndex.size() << " track associations for " << recoData.getPrimaryVertices().size()
diff --git a/Detectors/GlobalTrackingWorkflow/study/src/TPCDataFilter.cxx b/Detectors/GlobalTrackingWorkflow/study/src/TPCDataFilter.cxx
index ed06e014184b6..8a962686d743e 100644
--- a/Detectors/GlobalTrackingWorkflow/study/src/TPCDataFilter.cxx
+++ b/Detectors/GlobalTrackingWorkflow/study/src/TPCDataFilter.cxx
@@ -107,10 +107,10 @@ void TPCDataFilter::run(ProcessingContext& pc)
 void TPCDataFilter::sendOutput(ProcessingContext& pc)
 {
 
-  pc.outputs().snapshot(Output{"TPC", "TRACKSF", 0, Lifetime::Timeframe}, mTracksFiltered);
-  pc.outputs().snapshot(Output{"TPC", "CLUSREFSF", 0, Lifetime::Timeframe}, mTrackClusIdxFiltered);
+  pc.outputs().snapshot(Output{"TPC", "TRACKSF", 0}, mTracksFiltered);
+  pc.outputs().snapshot(Output{"TPC", "CLUSREFSF", 0}, mTrackClusIdxFiltered);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"TPC", "TRACKSMCLBLF", 0, Lifetime::Timeframe}, mTPCTrkLabelsFiltered);
+    pc.outputs().snapshot(Output{"TPC", "TRACKSMCLBLF", 0}, mTPCTrkLabelsFiltered);
   }
 
   o2::tpc::TPCSectorHeader clusterOutputSectorHeader{0};
@@ -118,7 +118,7 @@ void TPCDataFilter::sendOutput(ProcessingContext& pc)
   for (int i = 0; i < o2::tpc::constants::MAXSECTOR; i++) {
     clusterOutputSectorHeader.sectorBits = (1ul << i);
     o2::header::DataHeader::SubSpecificationType subspec = i;
-    char* buffer = pc.outputs().make<char>({o2::header::gDataOriginTPC, "CLUSTERNATIVEF", subspec, Lifetime::Timeframe, {clusterOutputSectorHeader}},
+    char* buffer = pc.outputs().make<char>({o2::header::gDataOriginTPC, "CLUSTERNATIVEF", subspec, {clusterOutputSectorHeader}},
                                            mClusFiltered.nClustersSector[i] * sizeof(*mClusFiltered.clustersLinear) + sizeof(o2::tpc::ClusterCountIndex))
                      .data();
     o2::tpc::ClusterCountIndex* outIndex = reinterpret_cast<o2::tpc::ClusterCountIndex*>(buffer);
@@ -138,7 +138,7 @@ void TPCDataFilter::sendOutput(ProcessingContext& pc)
       }
       o2::dataformats::ConstMCLabelContainer contflat;
       cont.flatten_to(contflat);
-      pc.outputs().snapshot({o2::header::gDataOriginTPC, "CLNATIVEMCLBLF", subspec, Lifetime::Timeframe, {clusterOutputSectorHeader}}, contflat);
+      pc.outputs().snapshot({o2::header::gDataOriginTPC, "CLNATIVEMCLBLF", subspec, {clusterOutputSectorHeader}}, contflat);
     }
   }
 }
diff --git a/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCInterpolationSpec.cxx b/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCInterpolationSpec.cxx
index 69554d3d495a4..89e3ae19cf65b 100644
--- a/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCInterpolationSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCInterpolationSpec.cxx
@@ -133,18 +133,18 @@ void TPCInterpolationDPL::run(ProcessingContext& pc)
   LOGF(info, "TPC interpolation timing: Cpu: %.3e Real: %.3e s", mTimer.CpuTime(), mTimer.RealTime());
   if (SpacePointsCalibConfParam::Instance().writeUnfiltered) {
     // these are the residuals and tracks before outlier rejection; they are not used in production
-    pc.outputs().snapshot(Output{"GLO", "TPCINT_RES", 0, Lifetime::Timeframe}, mInterpolation.getClusterResidualsUnfiltered());
+    pc.outputs().snapshot(Output{"GLO", "TPCINT_RES", 0}, mInterpolation.getClusterResidualsUnfiltered());
     if (mSendTrackData) {
-      pc.outputs().snapshot(Output{"GLO", "TPCINT_TRK", 0, Lifetime::Timeframe}, mInterpolation.getReferenceTracksUnfiltered());
+      pc.outputs().snapshot(Output{"GLO", "TPCINT_TRK", 0}, mInterpolation.getReferenceTracksUnfiltered());
     }
   }
-  pc.outputs().snapshot(Output{"GLO", "UNBINNEDRES", 0, Lifetime::Timeframe}, mInterpolation.getClusterResiduals());
-  pc.outputs().snapshot(Output{"GLO", "TRKREFS", 0, Lifetime::Timeframe}, mInterpolation.getTrackDataCompact());
+  pc.outputs().snapshot(Output{"GLO", "UNBINNEDRES", 0}, mInterpolation.getClusterResiduals());
+  pc.outputs().snapshot(Output{"GLO", "TRKREFS", 0}, mInterpolation.getTrackDataCompact());
   if (mSendTrackData) {
-    pc.outputs().snapshot(Output{"GLO", "TRKDATA", 0, Lifetime::Timeframe}, mInterpolation.getReferenceTracks());
+    pc.outputs().snapshot(Output{"GLO", "TRKDATA", 0}, mInterpolation.getReferenceTracks());
   }
   if (mDebugOutput) {
-    pc.outputs().snapshot(Output{"GLO", "TRKDATAEXT", 0, Lifetime::Timeframe}, mInterpolation.getTrackDataExtended());
+    pc.outputs().snapshot(Output{"GLO", "TRKDATAEXT", 0}, mInterpolation.getTrackDataExtended());
   }
   mInterpolation.reset();
 }
diff --git a/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCUnbinnedResidualReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCUnbinnedResidualReaderSpec.cxx
index af50b32bc8bba..55da5a5e71e44 100644
--- a/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCUnbinnedResidualReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCUnbinnedResidualReaderSpec.cxx
@@ -56,11 +56,11 @@ void TPCUnbinnedResidualReader::run(ProcessingContext& pc)
   assert(currEntry < mTreeIn->GetEntries()); // this should not happen
   mTreeIn->GetEntry(currEntry);
   LOG(info) << "Pushing " << mUnbinnedResid.size() << " unbinned residuals at entry " << currEntry;
-  pc.outputs().snapshot(Output{"GLO", "UNBINNEDRES", 0, Lifetime::Timeframe}, mUnbinnedResid);
-  pc.outputs().snapshot(Output{"GLO", "TRKREFS", 0, Lifetime::Timeframe}, mTrackDataCompact);
+  pc.outputs().snapshot(Output{"GLO", "UNBINNEDRES", 0}, mUnbinnedResid);
+  pc.outputs().snapshot(Output{"GLO", "TRKREFS", 0}, mTrackDataCompact);
   if (mTrackInput) {
     LOG(info) << "Pushing " << mTrackData.size() << " reference tracks for these residuals";
-    pc.outputs().snapshot(Output{"GLO", "TRKDATA", 0, Lifetime::Timeframe}, mTrackData);
+    pc.outputs().snapshot(Output{"GLO", "TRKDATA", 0}, mTrackData);
   }
 
   if (mTreeIn->GetReadEntry() + 1 >= mTreeIn->GetEntries()) {
diff --git a/Detectors/HMPID/workflow/src/ClustersReaderSpec.cxx b/Detectors/HMPID/workflow/src/ClustersReaderSpec.cxx
index e71cd42845dd4..9ac5074acb505 100644
--- a/Detectors/HMPID/workflow/src/ClustersReaderSpec.cxx
+++ b/Detectors/HMPID/workflow/src/ClustersReaderSpec.cxx
@@ -71,8 +71,8 @@ void ClusterReaderTask::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
 
-  pc.outputs().snapshot(Output{"HMP", "CLUSTERS", 0, Lifetime::Timeframe}, mClustersFromFile);
-  pc.outputs().snapshot(Output{"HMP", "INTRECORDS1", 0, Lifetime::Timeframe}, mClusterTriggersFromFile);
+  pc.outputs().snapshot(Output{"HMP", "CLUSTERS", 0}, mClustersFromFile);
+  pc.outputs().snapshot(Output{"HMP", "INTRECORDS1", 0}, mClusterTriggersFromFile);
   mClustersReceived += mClustersFromFile.size();
   LOG(info) << "[HMPID ClusterReader - run() ] clusters  = " << mClustersFromFile.size();
 
diff --git a/Detectors/HMPID/workflow/src/DataDecoderSpec.cxx b/Detectors/HMPID/workflow/src/DataDecoderSpec.cxx
index 3a3b5a6338d81..1a6509fc75654 100644
--- a/Detectors/HMPID/workflow/src/DataDecoderSpec.cxx
+++ b/Detectors/HMPID/workflow/src/DataDecoderSpec.cxx
@@ -87,8 +87,8 @@ void DataDecoderTask::run(framework::ProcessingContext& pc)
   //  decodeReadout(pc);
   // decodeRawFile(pc);
 
-  pc.outputs().snapshot(o2::framework::Output{"HMP", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, mDeco->mDigits);
-  pc.outputs().snapshot(o2::framework::Output{"HMP", "INTRECORDS", 0, o2::framework::Lifetime::Timeframe}, mDeco->mIntReco);
+  pc.outputs().snapshot(o2::framework::Output{"HMP", "DIGITS", 0}, mDeco->mDigits);
+  pc.outputs().snapshot(o2::framework::Output{"HMP", "INTRECORDS", 0}, mDeco->mIntReco);
 
   LOG(debug) << "Writing   Digitis=" << mDeco->mDigits.size() << "/" << mTotalDigits << " Frame=" << mTotalFrames << " IntRec " << mDeco->mIntReco;
   mExTimer.elapseMes("Decoding... Digits decoded = " + std::to_string(mTotalDigits) + " Frames received = " + std::to_string(mTotalFrames));
diff --git a/Detectors/HMPID/workflow/src/DataDecoderSpec2.cxx b/Detectors/HMPID/workflow/src/DataDecoderSpec2.cxx
index 61051ab48577e..92a966938786a 100644
--- a/Detectors/HMPID/workflow/src/DataDecoderSpec2.cxx
+++ b/Detectors/HMPID/workflow/src/DataDecoderSpec2.cxx
@@ -93,8 +93,8 @@ void DataDecoderTask2::run(framework::ProcessingContext& pc)
 
   // Output the Digits/Triggers vector
   orderTriggers();
-  pc.outputs().snapshot(o2::framework::Output{"HMP", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, mDeco->mDigits);
-  pc.outputs().snapshot(o2::framework::Output{"HMP", "INTRECORDS", 0, o2::framework::Lifetime::Timeframe}, mTriggers);
+  pc.outputs().snapshot(o2::framework::Output{"HMP", "DIGITS", 0}, mDeco->mDigits);
+  pc.outputs().snapshot(o2::framework::Output{"HMP", "INTRECORDS", 0}, mTriggers);
 
   mExTimer.elapseMes("Decoding... Digits decoded = " + std::to_string(mTotalDigits) + " Frames received = " + std::to_string(mTotalFrames));
   return;
diff --git a/Detectors/HMPID/workflow/src/DigitsReaderSpec.cxx b/Detectors/HMPID/workflow/src/DigitsReaderSpec.cxx
index ed83db476b01a..88f6df2bce2e7 100644
--- a/Detectors/HMPID/workflow/src/DigitsReaderSpec.cxx
+++ b/Detectors/HMPID/workflow/src/DigitsReaderSpec.cxx
@@ -115,8 +115,8 @@ void DigitReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
 
-  pc.outputs().snapshot(Output{"HMP", "DIGITS", 0, Lifetime::Timeframe}, mDigitsFromFile);
-  pc.outputs().snapshot(Output{"HMP", "INTRECORDS", 0, Lifetime::Timeframe}, mTriggersFromFile);
+  pc.outputs().snapshot(Output{"HMP", "DIGITS", 0}, mDigitsFromFile);
+  pc.outputs().snapshot(Output{"HMP", "INTRECORDS", 0}, mTriggersFromFile);
   mDigitsReceived += mDigitsFromFile.size();
   LOG(info) << "[HMPID DigitsReader - run() ] digits  = " << mDigitsFromFile.size();
 
diff --git a/Detectors/HMPID/workflow/src/DigitsToClustersSpec.cxx b/Detectors/HMPID/workflow/src/DigitsToClustersSpec.cxx
index 377bc0f7af45e..81c04a9875c51 100644
--- a/Detectors/HMPID/workflow/src/DigitsToClustersSpec.cxx
+++ b/Detectors/HMPID/workflow/src/DigitsToClustersSpec.cxx
@@ -116,8 +116,8 @@ void DigitsToClustersTask::run(framework::ProcessingContext& pc)
   mDigitsReceived += digits.size();
   mClustersReceived += clusters.size();
 
-  pc.outputs().snapshot(o2::framework::Output{"HMP", "CLUSTERS", 0, o2::framework::Lifetime::Timeframe}, clusters);
-  pc.outputs().snapshot(o2::framework::Output{"HMP", "INTRECORDS1", 0, o2::framework::Lifetime::Timeframe}, clusterTriggers);
+  pc.outputs().snapshot(o2::framework::Output{"HMP", "CLUSTERS", 0}, clusters);
+  pc.outputs().snapshot(o2::framework::Output{"HMP", "INTRECORDS1", 0}, clusterTriggers);
 
   mExTimer.elapseMes("Clusterization of Digits received = " + std::to_string(mDigitsReceived));
   mExTimer.elapseMes("Clusterization of Clusters received = " + std::to_string(mClustersReceived));
diff --git a/Detectors/HMPID/workflow/src/EntropyEncoderSpec.cxx b/Detectors/HMPID/workflow/src/EntropyEncoderSpec.cxx
index 15bd0987a0ccc..95723f42d0fd6 100644
--- a/Detectors/HMPID/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/HMPID/workflow/src/EntropyEncoderSpec.cxx
@@ -72,7 +72,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"HMP", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"HMP", "CTFDATA", 0});
 
   auto iosize = mCTFCoder.encode(buffer, triggers, digits);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
diff --git a/Detectors/HMPID/workflow/src/HMPMatchedReaderSpec.cxx b/Detectors/HMPID/workflow/src/HMPMatchedReaderSpec.cxx
index e466a8a4e00c4..3ddb71e282a96 100644
--- a/Detectors/HMPID/workflow/src/HMPMatchedReaderSpec.cxx
+++ b/Detectors/HMPID/workflow/src/HMPMatchedReaderSpec.cxx
@@ -62,9 +62,9 @@ void HMPMatchedReader::run(ProcessingContext& pc)
   mTree->GetEntry(currEntry);
   LOG(debug) << "Pushing " << mMatches.size() << " HMP matchings at entry " << currEntry;
 
-  pc.outputs().snapshot(Output{o2::header::gDataOriginHMP, "MATCHES", 0, Lifetime::Timeframe}, mMatches);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginHMP, "MATCHES", 0}, mMatches);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginHMP, "MCLABELS", 0, Lifetime::Timeframe}, mLabelHMP);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginHMP, "MCLABELS", 0}, mLabelHMP);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/ITSMFT/ITS/QC/TestDataReaderWorkflow/src/TestDataReader.cxx b/Detectors/ITSMFT/ITS/QC/TestDataReaderWorkflow/src/TestDataReader.cxx
index d6ef8c947ed49..964f342c58b15 100644
--- a/Detectors/ITSMFT/ITS/QC/TestDataReaderWorkflow/src/TestDataReader.cxx
+++ b/Detectors/ITSMFT/ITS/QC/TestDataReaderWorkflow/src/TestDataReader.cxx
@@ -139,14 +139,14 @@ void TestDataReader::run(ProcessingContext& pc)
   if (mDiffFolderName.size() == 0) {
     cout << "No New Run -- No Need to Reset" << endl;
     mResetCommand = 0;
-    pc.outputs().snapshot(Output{"ITS", "TEST", 0, Lifetime::Timeframe}, mResetCommand);
+    pc.outputs().snapshot(Output{"ITS", "TEST", 0}, mResetCommand);
   }
 
   // New folders found, send the reset signal and reload configuration
   if (mDiffFolderName.size() > 0) {
     cout << "New Run Started -- Reset All Histograms" << endl;
     mResetCommand = 1;
-    pc.outputs().snapshot(Output{"ITS", "TEST", 0, Lifetime::Timeframe}, mResetCommand);
+    pc.outputs().snapshot(Output{"ITS", "TEST", 0}, mResetCommand);
     for (int i = 0; i < sNError; i++) {
       mErrors[i] = 0;
     }
@@ -264,11 +264,11 @@ void TestDataReader::run(ProcessingContext& pc)
         mErrorsVecTest.push_back(mErrors);
         mFileDone = 1;
         mFileInfo = mFileDone + mFileRemain * 10;
-        pc.outputs().snapshot(Output{"ITS", "Run", 0, Lifetime::Timeframe}, mRunNumber);
-        pc.outputs().snapshot(Output{"ITS", "File", 0, Lifetime::Timeframe}, mFileID);
-        pc.outputs().snapshot(Output{"ITS", "Error", 0, Lifetime::Timeframe}, mErrorsVecTest[0]);
-        pc.outputs().snapshot(Output{"ITS", "Finish", 0, Lifetime::Timeframe}, mFileInfo);
-        pc.outputs().snapshot(Output{"ITS", "DIGITS", 0, Lifetime::Timeframe}, mMultiDigitsTest);
+        pc.outputs().snapshot(Output{"ITS", "Run", 0}, mRunNumber);
+        pc.outputs().snapshot(Output{"ITS", "File", 0}, mFileID);
+        pc.outputs().snapshot(Output{"ITS", "Error", 0}, mErrorsVecTest[0]);
+        pc.outputs().snapshot(Output{"ITS", "Finish", 0}, mFileInfo);
+        pc.outputs().snapshot(Output{"ITS", "DIGITS", 0}, mMultiDigitsTest);
         mNewFileInj = 0;
         mErrorsVecTest.clear();
         mDigitsTest.clear();
@@ -422,10 +422,10 @@ void TestDataReader::run(ProcessingContext& pc)
 
     cout << "RunIDS = " << mRunNumber << "   FileIDS = " << mFileID << endl;
 
-    pc.outputs().snapshot(Output{"ITS", "Run", 0, Lifetime::Timeframe}, mRunNumber);
-    pc.outputs().snapshot(Output{"ITS", "File", 0, Lifetime::Timeframe}, mFileID);
+    pc.outputs().snapshot(Output{"ITS", "Run", 0}, mRunNumber);
+    pc.outputs().snapshot(Output{"ITS", "File", 0}, mFileID);
 
-    pc.outputs().snapshot(Output{"ITS", "Error", 0, Lifetime::Timeframe}, mErrorsVec[j]);
+    pc.outputs().snapshot(Output{"ITS", "Error", 0}, mErrorsVec[j]);
     mIndexPushEx = mIndexPush + mNDigits[j];
     LOG(debug) << "IndexPushEx = " << mIndexPushEx << "  mDigits.size() " << mDigits.size();
     if (mIndexPushEx > mDigits.size() - 5) {
@@ -436,11 +436,11 @@ void TestDataReader::run(ProcessingContext& pc)
 
     mFileInfo = mFileDone + mFileRemain * 10;
 
-    pc.outputs().snapshot(Output{"ITS", "Finish", 0, Lifetime::Timeframe}, mFileInfo);
+    pc.outputs().snapshot(Output{"ITS", "Finish", 0}, mFileInfo);
 
     LOG(debug) << "mIndexPush = " << mIndexPush << "    Chip ID Pushing " << mDigits[mIndexPush].getChipIndex();
 
-    pc.outputs().snapshot(Output{"ITS", "DIGITS", 0, Lifetime::Timeframe}, mMultiDigits);
+    pc.outputs().snapshot(Output{"ITS", "DIGITS", 0}, mMultiDigits);
 
     mMultiDigits.clear();
     mIndexPush = mIndexPush + mNDigits[j];
@@ -453,7 +453,7 @@ void TestDataReader::run(ProcessingContext& pc)
   //            << "mIndexPush = " << mIndexPush << "     mDigits.size() = " << mDigits.size();
   // while (mIndexPush < mDigits.size()) {
   // LOG(debug) << "mDigits.size() = " << mDigits.size();
-  //   pc.outputs().snapshot(Output{"ITS", "DIGITS", 0, Lifetime::Timeframe}, mDigits[mIndexPush++]);
+  //   pc.outputs().snapshot(Output{"ITS", "DIGITS", 0}, mDigits[mIndexPush++]);
   //   if (mIndexPush % 100000 == 0)
   //     LOG(debug) << "mIndexPush = " << mIndexPush << "    Chip ID Pushing " << mDigits[mIndexPush].getChipIndex();
   // }
@@ -483,7 +483,7 @@ void TestDataReader::run(ProcessingContext& pc)
     j = 0;
     mNDigits.clear();
     mFileDone = 1;
-    pc.outputs().snapshot(Output{"TST", "Finish", 0, Lifetime::Timeframe}, mFileDone);
+    pc.outputs().snapshot(Output{"TST", "Finish", 0}, mFileDone);
     PercentDone = 0;
     mErrorsVec.clear();
   }
diff --git a/Detectors/ITSMFT/ITS/workflow/src/ClustererSpec.cxx b/Detectors/ITSMFT/ITS/workflow/src/ClustererSpec.cxx
index 89596e8357b74..71987d0480566 100644
--- a/Detectors/ITSMFT/ITS/workflow/src/ClustererSpec.cxx
+++ b/Detectors/ITSMFT/ITS/workflow/src/ClustererSpec.cxx
@@ -88,17 +88,17 @@ void ClustererDPL::run(ProcessingContext& pc)
     clusterLabels = std::make_unique<o2::dataformats::MCTruthContainer<o2::MCCompLabel>>();
   }
   mClusterer->process(mNThreads, reader, &clusCompVec, &clusPattVec, &clusROFVec, clusterLabels.get());
-  pc.outputs().snapshot(Output{orig, "COMPCLUSTERS", 0, Lifetime::Timeframe}, clusCompVec);
-  pc.outputs().snapshot(Output{orig, "CLUSTERSROF", 0, Lifetime::Timeframe}, clusROFVec);
-  pc.outputs().snapshot(Output{orig, "PATTERNS", 0, Lifetime::Timeframe}, clusPattVec);
+  pc.outputs().snapshot(Output{orig, "COMPCLUSTERS", 0}, clusCompVec);
+  pc.outputs().snapshot(Output{orig, "CLUSTERSROF", 0}, clusROFVec);
+  pc.outputs().snapshot(Output{orig, "PATTERNS", 0}, clusPattVec);
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{orig, "CLUSTERSMCTR", 0, Lifetime::Timeframe}, *clusterLabels.get()); // at the moment requires snapshot
+    pc.outputs().snapshot(Output{orig, "CLUSTERSMCTR", 0}, *clusterLabels.get()); // at the moment requires snapshot
     std::vector<o2::itsmft::MC2ROFRecord> clusterMC2ROframes(mc2rofs.size());
     for (int i = mc2rofs.size(); i--;) {
       clusterMC2ROframes[i] = mc2rofs[i]; // Simply, replicate it from digits ?
     }
-    pc.outputs().snapshot(Output{orig, "CLUSTERSMC2ROF", 0, Lifetime::Timeframe}, clusterMC2ROframes);
+    pc.outputs().snapshot(Output{orig, "CLUSTERSMC2ROF", 0}, clusterMC2ROframes);
   }
 
   // TODO: in principle, after masking "overflow" pixels the MC2ROFRecord maxROF supposed to change, nominally to minROF
diff --git a/Detectors/ITSMFT/ITS/workflow/src/CookedTrackerSpec.cxx b/Detectors/ITSMFT/ITS/workflow/src/CookedTrackerSpec.cxx
index e3f585c81cd97..e4c9d43541e1f 100644
--- a/Detectors/ITSMFT/ITS/workflow/src/CookedTrackerSpec.cxx
+++ b/Detectors/ITSMFT/ITS/workflow/src/CookedTrackerSpec.cxx
@@ -95,7 +95,7 @@ void CookedTrackerDPL::run(ProcessingContext& pc)
     physTriggers = pc.inputs().get<gsl::span<o2::itsmft::PhysTrigger>>("phystrig");
   }
 
-  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "ITSTrackROF", 0, Lifetime::Timeframe}, rofsinput.begin(), rofsinput.end());
+  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "ITSTrackROF", 0}, rofsinput.begin(), rofsinput.end());
 
   std::unique_ptr<const o2::dataformats::MCTruthContainer<o2::MCCompLabel>> labels;
   gsl::span<itsmft::MC2ROFRecord const> mc2rofs;
@@ -116,11 +116,11 @@ void CookedTrackerDPL::run(ProcessingContext& pc)
   o2::its::ROframe event(0, 7);
   mVertexerPtr->adoptTimeFrame(mTimeFrame);
 
-  auto& vertROFvec = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "VERTICESROF", 0, Lifetime::Timeframe});
-  auto& vertices = pc.outputs().make<std::vector<Vertex>>(Output{"ITS", "VERTICES", 0, Lifetime::Timeframe});
-  auto& tracks = pc.outputs().make<std::vector<o2::its::TrackITS>>(Output{"ITS", "TRACKS", 0, Lifetime::Timeframe});
-  auto& clusIdx = pc.outputs().make<std::vector<int>>(Output{"ITS", "TRACKCLSID", 0, Lifetime::Timeframe});
-  auto& irFrames = pc.outputs().make<std::vector<o2::dataformats::IRFrame>>(Output{"ITS", "IRFRAMES", 0, Lifetime::Timeframe});
+  auto& vertROFvec = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "VERTICESROF", 0});
+  auto& vertices = pc.outputs().make<std::vector<Vertex>>(Output{"ITS", "VERTICES", 0});
+  auto& tracks = pc.outputs().make<std::vector<o2::its::TrackITS>>(Output{"ITS", "TRACKS", 0});
+  auto& clusIdx = pc.outputs().make<std::vector<int>>(Output{"ITS", "TRACKCLSID", 0});
+  auto& irFrames = pc.outputs().make<std::vector<o2::dataformats::IRFrame>>(Output{"ITS", "IRFRAMES", 0});
 
   const auto& alpParams = o2::itsmft::DPLAlpideParam<o2::detectors::DetID::ITS>::Instance(); // RS: this should come from CCDB
   int nBCPerTF = mTracker.getContinuousMode() ? alpParams.roFrameLengthInBC : alpParams.roFrameLengthTrig;
@@ -201,8 +201,8 @@ void CookedTrackerDPL::run(ProcessingContext& pc)
   LOG(info) << "ITSCookedTracker pushed " << tracks.size() << " tracks and " << vertices.size() << " vertices";
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"ITS", "TRACKSMCTR", 0, Lifetime::Timeframe}, trackLabels);
-    pc.outputs().snapshot(Output{"ITS", "ITSTrackMC2ROF", 0, Lifetime::Timeframe}, mc2rofs);
+    pc.outputs().snapshot(Output{"ITS", "TRACKSMCTR", 0}, trackLabels);
+    pc.outputs().snapshot(Output{"ITS", "ITSTrackMC2ROF", 0}, mc2rofs);
   }
   mTimer.Stop();
 }
diff --git a/Detectors/ITSMFT/ITS/workflow/src/DCSGeneratorSpec.cxx b/Detectors/ITSMFT/ITS/workflow/src/DCSGeneratorSpec.cxx
index 48bbaed171a85..2ea0c833f3871 100644
--- a/Detectors/ITSMFT/ITS/workflow/src/DCSGeneratorSpec.cxx
+++ b/Detectors/ITSMFT/ITS/workflow/src/DCSGeneratorSpec.cxx
@@ -117,7 +117,7 @@ void ITSDCSDataGenerator::run(o2::framework::ProcessingContext& pc)
   auto timeNow = std::chrono::system_clock::now();
   timingInfo.creation = std::chrono::duration_cast<std::chrono::milliseconds>(timeNow.time_since_epoch()).count(); // in ms
 
-  pc.outputs().snapshot(Output{"ITS", mDataDescription, 0, Lifetime::Timeframe}, dpcoms);
+  pc.outputs().snapshot(Output{"ITS", mDataDescription, 0}, dpcoms);
   mTFs++;
 }
 } // namespace
diff --git a/Detectors/ITSMFT/ITS/workflow/src/TrackReaderSpec.cxx b/Detectors/ITSMFT/ITS/workflow/src/TrackReaderSpec.cxx
index 416ba5b3b07cc..8e72faae9fd37 100644
--- a/Detectors/ITSMFT/ITS/workflow/src/TrackReaderSpec.cxx
+++ b/Detectors/ITSMFT/ITS/workflow/src/TrackReaderSpec.cxx
@@ -44,14 +44,14 @@ void TrackReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mTracks.size() << " track in " << mROFRec.size() << " ROFs at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "ITSTrackROF", 0, Lifetime::Timeframe}, mROFRec);
-  pc.outputs().snapshot(Output{mOrigin, "TRACKS", 0, Lifetime::Timeframe}, mTracks);
-  pc.outputs().snapshot(Output{mOrigin, "TRACKCLSID", 0, Lifetime::Timeframe}, mClusInd);
-  pc.outputs().snapshot(Output{"ITS", "VERTICES", 0, Lifetime::Timeframe}, mVertices);
-  pc.outputs().snapshot(Output{"ITS", "VERTICESROF", 0, Lifetime::Timeframe}, mVerticesROFRec);
+  pc.outputs().snapshot(Output{mOrigin, "ITSTrackROF", 0}, mROFRec);
+  pc.outputs().snapshot(Output{mOrigin, "TRACKS", 0}, mTracks);
+  pc.outputs().snapshot(Output{mOrigin, "TRACKCLSID", 0}, mClusInd);
+  pc.outputs().snapshot(Output{"ITS", "VERTICES", 0}, mVertices);
+  pc.outputs().snapshot(Output{"ITS", "VERTICESROF", 0}, mVerticesROFRec);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{mOrigin, "TRACKSMCTR", 0, Lifetime::Timeframe}, mMCTruth);
-    pc.outputs().snapshot(Output{mOrigin, "VERTICESMCTR", 0, Lifetime::Timeframe}, mMCVertTruth);
+    pc.outputs().snapshot(Output{mOrigin, "TRACKSMCTR", 0}, mMCTruth);
+    pc.outputs().snapshot(Output{mOrigin, "VERTICESMCTR", 0}, mMCVertTruth);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx b/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx
index 35671f646bd04..331e42907530a 100644
--- a/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx
+++ b/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx
@@ -150,8 +150,8 @@ void TrackerDPL::run(ProcessingContext& pc)
   }
 
   auto rofsinput = pc.inputs().get<gsl::span<o2::itsmft::ROFRecord>>("ROframes");
-  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "ITSTrackROF", 0, Lifetime::Timeframe}, rofsinput.begin(), rofsinput.end());
-  auto& irFrames = pc.outputs().make<std::vector<o2::dataformats::IRFrame>>(Output{"ITS", "IRFRAMES", 0, Lifetime::Timeframe});
+  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "ITSTrackROF", 0}, rofsinput.begin(), rofsinput.end());
+  auto& irFrames = pc.outputs().make<std::vector<o2::dataformats::IRFrame>>(Output{"ITS", "IRFRAMES", 0});
   const auto& alpParams = o2::itsmft::DPLAlpideParam<o2::detectors::DetID::ITS>::Instance(); // RS: this should come from CCDB
 
   irFrames.reserve(rofs.size());
@@ -164,19 +164,19 @@ void TrackerDPL::run(ProcessingContext& pc)
   if (mIsMC) {
     labels = pc.inputs().get<const dataformats::MCTruthContainer<MCCompLabel>*>("itsmclabels").release();
     // get the array as read-only span, a snapshot is sent forward
-    pc.outputs().snapshot(Output{"ITS", "ITSTrackMC2ROF", 0, Lifetime::Timeframe}, pc.inputs().get<gsl::span<itsmft::MC2ROFRecord>>("ITSMC2ROframes"));
+    pc.outputs().snapshot(Output{"ITS", "ITSTrackMC2ROF", 0}, pc.inputs().get<gsl::span<itsmft::MC2ROFRecord>>("ITSMC2ROframes"));
     LOG(info) << labels->getIndexedSize() << " MC label objects , in " << mc2rofs.size() << " MC events";
   }
 
-  auto& allClusIdx = pc.outputs().make<std::vector<int>>(Output{"ITS", "TRACKCLSID", 0, Lifetime::Timeframe});
-  auto& allTracks = pc.outputs().make<std::vector<o2::its::TrackITS>>(Output{"ITS", "TRACKS", 0, Lifetime::Timeframe});
-  auto& vertROFvec = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "VERTICESROF", 0, Lifetime::Timeframe});
-  auto& vertices = pc.outputs().make<std::vector<Vertex>>(Output{"ITS", "VERTICES", 0, Lifetime::Timeframe});
+  auto& allClusIdx = pc.outputs().make<std::vector<int>>(Output{"ITS", "TRACKCLSID", 0});
+  auto& allTracks = pc.outputs().make<std::vector<o2::its::TrackITS>>(Output{"ITS", "TRACKS", 0});
+  auto& vertROFvec = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "VERTICESROF", 0});
+  auto& vertices = pc.outputs().make<std::vector<Vertex>>(Output{"ITS", "VERTICES", 0});
 
   // MC
   static pmr::vector<o2::MCCompLabel> dummyMCLabTracks, dummyMCLabVerts;
-  auto& allTrackLabels = mIsMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"ITS", "TRACKSMCTR", 0, Lifetime::Timeframe}) : dummyMCLabTracks;
-  auto& allVerticesLabels = mIsMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"ITS", "VERTICESMCTR", 0, Lifetime::Timeframe}) : dummyMCLabVerts;
+  auto& allTrackLabels = mIsMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"ITS", "TRACKSMCTR", 0}) : dummyMCLabTracks;
+  auto& allVerticesLabels = mIsMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"ITS", "VERTICESMCTR", 0}) : dummyMCLabVerts;
 
   std::uint32_t roFrame = 0;
 
diff --git a/Detectors/ITSMFT/ITS/workflow/src/VertexReaderSpec.cxx b/Detectors/ITSMFT/ITS/workflow/src/VertexReaderSpec.cxx
index bbf033a31abee..e92f08af23c0d 100644
--- a/Detectors/ITSMFT/ITS/workflow/src/VertexReaderSpec.cxx
+++ b/Detectors/ITSMFT/ITS/workflow/src/VertexReaderSpec.cxx
@@ -41,8 +41,8 @@ void VertexReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mVerticesPtr->size() << " vertices in " << mVerticesROFRecPtr->size()
             << " ROFs at entry " << ent;
-  pc.outputs().snapshot(Output{"ITS", "VERTICES", 0, Lifetime::Timeframe}, mVertices);
-  pc.outputs().snapshot(Output{"ITS", "VERTICESROF", 0, Lifetime::Timeframe}, mVerticesROFRec);
+  pc.outputs().snapshot(Output{"ITS", "VERTICES", 0}, mVertices);
+  pc.outputs().snapshot(Output{"ITS", "VERTICESROF", 0}, mVerticesROFRec);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/ITSMFT/MFT/workflow/src/ClustererSpec.cxx b/Detectors/ITSMFT/MFT/workflow/src/ClustererSpec.cxx
index f6279be58cfd9..bc8568112e447 100644
--- a/Detectors/ITSMFT/MFT/workflow/src/ClustererSpec.cxx
+++ b/Detectors/ITSMFT/MFT/workflow/src/ClustererSpec.cxx
@@ -89,17 +89,17 @@ void ClustererDPL::run(ProcessingContext& pc)
     clusterLabels = std::make_unique<o2::dataformats::MCTruthContainer<o2::MCCompLabel>>();
   }
   mClusterer->process(mNThreads, reader, &clusCompVec, &clusPattVec, &clusROFVec, clusterLabels.get());
-  pc.outputs().snapshot(Output{orig, "COMPCLUSTERS", 0, Lifetime::Timeframe}, clusCompVec);
-  pc.outputs().snapshot(Output{orig, "CLUSTERSROF", 0, Lifetime::Timeframe}, clusROFVec);
-  pc.outputs().snapshot(Output{orig, "PATTERNS", 0, Lifetime::Timeframe}, clusPattVec);
+  pc.outputs().snapshot(Output{orig, "COMPCLUSTERS", 0}, clusCompVec);
+  pc.outputs().snapshot(Output{orig, "CLUSTERSROF", 0}, clusROFVec);
+  pc.outputs().snapshot(Output{orig, "PATTERNS", 0}, clusPattVec);
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{orig, "CLUSTERSMCTR", 0, Lifetime::Timeframe}, *clusterLabels.get()); // at the moment requires snapshot
+    pc.outputs().snapshot(Output{orig, "CLUSTERSMCTR", 0}, *clusterLabels.get()); // at the moment requires snapshot
     std::vector<o2::itsmft::MC2ROFRecord> clusterMC2ROframes(mc2rofs.size());
     for (int i = mc2rofs.size(); i--;) {
       clusterMC2ROframes[i] = mc2rofs[i]; // Simply, replicate it from digits ?
     }
-    pc.outputs().snapshot(Output{orig, "CLUSTERSMC2ROF", 0, Lifetime::Timeframe}, clusterMC2ROframes);
+    pc.outputs().snapshot(Output{orig, "CLUSTERSMC2ROF", 0}, clusterMC2ROframes);
   }
 
   // TODO: in principle, after masking "overflow" pixels the MC2ROFRecord maxROF supposed to change, nominally to minROF
diff --git a/Detectors/ITSMFT/MFT/workflow/src/TrackReaderSpec.cxx b/Detectors/ITSMFT/MFT/workflow/src/TrackReaderSpec.cxx
index 8d9b889d4d2aa..1a2ae573af536 100644
--- a/Detectors/ITSMFT/MFT/workflow/src/TrackReaderSpec.cxx
+++ b/Detectors/ITSMFT/MFT/workflow/src/TrackReaderSpec.cxx
@@ -45,11 +45,11 @@ void TrackReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mTracks.size() << " track in " << mROFRec.size() << " ROFs at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "MFTTrackROF", 0, Lifetime::Timeframe}, mROFRec);
-  pc.outputs().snapshot(Output{mOrigin, "TRACKS", 0, Lifetime::Timeframe}, mTracks);
-  pc.outputs().snapshot(Output{mOrigin, "TRACKCLSID", 0, Lifetime::Timeframe}, mClusInd);
+  pc.outputs().snapshot(Output{mOrigin, "MFTTrackROF", 0}, mROFRec);
+  pc.outputs().snapshot(Output{mOrigin, "TRACKS", 0}, mTracks);
+  pc.outputs().snapshot(Output{mOrigin, "TRACKCLSID", 0}, mClusInd);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{mOrigin, "TRACKSMCTR", 0, Lifetime::Timeframe}, mMCTruth);
+    pc.outputs().snapshot(Output{mOrigin, "TRACKSMCTR", 0}, mMCTruth);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/ITSMFT/MFT/workflow/src/TrackerSpec.cxx b/Detectors/ITSMFT/MFT/workflow/src/TrackerSpec.cxx
index 7bf7d27a69e4c..5d22a6f2d2306 100644
--- a/Detectors/ITSMFT/MFT/workflow/src/TrackerSpec.cxx
+++ b/Detectors/ITSMFT/MFT/workflow/src/TrackerSpec.cxx
@@ -72,7 +72,7 @@ void TrackerDPL::run(ProcessingContext& pc)
   // the output vector however is created directly inside the message memory thus avoiding copy by
   // snapshot
   auto rofsinput = pc.inputs().get<const std::vector<o2::itsmft::ROFRecord>>("ROframes");
-  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"MFT", "MFTTrackROF", 0, Lifetime::Timeframe}, rofsinput.begin(), rofsinput.end());
+  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"MFT", "MFTTrackROF", 0}, rofsinput.begin(), rofsinput.end());
 
   ROFFilter filter = [](const o2::itsmft::ROFRecord& r) { return true; };
 
@@ -104,12 +104,12 @@ void TrackerDPL::run(ProcessingContext& pc)
     LOG(info) << labels->getIndexedSize() << " MC label objects , in " << mc2rofs.size() << " MC events";
   }
 
-  auto& allClusIdx = pc.outputs().make<std::vector<int>>(Output{"MFT", "TRACKCLSID", 0, Lifetime::Timeframe});
+  auto& allClusIdx = pc.outputs().make<std::vector<int>>(Output{"MFT", "TRACKCLSID", 0});
   std::vector<o2::MCCompLabel> trackLabels;
   std::vector<o2::MCCompLabel> allTrackLabels;
   std::vector<o2::mft::TrackLTF> tracks;
   std::vector<o2::mft::TrackLTFL> tracksL;
-  auto& allTracksMFT = pc.outputs().make<std::vector<o2::mft::TrackMFT>>(Output{"MFT", "TRACKS", 0, Lifetime::Timeframe});
+  auto& allTracksMFT = pc.outputs().make<std::vector<o2::mft::TrackMFT>>(Output{"MFT", "TRACKS", 0});
 
   std::uint32_t roFrameId = 0;
   int nROFs = rofs.size();
@@ -327,8 +327,8 @@ void TrackerDPL::run(ProcessingContext& pc)
   LOG(info) << "MFTTracker pushed " << allTracksMFT.size() << " tracks";
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"MFT", "TRACKSMCTR", 0, Lifetime::Timeframe}, allTrackLabels);
-    pc.outputs().snapshot(Output{"MFT", "TRACKSMC2ROF", 0, Lifetime::Timeframe}, mc2rofs);
+    pc.outputs().snapshot(Output{"MFT", "TRACKSMCTR", 0}, allTrackLabels);
+    pc.outputs().snapshot(Output{"MFT", "TRACKSMC2ROF", 0}, mc2rofs);
   }
 
   mTimer[SWTot].Stop();
diff --git a/Detectors/ITSMFT/common/workflow/src/ClusterReaderSpec.cxx b/Detectors/ITSMFT/common/workflow/src/ClusterReaderSpec.cxx
index ad7f26b1b4680..ea906056c7898 100644
--- a/Detectors/ITSMFT/common/workflow/src/ClusterReaderSpec.cxx
+++ b/Detectors/ITSMFT/common/workflow/src/ClusterReaderSpec.cxx
@@ -58,18 +58,18 @@ void ClusterReader::run(ProcessingContext& pc)
 
   // This is a very ugly way of providing DataDescription, which anyway does not need to contain detector name.
   // To be fixed once the names-definition class is ready
-  pc.outputs().snapshot(Output{mOrigin, "CLUSTERSROF", 0, Lifetime::Timeframe}, mClusROFRec);
-  pc.outputs().snapshot(Output{mOrigin, "COMPCLUSTERS", 0, Lifetime::Timeframe}, mClusterCompArray);
+  pc.outputs().snapshot(Output{mOrigin, "CLUSTERSROF", 0}, mClusROFRec);
+  pc.outputs().snapshot(Output{mOrigin, "COMPCLUSTERS", 0}, mClusterCompArray);
   if (mUsePatterns) {
-    pc.outputs().snapshot(Output{mOrigin, "PATTERNS", 0, Lifetime::Timeframe}, mPatternsArray);
+    pc.outputs().snapshot(Output{mOrigin, "PATTERNS", 0}, mPatternsArray);
   }
   if (mUseMC) {
-    pc.outputs().snapshot(Output{mOrigin, "CLUSTERSMCTR", 0, Lifetime::Timeframe}, mClusterMCTruth);
-    pc.outputs().snapshot(Output{mOrigin, "CLUSTERSMC2ROF", 0, Lifetime::Timeframe}, mClusMC2ROFs);
+    pc.outputs().snapshot(Output{mOrigin, "CLUSTERSMCTR", 0}, mClusterMCTruth);
+    pc.outputs().snapshot(Output{mOrigin, "CLUSTERSMC2ROF", 0}, mClusMC2ROFs);
   }
   if (mTriggerOut) {
     std::vector<o2::itsmft::PhysTrigger> dummyTrig;
-    pc.outputs().snapshot(Output{mOrigin, "PHYSTRIG", 0, Lifetime::Timeframe}, dummyTrig);
+    pc.outputs().snapshot(Output{mOrigin, "PHYSTRIG", 0}, dummyTrig);
   }
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/ITSMFT/common/workflow/src/DigitReaderSpec.cxx b/Detectors/ITSMFT/common/workflow/src/DigitReaderSpec.cxx
index fbdd63c73d782..ef9494c85bf4b 100644
--- a/Detectors/ITSMFT/common/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/ITSMFT/common/workflow/src/DigitReaderSpec.cxx
@@ -74,20 +74,20 @@ void DigitReader::run(ProcessingContext& pc)
 
   // This is a very ugly way of providing DataDescription, which anyway does not need to contain detector name.
   // To be fixed once the names-definition class is ready
-  pc.outputs().snapshot(Output{mOrigin, "DIGITSROF", 0, Lifetime::Timeframe}, mDigROFRec);
-  pc.outputs().snapshot(Output{mOrigin, "DIGITS", 0, Lifetime::Timeframe}, mDigits);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITSROF", 0}, mDigROFRec);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITS", 0}, mDigits);
   if (mUseCalib) {
-    pc.outputs().snapshot(Output{mOrigin, "GBTCALIB", 0, Lifetime::Timeframe}, mCalib);
+    pc.outputs().snapshot(Output{mOrigin, "GBTCALIB", 0}, mCalib);
   }
   if (mTriggerOut) {
     std::vector<o2::itsmft::PhysTrigger> dummyTrig;
-    pc.outputs().snapshot(Output{mOrigin, "PHYSTRIG", 0, Lifetime::Timeframe}, dummyTrig);
+    pc.outputs().snapshot(Output{mOrigin, "PHYSTRIG", 0}, dummyTrig);
   }
   if (mUseMC) {
-    auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{mOrigin, "DIGITSMCTR", 0, Lifetime::Timeframe});
+    auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{mOrigin, "DIGITSMCTR", 0});
     plabels->copyandflatten(sharedlabels);
     delete plabels;
-    pc.outputs().snapshot(Output{mOrigin, "DIGITSMC2ROF", 0, Lifetime::Timeframe}, mDigMC2ROFs);
+    pc.outputs().snapshot(Output{mOrigin, "DIGITSMC2ROF", 0}, mDigMC2ROFs);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/ITSMFT/common/workflow/src/EntropyEncoderSpec.cxx b/Detectors/ITSMFT/common/workflow/src/EntropyEncoderSpec.cxx
index ccb6f0539909e..4b35f6cc44e39 100644
--- a/Detectors/ITSMFT/common/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/ITSMFT/common/workflow/src/EntropyEncoderSpec.cxx
@@ -56,7 +56,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{mOrigin, "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{mOrigin, "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, rofs, compClusters, pspan, mPattIdConverter, mStrobeLength);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   if (mSelIR) {
diff --git a/Detectors/ITSMFT/common/workflow/src/STFDecoderSpec.cxx b/Detectors/ITSMFT/common/workflow/src/STFDecoderSpec.cxx
index 73132163d102c..a31ec0f1a4af2 100644
--- a/Detectors/ITSMFT/common/workflow/src/STFDecoderSpec.cxx
+++ b/Detectors/ITSMFT/common/workflow/src/STFDecoderSpec.cxx
@@ -139,7 +139,7 @@ void STFDecoder<Mapping>::run(ProcessingContext& pc)
   std::vector<Digit> digVec;
   std::vector<GBTCalibData> calVec;
   std::vector<ROFRecord> digROFVec;
-  auto& chipStatus = pc.outputs().make<std::vector<char>>(Output{orig, "CHIPSSTATUS", 0, Lifetime::Timeframe}, (size_t)Mapping::getNChips());
+  auto& chipStatus = pc.outputs().make<std::vector<char>>(Output{orig, "CHIPSSTATUS", 0}, (size_t)Mapping::getNChips());
 
   try {
     mDecoder->startNewTF(pc.inputs());
@@ -202,29 +202,29 @@ void STFDecoder<Mapping>::run(ProcessingContext& pc)
     }
   }
   if (mDoDigits) {
-    pc.outputs().snapshot(Output{orig, "DIGITS", 0, Lifetime::Timeframe}, digVec);
-    pc.outputs().snapshot(Output{orig, "DIGITSROF", 0, Lifetime::Timeframe}, digROFVec);
+    pc.outputs().snapshot(Output{orig, "DIGITS", 0}, digVec);
+    pc.outputs().snapshot(Output{orig, "DIGITSROF", 0}, digROFVec);
     mEstNDig = std::max(mEstNDig, size_t(digVec.size() * 1.2));
     mEstNROF = std::max(mEstNROF, size_t(digROFVec.size() * 1.2));
     if (mDoCalibData) {
-      pc.outputs().snapshot(Output{orig, "GBTCALIB", 0, Lifetime::Timeframe}, calVec);
+      pc.outputs().snapshot(Output{orig, "GBTCALIB", 0}, calVec);
       mEstNCalib = std::max(mEstNCalib, size_t(calVec.size() * 1.2));
     }
   }
 
   if (mDoClusters) { // we are not obliged to create vectors which are not requested, but other devices might not know the options of this one
-    pc.outputs().snapshot(Output{orig, "COMPCLUSTERS", 0, Lifetime::Timeframe}, clusCompVec);
-    pc.outputs().snapshot(Output{orig, "PATTERNS", 0, Lifetime::Timeframe}, clusPattVec);
-    pc.outputs().snapshot(Output{orig, "CLUSTERSROF", 0, Lifetime::Timeframe}, clusROFVec);
+    pc.outputs().snapshot(Output{orig, "COMPCLUSTERS", 0}, clusCompVec);
+    pc.outputs().snapshot(Output{orig, "PATTERNS", 0}, clusPattVec);
+    pc.outputs().snapshot(Output{orig, "CLUSTERSROF", 0}, clusROFVec);
     mEstNClus = std::max(mEstNClus, size_t(clusCompVec.size() * 1.2));
     mEstNClusPatt = std::max(mEstNClusPatt, size_t(clusPattVec.size() * 1.2));
     mEstNROF = std::max(mEstNROF, size_t(clusROFVec.size() * 1.2));
   }
-  auto& linkErrors = pc.outputs().make<std::vector<GBTLinkDecodingStat>>(Output{orig, "LinkErrors", 0, Lifetime::Timeframe});
-  auto& decErrors = pc.outputs().make<std::vector<ChipError>>(Output{orig, "ChipErrors", 0, Lifetime::Timeframe});
+  auto& linkErrors = pc.outputs().make<std::vector<GBTLinkDecodingStat>>(Output{orig, "LinkErrors", 0});
+  auto& decErrors = pc.outputs().make<std::vector<ChipError>>(Output{orig, "ChipErrors", 0});
   mDecoder->collectDecodingErrors(linkErrors, decErrors);
 
-  pc.outputs().snapshot(Output{orig, "PHYSTRIG", 0, Lifetime::Timeframe}, mDecoder->getExternalTriggers());
+  pc.outputs().snapshot(Output{orig, "PHYSTRIG", 0}, mDecoder->getExternalTriggers());
 
   if (mDumpOnError != int(GBTLink::RawDataDumps::DUMP_NONE)) {
     mDecoder->produceRawDataDumps(mDumpOnError, pc.services().get<o2::framework::TimingInfo>());
diff --git a/Detectors/MUON/MCH/IO/src/DigitReaderSpec.cxx b/Detectors/MUON/MCH/IO/src/DigitReaderSpec.cxx
index da4c37035c82b..541e6c5fa26c0 100644
--- a/Detectors/MUON/MCH/IO/src/DigitReaderSpec.cxx
+++ b/Detectors/MUON/MCH/IO/src/DigitReaderSpec.cxx
@@ -54,18 +54,18 @@ class DigitsReaderDeviceDPL
       mReader = std::make_unique<RootTreeReader>("o2sim", filename.c_str(), -1,
                                                  RootTreeReader::PublishingMode::Single,
                                                  RootTreeReader::BranchDefinition<std::vector<Digit>>{
-                                                   Output{header::gDataOriginMCH, mDescriptions[0], 0, Lifetime::Timeframe}, "MCHDigit"},
+                                                   Output{header::gDataOriginMCH, mDescriptions[0], 0}, "MCHDigit"},
                                                  RootTreeReader::BranchDefinition<std::vector<ROFRecord>>{
-                                                   Output{header::gDataOriginMCH, mDescriptions[1], 0, Lifetime::Timeframe}, "MCHROFRecords"},
+                                                   Output{header::gDataOriginMCH, mDescriptions[1], 0}, "MCHROFRecords"},
                                                  RootTreeReader::BranchDefinition<dataformats::MCTruthContainer<MCCompLabel>>{
-                                                   Output{header::gDataOriginMCH, mDescriptions[2], 0, Lifetime::Timeframe}, "MCHMCLabels"});
+                                                   Output{header::gDataOriginMCH, mDescriptions[2], 0}, "MCHMCLabels"});
     } else {
       mReader = std::make_unique<RootTreeReader>("o2sim", filename.c_str(), -1,
                                                  RootTreeReader::PublishingMode::Single,
                                                  RootTreeReader::BranchDefinition<std::vector<Digit>>{
-                                                   Output{header::gDataOriginMCH, mDescriptions[0], 0, Lifetime::Timeframe}, "MCHDigit"},
+                                                   Output{header::gDataOriginMCH, mDescriptions[0], 0}, "MCHDigit"},
                                                  RootTreeReader::BranchDefinition<std::vector<ROFRecord>>{
-                                                   Output{header::gDataOriginMCH, mDescriptions[1], 0, Lifetime::Timeframe}, "MCHROFRecords"});
+                                                   Output{header::gDataOriginMCH, mDescriptions[1], 0}, "MCHROFRecords"});
     }
   }
 
diff --git a/Detectors/MUON/MCH/Workflow/src/TrackAtVertexSpec.cxx b/Detectors/MUON/MCH/Workflow/src/TrackAtVertexSpec.cxx
index 080c36706a0eb..8b981d4ee4abd 100644
--- a/Detectors/MUON/MCH/Workflow/src/TrackAtVertexSpec.cxx
+++ b/Detectors/MUON/MCH/Workflow/src/TrackAtVertexSpec.cxx
@@ -139,7 +139,7 @@ class TrackAtVertexTask
     }
 
     // create the output message
-    auto msgOut = pc.outputs().make<char>(Output{"MCH", "TRACKSATVERTEX", 0, Lifetime::Timeframe},
+    auto msgOut = pc.outputs().make<char>(Output{"MCH", "TRACKSATVERTEX", 0},
                                           mTracksAtVtx.size() * sizeof(int) + nTracksTot * sizeof(TrackAtVtxStruct));
 
     // write the tracks
diff --git a/Detectors/MUON/MCH/Workflow/src/entropy-encoder-workflow.cxx b/Detectors/MUON/MCH/Workflow/src/entropy-encoder-workflow.cxx
index 50e085e238d55..058202dfb802b 100644
--- a/Detectors/MUON/MCH/Workflow/src/entropy-encoder-workflow.cxx
+++ b/Detectors/MUON/MCH/Workflow/src/entropy-encoder-workflow.cxx
@@ -72,7 +72,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"MCH", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"MCH", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, rofs, digits);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   mTimer.Stop();
diff --git a/Detectors/MUON/MID/Workflow/src/ClusterizerSpec.cxx b/Detectors/MUON/MID/Workflow/src/ClusterizerSpec.cxx
index 5657fd375f118..c544ce19fcdea 100644
--- a/Detectors/MUON/MID/Workflow/src/ClusterizerSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/ClusterizerSpec.cxx
@@ -99,13 +99,13 @@ class ClusterizerDeviceDPL
       // Clear the index correlations that will be used in the next cluster processing
       mCorrelation.clear();
 
-      pc.outputs().snapshot(of::Output{"MID", "CLUSTERSLABELS", 0, of::Lifetime::Timeframe}, mClusterLabeler.getContainer());
+      pc.outputs().snapshot(of::Output{"MID", "CLUSTERSLABELS", 0}, mClusterLabeler.getContainer());
       LOG(debug) << "Sent " << mClusterLabeler.getContainer().getIndexedSize() << " indexed clusters";
     }
 
-    pc.outputs().snapshot(of::Output{"MID", "CLUSTERS", 0, of::Lifetime::Timeframe}, mClusterizer.getClusters());
+    pc.outputs().snapshot(of::Output{"MID", "CLUSTERS", 0}, mClusterizer.getClusters());
     LOG(debug) << "Sent " << mClusterizer.getClusters().size() << " clusters";
-    pc.outputs().snapshot(of::Output{"MID", "CLUSTERSROF", 0, of::Lifetime::Timeframe}, mClusterizer.getROFRecords());
+    pc.outputs().snapshot(of::Output{"MID", "CLUSTERSROF", 0}, mClusterizer.getROFRecords());
     LOG(debug) << "Sent " << mClusterizer.getROFRecords().size() << " ROF";
 
     mTimer += std::chrono::high_resolution_clock::now() - tStart;
@@ -141,4 +141,4 @@ framework::DataProcessorSpec getClusterizerSpec(bool isMC, std::string_view inDa
     of::AlgorithmSpec{of::adaptFromTask<o2::mid::ClusterizerDeviceDPL>(isMC)}};
 }
 } // namespace mid
-} // namespace o2
\ No newline at end of file
+} // namespace o2
diff --git a/Detectors/MUON/MID/Workflow/src/ColumnDataSpecsUtils.cxx b/Detectors/MUON/MID/Workflow/src/ColumnDataSpecsUtils.cxx
index 98f9b8c08c654..e0d41cd8d91d2 100644
--- a/Detectors/MUON/MID/Workflow/src/ColumnDataSpecsUtils.cxx
+++ b/Detectors/MUON/MID/Workflow/src/ColumnDataSpecsUtils.cxx
@@ -127,7 +127,7 @@ std::vector<framework::Output> buildOutputs(std::vector<framework::OutputSpec> o
   std::vector<framework::Output> outputs;
   for (auto& outSpec : outputSpecs) {
     auto matcher = framework::DataSpecUtils::asConcreteDataMatcher(outSpec);
-    outputs.emplace_back(framework::Output{matcher.origin, matcher.description, matcher.subSpec, framework::Lifetime::Timeframe});
+    outputs.emplace_back(framework::Output{matcher.origin, matcher.description, matcher.subSpec});
   }
   return outputs;
 }
@@ -161,4 +161,4 @@ std::unique_ptr<const o2::dataformats::MCTruthContainer<MCLabel>> getLabels(fram
 
 } // namespace specs
 } // namespace mid
-} // namespace o2
\ No newline at end of file
+} // namespace o2
diff --git a/Detectors/MUON/MID/Workflow/src/DecodedDataAggregatorSpec.cxx b/Detectors/MUON/MID/Workflow/src/DecodedDataAggregatorSpec.cxx
index f03eb6715798a..192b4c52be9cc 100644
--- a/Detectors/MUON/MID/Workflow/src/DecodedDataAggregatorSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/DecodedDataAggregatorSpec.cxx
@@ -60,8 +60,8 @@ class DecodedDataAggregatorDeviceDPL
 
     for (o2::header::DataHeader::SubSpecificationType subSpec = 0; subSpec < 3; ++subSpec) {
       EventType evtType = static_cast<EventType>(subSpec);
-      pc.outputs().snapshot(of::Output{o2::header::gDataOriginMID, "DATA", subSpec, of::Lifetime::Timeframe}, mAggregator.getData(evtType));
-      pc.outputs().snapshot(of::Output{o2::header::gDataOriginMID, "DATAROF", subSpec, of::Lifetime::Timeframe}, mAggregator.getROFRecords(evtType));
+      pc.outputs().snapshot(of::Output{o2::header::gDataOriginMID, "DATA", subSpec}, mAggregator.getData(evtType));
+      pc.outputs().snapshot(of::Output{o2::header::gDataOriginMID, "DATAROF", subSpec}, mAggregator.getROFRecords(evtType));
     }
 
     mTimer += std::chrono::high_resolution_clock::now() - tStart;
diff --git a/Detectors/MUON/MID/Workflow/src/DigitReaderSpec.cxx b/Detectors/MUON/MID/Workflow/src/DigitReaderSpec.cxx
index 3445fad66642e..7f380d1afe13a 100644
--- a/Detectors/MUON/MID/Workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/DigitReaderSpec.cxx
@@ -55,19 +55,19 @@ class DigitsReaderDeviceDPL
       mReader = std::make_unique<RootTreeReader>("o2sim", filename.c_str(), -1,
                                                  RootTreeReader::PublishingMode::Single,
                                                  RootTreeReader::BranchDefinition<std::vector<ColumnData>>{
-                                                   Output{header::gDataOriginMID, mDescriptions[0], 0, Lifetime::Timeframe}, "MIDDigit"},
+                                                   Output{header::gDataOriginMID, mDescriptions[0], 0}, "MIDDigit"},
                                                  RootTreeReader::BranchDefinition<std::vector<ROFRecord>>{
-                                                   Output{header::gDataOriginMID, mDescriptions[1], 0, Lifetime::Timeframe}, "MIDROFRecords"},
+                                                   Output{header::gDataOriginMID, mDescriptions[1], 0}, "MIDROFRecords"},
                                                  RootTreeReader::BranchDefinition<dataformats::MCTruthContainer<MCLabel>>{
-                                                   Output{header::gDataOriginMID, mDescriptions[2], 0, Lifetime::Timeframe}, "MIDDigitMCLabels"},
+                                                   Output{header::gDataOriginMID, mDescriptions[2], 0}, "MIDDigitMCLabels"},
                                                  &mPublishDigits);
     } else {
       mReader = std::make_unique<RootTreeReader>("o2sim", filename.c_str(), -1,
                                                  RootTreeReader::PublishingMode::Single,
                                                  RootTreeReader::BranchDefinition<std::vector<ColumnData>>{
-                                                   Output{header::gDataOriginMID, mDescriptions[0], 0, Lifetime::Timeframe}, "MIDDigit"},
+                                                   Output{header::gDataOriginMID, mDescriptions[0], 0}, "MIDDigit"},
                                                  RootTreeReader::BranchDefinition<std::vector<ROFRecord>>{
-                                                   Output{header::gDataOriginMID, mDescriptions[1], 0, Lifetime::Timeframe}, "MIDROFRecords"},
+                                                   Output{header::gDataOriginMID, mDescriptions[1], 0}, "MIDROFRecords"},
                                                  &mPublishDigits);
     }
   }
diff --git a/Detectors/MUON/MID/Workflow/src/EntropyDecoderSpec.cxx b/Detectors/MUON/MID/Workflow/src/EntropyDecoderSpec.cxx
index e0030eb102c8d..5a8df6f8e81cb 100644
--- a/Detectors/MUON/MID/Workflow/src/EntropyDecoderSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/EntropyDecoderSpec.cxx
@@ -66,9 +66,9 @@ void EntropyDecoderSpec::run(ProcessingContext& pc)
   size_t insize = 0;
   for (uint32_t it = 0; it < NEvTypes; it++) {
     insize += cols[it].size() * sizeof(o2::mid::ColumnData);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginMID, "DATA", it, Lifetime::Timeframe}, cols[it]);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginMID, "DATA", it}, cols[it]);
     insize += rofs[it].size() * sizeof(o2::mid::ROFRecord);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginMID, "DATAROF", it, Lifetime::Timeframe}, rofs[it]);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginMID, "DATAROF", it}, rofs[it]);
   }
   iosize.rawIn = insize;
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
diff --git a/Detectors/MUON/MID/Workflow/src/EntropyEncoderSpec.cxx b/Detectors/MUON/MID/Workflow/src/EntropyEncoderSpec.cxx
index 44bf3dcc0ae6f..a472d6e28ff16 100644
--- a/Detectors/MUON/MID/Workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/EntropyEncoderSpec.cxx
@@ -83,7 +83,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   // build references for looping over the data in BC increasing direction
   tfData.buildReferences(mCTFCoder.getIRFramesSelector());
 
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{header::gDataOriginMID, "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{header::gDataOriginMID, "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, tfData);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   iosize.rawIn = insize;
diff --git a/Detectors/MUON/MID/Workflow/src/MaskMakerSpec.cxx b/Detectors/MUON/MID/Workflow/src/MaskMakerSpec.cxx
index 99acaa08ac5f0..28d2ff953ea23 100644
--- a/Detectors/MUON/MID/Workflow/src/MaskMakerSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/MaskMakerSpec.cxx
@@ -117,7 +117,7 @@ class MaskMakerDeviceDPL
     if (mCounterSinceReset >= mNReset) {
       for (size_t itype = 0; itype < 2; ++itype) {
         auto masks = o2::mid::makeMasks(mScalers[itype], mCounterSinceReset, mThreshold, mRefMasks);
-        pc.outputs().snapshot(of::Output{header::gDataOriginMID, "MASKS", static_cast<header::DataHeader::SubSpecificationType>(itype + 1), of::Lifetime::Timeframe}, masks);
+        pc.outputs().snapshot(of::Output{header::gDataOriginMID, "MASKS", static_cast<header::DataHeader::SubSpecificationType>(itype + 1)}, masks);
       }
       mCounterSinceReset = 0;
       for (auto& scaler : mScalers) {
@@ -160,4 +160,4 @@ framework::DataProcessorSpec getMaskMakerSpec(const FEEIdConfig& feeIdConfig, co
     of::Options{{"mid-mask-threshold", of::VariantType::Double, 0.9, {"Tolerated occupancy before producing a map"}}, {"mid-mask-reset", of::VariantType::Int, 100, {"Number of calibration events to be checked before resetting the scalers"}}}};
 }
 } // namespace mid
-} // namespace o2
\ No newline at end of file
+} // namespace o2
diff --git a/Detectors/MUON/MID/Workflow/src/RawGBTDecoderSpec.cxx b/Detectors/MUON/MID/Workflow/src/RawGBTDecoderSpec.cxx
index 6d23ce9cdab1f..ba15644e2b17a 100644
--- a/Detectors/MUON/MID/Workflow/src/RawGBTDecoderSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/RawGBTDecoderSpec.cxx
@@ -82,8 +82,8 @@ class RawGBTDecoderDeviceDPL
 
     mTimerAlgo += std::chrono::high_resolution_clock::now() - tAlgoStart;
 
-    pc.outputs().snapshot(o2::framework::Output{header::gDataOriginMID, "DECODED", dh->subSpecification, o2::framework::Lifetime::Timeframe}, data);
-    pc.outputs().snapshot(o2::framework::Output{header::gDataOriginMID, "DECODEDROF", dh->subSpecification, o2::framework::Lifetime::Timeframe}, rofRecords);
+    pc.outputs().snapshot(o2::framework::Output{header::gDataOriginMID, "DECODED", dh->subSpecification}, data);
+    pc.outputs().snapshot(o2::framework::Output{header::gDataOriginMID, "DECODEDROF", dh->subSpecification}, rofRecords);
 
     mTimer += std::chrono::high_resolution_clock::now() - tStart;
     mNROFs += rofRecords.size();
diff --git a/Detectors/MUON/MID/Workflow/src/TrackerSpec.cxx b/Detectors/MUON/MID/Workflow/src/TrackerSpec.cxx
index be1dd9124315c..e1044768e8ea6 100644
--- a/Detectors/MUON/MID/Workflow/src/TrackerSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/TrackerSpec.cxx
@@ -80,20 +80,20 @@ class TrackerDeviceDPL
     if (mIsMC) {
       std::unique_ptr<const o2::dataformats::MCTruthContainer<MCClusterLabel>> labels = pc.inputs().get<const o2::dataformats::MCTruthContainer<MCClusterLabel>*>("mid_clusterlabels");
       mTrackLabeler.process(mTracker->getClusters(), tracks, *labels);
-      pc.outputs().snapshot(of::Output{"MID", "TRACKLABELS", 0, of::Lifetime::Timeframe}, mTrackLabeler.getTracksLabels());
+      pc.outputs().snapshot(of::Output{"MID", "TRACKLABELS", 0}, mTrackLabeler.getTracksLabels());
       LOG(debug) << "Sent " << mTrackLabeler.getTracksLabels().size() << " indexed tracks.";
-      pc.outputs().snapshot(of::Output{"MID", "TRCLUSLABELS", 0, of::Lifetime::Timeframe}, mTrackLabeler.getTrackClustersLabels());
+      pc.outputs().snapshot(of::Output{"MID", "TRCLUSLABELS", 0}, mTrackLabeler.getTrackClustersLabels());
       LOG(debug) << "Sent " << mTrackLabeler.getTrackClustersLabels().getIndexedSize() << " indexed track clusters.";
     }
 
-    pc.outputs().snapshot(of::Output{"MID", "TRACKS", 0, of::Lifetime::Timeframe}, tracks);
+    pc.outputs().snapshot(of::Output{"MID", "TRACKS", 0}, tracks);
     LOG(debug) << "Sent " << tracks.size() << " tracks.";
-    pc.outputs().snapshot(of::Output{"MID", "TRACKCLUSTERS", 0, of::Lifetime::Timeframe}, mTracker->getClusters());
+    pc.outputs().snapshot(of::Output{"MID", "TRACKCLUSTERS", 0}, mTracker->getClusters());
     LOG(debug) << "Sent " << mTracker->getClusters().size() << " track clusters.";
 
-    pc.outputs().snapshot(of::Output{"MID", "TRACKROFS", 0, of::Lifetime::Timeframe}, mTracker->getTrackROFRecords());
+    pc.outputs().snapshot(of::Output{"MID", "TRACKROFS", 0}, mTracker->getTrackROFRecords());
     LOG(debug) << "Sent " << mTracker->getTrackROFRecords().size() << " ROFs.";
-    pc.outputs().snapshot(of::Output{"MID", "TRCLUSROFS", 0, of::Lifetime::Timeframe}, mTracker->getClusterROFRecords());
+    pc.outputs().snapshot(of::Output{"MID", "TRCLUSROFS", 0}, mTracker->getClusterROFRecords());
     LOG(debug) << "Sent " << mTracker->getClusterROFRecords().size() << " ROFs.";
 
     mTimer += std::chrono::high_resolution_clock::now() - tStart;
diff --git a/Detectors/MUON/MID/Workflow/src/ZeroSuppressionSpec.cxx b/Detectors/MUON/MID/Workflow/src/ZeroSuppressionSpec.cxx
index ccfe1641b87b6..5d89eee81c629 100644
--- a/Detectors/MUON/MID/Workflow/src/ZeroSuppressionSpec.cxx
+++ b/Detectors/MUON/MID/Workflow/src/ZeroSuppressionSpec.cxx
@@ -90,7 +90,7 @@ class ZeroSuppressionDeviceDPL
     }
 
     if (mUseMC) {
-      pc.outputs().snapshot(of::Output{header::gDataOriginMID, "DATALABELS", 0, of::Lifetime::Timeframe}, outMCContainer);
+      pc.outputs().snapshot(of::Output{header::gDataOriginMID, "DATALABELS", 0}, outMCContainer);
     }
   }
 
@@ -113,4 +113,4 @@ framework::DataProcessorSpec getZeroSuppressionSpec(bool useMC, std::string_view
     of::AlgorithmSpec{of::adaptFromTask<o2::mid::ZeroSuppressionDeviceDPL>(useMC)}};
 }
 } // namespace mid
-} // namespace o2
\ No newline at end of file
+} // namespace o2
diff --git a/Detectors/PHOS/calib/src/PHOSBadMapCalibDevice.cxx b/Detectors/PHOS/calib/src/PHOSBadMapCalibDevice.cxx
index 226630a8b1d81..4f81300431934 100644
--- a/Detectors/PHOS/calib/src/PHOSBadMapCalibDevice.cxx
+++ b/Detectors/PHOS/calib/src/PHOSBadMapCalibDevice.cxx
@@ -177,7 +177,7 @@ void PHOSBadMapCalibDevice::sendOutput(DataAllocator& output)
 
   // Send change to QC
   LOG(info) << "[PHOSBadMapCalibDevice - run] Sending QC ";
-  output.snapshot(o2::framework::Output{"PHS", "BADMAPDIFF", 0, o2::framework::Lifetime::Timeframe}, mBadMapDiff);
+  output.snapshot(o2::framework::Output{"PHS", "BADMAPDIFF", 0}, mBadMapDiff);
 }
 
 bool PHOSBadMapCalibDevice::calculateBadMap()
diff --git a/Detectors/PHOS/calib/src/PHOSHGLGRatioCalibDevice.cxx b/Detectors/PHOS/calib/src/PHOSHGLGRatioCalibDevice.cxx
index 0a37589d3abe7..5c07c6fee8493 100644
--- a/Detectors/PHOS/calib/src/PHOSHGLGRatioCalibDevice.cxx
+++ b/Detectors/PHOS/calib/src/PHOSHGLGRatioCalibDevice.cxx
@@ -229,7 +229,7 @@ void PHOSHGLGRatioCalibDevice::sendOutput(DataAllocator& output)
   }
   // Anyway send change to QC
   LOG(info) << "[PHOSHGLGRatioCalibDevice - sendOutput] Sending QC ";
-  output.snapshot(o2::framework::Output{"PHS", "CALIBDIFF", 0, o2::framework::Lifetime::Timeframe}, mRatioDiff);
+  output.snapshot(o2::framework::Output{"PHS", "CALIBDIFF", 0}, mRatioDiff);
 }
 
 DataProcessorSpec o2::phos::getHGLGRatioCalibSpec(bool useCCDB, bool forceUpdate)
diff --git a/Detectors/PHOS/calib/src/PHOSPedestalCalibDevice.cxx b/Detectors/PHOS/calib/src/PHOSPedestalCalibDevice.cxx
index 96f680f710fb0..42289a910debb 100644
--- a/Detectors/PHOS/calib/src/PHOSPedestalCalibDevice.cxx
+++ b/Detectors/PHOS/calib/src/PHOSPedestalCalibDevice.cxx
@@ -135,7 +135,7 @@ void PHOSPedestalCalibDevice::sendOutput(DataAllocator& output)
   }
   // Anyway send change to QC
   LOG(info) << "[PHOSPedestalCalibDevice - run] Sending QC ";
-  output.snapshot(o2::framework::Output{"PHS", "CALIBDIFF", 0, o2::framework::Lifetime::Timeframe}, mPedDiff);
+  output.snapshot(o2::framework::Output{"PHS", "CALIBDIFF", 0}, mPedDiff);
 }
 
 void PHOSPedestalCalibDevice::calculatePedestals()
diff --git a/Detectors/PHOS/workflow/src/CellConverterSpec.cxx b/Detectors/PHOS/workflow/src/CellConverterSpec.cxx
index 330dd27c45251..370df48a8fc40 100644
--- a/Detectors/PHOS/workflow/src/CellConverterSpec.cxx
+++ b/Detectors/PHOS/workflow/src/CellConverterSpec.cxx
@@ -44,12 +44,12 @@ void CellConverterSpec::run(framework::ProcessingContext& ctx)
   auto digitsTR = ctx.inputs().get<std::vector<o2::phos::TriggerRecord>>("digitTriggerRecords");
   if (!digitsTR.size()) { // nothing to process
     mOutputCells.clear();
-    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLS", 0, o2::framework::Lifetime::Timeframe}, mOutputCells);
+    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLS", 0}, mOutputCells);
     mOutputCellTrigRecs.clear();
-    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", 0, o2::framework::Lifetime::Timeframe}, mOutputCellTrigRecs);
+    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", 0}, mOutputCellTrigRecs);
     if (mPropagateMC) {
       mOutputTruthCont.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLSMCTR", 0, o2::framework::Lifetime::Timeframe}, mOutputTruthCont);
+      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLSMCTR", 0}, mOutputTruthCont);
     }
     return;
   }
@@ -128,10 +128,10 @@ void CellConverterSpec::run(framework::ProcessingContext& ctx)
   }
   LOG(info) << "[PHOSCellConverter - run] Writing " << mOutputCells.size() << " cells, " << mOutputCellTrigRecs.size() << " Trig Records " << mOutputTruthCont.getNElements() << " PHOS labels ";
 
-  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLS", 0, o2::framework::Lifetime::Timeframe}, mOutputCells);
-  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", 0, o2::framework::Lifetime::Timeframe}, mOutputCellTrigRecs);
+  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLS", 0}, mOutputCells);
+  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", 0}, mOutputCellTrigRecs);
   if (mPropagateMC) {
-    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLSMCTR", 0, o2::framework::Lifetime::Timeframe}, mOutputTruthCont);
+    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLSMCTR", 0}, mOutputTruthCont);
   }
 }
 
diff --git a/Detectors/PHOS/workflow/src/CellReaderSpec.cxx b/Detectors/PHOS/workflow/src/CellReaderSpec.cxx
index 64f54c1baf189..c7d93fc20301f 100644
--- a/Detectors/PHOS/workflow/src/CellReaderSpec.cxx
+++ b/Detectors/PHOS/workflow/src/CellReaderSpec.cxx
@@ -44,10 +44,10 @@ void CellReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mCells.size() << " Cells in " << mTRs.size() << " TriggerRecords at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "CELLS", 0, Lifetime::Timeframe}, mCells);
-  pc.outputs().snapshot(Output{mOrigin, "CELLTRIGREC", 0, Lifetime::Timeframe}, mTRs);
+  pc.outputs().snapshot(Output{mOrigin, "CELLS", 0}, mCells);
+  pc.outputs().snapshot(Output{mOrigin, "CELLTRIGREC", 0}, mTRs);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{mOrigin, "CELLSMCTR", 0, Lifetime::Timeframe}, mMCTruth);
+    pc.outputs().snapshot(Output{mOrigin, "CELLSMCTR", 0}, mMCTruth);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/PHOS/workflow/src/ClusterizerSpec.cxx b/Detectors/PHOS/workflow/src/ClusterizerSpec.cxx
index 27bce0f8745f4..13aa21258125d 100644
--- a/Detectors/PHOS/workflow/src/ClusterizerSpec.cxx
+++ b/Detectors/PHOS/workflow/src/ClusterizerSpec.cxx
@@ -78,16 +78,16 @@ void ClusterizerSpec::run(framework::ProcessingContext& ctx)
     auto digitsTR = ctx.inputs().get<std::vector<o2::phos::TriggerRecord>>("digitTriggerRecords");
     if (!digitsTR.size()) { // nothing to process
       mOutputClusters.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERS", 0, o2::framework::Lifetime::Timeframe}, mOutputClusters);
+      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERS", 0}, mOutputClusters);
       if (mFullCluOutput) {
         mOutputCluElements.clear();
-        ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUELEMENTS", 0, o2::framework::Lifetime::Timeframe}, mOutputCluElements);
+        ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUELEMENTS", 0}, mOutputCluElements);
       }
       mOutputClusterTrigRecs.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERTRIGREC", 0, o2::framework::Lifetime::Timeframe}, mOutputClusterTrigRecs);
+      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERTRIGREC", 0}, mOutputClusterTrigRecs);
       if (mPropagateMC) {
         mOutputTruthCont.clear();
-        ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERTRUEMC", 0, o2::framework::Lifetime::Timeframe}, mOutputTruthCont);
+        ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERTRUEMC", 0}, mOutputTruthCont);
       }
       return;
     }
@@ -120,13 +120,13 @@ void ClusterizerSpec::run(framework::ProcessingContext& ctx)
   } else {
     LOG(debug) << "[PHOSClusterizer - run] Writing " << mOutputClusters.size() << " clusters and " << mOutputClusterTrigRecs.size() << " TR";
   }
-  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERS", 0, o2::framework::Lifetime::Timeframe}, mOutputClusters);
+  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERS", 0}, mOutputClusters);
   if (mFullCluOutput) {
-    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUELEMENTS", 0, o2::framework::Lifetime::Timeframe}, mOutputCluElements);
+    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUELEMENTS", 0}, mOutputCluElements);
   }
-  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERTRIGREC", 0, o2::framework::Lifetime::Timeframe}, mOutputClusterTrigRecs);
+  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERTRIGREC", 0}, mOutputClusterTrigRecs);
   if (mPropagateMC) {
-    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERTRUEMC", 0, o2::framework::Lifetime::Timeframe}, mOutputTruthCont);
+    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CLUSTERTRUEMC", 0}, mOutputTruthCont);
   }
 }
 
diff --git a/Detectors/PHOS/workflow/src/DigitReaderSpec.cxx b/Detectors/PHOS/workflow/src/DigitReaderSpec.cxx
index 737856051af0c..70f5077b2f0c9 100644
--- a/Detectors/PHOS/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/PHOS/workflow/src/DigitReaderSpec.cxx
@@ -44,10 +44,10 @@ void DigitReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mDigits.size() << " Digits in " << mTRs.size() << " TriggerRecords at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "DIGITS", 0, Lifetime::Timeframe}, mDigits);
-  pc.outputs().snapshot(Output{mOrigin, "DIGITTRIGREC", 0, Lifetime::Timeframe}, mTRs);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITS", 0}, mDigits);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITTRIGREC", 0}, mTRs);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{mOrigin, "DIGITSMCTR", 0, Lifetime::Timeframe}, mMCTruth);
+    pc.outputs().snapshot(Output{mOrigin, "DIGITSMCTR", 0}, mMCTruth);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/PHOS/workflow/src/EntropyEncoderSpec.cxx b/Detectors/PHOS/workflow/src/EntropyEncoderSpec.cxx
index 226184c1179c6..a932a45f1bb53 100644
--- a/Detectors/PHOS/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/PHOS/workflow/src/EntropyEncoderSpec.cxx
@@ -54,7 +54,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"PHS", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"PHS", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, triggers, cells);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   if (mSelIR) {
diff --git a/Detectors/PHOS/workflow/src/EventBuilderSpec.cxx b/Detectors/PHOS/workflow/src/EventBuilderSpec.cxx
index d6562fdadfee6..9a1289889424e 100644
--- a/Detectors/PHOS/workflow/src/EventBuilderSpec.cxx
+++ b/Detectors/PHOS/workflow/src/EventBuilderSpec.cxx
@@ -178,8 +178,8 @@ void EventBuilderSpec::run(framework::ProcessingContext& ctx)
     }
   }
 
-  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginPHS, "CELLS", 0, framework::Lifetime::Timeframe}, outputCells);
-  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginPHS, "CELLTRIGREC", 0, framework::Lifetime::Timeframe}, outputTriggers);
+  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginPHS, "CELLS", 0}, outputCells);
+  ctx.outputs().snapshot(framework::Output{o2::header::gDataOriginPHS, "CELLTRIGREC", 0}, outputTriggers);
 }
 
 o2::framework::DataProcessorSpec o2::phos::getEventBuilderSpec()
diff --git a/Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx b/Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx
index ea0336f0ab19a..292e083d4bb48 100644
--- a/Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx
+++ b/Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx
@@ -108,11 +108,11 @@ void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
              contDeadBeef == maxWarn ? fmt::format(". {} such inputs in row received, stopping reporting", contDeadBeef) : "");
       }
       mOutputCells.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLS", mflpId, o2::framework::Lifetime::Timeframe}, mOutputCells);
+      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLS", mflpId}, mOutputCells);
       mOutputTriggerRecords.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", mflpId, o2::framework::Lifetime::Timeframe}, mOutputTriggerRecords);
+      ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", mflpId}, mOutputTriggerRecords);
       mOutputHWErrors.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"PHS", "RAWHWERRORS", 0, o2::framework::Lifetime::Timeframe}, mOutputHWErrors);
+      ctx.outputs().snapshot(o2::framework::Output{"PHS", "RAWHWERRORS", 0}, mOutputHWErrors);
       if (mFillChi2) {
         mOutputFitChi.clear();
         ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLFITQA", 0, o2::framework::Lifetime::QA}, mOutputFitChi);
@@ -290,9 +290,9 @@ void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
   mLastSize = 1.1 * mOutputCells.size();
 
   LOG(debug) << "[PHOSRawToCellConverter - run] Writing " << mOutputCells.size() << " cells ...";
-  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLS", mflpId, o2::framework::Lifetime::Timeframe}, mOutputCells);
-  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", mflpId, o2::framework::Lifetime::Timeframe}, mOutputTriggerRecords);
-  ctx.outputs().snapshot(o2::framework::Output{"PHS", "RAWHWERRORS", 0, o2::framework::Lifetime::Timeframe}, mOutputHWErrors);
+  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLS", mflpId}, mOutputCells);
+  ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", mflpId}, mOutputTriggerRecords);
+  ctx.outputs().snapshot(o2::framework::Output{"PHS", "RAWHWERRORS", 0}, mOutputHWErrors);
   if (mFillChi2) {
     ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLFITQA", 0, o2::framework::Lifetime::QA}, mOutputFitChi);
   }
diff --git a/Detectors/PHOS/workflow/src/ReaderSpec.cxx b/Detectors/PHOS/workflow/src/ReaderSpec.cxx
index b68046b47c156..9a1fb23a553df 100644
--- a/Detectors/PHOS/workflow/src/ReaderSpec.cxx
+++ b/Detectors/PHOS/workflow/src/ReaderSpec.cxx
@@ -58,7 +58,6 @@ DataProcessorSpec getDigitsReaderSpec(bool propagateMC)
       processAttributes->terminateOnEod = ic.options().get<bool>("terminate-on-eod");
       processAttributes->finished = false;
       processAttributes->datatype = "PHOSDigit";
-      constexpr auto persistency = Lifetime::Timeframe;
       o2::header::DataHeader::SubSpecificationType subSpec = 0;
       if (propagateMC) {
         processAttributes->reader = std::make_shared<RootTreeReader>(treename.c_str(), // tree name
@@ -66,19 +65,19 @@ DataProcessorSpec getDigitsReaderSpec(bool propagateMC)
                                                                      nofEvents,        // number of entries to publish
                                                                      publishingMode,
                                                                      RootTreeReader::BranchDefinition<std::vector<o2::phos::Digit>>{
-                                                                       Output{"PHS", "DIGITS", subSpec, persistency}, "PHOSDigit"},
+                                                                       Output{"PHS", "DIGITS", subSpec}, "PHOSDigit"},
                                                                      RootTreeReader::BranchDefinition<std::vector<o2::phos::TriggerRecord>>{
-                                                                       Output{"PHS", "DIGITTRIGREC", subSpec, persistency}, "PHOSDigitTrigRecords"},
-                                                                     Output{"PHS", "DIGITSMCTR", subSpec, persistency}, "PHOSDigitMCTruth"); // name of mc label branch
+                                                                       Output{"PHS", "DIGITTRIGREC", subSpec}, "PHOSDigitTrigRecords"},
+                                                                     Output{"PHS", "DIGITSMCTR", subSpec}, "PHOSDigitMCTruth"); // name of mc label branch
       } else {
         processAttributes->reader = std::make_shared<RootTreeReader>(treename.c_str(), // tree name
                                                                      filename.c_str(), // input file name
                                                                      nofEvents,        // number of entries to publish
                                                                      publishingMode,
                                                                      RootTreeReader::BranchDefinition<std::vector<o2::phos::Digit>>{
-                                                                       Output{"PHS", "DIGITS", subSpec, persistency}, "PHOSDigit"},
+                                                                       Output{"PHS", "DIGITS", subSpec}, "PHOSDigit"},
                                                                      RootTreeReader::BranchDefinition<std::vector<o2::phos::TriggerRecord>>{
-                                                                       Output{"PHS", "DIGITTRIGREC", subSpec, persistency}, "PHOSDigitTrigRecords"});
+                                                                       Output{"PHS", "DIGITTRIGREC", subSpec}, "PHOSDigitTrigRecords"});
       }
     }
 
@@ -156,7 +155,6 @@ DataProcessorSpec getCellReaderSpec(bool propagateMC)
       processAttributes->terminateOnEod = ic.options().get<bool>("terminate-on-eod");
       processAttributes->finished = false;
       processAttributes->datatype = "PHOSCell";
-      constexpr auto persistency = Lifetime::Timeframe;
       o2::header::DataHeader::SubSpecificationType subSpec = 0;
       if (propagateMC) {
         processAttributes->reader = std::make_shared<RootTreeReader>(treename.c_str(), // tree name
@@ -164,10 +162,10 @@ DataProcessorSpec getCellReaderSpec(bool propagateMC)
                                                                      nofEvents,        // number of entries to publish
                                                                      publishingMode,
                                                                      RootTreeReader::BranchDefinition<std::vector<o2::phos::Cell>>{
-                                                                       Output{"PHS", "CELLS", subSpec, persistency}, "PHOSCell"},
+                                                                       Output{"PHS", "CELLS", subSpec}, "PHOSCell"},
                                                                      RootTreeReader::BranchDefinition<std::vector<o2::phos::TriggerRecord>>{
-                                                                       Output{"PHS", "CELLTRIGREC", subSpec, persistency}, "PHOSCellTrigRec"},
-                                                                     Output{"PHS", "CELLSMCTR", subSpec, persistency},
+                                                                       Output{"PHS", "CELLTRIGREC", subSpec}, "PHOSCellTrigRec"},
+                                                                     Output{"PHS", "CELLSMCTR", subSpec},
                                                                      "PHOSCellTrueMC"); // name of mc label branch
       } else {
         processAttributes->reader = std::make_shared<RootTreeReader>(treename.c_str(), // tree name
@@ -175,9 +173,9 @@ DataProcessorSpec getCellReaderSpec(bool propagateMC)
                                                                      nofEvents,        // number of entries to publish
                                                                      publishingMode,
                                                                      RootTreeReader::BranchDefinition<std::vector<o2::phos::Cell>>{
-                                                                       Output{"PHS", "CELLS", subSpec, persistency}, "PHOSCell"},
+                                                                       Output{"PHS", "CELLS", subSpec}, "PHOSCell"},
                                                                      RootTreeReader::BranchDefinition<std::vector<o2::phos::TriggerRecord>>{
-                                                                       Output{"PHS", "CELLTRIGREC", subSpec, persistency}, "PHOSCellTrigRec"});
+                                                                       Output{"PHS", "CELLTRIGREC", subSpec}, "PHOSCellTrigRec"});
       }
     }
 
diff --git a/Detectors/PHOS/workflow/src/StandaloneAODProducerSpec.cxx b/Detectors/PHOS/workflow/src/StandaloneAODProducerSpec.cxx
index 736a569cacd34..454be7a5fcb83 100644
--- a/Detectors/PHOS/workflow/src/StandaloneAODProducerSpec.cxx
+++ b/Detectors/PHOS/workflow/src/StandaloneAODProducerSpec.cxx
@@ -136,8 +136,8 @@ void StandaloneAODProducerSpec::run(ProcessingContext& pc)
   } // end of event loop
   // std::cout << "Finished cell loop" << std::endl;
 
-  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0, Lifetime::Timeframe}, tfNumber);
-  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0, Lifetime::Timeframe}, "");
+  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0}, tfNumber);
+  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0}, "");
 
   mTimer.Stop();
 }
diff --git a/Detectors/TOF/calibration/testWorkflow/TOFCalibCollectorSpec.h b/Detectors/TOF/calibration/testWorkflow/TOFCalibCollectorSpec.h
index f42c5676d55e5..4beef9541eb55 100644
--- a/Detectors/TOF/calibration/testWorkflow/TOFCalibCollectorSpec.h
+++ b/Detectors/TOF/calibration/testWorkflow/TOFCalibCollectorSpec.h
@@ -113,8 +113,8 @@ class TOFCalibCollectorDevice : public o2::framework::Task
       auto entries = collectedInfo.size();
       // this means that we are ready to send the output
       auto entriesPerChannel = mCollector->getEntriesPerChannel();
-      output.snapshot(Output{o2::header::gDataOriginTOF, "COLLECTEDINFO", 0, Lifetime::Timeframe}, collectedInfo);
-      output.snapshot(Output{o2::header::gDataOriginTOF, "ENTRIESCH", 0, Lifetime::Timeframe}, entriesPerChannel);
+      output.snapshot(Output{o2::header::gDataOriginTOF, "COLLECTEDINFO", 0}, collectedInfo);
+      output.snapshot(Output{o2::header::gDataOriginTOF, "ENTRIESCH", 0}, entriesPerChannel);
       mCollector->initOutput(); // reset the output for the next round
     }
   }
diff --git a/Detectors/TOF/workflow/src/CompressedDecodingTask.cxx b/Detectors/TOF/workflow/src/CompressedDecodingTask.cxx
index ebc61f05e85c0..fc0e2b9b2c30d 100644
--- a/Detectors/TOF/workflow/src/CompressedDecodingTask.cxx
+++ b/Detectors/TOF/workflow/src/CompressedDecodingTask.cxx
@@ -109,18 +109,18 @@ void CompressedDecodingTask::postData(ProcessingContext& pc)
   // LOG(info) << "TOF: N tof window decoded = " << n_tof_window << "(orbits = " << n_orbits << ") with " << digit_size << " digits";
 
   // add digits in the output snapshot
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITS", 0, Lifetime::Timeframe}, *alldigits);
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "READOUTWINDOW", 0, Lifetime::Timeframe}, *row);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITS", 0}, *alldigits);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "READOUTWINDOW", 0}, *row);
 
   std::vector<uint8_t>& patterns = mDecoder.getPatterns();
 
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "PATTERNS", 0, Lifetime::Timeframe}, patterns);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "PATTERNS", 0}, patterns);
 
   std::vector<uint64_t>& errors = mDecoder.getErrors();
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "ERRORS", 0, Lifetime::Timeframe}, errors);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "ERRORS", 0}, errors);
 
   DigitHeader& digitH = mDecoder.getDigitHeader();
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITHEADER", 0, Lifetime::Timeframe}, digitH);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITHEADER", 0}, digitH);
 
   auto diagnosticFrequency = mDecoder.getDiagnosticFrequency();
   diagnosticFrequency.setTimeStamp(mCreationTime / 1000);
@@ -130,7 +130,7 @@ void CompressedDecodingTask::postData(ProcessingContext& pc)
   diagnosticFrequency.setTFIDInfo(tfinfo);
 
   //diagnosticFrequency.print();
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIAFREQ", 0, Lifetime::Timeframe}, diagnosticFrequency);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIAFREQ", 0}, diagnosticFrequency);
 
   mDecoder.clear();
 
diff --git a/Detectors/TOF/workflow/src/EntropyDecoderSpec.cxx b/Detectors/TOF/workflow/src/EntropyDecoderSpec.cxx
index b2c735af87241..400914c64021f 100644
--- a/Detectors/TOF/workflow/src/EntropyDecoderSpec.cxx
+++ b/Detectors/TOF/workflow/src/EntropyDecoderSpec.cxx
@@ -81,7 +81,7 @@ void EntropyDecoderSpec::run(ProcessingContext& pc)
   o2::dataformats::TFIDInfo tfinfo;
   o2::base::TFIDInfoHelper::fillTFIDInfo(pc, tfinfo);
   diagnostic.setTFIDInfo(tfinfo);
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIAFREQ", 0, Lifetime::Timeframe}, diagnostic);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIAFREQ", 0}, diagnostic);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   mTimer.Stop();
   LOG(info) << "Decoded " << digits.size() << " digits in " << row.size() << " ROF, (" << iosize.asString() << ") in " << mTimer.CpuTime() - cput << " s";
diff --git a/Detectors/TOF/workflow/src/EntropyEncoderSpec.cxx b/Detectors/TOF/workflow/src/EntropyEncoderSpec.cxx
index 8f215a0572961..3fc47955f53c0 100644
--- a/Detectors/TOF/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/TOF/workflow/src/EntropyEncoderSpec.cxx
@@ -55,7 +55,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{o2::header::gDataOriginTOF, "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{o2::header::gDataOriginTOF, "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, rofs, compDigits, pspan);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   if (mSelIR) {
diff --git a/Detectors/TOF/workflow/src/TOFClusterizerSpec.cxx b/Detectors/TOF/workflow/src/TOFClusterizerSpec.cxx
index 4d3e0589c186d..48b1508692c24 100644
--- a/Detectors/TOF/workflow/src/TOFClusterizerSpec.cxx
+++ b/Detectors/TOF/workflow/src/TOFClusterizerSpec.cxx
@@ -224,25 +224,25 @@ class TOFDPLClustererTask
     }
 
     // send clusters
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERS", 0, Lifetime::Timeframe}, mClustersArray);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERSMULT", 0, Lifetime::Timeframe}, mMultPerLongBC);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERS", 0}, mClustersArray);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERSMULT", 0}, mMultPerLongBC);
     // send labels
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERSMCTR", 0, Lifetime::Timeframe}, mClsLabels);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERSMCTR", 0}, mClsLabels);
     }
 
     if (mIsCalib) {
       std::vector<CalibInfoCluster>* clusterCalInfo = mClusterer.getInfoFromCluster();
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOCALCLUS", 0, Lifetime::Timeframe}, *clusterCalInfo);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOCALCLUS", 0}, *clusterCalInfo);
     }
 
     if (mIsCosmic) {
       std::vector<CosmicInfo>* cosmicInfo = mCosmicProcessor.getCosmicInfo();
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOCOSMICS", 0, Lifetime::Timeframe}, *cosmicInfo);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOCOSMICS", 0}, *cosmicInfo);
       std::vector<CalibInfoTrackCl>* cosmicTrack = mCosmicProcessor.getCosmicTrack();
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOTRACKCOS", 0, Lifetime::Timeframe}, *cosmicTrack);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOTRACKCOS", 0}, *cosmicTrack);
       std::vector<int>* cosmicTrackSize = mCosmicProcessor.getCosmicTrackSize();
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOTRACKSIZE", 0, Lifetime::Timeframe}, *cosmicTrackSize);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOTRACKSIZE", 0}, *cosmicTrackSize);
     }
 
     mTimer.Stop();
diff --git a/Detectors/TOF/workflowIO/src/CalibClusReaderSpec.cxx b/Detectors/TOF/workflowIO/src/CalibClusReaderSpec.cxx
index 9be38858250ca..116f93a06c208 100644
--- a/Detectors/TOF/workflowIO/src/CalibClusReaderSpec.cxx
+++ b/Detectors/TOF/workflowIO/src/CalibClusReaderSpec.cxx
@@ -39,13 +39,13 @@ void CalibClusReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(debug) << "Pushing " << mPclusInfos->size() << " TOF clusters calib info at entry " << ent;
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOCALCLUS", 0, Lifetime::Timeframe}, mClusInfos);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOCALCLUS", 0}, mClusInfos);
 
   if (mIsCosmics) {
     LOG(debug) << "Pushing " << mPcosmicInfo->size() << " TOF cosmics info at entry " << ent;
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOCOSMICS", 0, Lifetime::Timeframe}, mCosmicInfo);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOTRACKCOS", 0, Lifetime::Timeframe}, mCosmicTrack);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOTRACKSIZE", 0, Lifetime::Timeframe}, mCosmicTrackSize);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOCOSMICS", 0}, mCosmicInfo);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOTRACKCOS", 0}, mCosmicTrack);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "INFOTRACKSIZE", 0}, mCosmicTrackSize);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/TOF/workflowIO/src/CalibInfoReaderSpec.cxx b/Detectors/TOF/workflowIO/src/CalibInfoReaderSpec.cxx
index d81905fb38888..1854be2f1b88f 100644
--- a/Detectors/TOF/workflowIO/src/CalibInfoReaderSpec.cxx
+++ b/Detectors/TOF/workflowIO/src/CalibInfoReaderSpec.cxx
@@ -90,9 +90,9 @@ void CalibInfoReader::run(ProcessingContext& pc)
       LOG(debug) << "Current entry " << mCurrentEntry;
       LOG(debug) << "Send " << mVect.size() << " calib infos";
 
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, mTOFTPC ? ddCalib_tpc : ddCalib, 0, Lifetime::Timeframe}, mVect);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, mTOFTPC ? ddCalib_tpc : ddCalib, 0}, mVect);
 
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, ddDia, 0, Lifetime::Timeframe}, mDia);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, ddDia, 0}, mDia);
       usleep(100);
     }
     mGlobalEntry++;
diff --git a/Detectors/TOF/workflowIO/src/ClusterReaderSpec.cxx b/Detectors/TOF/workflowIO/src/ClusterReaderSpec.cxx
index 70f10b109a70e..e2979a8fc0dbf 100644
--- a/Detectors/TOF/workflowIO/src/ClusterReaderSpec.cxx
+++ b/Detectors/TOF/workflowIO/src/ClusterReaderSpec.cxx
@@ -44,10 +44,10 @@ void ClusterReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(debug) << "Pushing " << mClustersPtr->size() << " TOF clusters at entry " << ent;
 
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERS", 0, Lifetime::Timeframe}, mClusters);
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERSMULT", 0, Lifetime::Timeframe}, mClustersMult);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERS", 0}, mClusters);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERSMULT", 0}, mClustersMult);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERSMCTR", 0, Lifetime::Timeframe}, mLabels);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CLUSTERSMCTR", 0}, mLabels);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/TOF/workflowIO/src/DigitReaderSpec.cxx b/Detectors/TOF/workflowIO/src/DigitReaderSpec.cxx
index f9fd57cf6b58f..30ddea9d5b9e7 100644
--- a/Detectors/TOF/workflowIO/src/DigitReaderSpec.cxx
+++ b/Detectors/TOF/workflowIO/src/DigitReaderSpec.cxx
@@ -81,18 +81,18 @@ void DigitReader::run(ProcessingContext& pc)
     mDiagnostic = mFiller.getDiagnosticFrequency();
 
     // add digits loaded in the output snapshot
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITS", 0, Lifetime::Timeframe}, mDigits);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "READOUTWINDOW", 0, Lifetime::Timeframe}, mRow);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "PATTERNS", 0, Lifetime::Timeframe}, mPatterns);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIAFREQ", 0, Lifetime::Timeframe}, mDiagnostic);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITS", 0}, mDigits);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "READOUTWINDOW", 0}, mRow);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "PATTERNS", 0}, mPatterns);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIAFREQ", 0}, mDiagnostic);
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITSMCTR", 0, Lifetime::Timeframe}, mLabels);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITSMCTR", 0}, mLabels);
     }
 
     static o2::parameters::GRPObject::ROMode roMode = o2::parameters::GRPObject::CONTINUOUS;
 
     LOG(debug) << "TOF: Sending ROMode= " << roMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "ROMode", 0, Lifetime::Timeframe}, roMode);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "ROMode", 0}, roMode);
   } else {
     LOG(error) << "Cannot read the TOF digits !";
     return;
diff --git a/Detectors/TOF/workflowIO/src/TOFMatchedReaderSpec.cxx b/Detectors/TOF/workflowIO/src/TOFMatchedReaderSpec.cxx
index 43264eb70cb58..48ccb83ca3cc3 100644
--- a/Detectors/TOF/workflowIO/src/TOFMatchedReaderSpec.cxx
+++ b/Detectors/TOF/workflowIO/src/TOFMatchedReaderSpec.cxx
@@ -74,12 +74,12 @@ void TOFMatchedReader::run(ProcessingContext& pc)
   LOG(debug) << "Pushing " << mMatches.size() << " TOF matchings at entry " << currEntry;
 
   uint32_t tpcMatchSS = o2::globaltracking::getSubSpec(mSubSpecStrict && (!mMode) ? o2::globaltracking::MatchingType::Strict : o2::globaltracking::MatchingType::Standard);
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, ddMatchInfo[mMode], tpcMatchSS, Lifetime::Timeframe}, mMatches);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, ddMatchInfo[mMode], tpcMatchSS}, mMatches);
   if (mReadTracks && (!mMode)) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "TOFTRACKS_TPC", tpcMatchSS, Lifetime::Timeframe}, mTracks);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "TOFTRACKS_TPC", tpcMatchSS}, mTracks);
   }
   if (mUseMC) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, ddMCMatchTOF[mMode], tpcMatchSS, Lifetime::Timeframe}, mLabelTOF);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, ddMCMatchTOF[mMode], tpcMatchSS}, mLabelTOF);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFLPIDCSpec.h b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFLPIDCSpec.h
index 6316824966856..02857398f0e07 100644
--- a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFLPIDCSpec.h
+++ b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFLPIDCSpec.h
@@ -195,14 +195,14 @@ class TPCFLPIDCDevice : public o2::framework::Task
 
     fill1DIDCs(cru);
     LOGP(debug, "Sending 1D-IDCs to EPNs of size {} and weights of size {}", mOneDIDCs.first.size(), mOneDIDCs.second.size());
-    output.snapshot(Output{gDataOriginTPC, getDataDescription1DIDCEPN(), subSpec, Lifetime::Timeframe}, mOneDIDCs.first);
-    output.snapshot(Output{gDataOriginTPC, getDataDescription1DIDCEPNWeights(), subSpec, Lifetime::Timeframe}, mOneDIDCs.second);
+    output.snapshot(Output{gDataOriginTPC, getDataDescription1DIDCEPN(), subSpec}, mOneDIDCs.first);
+    output.snapshot(Output{gDataOriginTPC, getDataDescription1DIDCEPNWeights(), subSpec}, mOneDIDCs.second);
   }
 
   void sendOutput(DataAllocator& output, const uint32_t cru)
   {
     const header::DataHeader::SubSpecificationType subSpec{cru << 7};
-    output.adoptContainer(Output{gDataOriginTPC, getDataDescriptionIDCGroup(CRU(cru).side()), subSpec, Lifetime::Timeframe}, std::move(mIDCs[cru]));
+    output.adoptContainer(Output{gDataOriginTPC, getDataDescriptionIDCGroup(CRU(cru).side()), subSpec}, std::move(mIDCs[cru]));
   }
 
   void fill1DIDCs(const uint32_t cru)
diff --git a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformEPNSpec.h b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformEPNSpec.h
index 7fd4c923feb19..e52234b3bb9ef 100644
--- a/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformEPNSpec.h
+++ b/Detectors/TPC/workflow/include/TPCWorkflow/TPCFourierTransformEPNSpec.h
@@ -120,7 +120,7 @@ class TPCFourierTransformEPNSpec : public o2::framework::Task
 
   void sendOutput(DataAllocator& output, const Side side)
   {
-    output.snapshot(Output{gDataOriginTPC, getDataDescription(), header::DataHeader::SubSpecificationType{side}, Lifetime::Timeframe}, mIDCFourierTransform.getFourierCoefficients().getFourierCoefficients());
+    output.snapshot(Output{gDataOriginTPC, getDataDescription(), header::DataHeader::SubSpecificationType{side}}, mIDCFourierTransform.getFourierCoefficients().getFourierCoefficients());
   }
 };
 
diff --git a/Detectors/TPC/workflow/include/TPCWorkflow/TPCIntegrateIDCSpec.h b/Detectors/TPC/workflow/include/TPCWorkflow/TPCIntegrateIDCSpec.h
index 99d64220a2238..3ce0553f8dd7f 100644
--- a/Detectors/TPC/workflow/include/TPCWorkflow/TPCIntegrateIDCSpec.h
+++ b/Detectors/TPC/workflow/include/TPCWorkflow/TPCIntegrateIDCSpec.h
@@ -92,14 +92,14 @@ class TPCIntegrateIDCDevice : public o2::framework::Task
     for (const auto& idcs : mIDCs[sector].get()) {
       const header::DataHeader::SubSpecificationType subSpec{cru << 7};
       if (mIDCFormat == IDCFormat::Sim) {
-        output.snapshot(Output{gDataOriginTPC, getDataDescription(mIDCFormat), subSpec, Lifetime::Timeframe}, idcs);
+        output.snapshot(Output{gDataOriginTPC, getDataDescription(mIDCFormat), subSpec}, idcs);
       } else {
         // TODO
         // convert to format from thorsten here
         // send.......
         // DUMMY FOR NOW
         // const TPCCRUHeader cruheader{cru, mIntegrationIntervalsPerTF};
-        output.snapshot(Output{gDataOriginTPC, getDataDescription(mIDCFormat), subSpec, Lifetime::Timeframe}, idcs);
+        output.snapshot(Output{gDataOriginTPC, getDataDescription(mIDCFormat), subSpec}, idcs);
       }
       ++cru;
     }
diff --git a/Detectors/TPC/workflow/readers/include/TPCReaderWorkflow/PublisherSpec.h b/Detectors/TPC/workflow/readers/include/TPCReaderWorkflow/PublisherSpec.h
index 531bf635a1e5b..d73f80adf8b4b 100644
--- a/Detectors/TPC/workflow/readers/include/TPCReaderWorkflow/PublisherSpec.h
+++ b/Detectors/TPC/workflow/readers/include/TPCReaderWorkflow/PublisherSpec.h
@@ -62,22 +62,21 @@ framework::DataProcessorSpec getPublisherSpec(PublisherConf const& config, bool
 
   // a creator callback for the actual reader instance
   auto creator = [dto, mco, propagateMC](const char* treename, const char* filename, int nofEvents, Reader::PublishingMode publishingMode, o2::header::DataHeader::SubSpecificationType subSpec, const char* branchname, const char* mcbranchname, Reader::SpecialPublishHook* publishhook = nullptr) {
-    constexpr auto persistency = o2::framework::Lifetime::Timeframe;
     if (propagateMC) {
       return std::make_shared<Reader>(treename,
                                       filename,
                                       nofEvents,
                                       publishingMode,
-                                      Output{mco.origin, mco.description, subSpec, persistency},
+                                      Output{mco.origin, mco.description, subSpec},
                                       mcbranchname,
-                                      Reader::BranchDefinition<T>{Output{dto.origin, dto.description, subSpec, persistency}, branchname},
+                                      Reader::BranchDefinition<T>{Output{dto.origin, dto.description, subSpec}, branchname},
                                       publishhook);
     } else {
       return std::make_shared<Reader>(treename,
                                       filename,
                                       nofEvents,
                                       publishingMode,
-                                      Reader::BranchDefinition<T>{Output{dto.origin, dto.description, subSpec, persistency}, branchname},
+                                      Reader::BranchDefinition<T>{Output{dto.origin, dto.description, subSpec}, branchname},
                                       publishhook);
     }
   };
diff --git a/Detectors/TPC/workflow/readers/src/PublisherSpec.cxx b/Detectors/TPC/workflow/readers/src/PublisherSpec.cxx
index e02a78b3dca3a..7abd744440517 100644
--- a/Detectors/TPC/workflow/readers/src/PublisherSpec.cxx
+++ b/Detectors/TPC/workflow/readers/src/PublisherSpec.cxx
@@ -204,9 +204,9 @@ DataProcessorSpec createPublisherSpec(PublisherConf const& config, bool propagat
         header.sectorBits = 0;
         header.activeSectors = processAttributes->activeSectors;
         for (auto const& subSpec : processAttributes->zeroLengthOutputs) {
-          pc.outputs().make<char>({dto.origin, dto.description, subSpec, Lifetime::Timeframe, {header}});
+          pc.outputs().make<char>({dto.origin, dto.description, subSpec, {header}});
           if (pc.outputs().isAllowed({mco.origin, mco.description, subSpec})) {
-            pc.outputs().make<char>({mco.origin, mco.description, subSpec, Lifetime::Timeframe, {header}});
+            pc.outputs().make<char>({mco.origin, mco.description, subSpec, {header}});
           }
         }
       }
diff --git a/Detectors/TPC/workflow/readers/src/TrackReaderSpec.cxx b/Detectors/TPC/workflow/readers/src/TrackReaderSpec.cxx
index 66575dd843458..bfe1644bd8b5d 100644
--- a/Detectors/TPC/workflow/readers/src/TrackReaderSpec.cxx
+++ b/Detectors/TPC/workflow/readers/src/TrackReaderSpec.cxx
@@ -68,10 +68,10 @@ void TrackReader::run(ProcessingContext& pc)
     }
   }
 
-  pc.outputs().snapshot(Output{"TPC", "TRACKS", 0, Lifetime::Timeframe}, mTracksOut);
-  pc.outputs().snapshot(Output{"TPC", "CLUSREFS", 0, Lifetime::Timeframe}, mCluRefVecOut);
+  pc.outputs().snapshot(Output{"TPC", "TRACKS", 0}, mTracksOut);
+  pc.outputs().snapshot(Output{"TPC", "CLUSREFS", 0}, mCluRefVecOut);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"TPC", "TRACKSMCLBL", 0, Lifetime::Timeframe}, mMCTruthOut);
+    pc.outputs().snapshot(Output{"TPC", "TRACKSMCLBL", 0}, mMCTruthOut);
   }
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/TPC/workflow/readers/src/TriggerReaderSpec.cxx b/Detectors/TPC/workflow/readers/src/TriggerReaderSpec.cxx
index 2c363f4f7ed93..5dd3c16b3b1a8 100644
--- a/Detectors/TPC/workflow/readers/src/TriggerReaderSpec.cxx
+++ b/Detectors/TPC/workflow/readers/src/TriggerReaderSpec.cxx
@@ -36,7 +36,7 @@ void TriggerReader::run(ProcessingContext& pc)
   auto ent = mTree->GetReadEntry() + 1;
   mTree->GetEntry(ent);
 
-  pc.outputs().snapshot(Output{"TPC", "TRIGGERWORDS", 0, Lifetime::Timeframe}, *mTrig);
+  pc.outputs().snapshot(Output{"TPC", "TRIGGERWORDS", 0}, *mTrig);
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Detectors/TPC/workflow/src/ChunkedDigitPublisher.cxx b/Detectors/TPC/workflow/src/ChunkedDigitPublisher.cxx
index cf11e3c7b39ce..adf0cba944c03 100644
--- a/Detectors/TPC/workflow/src/ChunkedDigitPublisher.cxx
+++ b/Detectors/TPC/workflow/src/ChunkedDigitPublisher.cxx
@@ -111,8 +111,7 @@ auto makePublishBuffer(framework::ProcessingContext& pc, int sector, uint64_t ac
 
   o2::tpc::TPCSectorHeader header{sector};
   header.activeSectors = activeSectors;
-  return &pc.outputs().make<T>(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(sector), Lifetime::Timeframe,
-                                      header});
+  return &pc.outputs().make<T>(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(sector), header});
 }
 
 template <>
@@ -138,7 +137,7 @@ void publishBuffer<MCTruthContainer>(framework::ProcessingContext& pc, int secto
   LabelType* sharedlabels;
 #pragma omp critical
   sharedlabels = &pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(
-    Output{"TPC", "DIGITSMCTR", static_cast<SubSpecificationType>(sector), Lifetime::Timeframe, header});
+    Output{"TPC", "DIGITSMCTR", static_cast<SubSpecificationType>(sector), header});
 
   accum->flatten_to(*sharedlabels);
   delete accum;
diff --git a/Detectors/TPC/workflow/src/ClusterDecoderRawSpec.cxx b/Detectors/TPC/workflow/src/ClusterDecoderRawSpec.cxx
index bb20bf86d531a..9e2e70e8dde46 100644
--- a/Detectors/TPC/workflow/src/ClusterDecoderRawSpec.cxx
+++ b/Detectors/TPC/workflow/src/ClusterDecoderRawSpec.cxx
@@ -88,7 +88,7 @@ DataProcessorSpec getClusterDecoderRawSpec(bool sendMC)
           o2::header::Stack actual{*sectorHeaderMC};
           std::swap(mcHeaderStack, actual);
           if (sectorHeaderMC->sector() < 0) {
-            pc.outputs().snapshot(Output{gDataOriginTPC, DataDescription("CLNATIVEMCLBL"), fanSpec, Lifetime::Timeframe, std::move(mcHeaderStack)}, fanSpec);
+            pc.outputs().snapshot(Output{gDataOriginTPC, DataDescription("CLNATIVEMCLBL"), fanSpec, std::move(mcHeaderStack)}, fanSpec);
           }
         }
       }
@@ -97,7 +97,7 @@ DataProcessorSpec getClusterDecoderRawSpec(bool sendMC)
         o2::header::Stack actual{*sectorHeader};
         std::swap(rawHeaderStack, actual);
         if (sectorHeader->sector() < 0) {
-          pc.outputs().snapshot(Output{gDataOriginTPC, DataDescription("CLUSTERNATIVE"), fanSpec, Lifetime::Timeframe, std::move(rawHeaderStack)}, fanSpec);
+          pc.outputs().snapshot(Output{gDataOriginTPC, DataDescription("CLUSTERNATIVE"), fanSpec, std::move(rawHeaderStack)}, fanSpec);
           return;
         }
       }
@@ -167,7 +167,7 @@ DataProcessorSpec getClusterDecoderRawSpec(bool sendMC)
       // containers are created for clusters and MC labels per (sector,globalPadRow) address
       char* outputBuffer = nullptr;
       auto outputAllocator = [&pc, &fanSpec, &outputBuffer, &rawHeaderStack](size_t size) -> char* {
-        outputBuffer = pc.outputs().newChunk(Output{gDataOriginTPC, DataDescription("CLUSTERNATIVE"), fanSpec, Lifetime::Timeframe, std::move(rawHeaderStack)}, size).data();
+        outputBuffer = pc.outputs().newChunk(Output{gDataOriginTPC, DataDescription("CLUSTERNATIVE"), fanSpec, std::move(rawHeaderStack)}, size).data();
         return outputBuffer;
       };
       MCLabelContainer mcout;
@@ -188,7 +188,7 @@ DataProcessorSpec getClusterDecoderRawSpec(bool sendMC)
         // serialize the complete list of MC label containers
         ConstMCLabelContainer labelsFlat;
         mcout.flatten_to(labelsFlat);
-        pc.outputs().snapshot(Output{gDataOriginTPC, DataDescription("CLNATIVEMCLBL"), fanSpec, Lifetime::Timeframe, std::move(mcHeaderStack)}, labelsFlat);
+        pc.outputs().snapshot(Output{gDataOriginTPC, DataDescription("CLNATIVEMCLBL"), fanSpec, std::move(mcHeaderStack)}, labelsFlat);
       }
     };
 
diff --git a/Detectors/TPC/workflow/src/ClustererSpec.cxx b/Detectors/TPC/workflow/src/ClustererSpec.cxx
index 71043db3935be..310551a67753e 100644
--- a/Detectors/TPC/workflow/src/ClustererSpec.cxx
+++ b/Detectors/TPC/workflow/src/ClustererSpec.cxx
@@ -81,9 +81,9 @@ DataProcessorSpec getClustererSpec(bool sendMC)
         // forward the control information
         // FIXME define and use flags in TPCSectorHeader
         o2::tpc::TPCSectorHeader header{sector};
-        pc.outputs().snapshot(Output{gDataOriginTPC, "CLUSTERHW", fanSpec, Lifetime::Timeframe, {header}}, fanSpec);
+        pc.outputs().snapshot(Output{gDataOriginTPC, "CLUSTERHW", fanSpec, {header}}, fanSpec);
         if (DataRefUtils::isValid(mclabelref)) {
-          pc.outputs().snapshot(Output{gDataOriginTPC, "CLUSTERHWMCLBL", fanSpec, Lifetime::Timeframe, {header}}, fanSpec);
+          pc.outputs().snapshot(Output{gDataOriginTPC, "CLUSTERHWMCLBL", fanSpec, {header}}, fanSpec);
         }
         return;
       }
@@ -131,12 +131,12 @@ DataProcessorSpec getClustererSpec(bool sendMC)
       }
       // FIXME: that should be a case for pmr, want to send the content of the vector as a binary
       // block by using move semantics
-      auto outputPages = pc.outputs().make<ClusterHardwareContainer8kb>(Output{gDataOriginTPC, "CLUSTERHW", fanSpec, Lifetime::Timeframe, {*sectorHeader}}, clusterArray.size());
+      auto outputPages = pc.outputs().make<ClusterHardwareContainer8kb>(Output{gDataOriginTPC, "CLUSTERHW", fanSpec, {*sectorHeader}}, clusterArray.size());
       std::copy(clusterArray.begin(), clusterArray.end(), outputPages.begin());
       if (DataRefUtils::isValid(mclabelref)) {
         ConstMCLabelContainer mcflat;
         mctruthArray.flatten_to(mcflat);
-        pc.outputs().snapshot(Output{gDataOriginTPC, "CLUSTERHWMCLBL", fanSpec, Lifetime::Timeframe, {*sectorHeader}}, mcflat);
+        pc.outputs().snapshot(Output{gDataOriginTPC, "CLUSTERHWMCLBL", fanSpec, {*sectorHeader}}, mcflat);
       }
     };
 
diff --git a/Detectors/TPC/workflow/src/EntropyEncoderSpec.cxx b/Detectors/TPC/workflow/src/EntropyEncoderSpec.cxx
index 7e03748da0bc4..75fedb4a0effe 100644
--- a/Detectors/TPC/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/TPC/workflow/src/EntropyEncoderSpec.cxx
@@ -133,7 +133,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   auto triggers = pc.inputs().get<gsl::span<o2::tpc::TriggerInfoDLBZS>>("trigger");
   auto cput = mTimer.CpuTime();
   mTimer.Start(false);
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"TPC", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"TPC", "CTFDATA", 0});
   std::vector<bool> rejectHits, rejectTracks, rejectTrackHits, rejectTrackHitsReduced;
   CompressedClusters clustersFiltered = clusters;
   std::vector<std::pair<std::vector<unsigned int>, std::vector<unsigned short>>> tmpBuffer(std::max<int>(mNThreads, 1));
diff --git a/Detectors/TPC/workflow/src/KryptonClustererSpec.cxx b/Detectors/TPC/workflow/src/KryptonClustererSpec.cxx
index d0119f719193c..e946488073bf9 100644
--- a/Detectors/TPC/workflow/src/KryptonClustererSpec.cxx
+++ b/Detectors/TPC/workflow/src/KryptonClustererSpec.cxx
@@ -75,7 +75,7 @@ class KrBoxClusterFinderDevice : public o2::framework::Task
   {
     o2::tpc::TPCSectorHeader header{sector};
     header.activeSectors = (0x1 << sector);
-    output.snapshot(Output{gDataOriginTPC, "KRCLUSTERS", static_cast<SubSpecificationType>(sector), Lifetime::Timeframe, header}, clusters);
+    output.snapshot(Output{gDataOriginTPC, "KRCLUSTERS", static_cast<SubSpecificationType>(sector), header}, clusters);
   }
 };
 
diff --git a/Detectors/TPC/workflow/src/KryptonRawFilterSpec.cxx b/Detectors/TPC/workflow/src/KryptonRawFilterSpec.cxx
index 36dfa5edc65f7..d968876eefccb 100644
--- a/Detectors/TPC/workflow/src/KryptonRawFilterSpec.cxx
+++ b/Detectors/TPC/workflow/src/KryptonRawFilterSpec.cxx
@@ -223,7 +223,7 @@ class KrRawFilterDevice : public o2::framework::Task
   {
     o2::tpc::TPCSectorHeader header{sector};
     header.activeSectors = (0x1 << sector);
-    output.snapshot(Output{gDataOriginTPC, "FILTERDIG", static_cast<SubSpecificationType>(sector), Lifetime::Timeframe, header}, digits);
+    output.snapshot(Output{gDataOriginTPC, "FILTERDIG", static_cast<SubSpecificationType>(sector), header}, digits);
   }
 };
 
diff --git a/Detectors/TPC/workflow/src/LinkZSToDigitsSpec.cxx b/Detectors/TPC/workflow/src/LinkZSToDigitsSpec.cxx
index c7758d9f2bb4e..c42a93f7a7f04 100644
--- a/Detectors/TPC/workflow/src/LinkZSToDigitsSpec.cxx
+++ b/Detectors/TPC/workflow/src/LinkZSToDigitsSpec.cxx
@@ -110,7 +110,7 @@ o2::framework::DataProcessorSpec getLinkZSToDigitsSpec(int channel, const std::s
         header.activeSectors = processAttributes->activeSectors;
         // digit for now are transported per sector, not per lane
         // pc.outputs().snapshot(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(channel), Lifetime::Timeframe, header},
-        pc.outputs().snapshot(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(sector), Lifetime::Timeframe, header},
+        pc.outputs().snapshot(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(sector), header},
                               const_cast<std::vector<o2::tpc::Digit>&>(digits));
       };
 
diff --git a/Detectors/TPC/workflow/src/MIPTrackFilterSpec.cxx b/Detectors/TPC/workflow/src/MIPTrackFilterSpec.cxx
index d4291552ec151..2e60ba0bc72f6 100644
--- a/Detectors/TPC/workflow/src/MIPTrackFilterSpec.cxx
+++ b/Detectors/TPC/workflow/src/MIPTrackFilterSpec.cxx
@@ -141,7 +141,7 @@ void MIPTrackFilterDevice::run(ProcessingContext& pc)
   mMIPTracks.clear();
 }
 
-void MIPTrackFilterDevice::sendOutput(DataAllocator& output) { output.snapshot(Output{header::gDataOriginTPC, "MIPS", 0, Lifetime::Timeframe}, mMIPTracks); }
+void MIPTrackFilterDevice::sendOutput(DataAllocator& output) { output.snapshot(Output{header::gDataOriginTPC, "MIPS", 0}, mMIPTracks); }
 
 void MIPTrackFilterDevice::endOfStream(EndOfStreamContext& eos)
 {
diff --git a/Detectors/TPC/workflow/src/RawToDigitsSpec.cxx b/Detectors/TPC/workflow/src/RawToDigitsSpec.cxx
index 5c465be53b8c8..658a60fb75fbf 100644
--- a/Detectors/TPC/workflow/src/RawToDigitsSpec.cxx
+++ b/Detectors/TPC/workflow/src/RawToDigitsSpec.cxx
@@ -226,10 +226,10 @@ class TPCDigitDumpDevice : public o2::framework::Task
       o2::tpc::TPCSectorHeader header{isector};
       header.activeSectors = mActiveSectors;
       // digit for now are transported per sector, not per lane
-      output.snapshot(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(isector), Lifetime::Timeframe, header},
+      output.snapshot(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(isector), header},
                       mDigitDump.getDigits(isector));
       if (mSendCEdigits) {
-        output.snapshot(Output{"TPC", "CEDIGITS", static_cast<SubSpecificationType>(isector), Lifetime::Timeframe, header},
+        output.snapshot(Output{"TPC", "CEDIGITS", static_cast<SubSpecificationType>(isector), header},
                         ceDigits[isector]);
       }
     }
diff --git a/Detectors/TPC/workflow/src/SACProcessorSpec.cxx b/Detectors/TPC/workflow/src/SACProcessorSpec.cxx
index 226bd8a410dc9..e69533a0bb6d3 100644
--- a/Detectors/TPC/workflow/src/SACProcessorSpec.cxx
+++ b/Detectors/TPC/workflow/src/SACProcessorSpec.cxx
@@ -129,8 +129,8 @@ class SACProcessorDevice : public Task
 
   void sendData(DataAllocator& output)
   {
-    output.snapshot(Output{"TPC", "REFTIMESAC", 0, Lifetime::Timeframe}, mDecoder.getDecodedData().referenceTime);
-    output.snapshot(Output{"TPC", "DECODEDSAC", 0, Lifetime::Timeframe}, mDecoder.getDecodedData().getGoodData());
+    output.snapshot(Output{"TPC", "REFTIMESAC", 0}, mDecoder.getDecodedData().referenceTime);
+    output.snapshot(Output{"TPC", "DECODEDSAC", 0}, mDecoder.getDecodedData().getGoodData());
     mDecoder.clearDecodedData();
   }
 
diff --git a/Detectors/TPC/workflow/src/ZSSpec.cxx b/Detectors/TPC/workflow/src/ZSSpec.cxx
index c73d2ae406ebd..00c78ba463507 100644
--- a/Detectors/TPC/workflow/src/ZSSpec.cxx
+++ b/Detectors/TPC/workflow/src/ZSSpec.cxx
@@ -133,8 +133,8 @@ DataProcessorSpec getZSEncoderSpec(std::vector<int> const& tpcSectors, bool outR
       }
       o2::tpc::TPCSectorHeader sh{0};
       gsl::span<ZeroSuppressedContainer8kb> outp(&page[0], offset);
-      pc.outputs().snapshot(Output{gDataOriginTPC, "TPCZS", 0, Lifetime::Timeframe, sh}, outp);
-      pc.outputs().snapshot(Output{gDataOriginTPC, "ZSSIZES", 0, Lifetime::Timeframe, sh}, sizes);
+      pc.outputs().snapshot(Output{gDataOriginTPC, "TPCZS", 0, sh}, outp);
+      pc.outputs().snapshot(Output{gDataOriginTPC, "ZSSIZES", 0, sh}, sizes);
 
       if (outRaw) {
         // ===| set up raw writer |===================================================
@@ -301,7 +301,7 @@ DataProcessorSpec getZStoDigitsSpec(std::vector<int> const& tpcSectors)
       for (int i = 0; i < NSectors; i++) {
         LOG(info) << "digits in sector " << i << " : " << outDigits[i].size();
         o2::tpc::TPCSectorHeader sh{i};
-        pc.outputs().snapshot(Output{gDataOriginTPC, "DIGITS", (unsigned int)i, Lifetime::Timeframe, sh}, outDigits[i]);
+        pc.outputs().snapshot(Output{gDataOriginTPC, "DIGITS", (unsigned int)i, sh}, outDigits[i]);
       }
     };
     return processingFct;
diff --git a/Detectors/TPC/workflow/test/test_ft_EPN_Aggregator.cxx b/Detectors/TPC/workflow/test/test_ft_EPN_Aggregator.cxx
index ed4be20a22569..e8255f98eff6c 100644
--- a/Detectors/TPC/workflow/test/test_ft_EPN_Aggregator.cxx
+++ b/Detectors/TPC/workflow/test/test_ft_EPN_Aggregator.cxx
@@ -243,7 +243,7 @@ DataProcessorSpec generateIDCsCRU(int lane, const unsigned int maxTFs, const std
               }
             }
           }
-          ctx.outputs().adoptContainer(Output{gDataOriginTPC, TPCIntegrateIDCDevice::getDataDescription(TPCIntegrateIDCDevice::IDCFormat::Sim), o2::header::DataHeader::SubSpecificationType{icru << 7}, Lifetime::Timeframe}, std::move(idcs));
+          ctx.outputs().adoptContainer(Output{gDataOriginTPC, TPCIntegrateIDCDevice::getDataDescription(TPCIntegrateIDCDevice::IDCFormat::Sim), o2::header::DataHeader::SubSpecificationType{icru << 7}}, std::move(idcs));
         }
 
         if (delay) {
diff --git a/Detectors/TRD/reconstruction/src/EventRecord.cxx b/Detectors/TRD/reconstruction/src/EventRecord.cxx
index 147b052a8ca3f..0e1a134e45b8f 100644
--- a/Detectors/TRD/reconstruction/src/EventRecord.cxx
+++ b/Detectors/TRD/reconstruction/src/EventRecord.cxx
@@ -77,16 +77,16 @@ void EventRecordContainer::sendData(o2::framework::ProcessingContext& pc, bool g
     counters.push_back(event.getCounters());
   }
 
-  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "DIGITS", 0, o2::framework::Lifetime::Timeframe}, digits);
-  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "TRACKLETS", 0, o2::framework::Lifetime::Timeframe}, tracklets);
-  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "TRKTRGRD", 0, o2::framework::Lifetime::Timeframe}, triggers);
+  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "DIGITS", 0}, digits);
+  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "TRACKLETS", 0}, tracklets);
+  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "TRKTRGRD", 0}, triggers);
   if (generatestats) {
     accumulateStats();
     o2::base::TFIDInfoHelper::fillTFIDInfo(pc, mTFStats.mTFIDInfo);
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "RAWSTATS", 0, o2::framework::Lifetime::Timeframe}, mTFStats);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "RAWSTATS", 0}, mTFStats);
   }
   if (sendLinkStats) {
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "LINKSTATS", 0, o2::framework::Lifetime::Timeframe}, counters);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginTRD, "LINKSTATS", 0}, counters);
   }
 
   std::chrono::duration<double, std::micro> dataReadTime = std::chrono::high_resolution_clock::now() - dataReadStart;
diff --git a/Detectors/TRD/workflow/include/TRDWorkflow/KrClustererSpec.h b/Detectors/TRD/workflow/include/TRDWorkflow/KrClustererSpec.h
index 26be31d51c084..8f50c1af91183 100644
--- a/Detectors/TRD/workflow/include/TRDWorkflow/KrClustererSpec.h
+++ b/Detectors/TRD/workflow/include/TRDWorkflow/KrClustererSpec.h
@@ -67,8 +67,8 @@ void TRDKrClustererDevice::run(ProcessingContext& pc)
   LOGP(info, "Found {} Kr clusters in {} input trigger records. Timing: CPU: {}, Real: {}",
        mKrClFinder.getKrClusters().size(), triggerRecords.size(), timer.CpuTime(), timer.RealTime());
 
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "KRCLUSTER", 0, Lifetime::Timeframe}, mKrClFinder.getKrClusters());
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGKRCLS", 0, Lifetime::Timeframe}, mKrClFinder.getKrTrigRecs());
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "KRCLUSTER", 0}, mKrClFinder.getKrClusters());
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGKRCLS", 0}, mKrClFinder.getKrTrigRecs());
 }
 
 void TRDKrClustererDevice::endOfStream(EndOfStreamContext& ec)
diff --git a/Detectors/TRD/workflow/include/TRDWorkflow/TRDGlobalTrackingQCSpec.h b/Detectors/TRD/workflow/include/TRDWorkflow/TRDGlobalTrackingQCSpec.h
index 375b36c54af17..a3c5b28aa8b63 100644
--- a/Detectors/TRD/workflow/include/TRDWorkflow/TRDGlobalTrackingQCSpec.h
+++ b/Detectors/TRD/workflow/include/TRDWorkflow/TRDGlobalTrackingQCSpec.h
@@ -63,7 +63,7 @@ class TRDGlobalTrackingQC : public Task
     mQC.reset();
     mQC.setInput(recoData);
     mQC.run();
-    pc.outputs().snapshot(Output{"TRD", "TRACKINGQC", 0, Lifetime::Timeframe}, mQC.getTrackQC());
+    pc.outputs().snapshot(Output{"TRD", "TRACKINGQC", 0}, mQC.getTrackQC());
   }
   void endOfStream(framework::EndOfStreamContext& ec) final {}
   void finaliseCCDB(framework::ConcreteDataMatcher& matcher, void* obj) final
diff --git a/Detectors/TRD/workflow/include/TRDWorkflow/TRDPulseHeightSpec.h b/Detectors/TRD/workflow/include/TRDWorkflow/TRDPulseHeightSpec.h
index 6ba185ee28ccb..a1e35f3a3482e 100644
--- a/Detectors/TRD/workflow/include/TRDWorkflow/TRDPulseHeightSpec.h
+++ b/Detectors/TRD/workflow/include/TRDWorkflow/TRDPulseHeightSpec.h
@@ -52,7 +52,7 @@ class PuseHeightDevice : public o2::framework::Task
     }
     if (mRunStopRequested) {
       std::vector<PHData> mPHValues{}; // the calibration expects data at every TF, so inject dummy
-      pc.outputs().snapshot(Output{"TRD", "PULSEHEIGHT", 0, Lifetime::Timeframe}, mPHValues);
+      pc.outputs().snapshot(Output{"TRD", "PULSEHEIGHT", 0}, mPHValues);
       return;
     }
     RecoContainer recoData;
@@ -61,7 +61,7 @@ class PuseHeightDevice : public o2::framework::Task
     mPulseHeight->setInput(recoData, &digits);
     mPulseHeight->reset();
     mPulseHeight->process();
-    pc.outputs().snapshot(Output{"TRD", "PULSEHEIGHT", 0, Lifetime::Timeframe}, mPulseHeight->getPHData());
+    pc.outputs().snapshot(Output{"TRD", "PULSEHEIGHT", 0}, mPulseHeight->getPHData());
     if (pc.transitionState() == TransitionHandlingState::Requested) {
       LOG(info) << "Run stop requested, finalizing";
       mRunStopRequested = true;
diff --git a/Detectors/TRD/workflow/io/src/TRDCalibReaderSpec.cxx b/Detectors/TRD/workflow/io/src/TRDCalibReaderSpec.cxx
index d3a565b4c9804..cb76bb8bbad69 100644
--- a/Detectors/TRD/workflow/io/src/TRDCalibReaderSpec.cxx
+++ b/Detectors/TRD/workflow/io/src/TRDCalibReaderSpec.cxx
@@ -53,7 +53,7 @@ void TRDCalibReader::run(ProcessingContext& pc)
   assert(currEntry < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(currEntry);
   LOG(info) << "Pushing angular residual histograms filled with " << mAngResids.getNEntries() << " entries at tree entry " << currEntry;
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "ANGRESHISTS", 0, Lifetime::Timeframe}, mAngResids);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "ANGRESHISTS", 0}, mAngResids);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/TRD/workflow/io/src/TRDDigitReaderSpec.cxx b/Detectors/TRD/workflow/io/src/TRDDigitReaderSpec.cxx
index 4ca6fd46e4492..92d001bc5120a 100644
--- a/Detectors/TRD/workflow/io/src/TRDDigitReaderSpec.cxx
+++ b/Detectors/TRD/workflow/io/src/TRDDigitReaderSpec.cxx
@@ -58,13 +58,13 @@ void TRDDigitReaderSpec::run(ProcessingContext& pc)
   assert(currEntry < mTreeDigits->GetEntries()); // this should not happen
   mTreeDigits->GetEntry(currEntry);
   LOGP(info, "Pushing {} digits for tree entry {}", mDigits.size(), currEntry);
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "DIGITS", mSubSpec, Lifetime::Timeframe}, mDigits);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "DIGITS", mSubSpec}, mDigits);
   if (mUseTriggerRecords) {
     LOGP(info, "Pushing {} trigger records for tree entry {}", mTriggerRecords.size(), currEntry);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRKTRGRD", mSubSpec, Lifetime::Timeframe}, mTriggerRecords);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRKTRGRD", mSubSpec}, mTriggerRecords);
   }
   if (mUseMC) {
-    auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{o2::header::gDataOriginTRD, "LABELS", 0, Lifetime::Timeframe});
+    auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{o2::header::gDataOriginTRD, "LABELS", 0});
     mLabels->copyandflatten(sharedlabels);
   }
   if (mTreeDigits->GetReadEntry() + 1 >= mTreeDigits->GetEntries()) {
diff --git a/Detectors/TRD/workflow/io/src/TRDPHReaderSpec.cxx b/Detectors/TRD/workflow/io/src/TRDPHReaderSpec.cxx
index 235c58c2ad302..7f873beb2ece1 100644
--- a/Detectors/TRD/workflow/io/src/TRDPHReaderSpec.cxx
+++ b/Detectors/TRD/workflow/io/src/TRDPHReaderSpec.cxx
@@ -54,7 +54,7 @@ void TRDPHReader::run(ProcessingContext& pc)
   mTree->GetEntry(currEntry);
 
   LOG(info) << "Pushing vector of PH values filled with " << mPHValues.size() << " entries at tree entry " << currEntry;
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "PULSEHEIGHT", 0, Lifetime::Timeframe}, mPHValues);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "PULSEHEIGHT", 0}, mPHValues);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/TRD/workflow/io/src/TRDTrackReaderSpec.cxx b/Detectors/TRD/workflow/io/src/TRDTrackReaderSpec.cxx
index 20a7b5d5ac234..cd9702a3d2385 100644
--- a/Detectors/TRD/workflow/io/src/TRDTrackReaderSpec.cxx
+++ b/Detectors/TRD/workflow/io/src/TRDTrackReaderSpec.cxx
@@ -50,18 +50,18 @@ void TRDTrackReader::run(ProcessingContext& pc)
 
   if (mMode == Mode::TPCTRD) {
     uint32_t ss = o2::globaltracking::getSubSpec(mSubSpecStrict ? o2::globaltracking::MatchingType::Strict : o2::globaltracking::MatchingType::Standard);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MATCH_TPC", ss, Lifetime::Timeframe}, mTracks);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGREC_TPC", ss, Lifetime::Timeframe}, mTrigRec);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MATCH_TPC", ss}, mTracks);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGREC_TPC", ss}, mTrigRec);
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_TPC", ss, Lifetime::Timeframe}, mLabelsMatch);
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_TPC_TRD", ss, Lifetime::Timeframe}, mLabelsTrd);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_TPC", ss}, mLabelsMatch);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_TPC_TRD", ss}, mLabelsTrd);
     }
   } else if (mMode == Mode::ITSTPCTRD) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MATCH_ITSTPC", 0, Lifetime::Timeframe}, mTracks);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGREC_ITSTPC", 0, Lifetime::Timeframe}, mTrigRec);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MATCH_ITSTPC", 0}, mTracks);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGREC_ITSTPC", 0}, mTrigRec);
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_ITSTPC", 0, Lifetime::Timeframe}, mLabelsMatch);
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_ITSTPC_TRD", 0, Lifetime::Timeframe}, mLabelsTrd);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_ITSTPC", 0}, mLabelsMatch);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_ITSTPC_TRD", 0}, mLabelsTrd);
     }
   }
 
diff --git a/Detectors/TRD/workflow/io/src/TRDTrackletReaderSpec.cxx b/Detectors/TRD/workflow/io/src/TRDTrackletReaderSpec.cxx
index c588da696e321..864cc3d665af5 100644
--- a/Detectors/TRD/workflow/io/src/TRDTrackletReaderSpec.cxx
+++ b/Detectors/TRD/workflow/io/src/TRDTrackletReaderSpec.cxx
@@ -72,20 +72,20 @@ void TRDTrackletReader::run(ProcessingContext& pc)
   mTreeTrklt->GetEntry(currEntry);
   LOG(info) << "Pushing " << mTriggerRecords.size() << " TRD trigger records at entry " << currEntry;
   LOG(info) << "Pushing " << mTracklets.size() << " uncalibrated TRD tracklets for these trigger records";
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRACKLETS", 0, Lifetime::Timeframe}, mTracklets);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRACKLETS", 0}, mTracklets);
   if (mUseTrackletTransform) {
     assert(mTreeTrklt->GetEntries() == mTreeCTrklt->GetEntries());
     mTreeCTrklt->GetEntry(currEntry);
     LOG(info) << "Pushing " << mTrackletsCal.size() << " calibrated TRD tracklets for these trigger records";
     LOG(info) << "Pushing " << mTrigRecMask.size() << " flags for the given TRD trigger records";
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "CTRACKLETS", 0, Lifetime::Timeframe}, mTrackletsCal);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRIGRECMASK", 0, Lifetime::Timeframe}, mTrigRecMask);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "CTRACKLETS", 0}, mTrackletsCal);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRIGRECMASK", 0}, mTrigRecMask);
   }
 
-  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRKTRGRD", 0, Lifetime::Timeframe}, mTriggerRecords);
+  pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRKTRGRD", 0}, mTriggerRecords);
   if (mUseMC) {
     LOG(info) << "Pushing " << mLabels.getNElements() << " TRD tracklet labels";
-    pc.outputs().snapshot(Output{"TRD", "TRKLABELS", 0, Lifetime::Timeframe}, mLabels);
+    pc.outputs().snapshot(Output{"TRD", "TRKLABELS", 0}, mLabels);
   }
 
   if (mTreeTrklt->GetReadEntry() + 1 >= mTreeTrklt->GetEntries()) {
diff --git a/Detectors/TRD/workflow/src/EntropyEncoderSpec.cxx b/Detectors/TRD/workflow/src/EntropyEncoderSpec.cxx
index 07243d2b316cb..d345dd74141ed 100644
--- a/Detectors/TRD/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/TRD/workflow/src/EntropyEncoderSpec.cxx
@@ -76,7 +76,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
   if (mSelIR) {
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"TRD", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"TRD", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, triggers, tracklets, digits);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   if (mSelIR) {
diff --git a/Detectors/TRD/workflow/src/TRDDigitizerSpec.cxx b/Detectors/TRD/workflow/src/TRDDigitizerSpec.cxx
index c0a4d85acd1bc..1cd145a6c2e36 100644
--- a/Detectors/TRD/workflow/src/TRDDigitizerSpec.cxx
+++ b/Detectors/TRD/workflow/src/TRDDigitizerSpec.cxx
@@ -177,17 +177,17 @@ class TRDDPLDigitizerTask : public o2::base::BaseDPLDigitizer
     LOGF(info, "TRD digitization timing: Cpu: %.3e Real: %.3e s", timer.CpuTime(), timer.RealTime());
 
     LOG(info) << "TRD: Sending " << digitsAccum.size() << " digits";
-    pc.outputs().snapshot(Output{"TRD", "DIGITS", 1, Lifetime::Timeframe}, digitsAccum);
+    pc.outputs().snapshot(Output{"TRD", "DIGITS", 1}, digitsAccum);
     if (mctruth) {
       LOG(info) << "TRD: Sending " << labelsAccum.getNElements() << " labels";
       // we are flattening the labels and write to managed shared memory container for further communication
-      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{"TRD", "LABELS", 0, Lifetime::Timeframe});
+      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{"TRD", "LABELS", 0});
       labelsAccum.flatten_to(sharedlabels);
     }
     LOG(info) << "TRD: Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{"TRD", "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{"TRD", "ROMode", 0}, mROMode);
     LOG(info) << "TRD: Sending trigger records";
-    pc.outputs().snapshot(Output{"TRD", "TRKTRGRD", 1, Lifetime::Timeframe}, triggers);
+    pc.outputs().snapshot(Output{"TRD", "TRKTRGRD", 1}, triggers);
     // we should be only called once; tell DPL that this process is ready to exit
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
     finished = true;
diff --git a/Detectors/TRD/workflow/src/TRDGlobalTrackingSpec.cxx b/Detectors/TRD/workflow/src/TRDGlobalTrackingSpec.cxx
index 1be328b4bcbe9..abe3a6f45ec70 100644
--- a/Detectors/TRD/workflow/src/TRDGlobalTrackingSpec.cxx
+++ b/Detectors/TRD/workflow/src/TRDGlobalTrackingSpec.cxx
@@ -496,19 +496,19 @@ void TRDGlobalTracking::run(ProcessingContext& pc)
 
   uint32_t ss = o2::globaltracking::getSubSpec(mStrict ? o2::globaltracking::MatchingType::Strict : o2::globaltracking::MatchingType::Standard);
   if (GTrackID::includesSource(GTrackID::Source::ITSTPC, mTrkMask)) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MATCH_ITSTPC", 0, Lifetime::Timeframe}, tracksOutITSTPC);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGREC_ITSTPC", 0, Lifetime::Timeframe}, trackTrigRecITSTPC);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MATCH_ITSTPC", 0}, tracksOutITSTPC);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGREC_ITSTPC", 0}, trackTrigRecITSTPC);
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_ITSTPC", 0, Lifetime::Timeframe}, matchLabelsITSTPC);
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_ITSTPC_TRD", 0, Lifetime::Timeframe}, trdLabelsITSTPC);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_ITSTPC", 0}, matchLabelsITSTPC);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_ITSTPC_TRD", 0}, trdLabelsITSTPC);
     }
   }
   if (GTrackID::includesSource(GTrackID::Source::TPC, mTrkMask)) {
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MATCH_TPC", ss, Lifetime::Timeframe}, tracksOutTPC);
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGREC_TPC", ss, Lifetime::Timeframe}, trackTrigRecTPC);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MATCH_TPC", ss}, tracksOutTPC);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "TRGREC_TPC", ss}, trackTrigRecTPC);
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_TPC", ss, Lifetime::Timeframe}, matchLabelsTPC);
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_TPC_TRD", ss, Lifetime::Timeframe}, trdLabelsTPC);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_TPC", ss}, matchLabelsTPC);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "MCLB_TPC_TRD", ss}, trdLabelsTPC);
     }
   }
 
diff --git a/Detectors/TRD/workflow/src/TRDTrackletTransformerSpec.cxx b/Detectors/TRD/workflow/src/TRDTrackletTransformerSpec.cxx
index bb6d552cf3fee..0ac920c6e49c6 100644
--- a/Detectors/TRD/workflow/src/TRDTrackletTransformerSpec.cxx
+++ b/Detectors/TRD/workflow/src/TRDTrackletTransformerSpec.cxx
@@ -113,8 +113,8 @@ void TRDTrackletTransformerSpec::run(o2::framework::ProcessingContext& pc)
 
   LOGF(info, "Found %lu tracklets in %lu trigger records. Applied filter for ITS IR frames: %i. Transformed %i tracklets.", tracklets.size(), trigRecs.size(), mTrigRecFilterActive, nTrackletsTransformed);
 
-  pc.outputs().snapshot(Output{"TRD", "CTRACKLETS", 0, Lifetime::Timeframe}, calibratedTracklets);
-  pc.outputs().snapshot(Output{"TRD", "TRIGRECMASK", 0, Lifetime::Timeframe}, trigRecBitfield);
+  pc.outputs().snapshot(Output{"TRD", "CTRACKLETS", 0}, calibratedTracklets);
+  pc.outputs().snapshot(Output{"TRD", "TRIGRECMASK", 0}, trigRecBitfield);
 }
 
 void TRDTrackletTransformerSpec::updateTimeDependentParams(ProcessingContext& pc)
diff --git a/Detectors/TRD/workflow/src/TRDTrapSimulatorSpec.cxx b/Detectors/TRD/workflow/src/TRDTrapSimulatorSpec.cxx
index 4c6ac24a0dc43..20d0b719b9bf7 100644
--- a/Detectors/TRD/workflow/src/TRDTrapSimulatorSpec.cxx
+++ b/Detectors/TRD/workflow/src/TRDTrapSimulatorSpec.cxx
@@ -328,11 +328,11 @@ void TRDDPLTrapSimulatorTask::run(o2::framework::ProcessingContext& pc)
   LOG(info) << "Digit Sorting took: " << std::chrono::duration_cast<std::chrono::milliseconds>(sortTime).count() << "ms";
   LOG(info) << "Processing time for parallel region: " << std::chrono::duration_cast<std::chrono::milliseconds>(parallelTime).count() << "ms";
 
-  pc.outputs().snapshot(Output{"TRD", "TRACKLETS", 0, Lifetime::Timeframe}, tracklets);
-  pc.outputs().snapshot(Output{"TRD", "TRKTRGRD", 0, Lifetime::Timeframe}, triggerRecords);
-  pc.outputs().snapshot(Output{"TRD", "DIGITS", 0, Lifetime::Timeframe}, digitsOut);
+  pc.outputs().snapshot(Output{"TRD", "TRACKLETS", 0}, tracklets);
+  pc.outputs().snapshot(Output{"TRD", "TRKTRGRD", 0}, triggerRecords);
+  pc.outputs().snapshot(Output{"TRD", "DIGITS", 0}, digitsOut);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"TRD", "TRKLABELS", 0, Lifetime::Timeframe}, lblTracklets);
+    pc.outputs().snapshot(Output{"TRD", "TRKLABELS", 0}, lblTracklets);
   }
 
   LOG(debug) << "TRD Trap Simulator Device exiting";
diff --git a/Detectors/TRD/workflow/src/TrackBasedCalibSpec.cxx b/Detectors/TRD/workflow/src/TrackBasedCalibSpec.cxx
index 6011e194685ad..4418b9af61300 100644
--- a/Detectors/TRD/workflow/src/TrackBasedCalibSpec.cxx
+++ b/Detectors/TRD/workflow/src/TrackBasedCalibSpec.cxx
@@ -75,12 +75,12 @@ void TRDTrackBasedCalibDevice::run(ProcessingContext& pc)
 
   if (mDoVdExBCalib) {
     mCalibrator.calculateAngResHistos();
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "ANGRESHISTS", 0, Lifetime::Timeframe}, mCalibrator.getAngResHistos());
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "ANGRESHISTS", 0}, mCalibrator.getAngResHistos());
   }
 
   if (mDoGainCalib) {
     mCalibrator.calculateGainCalibObjs();
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "GAINCALIBHISTS", 0, Lifetime::Timeframe}, mCalibrator.getGainCalibHistos());
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "GAINCALIBHISTS", 0}, mCalibrator.getGainCalibHistos());
   }
 
   mCalibrator.reset();
diff --git a/Detectors/Upgrades/ITS3/workflow/src/ClustererSpec.cxx b/Detectors/Upgrades/ITS3/workflow/src/ClustererSpec.cxx
index e6c9b5bc4b910..5512964eec950 100644
--- a/Detectors/Upgrades/ITS3/workflow/src/ClustererSpec.cxx
+++ b/Detectors/Upgrades/ITS3/workflow/src/ClustererSpec.cxx
@@ -109,17 +109,17 @@ void ClustererDPL::run(ProcessingContext& pc)
     clusterLabels = std::make_unique<o2::dataformats::MCTruthContainer<o2::MCCompLabel>>();
   }
   mClusterer->process(mNThreads, reader, &clusCompVec, mPatterns ? &clusPattVec : nullptr, &clusROFVec, clusterLabels.get());
-  pc.outputs().snapshot(Output{orig, "COMPCLUSTERS", 0, Lifetime::Timeframe}, clusCompVec);
-  pc.outputs().snapshot(Output{orig, "CLUSTERSROF", 0, Lifetime::Timeframe}, clusROFVec);
-  pc.outputs().snapshot(Output{orig, "PATTERNS", 0, Lifetime::Timeframe}, clusPattVec);
+  pc.outputs().snapshot(Output{orig, "COMPCLUSTERS", 0}, clusCompVec);
+  pc.outputs().snapshot(Output{orig, "CLUSTERSROF", 0}, clusROFVec);
+  pc.outputs().snapshot(Output{orig, "PATTERNS", 0}, clusPattVec);
 
   if (mUseMC) {
-    pc.outputs().snapshot(Output{orig, "CLUSTERSMCTR", 0, Lifetime::Timeframe}, *clusterLabels.get()); // at the moment requires snapshot
+    pc.outputs().snapshot(Output{orig, "CLUSTERSMCTR", 0}, *clusterLabels.get()); // at the moment requires snapshot
     std::vector<o2::itsmft::MC2ROFRecord> clusterMC2ROframes(mc2rofs.size());
     for (int i = mc2rofs.size(); i--;) {
       clusterMC2ROframes[i] = mc2rofs[i]; // Simply, replicate it from digits ?
     }
-    pc.outputs().snapshot(Output{orig, "CLUSTERSMC2ROF", 0, Lifetime::Timeframe}, clusterMC2ROframes);
+    pc.outputs().snapshot(Output{orig, "CLUSTERSMC2ROF", 0}, clusterMC2ROframes);
   }
 
   // TODO: in principle, after masking "overflow" pixels the MC2ROFRecord maxROF supposed to change, nominally to minROF
diff --git a/Detectors/Upgrades/ITS3/workflow/src/DigitReaderSpec.cxx b/Detectors/Upgrades/ITS3/workflow/src/DigitReaderSpec.cxx
index a679beed2c530..c2380be77f956 100644
--- a/Detectors/Upgrades/ITS3/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/Upgrades/ITS3/workflow/src/DigitReaderSpec.cxx
@@ -71,17 +71,17 @@ void DigitReader::run(ProcessingContext& pc)
 
   // This is a very ugly way of providing DataDescription, which anyway does not need to contain detector name.
   // To be fixed once the names-definition class is ready
-  pc.outputs().snapshot(Output{mOrigin, "DIGITSROF", 0, Lifetime::Timeframe}, mDigROFRec);
-  pc.outputs().snapshot(Output{mOrigin, "DIGITS", 0, Lifetime::Timeframe}, mDigits);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITSROF", 0}, mDigROFRec);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITS", 0}, mDigits);
   if (mUseCalib) {
-    pc.outputs().snapshot(Output{mOrigin, "GBTCALIB", 0, Lifetime::Timeframe}, mCalib);
+    pc.outputs().snapshot(Output{mOrigin, "GBTCALIB", 0}, mCalib);
   }
 
   if (mUseMC) {
-    auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{mOrigin, "DIGITSMCTR", 0, Lifetime::Timeframe});
+    auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{mOrigin, "DIGITSMCTR", 0});
     plabels->copyandflatten(sharedlabels);
     delete plabels;
-    pc.outputs().snapshot(Output{mOrigin, "DIGITSMC2ROF", 0, Lifetime::Timeframe}, mDigMC2ROFs);
+    pc.outputs().snapshot(Output{mOrigin, "DIGITSMC2ROF", 0}, mDigMC2ROFs);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/Upgrades/ITS3/workflow/src/TrackReaderSpec.cxx b/Detectors/Upgrades/ITS3/workflow/src/TrackReaderSpec.cxx
index 3d048bf981ea3..54d8951b5a0d0 100644
--- a/Detectors/Upgrades/ITS3/workflow/src/TrackReaderSpec.cxx
+++ b/Detectors/Upgrades/ITS3/workflow/src/TrackReaderSpec.cxx
@@ -44,14 +44,14 @@ void TrackReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mTracks.size() << " track in " << mROFRec.size() << " ROFs at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "IT3TrackROF", 0, Lifetime::Timeframe}, mROFRec);
-  pc.outputs().snapshot(Output{mOrigin, "TRACKS", 0, Lifetime::Timeframe}, mTracks);
-  pc.outputs().snapshot(Output{mOrigin, "TRACKCLSID", 0, Lifetime::Timeframe}, mClusInd);
-  pc.outputs().snapshot(Output{"IT3", "VERTICES", 0, Lifetime::Timeframe}, mVertices);
-  pc.outputs().snapshot(Output{"IT3", "VERTICESROF", 0, Lifetime::Timeframe}, mVerticesROFRec);
+  pc.outputs().snapshot(Output{mOrigin, "IT3TrackROF", 0}, mROFRec);
+  pc.outputs().snapshot(Output{mOrigin, "TRACKS", 0}, mTracks);
+  pc.outputs().snapshot(Output{mOrigin, "TRACKCLSID", 0}, mClusInd);
+  pc.outputs().snapshot(Output{"IT3", "VERTICES", 0}, mVertices);
+  pc.outputs().snapshot(Output{"IT3", "VERTICESROF", 0}, mVerticesROFRec);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{mOrigin, "TRACKSMCTR", 0, Lifetime::Timeframe}, mMCTruth);
-    pc.outputs().snapshot(Output{mOrigin, "VERTICESMCTR", 0, Lifetime::Timeframe}, mMCVertTruth);
+    pc.outputs().snapshot(Output{mOrigin, "TRACKSMCTR", 0}, mMCTruth);
+    pc.outputs().snapshot(Output{mOrigin, "VERTICESMCTR", 0}, mMCVertTruth);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
diff --git a/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx b/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx
index e2078046d1966..e4240401928b6 100644
--- a/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx
+++ b/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx
@@ -171,9 +171,9 @@ void TrackerDPL::run(ProcessingContext& pc)
   // the output vector however is created directly inside the message memory thus avoiding copy by
   // snapshot
   auto rofsinput = pc.inputs().get<gsl::span<o2::itsmft::ROFRecord>>("ROframes");
-  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"IT3", "IT3TrackROF", 0, Lifetime::Timeframe}, rofsinput.begin(), rofsinput.end());
+  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"IT3", "IT3TrackROF", 0}, rofsinput.begin(), rofsinput.end());
 
-  auto& irFrames = pc.outputs().make<std::vector<o2::dataformats::IRFrame>>(Output{"IT3", "IRFRAMES", 0, Lifetime::Timeframe});
+  auto& irFrames = pc.outputs().make<std::vector<o2::dataformats::IRFrame>>(Output{"IT3", "IRFRAMES", 0});
 
   const auto& alpParams = o2::itsmft::DPLAlpideParam<o2::detectors::DetID::ITS>::Instance(); // RS: this should come from CCDB
   int nBCPerTF = alpParams.roFrameLengthInBC;
@@ -189,15 +189,15 @@ void TrackerDPL::run(ProcessingContext& pc)
     LOG(info) << labels->getIndexedSize() << " MC label objects , in " << mc2rofs.size() << " MC events";
   }
 
-  auto& allClusIdx = pc.outputs().make<std::vector<int>>(Output{"IT3", "TRACKCLSID", 0, Lifetime::Timeframe});
+  auto& allClusIdx = pc.outputs().make<std::vector<int>>(Output{"IT3", "TRACKCLSID", 0});
   std::vector<o2::MCCompLabel> trackLabels;
   std::vector<MCCompLabel> verticesLabels;
-  auto& allTracks = pc.outputs().make<std::vector<o2::its::TrackITS>>(Output{"IT3", "TRACKS", 0, Lifetime::Timeframe});
+  auto& allTracks = pc.outputs().make<std::vector<o2::its::TrackITS>>(Output{"IT3", "TRACKS", 0});
   std::vector<o2::MCCompLabel> allTrackLabels;
   std::vector<o2::MCCompLabel> allVerticesLabels;
 
-  auto& vertROFvec = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"IT3", "VERTICESROF", 0, Lifetime::Timeframe});
-  auto& vertices = pc.outputs().make<std::vector<Vertex>>(Output{"IT3", "VERTICES", 0, Lifetime::Timeframe});
+  auto& vertROFvec = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"IT3", "VERTICESROF", 0});
+  auto& vertices = pc.outputs().make<std::vector<Vertex>>(Output{"IT3", "VERTICES", 0});
 
   TimeFrame* timeFrame = mChainITS->GetITSTimeframe();
   timeFrame->resizeVectors(mNLayers);
@@ -314,9 +314,9 @@ void TrackerDPL::run(ProcessingContext& pc)
       LOGP(info, "ITS3Tracker pushed {} track labels", allTrackLabels.size());
       LOGP(info, "ITS3Tracker pushed {} vertex labels", allVerticesLabels.size());
 
-      pc.outputs().snapshot(Output{"IT3", "TRACKSMCTR", 0, Lifetime::Timeframe}, allTrackLabels);
-      pc.outputs().snapshot(Output{"IT3", "VERTICESMCTR", 0, Lifetime::Timeframe}, allVerticesLabels);
-      pc.outputs().snapshot(Output{"IT3", "IT3TrackMC2ROF", 0, Lifetime::Timeframe}, mc2rofs);
+      pc.outputs().snapshot(Output{"IT3", "TRACKSMCTR", 0}, allTrackLabels);
+      pc.outputs().snapshot(Output{"IT3", "VERTICESMCTR", 0}, allVerticesLabels);
+      pc.outputs().snapshot(Output{"IT3", "IT3TrackMC2ROF", 0}, mc2rofs);
     }
   }
   mTimer.Stop();
diff --git a/Detectors/Upgrades/ITS3/workflow/src/VertexReaderSpec.cxx b/Detectors/Upgrades/ITS3/workflow/src/VertexReaderSpec.cxx
index 015aa1a78977b..25fd3e530161a 100644
--- a/Detectors/Upgrades/ITS3/workflow/src/VertexReaderSpec.cxx
+++ b/Detectors/Upgrades/ITS3/workflow/src/VertexReaderSpec.cxx
@@ -41,8 +41,8 @@ void VertexReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
   LOG(info) << "Pushing " << mVerticesPtr->size() << " vertices in " << mVerticesROFRecPtr->size()
             << " ROFs at entry " << ent;
-  pc.outputs().snapshot(Output{"IT3", "VERTICES", 0, Lifetime::Timeframe}, mVertices);
-  pc.outputs().snapshot(Output{"IT3", "VERTICESROF", 0, Lifetime::Timeframe}, mVerticesROFRec);
+  pc.outputs().snapshot(Output{"IT3", "VERTICES", 0}, mVertices);
+  pc.outputs().snapshot(Output{"IT3", "VERTICESROF", 0}, mVerticesROFRec);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/Vertexing/src/SVertexer.cxx b/Detectors/Vertexing/src/SVertexer.cxx
index de167af74526e..120d49d44b4e5 100644
--- a/Detectors/Vertexing/src/SVertexer.cxx
+++ b/Detectors/Vertexing/src/SVertexer.cxx
@@ -117,15 +117,15 @@ void SVertexer::produceOutput(o2::framework::ProcessingContext& pc)
   std::sort(nbodySortID.begin(), nbodySortID.end(), [](const vid& a, const vid& b) { return a.vtxID < b.vtxID; });
 
   // dpl output
-  auto& v0sIdx = pc.outputs().make<std::vector<V0Index>>(o2f::Output{"GLO", "V0S_IDX", 0, o2f::Lifetime::Timeframe});
-  auto& cascsIdx = pc.outputs().make<std::vector<CascadeIndex>>(o2f::Output{"GLO", "CASCS_IDX", 0, o2f::Lifetime::Timeframe});
-  auto& body3Idx = pc.outputs().make<std::vector<Decay3BodyIndex>>(o2f::Output{"GLO", "DECAYS3BODY_IDX", 0, o2f::Lifetime::Timeframe});
-  auto& fullv0s = pc.outputs().make<std::vector<V0>>(o2f::Output{"GLO", "V0S", 0, o2f::Lifetime::Timeframe});
-  auto& fullcascs = pc.outputs().make<std::vector<Cascade>>(o2f::Output{"GLO", "CASCS", 0, o2f::Lifetime::Timeframe});
-  auto& full3body = pc.outputs().make<std::vector<Decay3Body>>(o2f::Output{"GLO", "DECAYS3BODY", 0, o2f::Lifetime::Timeframe});
-  auto& v0Refs = pc.outputs().make<std::vector<RRef>>(o2f::Output{"GLO", "PVTX_V0REFS", 0, o2f::Lifetime::Timeframe});
-  auto& cascRefs = pc.outputs().make<std::vector<RRef>>(o2f::Output{"GLO", "PVTX_CASCREFS", 0, o2f::Lifetime::Timeframe});
-  auto& vtx3bodyRefs = pc.outputs().make<std::vector<RRef>>(o2f::Output{"GLO", "PVTX_3BODYREFS", 0, o2f::Lifetime::Timeframe});
+  auto& v0sIdx = pc.outputs().make<std::vector<V0Index>>(o2f::Output{"GLO", "V0S_IDX", 0});
+  auto& cascsIdx = pc.outputs().make<std::vector<CascadeIndex>>(o2f::Output{"GLO", "CASCS_IDX", 0});
+  auto& body3Idx = pc.outputs().make<std::vector<Decay3BodyIndex>>(o2f::Output{"GLO", "DECAYS3BODY_IDX", 0});
+  auto& fullv0s = pc.outputs().make<std::vector<V0>>(o2f::Output{"GLO", "V0S", 0});
+  auto& fullcascs = pc.outputs().make<std::vector<Cascade>>(o2f::Output{"GLO", "CASCS", 0});
+  auto& full3body = pc.outputs().make<std::vector<Decay3Body>>(o2f::Output{"GLO", "DECAYS3BODY", 0});
+  auto& v0Refs = pc.outputs().make<std::vector<RRef>>(o2f::Output{"GLO", "PVTX_V0REFS", 0});
+  auto& cascRefs = pc.outputs().make<std::vector<RRef>>(o2f::Output{"GLO", "PVTX_CASCREFS", 0});
+  auto& vtx3bodyRefs = pc.outputs().make<std::vector<RRef>>(o2f::Output{"GLO", "PVTX_3BODYREFS", 0});
 
   // sorted V0s
   v0sIdx.reserve(mNV0s);
@@ -214,8 +214,8 @@ void SVertexer::produceOutput(o2::framework::ProcessingContext& pc)
       }
     }
 
-    auto& strTracksOut = pc.outputs().make<std::vector<o2::dataformats::StrangeTrack>>(o2f::Output{"GLO", "STRANGETRACKS", 0, o2f::Lifetime::Timeframe});
-    auto& strClustOut = pc.outputs().make<std::vector<o2::strangeness_tracking::ClusAttachments>>(o2f::Output{"GLO", "CLUSUPDATES", 0, o2f::Lifetime::Timeframe});
+    auto& strTracksOut = pc.outputs().make<std::vector<o2::dataformats::StrangeTrack>>(o2f::Output{"GLO", "STRANGETRACKS", 0});
+    auto& strClustOut = pc.outputs().make<std::vector<o2::strangeness_tracking::ClusAttachments>>(o2f::Output{"GLO", "CLUSUPDATES", 0});
     o2::pmr::vector<o2::MCCompLabel> mcLabsOut;
     strTracksOut.resize(mNStrangeTracks);
     strClustOut.resize(mNStrangeTracks);
@@ -239,7 +239,7 @@ void SVertexer::produceOutput(o2::framework::ProcessingContext& pc)
     }
 
     if (mStrTracker->getMCTruthOn()) {
-      auto& strTrMCLableOut = pc.outputs().make<std::vector<o2::MCCompLabel>>(o2f::Output{"GLO", "STRANGETRACKS_MC", 0, o2f::Lifetime::Timeframe});
+      auto& strTrMCLableOut = pc.outputs().make<std::vector<o2::MCCompLabel>>(o2f::Output{"GLO", "STRANGETRACKS_MC", 0});
       strTrMCLableOut.swap(mcLabsOut);
     }
   }
diff --git a/Detectors/ZDC/calib/src/InterCalibEPNSpec.cxx b/Detectors/ZDC/calib/src/InterCalibEPNSpec.cxx
index 80b23c696b5a4..a1a83aada50f0 100644
--- a/Detectors/ZDC/calib/src/InterCalibEPNSpec.cxx
+++ b/Detectors/ZDC/calib/src/InterCalibEPNSpec.cxx
@@ -101,16 +101,16 @@ void InterCalibEPNSpec::run(ProcessingContext& pc)
   mWorker.process(bcrec, energy, tdc, info);
 
   // Send intermediate calibration data and histograms
-  o2::framework::Output output("ZDC", "INTERCALIBDATA", 0, Lifetime::Timeframe);
+  o2::framework::Output output("ZDC", "INTERCALIBDATA", 0);
   pc.outputs().snapshot(output, mWorker.mData);
   for (int ih = 0; ih < (2 * InterCalibData::NH); ih++) {
     if (mWorker.mH[ih] != nullptr) {
-      o2::framework::Output output("ZDC", "INTER_1DH", ih, Lifetime::Timeframe);
+      o2::framework::Output output("ZDC", "INTER_1DH", ih);
       pc.outputs().snapshot(output, mWorker.mH[ih]->getBase());
     }
   }
   for (int ih = 0; ih < InterCalibData::NH; ih++) {
-    o2::framework::Output output("ZDC", "INTER_2DH", ih, Lifetime::Timeframe);
+    o2::framework::Output output("ZDC", "INTER_2DH", ih);
     pc.outputs().snapshot(output, mWorker.mC[ih]->getBase());
   }
 }
diff --git a/Detectors/ZDC/calib/src/TDCCalibEPNSpec.cxx b/Detectors/ZDC/calib/src/TDCCalibEPNSpec.cxx
index 8b90ee3b0af36..6125baaa197f8 100644
--- a/Detectors/ZDC/calib/src/TDCCalibEPNSpec.cxx
+++ b/Detectors/ZDC/calib/src/TDCCalibEPNSpec.cxx
@@ -114,10 +114,10 @@ void TDCCalibEPNSpec::run(ProcessingContext& pc)
     if (mVerbosity > DbgMedium && mModTF > 0) {
       LOG(info) << "Send intermediate calibration data mProcessed=" << mProcessed << " >= mModTF=" << mModTF;
     }
-    o2::framework::Output output("ZDC", "TDCCALIBDATA", 0, Lifetime::Timeframe);
+    o2::framework::Output output("ZDC", "TDCCALIBDATA", 0);
     pc.outputs().snapshot(output, mWorker.mData);
     for (int ih = 0; ih < TDCCalibData::NTDC; ih++) {
-      o2::framework::Output output("ZDC", "TDC_1DH", ih, Lifetime::Timeframe);
+      o2::framework::Output output("ZDC", "TDC_1DH", ih);
       pc.outputs().snapshot(output, mWorker.mTDC[ih]->getBase());
     }
     mWorker.clear();
diff --git a/Detectors/ZDC/calib/src/WaveformCalibEPNSpec.cxx b/Detectors/ZDC/calib/src/WaveformCalibEPNSpec.cxx
index 62ee862905524..72f606cc4f6ed 100644
--- a/Detectors/ZDC/calib/src/WaveformCalibEPNSpec.cxx
+++ b/Detectors/ZDC/calib/src/WaveformCalibEPNSpec.cxx
@@ -107,7 +107,7 @@ void WaveformCalibEPNSpec::run(ProcessingContext& pc)
   mWorker.process(bcrec, energy, tdc, info, wave);
 
   // Send intermediate calibration data
-  o2::framework::Output output("ZDC", "WAVECALIBDATA", 0, Lifetime::Timeframe);
+  o2::framework::Output output("ZDC", "WAVECALIBDATA", 0);
   pc.outputs().snapshot(output, mWorker.mData);
 }
 
diff --git a/Detectors/ZDC/raw/include/ZDCRaw/RawReaderZDC.h b/Detectors/ZDC/raw/include/ZDCRaw/RawReaderZDC.h
index b27f7e34353c0..70f7ce30647f5 100644
--- a/Detectors/ZDC/raw/include/ZDCRaw/RawReaderZDC.h
+++ b/Detectors/ZDC/raw/include/ZDCRaw/RawReaderZDC.h
@@ -95,9 +95,9 @@ class RawReaderZDC
 
   void makeSnapshot(o2::framework::ProcessingContext& pc)
   {
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginZDC, "DIGITSBC", 0, o2::framework::Lifetime::Timeframe}, mDigitsBC);
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginZDC, "DIGITSCH", 0, o2::framework::Lifetime::Timeframe}, mDigitsCh);
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginZDC, "DIGITSPD", 0, o2::framework::Lifetime::Timeframe}, mOrbitData);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginZDC, "DIGITSBC", 0}, mDigitsBC);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginZDC, "DIGITSCH", 0}, mDigitsCh);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginZDC, "DIGITSPD", 0}, mOrbitData);
   }
 };
 } // namespace zdc
diff --git a/Detectors/ZDC/workflow/src/DigitReaderSpec.cxx b/Detectors/ZDC/workflow/src/DigitReaderSpec.cxx
index 59b0e34d16ecd..e952111e0c6c3 100644
--- a/Detectors/ZDC/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/ZDC/workflow/src/DigitReaderSpec.cxx
@@ -69,11 +69,11 @@ void DigitReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "ZDCDigitReader pushed " << zdcOrbitData.size() << " orbits with " << zdcBCData.size() << " bcs and " << zdcChData.size() << " digits";
-  pc.outputs().snapshot(Output{"ZDC", "DIGITSPD", 0, Lifetime::Timeframe}, zdcOrbitData);
-  pc.outputs().snapshot(Output{"ZDC", "DIGITSBC", 0, Lifetime::Timeframe}, zdcBCData);
-  pc.outputs().snapshot(Output{"ZDC", "DIGITSCH", 0, Lifetime::Timeframe}, zdcChData);
+  pc.outputs().snapshot(Output{"ZDC", "DIGITSPD", 0}, zdcOrbitData);
+  pc.outputs().snapshot(Output{"ZDC", "DIGITSBC", 0}, zdcBCData);
+  pc.outputs().snapshot(Output{"ZDC", "DIGITSCH", 0}, zdcChData);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"ZDC", "DIGITSLBL", 0, Lifetime::Timeframe}, labels);
+    pc.outputs().snapshot(Output{"ZDC", "DIGITSLBL", 0}, labels);
   }
   uint64_t nextEntry = mTree->GetReadEntry() + 1;
   if (nextEntry >= mTree->GetEntries() || (mLastEntry >= 0 && nextEntry > mLastEntry)) {
diff --git a/Detectors/ZDC/workflow/src/DigitRecoSpec.cxx b/Detectors/ZDC/workflow/src/DigitRecoSpec.cxx
index d504d33483081..e9b63c5b49d5b 100644
--- a/Detectors/ZDC/workflow/src/DigitRecoSpec.cxx
+++ b/Detectors/ZDC/workflow/src/DigitRecoSpec.cxx
@@ -264,11 +264,11 @@ void DigitRecoSpec::run(ProcessingContext& pc)
   }
   // TODO: rate information for all channels
   // TODO: summary of reconstruction to be collected by DQM?
-  pc.outputs().snapshot(Output{"ZDC", "BCREC", 0, Lifetime::Timeframe}, recEvent.mRecBC);
-  pc.outputs().snapshot(Output{"ZDC", "ENERGY", 0, Lifetime::Timeframe}, recEvent.mEnergy);
-  pc.outputs().snapshot(Output{"ZDC", "TDCDATA", 0, Lifetime::Timeframe}, recEvent.mTDCData);
-  pc.outputs().snapshot(Output{"ZDC", "INFO", 0, Lifetime::Timeframe}, recEvent.mInfo);
-  pc.outputs().snapshot(Output{"ZDC", "WAVE", 0, Lifetime::Timeframe}, recEvent.mWaveform);
+  pc.outputs().snapshot(Output{"ZDC", "BCREC", 0}, recEvent.mRecBC);
+  pc.outputs().snapshot(Output{"ZDC", "ENERGY", 0}, recEvent.mEnergy);
+  pc.outputs().snapshot(Output{"ZDC", "TDCDATA", 0}, recEvent.mTDCData);
+  pc.outputs().snapshot(Output{"ZDC", "INFO", 0}, recEvent.mInfo);
+  pc.outputs().snapshot(Output{"ZDC", "WAVE", 0}, recEvent.mWaveform);
   mTimer.Stop();
 }
 
diff --git a/Detectors/ZDC/workflow/src/EntropyEncoderSpec.cxx b/Detectors/ZDC/workflow/src/EntropyEncoderSpec.cxx
index 7c30a0f34d151..abbd821fcb749 100644
--- a/Detectors/ZDC/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/ZDC/workflow/src/EntropyEncoderSpec.cxx
@@ -58,7 +58,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
     mCTFCoder.setSelectedIRFrames(pc.inputs().get<gsl::span<o2::dataformats::IRFrame>>("selIRFrames"));
   }
 
-  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"ZDC", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"ZDC", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, bcdata, chans, peds);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   if (mSelIR) {
diff --git a/Detectors/ZDC/workflow/src/RecEventReaderSpec.cxx b/Detectors/ZDC/workflow/src/RecEventReaderSpec.cxx
index acb0cef3e29be..18c620e427569 100644
--- a/Detectors/ZDC/workflow/src/RecEventReaderSpec.cxx
+++ b/Detectors/ZDC/workflow/src/RecEventReaderSpec.cxx
@@ -49,10 +49,10 @@ void RecEventReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
 
   LOG(info) << "ZDC RecEventReader pushes " << mBCRecData->size() << " events with " << mBCRecData->size() << " energy, " << mZDCTDCData->size() << " TDC and " << mZDCInfo->size() << " info records at entry " << ent;
-  pc.outputs().snapshot(Output{"ZDC", "BCREC", 0, Lifetime::Timeframe}, *mBCRecData);
-  pc.outputs().snapshot(Output{"ZDC", "ENERGY", 0, Lifetime::Timeframe}, *mZDCEnergy);
-  pc.outputs().snapshot(Output{"ZDC", "TDCDATA", 0, Lifetime::Timeframe}, *mZDCTDCData);
-  pc.outputs().snapshot(Output{"ZDC", "INFO", 0, Lifetime::Timeframe}, *mZDCInfo);
+  pc.outputs().snapshot(Output{"ZDC", "BCREC", 0}, *mBCRecData);
+  pc.outputs().snapshot(Output{"ZDC", "ENERGY", 0}, *mZDCEnergy);
+  pc.outputs().snapshot(Output{"ZDC", "TDCDATA", 0}, *mZDCTDCData);
+  pc.outputs().snapshot(Output{"ZDC", "INFO", 0}, *mZDCInfo);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
diff --git a/Detectors/ZDC/workflow/src/RecoReaderSpec.cxx b/Detectors/ZDC/workflow/src/RecoReaderSpec.cxx
index 20b9510243be4..ebea51c932c5c 100644
--- a/Detectors/ZDC/workflow/src/RecoReaderSpec.cxx
+++ b/Detectors/ZDC/workflow/src/RecoReaderSpec.cxx
@@ -64,10 +64,10 @@ void RecoReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(info) << "ZDCRecoReader pushed " << RecBC.size() << " b.c. " << Energy.size() << " Energies " << TDCData.size() << " TDCs " << Info.size() << " Infos";
-  pc.outputs().snapshot(Output{"ZDC", "BCREC", 0, Lifetime::Timeframe}, RecBC);
-  pc.outputs().snapshot(Output{"ZDC", "ENERGY", 0, Lifetime::Timeframe}, Energy);
-  pc.outputs().snapshot(Output{"ZDC", "TDCDATA", 0, Lifetime::Timeframe}, TDCData);
-  pc.outputs().snapshot(Output{"ZDC", "INFO", 0, Lifetime::Timeframe}, Info);
+  pc.outputs().snapshot(Output{"ZDC", "BCREC", 0}, RecBC);
+  pc.outputs().snapshot(Output{"ZDC", "ENERGY", 0}, Energy);
+  pc.outputs().snapshot(Output{"ZDC", "TDCDATA", 0}, TDCData);
+  pc.outputs().snapshot(Output{"ZDC", "INFO", 0}, Info);
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get<ControlService>().endOfStream();
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Framework/Core/include/Framework/Output.h b/Framework/Core/include/Framework/Output.h
index b7d26e8dfd7ee..9f85d6e8edb92 100644
--- a/Framework/Core/include/Framework/Output.h
+++ b/Framework/Core/include/Framework/Output.h
@@ -43,6 +43,11 @@ struct Output {
   {
   }
 
+  Output(header::DataOrigin o, header::DataDescription d, header::DataHeader::SubSpecificationType s, header::Stack&& stack)
+    : origin(o), description(d), subSpec(s), metaHeader(std::move(stack))
+  {
+  }
+
   Output(header::DataOrigin o, header::DataDescription d, header::DataHeader::SubSpecificationType s, Lifetime l,
          header::Stack&& stack)
     : origin(o), description(d), subSpec(s), lifetime(l), metaHeader(std::move(stack))
diff --git a/GPU/Workflow/src/GPUWorkflowITS.cxx b/GPU/Workflow/src/GPUWorkflowITS.cxx
index e82f2348255d7..1785ddd05f39c 100644
--- a/GPU/Workflow/src/GPUWorkflowITS.cxx
+++ b/GPU/Workflow/src/GPUWorkflowITS.cxx
@@ -134,8 +134,8 @@ int GPURecoWorkflowSpec::runITSTracking(o2::framework::ProcessingContext& pc)
 
   auto rofsinput = pc.inputs().get<gsl::span<o2::itsmft::ROFRecord>>("ROframes");
 
-  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "ITSTrackROF", 0, Lifetime::Timeframe}, rofsinput.begin(), rofsinput.end());
-  auto& irFrames = pc.outputs().make<std::vector<o2::dataformats::IRFrame>>(Output{"ITS", "IRFRAMES", 0, Lifetime::Timeframe});
+  auto& rofs = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "ITSTrackROF", 0}, rofsinput.begin(), rofsinput.end());
+  auto& irFrames = pc.outputs().make<std::vector<o2::dataformats::IRFrame>>(Output{"ITS", "IRFRAMES", 0});
   irFrames.reserve(rofs.size());
 
   const auto& alpParams = o2::itsmft::DPLAlpideParam<o2::detectors::DetID::ITS>::Instance(); // RS: this should come from CCDB
@@ -148,19 +148,19 @@ int GPURecoWorkflowSpec::runITSTracking(o2::framework::ProcessingContext& pc)
   if (mSpecConfig.processMC) {
     labels = pc.inputs().get<const dataformats::MCTruthContainer<MCCompLabel>*>("itsmclabels").release();
     // get the array as read-only span, a snapshot is sent forward
-    pc.outputs().snapshot(Output{"ITS", "ITSTrackMC2ROF", 0, Lifetime::Timeframe}, pc.inputs().get<gsl::span<itsmft::MC2ROFRecord>>("ITSMC2ROframes"));
+    pc.outputs().snapshot(Output{"ITS", "ITSTrackMC2ROF", 0}, pc.inputs().get<gsl::span<itsmft::MC2ROFRecord>>("ITSMC2ROframes"));
     LOG(info) << labels->getIndexedSize() << " MC label objects , in " << mc2rofs.size() << " MC events";
   }
 
-  auto& allClusIdx = pc.outputs().make<std::vector<int>>(Output{"ITS", "TRACKCLSID", 0, Lifetime::Timeframe});
-  auto& allTracks = pc.outputs().make<std::vector<o2::its::TrackITS>>(Output{"ITS", "TRACKS", 0, Lifetime::Timeframe});
-  auto& vertROFvec = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "VERTICESROF", 0, Lifetime::Timeframe});
-  auto& vertices = pc.outputs().make<std::vector<Vertex>>(Output{"ITS", "VERTICES", 0, Lifetime::Timeframe});
+  auto& allClusIdx = pc.outputs().make<std::vector<int>>(Output{"ITS", "TRACKCLSID", 0});
+  auto& allTracks = pc.outputs().make<std::vector<o2::its::TrackITS>>(Output{"ITS", "TRACKS", 0});
+  auto& vertROFvec = pc.outputs().make<std::vector<o2::itsmft::ROFRecord>>(Output{"ITS", "VERTICESROF", 0});
+  auto& vertices = pc.outputs().make<std::vector<Vertex>>(Output{"ITS", "VERTICES", 0});
 
   // MC
   static pmr::vector<o2::MCCompLabel> dummyMCLabTracks, dummyMCLabVerts;
-  auto& allTrackLabels = mSpecConfig.processMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"ITS", "TRACKSMCTR", 0, Lifetime::Timeframe}) : dummyMCLabTracks;
-  auto& allVerticesLabels = mSpecConfig.processMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"ITS", "VERTICESMCTR", 0, Lifetime::Timeframe}) : dummyMCLabVerts;
+  auto& allTrackLabels = mSpecConfig.processMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"ITS", "TRACKSMCTR", 0}) : dummyMCLabTracks;
+  auto& allVerticesLabels = mSpecConfig.processMC ? pc.outputs().make<std::vector<o2::MCCompLabel>>(Output{"ITS", "VERTICESMCTR", 0}) : dummyMCLabVerts;
 
   std::uint32_t roFrame = 0;
 
diff --git a/GPU/Workflow/src/GPUWorkflowPipeline.cxx b/GPU/Workflow/src/GPUWorkflowPipeline.cxx
index c50f8f67cbd32..0352151b342c6 100644
--- a/GPU/Workflow/src/GPUWorkflowPipeline.cxx
+++ b/GPU/Workflow/src/GPUWorkflowPipeline.cxx
@@ -177,7 +177,7 @@ int GPURecoWorkflowSpec::handlePipeline(ProcessingContext& pc, GPUTrackingInOutP
     ptrs.tpcZS = &tpcZS;
   }
   if (mSpecConfig.enableDoublePipeline == 2) {
-    auto prepareBuffer = pc.outputs().make<DataAllocator::UninitializedVector<char>>(Output{gDataOriginGPU, "PIPELINEPREPARE", 0, Lifetime::Timeframe}, 0u);
+    auto prepareBuffer = pc.outputs().make<DataAllocator::UninitializedVector<char>>(Output{gDataOriginGPU, "PIPELINEPREPARE", 0}, 0u);
 
     size_t ptrsTotal = 0;
     const void* firstPtr = nullptr;
diff --git a/GPU/Workflow/src/GPUWorkflowSpec.cxx b/GPU/Workflow/src/GPUWorkflowSpec.cxx
index 40b6c13495a66..fb482df5b11c0 100644
--- a/GPU/Workflow/src/GPUWorkflowSpec.cxx
+++ b/GPU/Workflow/src/GPUWorkflowSpec.cxx
@@ -734,7 +734,7 @@ void GPURecoWorkflowSpec::run(ProcessingContext& pc)
   };
 
   setOutputAllocator("COMPCLUSTERSFLAT", mSpecConfig.outputCompClustersFlat, outputRegions.compressedClusters, std::make_tuple(gDataOriginTPC, (DataDescription) "COMPCLUSTERSFLAT", 0));
-  setOutputAllocator("CLUSTERNATIVE", mClusterOutputIds.size() > 0, outputRegions.clustersNative, std::make_tuple(gDataOriginTPC, mSpecConfig.sendClustersPerSector ? (DataDescription) "CLUSTERNATIVETMP" : (DataDescription) "CLUSTERNATIVE", NSectors, Lifetime::Timeframe, clusterOutputSectorHeader), sizeof(o2::tpc::ClusterCountIndex));
+  setOutputAllocator("CLUSTERNATIVE", mClusterOutputIds.size() > 0, outputRegions.clustersNative, std::make_tuple(gDataOriginTPC, mSpecConfig.sendClustersPerSector ? (DataDescription) "CLUSTERNATIVETMP" : (DataDescription) "CLUSTERNATIVE", NSectors, clusterOutputSectorHeader), sizeof(o2::tpc::ClusterCountIndex));
   setOutputAllocator("CLSHAREDMAP", mSpecConfig.outputSharedClusterMap, outputRegions.sharedClusterMap, std::make_tuple(gDataOriginTPC, (DataDescription) "CLSHAREDMAP", 0));
   setOutputAllocator("TRACKS", mSpecConfig.outputTracks, outputRegions.tpcTracksO2, std::make_tuple(gDataOriginTPC, (DataDescription) "TRACKS", 0));
   setOutputAllocator("CLUSREFS", mSpecConfig.outputTracks, outputRegions.tpcTracksO2ClusRefs, std::make_tuple(gDataOriginTPC, (DataDescription) "CLUSREFS", 0));
@@ -909,7 +909,7 @@ void GPURecoWorkflowSpec::run(ProcessingContext& pc)
         if (mTPCSectorMask & (1ul << i)) {
           DataHeader::SubSpecificationType subspec = i;
           clusterOutputSectorHeader.sectorBits = (1ul << i);
-          char* buffer = pc.outputs().make<char>({gDataOriginTPC, "CLUSTERNATIVE", subspec, Lifetime::Timeframe, {clusterOutputSectorHeader}}, accessIndex.nClustersSector[i] * sizeof(*accessIndex.clustersLinear) + sizeof(o2::tpc::ClusterCountIndex)).data();
+          char* buffer = pc.outputs().make<char>({gDataOriginTPC, "CLUSTERNATIVE", subspec, {clusterOutputSectorHeader}}, accessIndex.nClustersSector[i] * sizeof(*accessIndex.clustersLinear) + sizeof(o2::tpc::ClusterCountIndex)).data();
           o2::tpc::ClusterCountIndex* outIndex = reinterpret_cast<o2::tpc::ClusterCountIndex*>(buffer);
           memset(outIndex, 0, sizeof(*outIndex));
           for (int j = 0; j < o2::tpc::constants::MAXGLOBALPADROW; j++) {
@@ -926,7 +926,7 @@ void GPURecoWorkflowSpec::run(ProcessingContext& pc)
             }
             ConstMCLabelContainer contflat;
             cont.flatten_to(contflat);
-            pc.outputs().snapshot({gDataOriginTPC, "CLNATIVEMCLBL", subspec, Lifetime::Timeframe, {clusterOutputSectorHeader}}, contflat);
+            pc.outputs().snapshot({gDataOriginTPC, "CLNATIVEMCLBL", subspec, {clusterOutputSectorHeader}}, contflat);
           }
         }
       }
@@ -937,7 +937,7 @@ void GPURecoWorkflowSpec::run(ProcessingContext& pc)
       static_assert(sizeof(o2::tpc::ClusterCountIndex) == sizeof(accessIndex.nClusters));
       memcpy(outIndex, &accessIndex.nClusters[0][0], sizeof(o2::tpc::ClusterCountIndex));
       if (mSpecConfig.processMC && mSpecConfig.caClusterer && accessIndex.clustersMCTruth) {
-        pc.outputs().snapshot({gDataOriginTPC, "CLNATIVEMCLBL", subspec, Lifetime::Timeframe, {clusterOutputSectorHeader}}, clustersMCBuffer.first);
+        pc.outputs().snapshot({gDataOriginTPC, "CLNATIVEMCLBL", subspec, {clusterOutputSectorHeader}}, clustersMCBuffer.first);
       }
     }
   }
@@ -952,17 +952,17 @@ void GPURecoWorkflowSpec::run(ProcessingContext& pc)
     if (sendQAOutput) {
       mQA->postprocessExternal(copy1, copy2, copy3, copy4, out, mQATaskMask ? mQATaskMask : -1);
     }
-    pc.outputs().snapshot({gDataOriginTPC, "TRACKINGQA", 0, Lifetime::Timeframe}, out);
+    pc.outputs().snapshot({gDataOriginTPC, "TRACKINGQA", 0}, out);
     if (sendQAOutput) {
       mQA->cleanup();
     }
   }
   if (mSpecConfig.outputErrorQA) {
-    pc.outputs().snapshot({gDataOriginGPU, "ERRORQA", 0, Lifetime::Timeframe}, mErrorQA);
+    pc.outputs().snapshot({gDataOriginGPU, "ERRORQA", 0}, mErrorQA);
     mErrorQA.clear(); // FIXME: This is a race condition once we run multi-threaded!
   }
   if (mSpecConfig.tpcTriggerHandling && !(mSpecConfig.zsOnTheFly || mSpecConfig.zsDecoder)) {
-    pc.outputs().make<DataAllocator::UninitializedVector<outputDataType>>(Output{gDataOriginTPC, "TRIGGERWORDS", 0, Lifetime::Timeframe}, 0u);
+    pc.outputs().make<DataAllocator::UninitializedVector<outputDataType>>(Output{gDataOriginTPC, "TRIGGERWORDS", 0}, 0u);
   }
   mTimer->Stop();
   LOG(info) << "GPU Reconstruction time for this TF " << mTimer->CpuTime() - cput << " s (cpu), " << mTimer->RealTime() - realt << " s (wall)";
diff --git a/Steer/DigitizerWorkflow/src/CPVDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/CPVDigitizerSpec.cxx
index 57fb11249b312..fc64106e39d79 100644
--- a/Steer/DigitizerWorkflow/src/CPVDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/CPVDigitizerSpec.cxx
@@ -171,15 +171,15 @@ void DigitizerSpec::run(framework::ProcessingContext& pc)
   }
   LOG(debug) << "Have " << mLabels.getNElements() << " CPV labels ";
   // here we have all digits and we can send them to consumer (aka snapshot it onto output)
-  pc.outputs().snapshot(Output{"CPV", "DIGITS", 0, Lifetime::Timeframe}, mDigitsOut);
-  pc.outputs().snapshot(Output{"CPV", "DIGITTRIGREC", 0, Lifetime::Timeframe}, triggers);
+  pc.outputs().snapshot(Output{"CPV", "DIGITS", 0}, mDigitsOut);
+  pc.outputs().snapshot(Output{"CPV", "DIGITTRIGREC", 0}, triggers);
   if (pc.outputs().isAllowed({"CPV", "DIGITSMCTR", 0})) {
-    pc.outputs().snapshot(Output{"CPV", "DIGITSMCTR", 0, Lifetime::Timeframe}, mLabels);
+    pc.outputs().snapshot(Output{"CPV", "DIGITSMCTR", 0}, mLabels);
   }
   // CPV is always a triggered detector
   const o2::parameters::GRPObject::ROMode roMode = o2::parameters::GRPObject::PRESENT;
   LOG(debug) << "CPV: Sending ROMode= " << roMode << " to GRPUpdater";
-  pc.outputs().snapshot(Output{"CPV", "ROMode", 0, Lifetime::Timeframe}, roMode);
+  pc.outputs().snapshot(Output{"CPV", "ROMode", 0}, roMode);
 
   timer.Stop();
   LOG(info) << "Digitization took " << timer.CpuTime() << "s";
diff --git a/Steer/DigitizerWorkflow/src/CTPDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/CTPDigitizerSpec.cxx
index 412e6d6129a76..a72b9e26581eb 100644
--- a/Steer/DigitizerWorkflow/src/CTPDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/CTPDigitizerSpec.cxx
@@ -79,9 +79,9 @@ class CTPDPLDigitizerTask : public o2::base::BaseDPLDigitizer
     auto digits = mDigitizer.process(ginputs);
     // send out to next stage
     LOG(info) << "CTP DIGITS being sent.";
-    pc.outputs().snapshot(Output{"CTP", "DIGITS", 0, Lifetime::Timeframe}, digits);
+    pc.outputs().snapshot(Output{"CTP", "DIGITS", 0}, digits);
     LOG(info) << "CTP PRESENT being sent.";
-    pc.outputs().snapshot(Output{"CTP", "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{"CTP", "ROMode", 0}, mROMode);
     timer.Stop();
     LOG(info) << "CTP Digitization took " << timer.CpuTime() << "s";
   }
diff --git a/Steer/DigitizerWorkflow/src/FDDDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/FDDDigitizerSpec.cxx
index 99150a3cb83fb..901c7e709d265 100644
--- a/Steer/DigitizerWorkflow/src/FDDDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/FDDDigitizerSpec.cxx
@@ -120,17 +120,17 @@ class FDDDPLDigitizerTask : public o2::base::BaseDPLDigitizer
     mDigitizer.flush(mDigitsBC, mDigitsCh, mDigitsTrig, labels);
 
     // send out to next stage
-    pc.outputs().snapshot(Output{"FDD", "DIGITSBC", 0, Lifetime::Timeframe}, mDigitsBC);
-    pc.outputs().snapshot(Output{"FDD", "DIGITSCH", 0, Lifetime::Timeframe}, mDigitsCh);
-    pc.outputs().snapshot(Output{"FDD", "TRIGGERINPUT", 0, Lifetime::Timeframe}, mDigitsTrig);
+    pc.outputs().snapshot(Output{"FDD", "DIGITSBC", 0}, mDigitsBC);
+    pc.outputs().snapshot(Output{"FDD", "DIGITSCH", 0}, mDigitsCh);
+    pc.outputs().snapshot(Output{"FDD", "TRIGGERINPUT", 0}, mDigitsTrig);
     if (pc.outputs().isAllowed({"FDD", "DIGITLBL", 0})) {
-      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::fdd::MCLabel>>(Output{"FDD", "DIGITLBL", 0, Lifetime::Timeframe});
+      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::fdd::MCLabel>>(Output{"FDD", "DIGITLBL", 0});
       labels.flatten_to(sharedlabels);
       labels.clear_andfreememory();
     }
 
     LOG(info) << "FDD: Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{"FDD", "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{"FDD", "ROMode", 0}, mROMode);
 
     // we should be only called once; tell DPL that this process is ready to exit
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Steer/DigitizerWorkflow/src/FT0DigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/FT0DigitizerSpec.cxx
index 4eda984358634..9098240d17627 100644
--- a/Steer/DigitizerWorkflow/src/FT0DigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/FT0DigitizerSpec.cxx
@@ -138,14 +138,14 @@ class FT0DPLDigitizerTask : public o2::base::BaseDPLDigitizer
     mDigitizer.flush_all(mDigitsBC, mDigitsCh, mDigitsTrig, labels);
 
     // send out to next stage
-    pc.outputs().snapshot(Output{"FT0", "DIGITSBC", 0, Lifetime::Timeframe}, mDigitsBC);
-    pc.outputs().snapshot(Output{"FT0", "DIGITSCH", 0, Lifetime::Timeframe}, mDigitsCh);
-    pc.outputs().snapshot(Output{"FT0", "TRIGGERINPUT", 0, Lifetime::Timeframe}, mDigitsTrig);
+    pc.outputs().snapshot(Output{"FT0", "DIGITSBC", 0}, mDigitsBC);
+    pc.outputs().snapshot(Output{"FT0", "DIGITSCH", 0}, mDigitsCh);
+    pc.outputs().snapshot(Output{"FT0", "TRIGGERINPUT", 0}, mDigitsTrig);
     if (pc.outputs().isAllowed({"FT0", "DIGITSMCTR", 0})) {
-      pc.outputs().snapshot(Output{"FT0", "DIGITSMCTR", 0, Lifetime::Timeframe}, labels);
+      pc.outputs().snapshot(Output{"FT0", "DIGITSMCTR", 0}, labels);
     }
     LOG(info) << "FT0: Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{"FT0", "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{"FT0", "ROMode", 0}, mROMode);
 
     timer.Stop();
     LOG(info) << "Digitization took " << timer.CpuTime() << "s";
diff --git a/Steer/DigitizerWorkflow/src/FV0DigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/FV0DigitizerSpec.cxx
index 151175e0c0758..bb5174dfcbb2c 100644
--- a/Steer/DigitizerWorkflow/src/FV0DigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/FV0DigitizerSpec.cxx
@@ -102,14 +102,14 @@ class FV0DPLDigitizerTask : public o2::base::BaseDPLDigitizer
     LOG(info) << "FV0: Sending " << mDigitsBC.size() << " digitsBC and " << mDigitsCh.size() << " digitsCh.";
 
     // send out to next stage
-    pc.outputs().snapshot(Output{"FV0", "DIGITSBC", 0, Lifetime::Timeframe}, mDigitsBC);
-    pc.outputs().snapshot(Output{"FV0", "DIGITSCH", 0, Lifetime::Timeframe}, mDigitsCh);
-    pc.outputs().snapshot(Output{"FV0", "TRIGGERINPUT", 0, Lifetime::Timeframe}, mDigitsTrig);
+    pc.outputs().snapshot(Output{"FV0", "DIGITSBC", 0}, mDigitsBC);
+    pc.outputs().snapshot(Output{"FV0", "DIGITSCH", 0}, mDigitsCh);
+    pc.outputs().snapshot(Output{"FV0", "TRIGGERINPUT", 0}, mDigitsTrig);
     if (pc.outputs().isAllowed({"FV0", "DIGITLBL", 0})) {
-      pc.outputs().snapshot(Output{"FV0", "DIGITLBL", 0, Lifetime::Timeframe}, mLabels);
+      pc.outputs().snapshot(Output{"FV0", "DIGITLBL", 0}, mLabels);
     }
     LOG(info) << "FV0: Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{"FV0", "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{"FV0", "ROMode", 0}, mROMode);
 
     // we should be only called once; tell DPL that this process is ready to exit
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Steer/DigitizerWorkflow/src/HMPIDDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/HMPIDDigitizerSpec.cxx
index 22ab3e5d7ef84..0adb9eb0a90df 100644
--- a/Steer/DigitizerWorkflow/src/HMPIDDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/HMPIDDigitizerSpec.cxx
@@ -132,13 +132,13 @@ class HMPIDDPLDigitizerTask : public o2::base::BaseDPLDigitizer
     flushDigitsAndLabels();
 
     // send out to next stage
-    pc.outputs().snapshot(Output{"HMP", "DIGITS", 0, Lifetime::Timeframe}, digitsAccum);
-    pc.outputs().snapshot(Output{"HMP", "INTRECORDS", 0, Lifetime::Timeframe}, mIntRecord);
+    pc.outputs().snapshot(Output{"HMP", "DIGITS", 0}, digitsAccum);
+    pc.outputs().snapshot(Output{"HMP", "INTRECORDS", 0}, mIntRecord);
     if (pc.outputs().isAllowed({"HMP", "DIGITLBL", 0})) {
-      pc.outputs().snapshot(Output{"HMP", "DIGITLBL", 0, Lifetime::Timeframe}, labelAccum);
+      pc.outputs().snapshot(Output{"HMP", "DIGITLBL", 0}, labelAccum);
     }
     LOG(info) << "HMP: Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{"HMP", "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{"HMP", "ROMode", 0}, mROMode);
 
     // we should be only called once; tell DPL that this process is ready to exit
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Steer/DigitizerWorkflow/src/ITS3DigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/ITS3DigitizerSpec.cxx
index 9f03bd6733729..21f808f1e1ed2 100644
--- a/Steer/DigitizerWorkflow/src/ITS3DigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/ITS3DigitizerSpec.cxx
@@ -143,7 +143,7 @@ class ITS3DPLDigitizerTask : BaseDPLDigitizer
     mDigitizer.setMCLabels(&mLabels);
 
     // digits are directly put into DPL owned resource
-    auto& digitsAccum = pc.outputs().make<std::vector<itsmft::Digit>>(Output{mOrigin, "DIGITS", 0, Lifetime::Timeframe});
+    auto& digitsAccum = pc.outputs().make<std::vector<itsmft::Digit>>(Output{mOrigin, "DIGITS", 0});
 
     auto accumulate = [this, &digitsAccum]() {
       // accumulate result of single event processing, called after processing every event supplied
@@ -216,17 +216,17 @@ class ITS3DPLDigitizerTask : BaseDPLDigitizer
 
     // here we have all digits and labels and we can send them to consumer (aka snapshot it onto output)
 
-    pc.outputs().snapshot(Output{mOrigin, "DIGITSROF", 0, Lifetime::Timeframe}, mROFRecordsAccum);
+    pc.outputs().snapshot(Output{mOrigin, "DIGITSROF", 0}, mROFRecordsAccum);
     if (mWithMCTruth) {
-      pc.outputs().snapshot(Output{mOrigin, "DIGITSMC2ROF", 0, Lifetime::Timeframe}, mMC2ROFRecordsAccum);
-      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{mOrigin, "DIGITSMCTR", 0, Lifetime::Timeframe});
+      pc.outputs().snapshot(Output{mOrigin, "DIGITSMC2ROF", 0}, mMC2ROFRecordsAccum);
+      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{mOrigin, "DIGITSMCTR", 0});
       mLabelsAccum.flatten_to(sharedlabels);
       // free space of existing label containers
       mLabels.clear_andfreememory();
       mLabelsAccum.clear_andfreememory();
     }
     LOG(info) << mID.getName() << ": Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{mOrigin, "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{mOrigin, "ROMode", 0}, mROMode);
 
     timer.Stop();
     LOG(info) << "Digitization took " << timer.CpuTime() << "s";
diff --git a/Steer/DigitizerWorkflow/src/ITSMFTDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/ITSMFTDigitizerSpec.cxx
index e1291fe0cb0d8..2a8c28bb4dbbd 100644
--- a/Steer/DigitizerWorkflow/src/ITSMFTDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/ITSMFTDigitizerSpec.cxx
@@ -82,7 +82,7 @@ class ITSMFTDPLDigitizerTask : BaseDPLDigitizer
     mDigitizer.setMCLabels(&mLabels);
 
     // digits are directly put into DPL owned resource
-    auto& digitsAccum = pc.outputs().make<std::vector<itsmft::Digit>>(Output{mOrigin, "DIGITS", 0, Lifetime::Timeframe});
+    auto& digitsAccum = pc.outputs().make<std::vector<itsmft::Digit>>(Output{mOrigin, "DIGITS", 0});
 
     auto accumulate = [this, &digitsAccum]() {
       // accumulate result of single event processing, called after processing every event supplied
@@ -160,17 +160,17 @@ class ITSMFTDPLDigitizerTask : BaseDPLDigitizer
 
     // here we have all digits and labels and we can send them to consumer (aka snapshot it onto output)
 
-    pc.outputs().snapshot(Output{mOrigin, "DIGITSROF", 0, Lifetime::Timeframe}, mROFRecordsAccum);
+    pc.outputs().snapshot(Output{mOrigin, "DIGITSROF", 0}, mROFRecordsAccum);
     if (mWithMCTruth) {
-      pc.outputs().snapshot(Output{mOrigin, "DIGITSMC2ROF", 0, Lifetime::Timeframe}, mMC2ROFRecordsAccum);
-      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{mOrigin, "DIGITSMCTR", 0, Lifetime::Timeframe});
+      pc.outputs().snapshot(Output{mOrigin, "DIGITSMC2ROF", 0}, mMC2ROFRecordsAccum);
+      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{mOrigin, "DIGITSMCTR", 0});
       mLabelsAccum.flatten_to(sharedlabels);
       // free space of existing label containers
       mLabels.clear_andfreememory();
       mLabelsAccum.clear_andfreememory();
     }
     LOG(info) << mID.getName() << ": Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{mOrigin, "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{mOrigin, "ROMode", 0}, mROMode);
 
     timer.Stop();
     LOG(info) << "Digitization took " << timer.CpuTime() << "s";
diff --git a/Steer/DigitizerWorkflow/src/MCHDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/MCHDigitizerSpec.cxx
index fc92020f25b21..187c24f962c1f 100644
--- a/Steer/DigitizerWorkflow/src/MCHDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/MCHDigitizerSpec.cxx
@@ -111,12 +111,12 @@ class MCHDPLDigitizerTask : public o2::base::BaseDPLDigitizer
     dataformats::MCLabelContainer labels{};
     auto nPileup = mDigitizer->digitize(rofs, digits, labels);
 
-    pc.outputs().snapshot(Output{"MCH", "DIGITS", 0, Lifetime::Timeframe}, digits);
-    pc.outputs().snapshot(Output{"MCH", "DIGITROFS", 0, Lifetime::Timeframe}, rofs);
+    pc.outputs().snapshot(Output{"MCH", "DIGITS", 0}, digits);
+    pc.outputs().snapshot(Output{"MCH", "DIGITROFS", 0}, rofs);
     if (pc.outputs().isAllowed({"MCH", "DIGITSLABELS", 0})) {
-      pc.outputs().snapshot(Output{"MCH", "DIGITSLABELS", 0, Lifetime::Timeframe}, labels);
+      pc.outputs().snapshot(Output{"MCH", "DIGITSLABELS", 0}, labels);
     }
-    pc.outputs().snapshot(Output{"MCH", "ROMode", 0, Lifetime::Timeframe},
+    pc.outputs().snapshot(Output{"MCH", "ROMode", 0},
                           DigitizerParam::Instance().continuous ? o2::parameters::GRPObject::CONTINUOUS : o2::parameters::GRPObject::TRIGGERING);
 
     // we should be only called once; tell DPL that this process is ready to exit
diff --git a/Steer/DigitizerWorkflow/src/MCTruthReaderSpec.h b/Steer/DigitizerWorkflow/src/MCTruthReaderSpec.h
index e3359af01b1e5..8204f4e437cc3 100644
--- a/Steer/DigitizerWorkflow/src/MCTruthReaderSpec.h
+++ b/Steer/DigitizerWorkflow/src/MCTruthReaderSpec.h
@@ -61,7 +61,7 @@ class MCTruthReaderTask : public o2::framework::Task
       br->GetEntry(0);
 
       // publish the labels in a const shared memory container
-      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{"TST", "LABELS2", 0, Lifetime::Timeframe});
+      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{"TST", "LABELS2", 0});
       iocontainer->copyandflatten(sharedlabels);
 
     } else {
@@ -74,7 +74,7 @@ class MCTruthReaderTask : public o2::framework::Task
       LOG(info) << "MCCONTAINER CHECK" << mccontainer->getNElements();
 
       // publish the original labels
-      pc.outputs().snapshot(Output{"TST", "LABELS2", 0, Lifetime::Timeframe}, *mccontainer);
+      pc.outputs().snapshot(Output{"TST", "LABELS2", 0}, *mccontainer);
     }
     // we should be only called once; tell DPL that this process is ready to exit
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Steer/DigitizerWorkflow/src/MCTruthSourceSpec.cxx b/Steer/DigitizerWorkflow/src/MCTruthSourceSpec.cxx
index 1b0f325cba9d6..6888b118fb06c 100644
--- a/Steer/DigitizerWorkflow/src/MCTruthSourceSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/MCTruthSourceSpec.cxx
@@ -54,12 +54,12 @@ class MCTruthSourceTask : public o2::framework::Task
     if (mNew) {
       LOG(info) << "New serialization";
       // we need to flatten it and write to managed shared memory container
-      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{"TST", "LABELS", 0, Lifetime::Timeframe});
+      auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{"TST", "LABELS", 0});
       container.flatten_to(sharedlabels);
       sleep(1);
     } else {
       LOG(info) << "Old serialization";
-      pc.outputs().snapshot({"TST", "LABELS", 0, Lifetime::Timeframe}, container);
+      pc.outputs().snapshot({"TST", "LABELS", 0}, container);
       sleep(1);
     }
 
diff --git a/Steer/DigitizerWorkflow/src/MCTruthWriterSpec.cxx b/Steer/DigitizerWorkflow/src/MCTruthWriterSpec.cxx
index 9ce78d97418c3..78fb8988a2419 100644
--- a/Steer/DigitizerWorkflow/src/MCTruthWriterSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/MCTruthWriterSpec.cxx
@@ -84,7 +84,7 @@ class MCTruthWriterTask : public o2::framework::Task
     }
     if (mIO) {
       // this triggers the reader process
-      pc.outputs().snapshot({"TST", "TRIGGERREAD", 0, Lifetime::Timeframe}, labelfilename);
+      pc.outputs().snapshot({"TST", "TRIGGERREAD", 0}, labelfilename);
     }
 
     // we should be only called once; tell DPL that this process is ready to exit
diff --git a/Steer/DigitizerWorkflow/src/MIDDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/MIDDigitizerSpec.cxx
index c0a3be730c411..1dbd392fb9f73 100644
--- a/Steer/DigitizerWorkflow/src/MIDDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/MIDDigitizerSpec.cxx
@@ -117,13 +117,13 @@ class MIDDPLDigitizerTask : public o2::base::BaseDPLDigitizer
     mDigitsMerger.process(digitsAccum, labelsAccum, rofRecords);
 
     LOG(debug) << "MID: Sending " << digitsAccum.size() << " digits.";
-    pc.outputs().snapshot(Output{"MID", "DIGITS", 0, Lifetime::Timeframe}, mDigitsMerger.getColumnData());
-    pc.outputs().snapshot(Output{"MID", "DIGITSROF", 0, Lifetime::Timeframe}, mDigitsMerger.getROFRecords());
+    pc.outputs().snapshot(Output{"MID", "DIGITS", 0}, mDigitsMerger.getColumnData());
+    pc.outputs().snapshot(Output{"MID", "DIGITSROF", 0}, mDigitsMerger.getROFRecords());
     if (pc.outputs().isAllowed({"MID", "DIGITLABELS", 0})) {
-      pc.outputs().snapshot(Output{"MID", "DIGITLABELS", 0, Lifetime::Timeframe}, mDigitsMerger.getMCContainer());
+      pc.outputs().snapshot(Output{"MID", "DIGITLABELS", 0}, mDigitsMerger.getMCContainer());
     }
     LOG(debug) << "MID: Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{"MID", "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{"MID", "ROMode", 0}, mROMode);
 
     // we should be only called once; tell DPL that this process is ready to exit
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Steer/DigitizerWorkflow/src/PHOSDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/PHOSDigitizerSpec.cxx
index db20e811941c2..0b7b968d22ff6 100644
--- a/Steer/DigitizerWorkflow/src/PHOSDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/PHOSDigitizerSpec.cxx
@@ -160,16 +160,16 @@ void DigitizerSpec::run(framework::ProcessingContext& pc)
   }
   LOG(debug) << "Have " << mLabels.getNElements() << " PHOS labels ";
   // here we have all digits and we can send them to consumer (aka snapshot it onto output)
-  pc.outputs().snapshot(Output{"PHS", "DIGITS", 0, Lifetime::Timeframe}, mDigitsOut);
-  pc.outputs().snapshot(Output{"PHS", "DIGITTRIGREC", 0, Lifetime::Timeframe}, triggers);
+  pc.outputs().snapshot(Output{"PHS", "DIGITS", 0}, mDigitsOut);
+  pc.outputs().snapshot(Output{"PHS", "DIGITTRIGREC", 0}, triggers);
   if (pc.outputs().isAllowed({"PHS", "DIGITSMCTR", 0})) {
-    pc.outputs().snapshot(Output{"PHS", "DIGITSMCTR", 0, Lifetime::Timeframe}, mLabels);
+    pc.outputs().snapshot(Output{"PHS", "DIGITSMCTR", 0}, mLabels);
   }
 
   // PHOS is always a triggering detector
   const o2::parameters::GRPObject::ROMode roMode = o2::parameters::GRPObject::TRIGGERING;
   LOG(debug) << "PHOS: Sending ROMode= " << roMode << " to GRPUpdater";
-  pc.outputs().snapshot(Output{"PHS", "ROMode", 0, Lifetime::Timeframe}, roMode);
+  pc.outputs().snapshot(Output{"PHS", "ROMode", 0}, roMode);
 
   timer.Stop();
   LOG(info) << "Digitization took " << timer.CpuTime() << "s";
diff --git a/Steer/DigitizerWorkflow/src/TOFDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/TOFDigitizerSpec.cxx
index c74e5ca6327fc..e512659686c86 100644
--- a/Steer/DigitizerWorkflow/src/TOFDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/TOFDigitizerSpec.cxx
@@ -259,21 +259,21 @@ class TOFDPLDigitizerTask : public o2::base::BaseDPLDigitizer
     LOG(info) << "Post " << digitsVector->size() << " digits in " << readoutwindow->size() << " RO windows";
 
     // here we have all digits and we can send them to consumer (aka snapshot it onto output)
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITS", 0, Lifetime::Timeframe}, *digitsVector);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITS", 0}, *digitsVector);
     if (pc.outputs().isAllowed({o2::header::gDataOriginTOF, "DIGITSMCTR", 0})) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITSMCTR", 0, Lifetime::Timeframe}, *mcLabVecOfVec);
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITSMCTR", 0}, *mcLabVecOfVec);
     }
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "READOUTWINDOW", 0, Lifetime::Timeframe}, *readoutwindow);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "READOUTWINDOW", 0}, *readoutwindow);
 
     // send empty pattern from digitizer (it may change in future)
     std::vector<uint8_t>& patterns = mDigitizer->getPatterns();
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "PATTERNS", 0, Lifetime::Timeframe}, patterns);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "PATTERNS", 0}, patterns);
 
     DigitHeader& digitH = mDigitizer->getDigitHeader();
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITHEADER", 0, Lifetime::Timeframe}, digitH);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "DIGITHEADER", 0}, digitH);
 
     LOG(info) << "TOF: Sending ROMode= " << roMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "ROMode", 0, Lifetime::Timeframe}, roMode);
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "ROMode", 0}, roMode);
 
     timer.Stop();
     LOG(info) << "Digitization took " << timer.CpuTime() << "s";
diff --git a/Steer/DigitizerWorkflow/src/TPCDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/TPCDigitizerSpec.cxx
index 59f8589a56449..3f24bb1a9ee12 100644
--- a/Steer/DigitizerWorkflow/src/TPCDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/TPCDigitizerSpec.cxx
@@ -282,7 +282,7 @@ class TPCDPLDigitizerTask : public BaseDPLDigitizer
       auto roMode = isContinuous ? o2::parameters::GRPObject::CONTINUOUS : o2::parameters::GRPObject::PRESENT;
       LOG(info) << "TPC: Sending ROMode= " << (mDigitizer.isContinuousReadout() ? "Continuous" : "Triggered")
                 << " to GRPUpdater from channel " << dh->subSpecification;
-      pc.outputs().snapshot(Output{"TPC", "ROMode", 0, Lifetime::Timeframe}, roMode);
+      pc.outputs().snapshot(Output{"TPC", "ROMode", 0}, roMode);
     }
     mWriteGRP = false;
 
@@ -309,7 +309,7 @@ class TPCDPLDigitizerTask : public BaseDPLDigitizer
         return ContainerType(nullptr);
       } else {
         // default case
-        return &pc.outputs().make<std::vector<o2::tpc::Digit>>(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(dh->subSpecification), Lifetime::Timeframe, header});
+        return &pc.outputs().make<std::vector<o2::tpc::Digit>>(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(dh->subSpecification), header});
       }
     };
     // lambda that snapshots the common mode vector to be sent out; prepares and attaches header with sector information
@@ -318,8 +318,7 @@ class TPCDPLDigitizerTask : public BaseDPLDigitizer
       header.activeSectors = activeSectors;
       if (!mInternalWriter) {
         // note that snapshoting only works with non-const references (to be fixed?)
-        pc.outputs().snapshot(Output{"TPC", "COMMONMODE", static_cast<SubSpecificationType>(dh->subSpecification), Lifetime::Timeframe,
-                                     header},
+        pc.outputs().snapshot(Output{"TPC", "COMMONMODE", static_cast<SubSpecificationType>(dh->subSpecification), header},
                               const_cast<std::vector<o2::tpc::CommonMode>&>(commonMode));
       }
     };
@@ -329,7 +328,7 @@ class TPCDPLDigitizerTask : public BaseDPLDigitizer
       header.activeSectors = activeSectors;
       if (mWithMCTruth) {
         if (!mInternalWriter) {
-          auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{"TPC", "DIGITSMCTR", static_cast<SubSpecificationType>(dh->subSpecification), Lifetime::Timeframe, header});
+          auto& sharedlabels = pc.outputs().make<o2::dataformats::ConstMCTruthContainer<o2::MCCompLabel>>(Output{"TPC", "DIGITSMCTR", static_cast<SubSpecificationType>(dh->subSpecification), header});
           labels.flatten_to(sharedlabels);
         }
       }
@@ -340,8 +339,7 @@ class TPCDPLDigitizerTask : public BaseDPLDigitizer
       header.activeSectors = activeSectors;
       if (!mInternalWriter) {
         LOG(info) << "TPC: Send TRIGGERS for sector " << sector << " channel " << dh->subSpecification << " | size " << events.size();
-        pc.outputs().snapshot(Output{"TPC", "DIGTRIGGERS", static_cast<SubSpecificationType>(dh->subSpecification), Lifetime::Timeframe,
-                                     header},
+        pc.outputs().snapshot(Output{"TPC", "DIGTRIGGERS", static_cast<SubSpecificationType>(dh->subSpecification), header},
                               const_cast<std::vector<DigiGroupRef>&>(events));
       }
     };
diff --git a/Steer/DigitizerWorkflow/src/ZDCDigitizerSpec.cxx b/Steer/DigitizerWorkflow/src/ZDCDigitizerSpec.cxx
index d616df2fadb35..cd07f6389c43f 100644
--- a/Steer/DigitizerWorkflow/src/ZDCDigitizerSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/ZDCDigitizerSpec.cxx
@@ -147,15 +147,15 @@ class ZDCDPLDigitizerTask : public o2::base::BaseDPLDigitizer
     mDigitizer.Finalize(mDigitsBC, mOrbitData);
 
     // send out to next stage
-    pc.outputs().snapshot(Output{"ZDC", "DIGITSBC", 0, Lifetime::Timeframe}, mDigitsBC);
-    pc.outputs().snapshot(Output{"ZDC", "DIGITSCH", 0, Lifetime::Timeframe}, mDigitsCh);
-    pc.outputs().snapshot(Output{"ZDC", "DIGITSPD", 0, Lifetime::Timeframe}, mOrbitData);
+    pc.outputs().snapshot(Output{"ZDC", "DIGITSBC", 0}, mDigitsBC);
+    pc.outputs().snapshot(Output{"ZDC", "DIGITSCH", 0}, mDigitsCh);
+    pc.outputs().snapshot(Output{"ZDC", "DIGITSPD", 0}, mOrbitData);
     if (pc.outputs().isAllowed({"ZDC", "DIGITSLBL", 0})) {
-      pc.outputs().snapshot(Output{"ZDC", "DIGITSLBL", 0, Lifetime::Timeframe}, mLabels);
+      pc.outputs().snapshot(Output{"ZDC", "DIGITSLBL", 0}, mLabels);
     }
 
     LOG(info) << "ZDC: Sending ROMode= " << mROMode << " to GRPUpdater";
-    pc.outputs().snapshot(Output{"ZDC", "ROMode", 0, Lifetime::Timeframe}, mROMode);
+    pc.outputs().snapshot(Output{"ZDC", "ROMode", 0}, mROMode);
 
     // we should be only called once; tell DPL that this process is ready to exit
     pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Utilities/DataSampling/src/DataSamplingPolicy.cxx b/Utilities/DataSampling/src/DataSamplingPolicy.cxx
index de7aa5d8f6865..44a7a63efcf51 100644
--- a/Utilities/DataSampling/src/DataSamplingPolicy.cxx
+++ b/Utilities/DataSampling/src/DataSamplingPolicy.cxx
@@ -128,7 +128,7 @@ Output DataSamplingPolicy::prepareOutput(const ConcreteDataMatcher& input, Lifet
   auto result = mPaths.find(input);
   if (result != mPaths.end()) {
     auto dataType = DataSpecUtils::asConcreteDataTypeMatcher(result->second);
-    return Output{dataType.origin, dataType.description, input.subSpec, lifetime};
+    return Output{dataType.origin, dataType.description, input.subSpec};
   } else {
     return Output{header::gDataOriginInvalid, header::gDataDescriptionInvalid};
   }
diff --git a/Utilities/DataSampling/src/Dispatcher.cxx b/Utilities/DataSampling/src/Dispatcher.cxx
index b32218ea43e58..de78993ff083e 100644
--- a/Utilities/DataSampling/src/Dispatcher.cxx
+++ b/Utilities/DataSampling/src/Dispatcher.cxx
@@ -114,7 +114,6 @@ void Dispatcher::run(ProcessingContext& ctx)
               routeAsConcreteDataType.origin,
               routeAsConcreteDataType.description,
               partInputHeader->subSpecification,
-              part.spec->lifetime,
               std::move(headerStack)};
             send(ctx.outputs(), part, output);
           }
diff --git a/run/dpl_eventgen.cxx b/run/dpl_eventgen.cxx
index 9700ac32c6938..3090944031ad3 100644
--- a/run/dpl_eventgen.cxx
+++ b/run/dpl_eventgen.cxx
@@ -69,8 +69,8 @@ struct GeneratorTask {
     for (auto i = 0; i < std::min((int)aggregate, nEvents - eventCounter); ++i) {
       mctracks.clear();
       genservice.generateEvent_MCTracks(mctracks, mcheader);
-      pc.outputs().snapshot(Output{"MC", "MCHEADER", 0, Lifetime::Timeframe}, mcheader);
-      pc.outputs().snapshot(Output{"MC", "MCTRACKS", 0, Lifetime::Timeframe}, mctracks);
+      pc.outputs().snapshot(Output{"MC", "MCHEADER", 0}, mcheader);
+      pc.outputs().snapshot(Output{"MC", "MCTRACKS", 0}, mctracks);
       ++eventCounter;
     }
     // report number of TFs injected for the rate limiter to work
diff --git a/run/o2sim_hepmc_publisher.cxx b/run/o2sim_hepmc_publisher.cxx
index 76fbddc1993f3..7172248ca68ac 100644
--- a/run/o2sim_hepmc_publisher.cxx
+++ b/run/o2sim_hepmc_publisher.cxx
@@ -135,8 +135,8 @@ struct O2simHepmcPublisher {
       }
 
       // add to the message
-      pc.outputs().snapshot(Output{"MC", "MCHEADER", 0, Lifetime::Timeframe}, mcHeader);
-      pc.outputs().snapshot(Output{"MC", "MCTRACKS", 0, Lifetime::Timeframe}, mcTracks);
+      pc.outputs().snapshot(Output{"MC", "MCHEADER", 0}, mcHeader);
+      pc.outputs().snapshot(Output{"MC", "MCTRACKS", 0}, mcTracks);
       mcTracks.clear();
       ++eventCounter;
     }
diff --git a/run/o2sim_kine_publisher.cxx b/run/o2sim_kine_publisher.cxx
index 35fd48af199e0..83f82fd69cca3 100644
--- a/run/o2sim_kine_publisher.cxx
+++ b/run/o2sim_kine_publisher.cxx
@@ -44,8 +44,8 @@ struct O2simKinePublisher {
     for (auto i = 0; i < std::min((int)aggregate, nEvents - eventCounter); ++i) {
       auto mcevent = mcKinReader->getMCEventHeader(0, eventCounter);
       auto mctracks = mcKinReader->getTracks(0, eventCounter);
-      pc.outputs().snapshot(Output{"MC", "MCHEADER", 0, Lifetime::Timeframe}, mcevent);
-      pc.outputs().snapshot(Output{"MC", "MCTRACKS", 0, Lifetime::Timeframe}, mctracks);
+      pc.outputs().snapshot(Output{"MC", "MCHEADER", 0}, mcevent);
+      pc.outputs().snapshot(Output{"MC", "MCTRACKS", 0}, mctracks);
       ++eventCounter;
     }
     // report number of TFs injected for the rate limiter to work
diff --git a/run/o2sim_mctracks_to_aod.cxx b/run/o2sim_mctracks_to_aod.cxx
index e5460e346c3f9..ca64ce0705f1a 100644
--- a/run/o2sim_mctracks_to_aod.cxx
+++ b/run/o2sim_mctracks_to_aod.cxx
@@ -171,8 +171,8 @@ struct MctracksToAod {
       }
     }
     ++timeframe;
-    pc.outputs().snapshot(Output{"TFF", "TFFilename", 0, Lifetime::Timeframe}, "");
-    pc.outputs().snapshot(Output{"TFN", "TFNumber", 0, Lifetime::Timeframe}, timeframe);
+    pc.outputs().snapshot(Output{"TFF", "TFFilename", 0}, "");
+    pc.outputs().snapshot(Output{"TFN", "TFNumber", 0}, timeframe);
   }
 };
 

From b0d60773d27fcbe609e426502ae3196f4e1cd8b4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 5 Dec 2023 11:56:45 +0100
Subject: [PATCH 153/726] Remove Lifetime specification from Output

The .lifetime field in Output is to be removed. This includes
cases in which the value is not the default.
---
 .../src/GlobalFwdMatchingAssessmentSpec.cxx               | 2 +-
 Detectors/ITSMFT/MFT/workflow/src/MFTAssessmentSpec.cxx   | 2 +-
 Detectors/PHOS/calib/src/PHOSL1phaseCalibDevice.cxx       | 2 +-
 Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx    | 4 ++--
 .../workflow/include/TPCWorkflow/TPCDistributeSACSpec.h   | 2 +-
 Detectors/TPC/workflow/src/OccupancyFilterSpec.cxx        | 2 +-
 Detectors/TPC/workflow/src/TPCVDriftTglCalibSpec.cxx      | 4 ++--
 Detectors/ZDC/calib/src/BaselineCalibEPNSpec.cxx          | 2 +-
 Detectors/ZDC/calib/src/NoiseCalibEPNSpec.cxx             | 8 ++++----
 9 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/Detectors/GlobalTrackingWorkflow/src/GlobalFwdMatchingAssessmentSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/GlobalFwdMatchingAssessmentSpec.cxx
index e4c045d2a7d93..e896c097cd03d 100644
--- a/Detectors/GlobalTrackingWorkflow/src/GlobalFwdMatchingAssessmentSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/src/GlobalFwdMatchingAssessmentSpec.cxx
@@ -109,7 +109,7 @@ void GlobalFwdAssessmentSpec::sendOutput(DataAllocator& output)
   TObjArray objar;
   mGloFwdAssessment->getHistos(objar);
 
-  output.snapshot(Output{"GLO", "FWDASSESSMENT", 0, Lifetime::Sporadic}, objar);
+  output.snapshot(Output{"GLO", "FWDASSESSMENT", 0}, objar);
 
   TFile* f = new TFile(Form("GlobalForwardAssessment.root"), "RECREATE");
   objar.Write();
diff --git a/Detectors/ITSMFT/MFT/workflow/src/MFTAssessmentSpec.cxx b/Detectors/ITSMFT/MFT/workflow/src/MFTAssessmentSpec.cxx
index 9778120a75389..d3e4c33457ae0 100644
--- a/Detectors/ITSMFT/MFT/workflow/src/MFTAssessmentSpec.cxx
+++ b/Detectors/ITSMFT/MFT/workflow/src/MFTAssessmentSpec.cxx
@@ -90,7 +90,7 @@ void MFTAssessmentSpec::sendOutput(DataAllocator& output)
   TObjArray objar;
   mMFTAssessment->getHistos(objar);
 
-  output.snapshot(Output{"MFT", "MFTASSESSMENT", 0, Lifetime::Sporadic}, objar);
+  output.snapshot(Output{"MFT", "MFTASSESSMENT", 0}, objar);
 
   TFile* f = new TFile(Form("MFTAssessment.root"), "RECREATE");
   objar.Write();
diff --git a/Detectors/PHOS/calib/src/PHOSL1phaseCalibDevice.cxx b/Detectors/PHOS/calib/src/PHOSL1phaseCalibDevice.cxx
index 1ed04baa77892..9ce61d0af38cb 100644
--- a/Detectors/PHOS/calib/src/PHOSL1phaseCalibDevice.cxx
+++ b/Detectors/PHOS/calib/src/PHOSL1phaseCalibDevice.cxx
@@ -71,7 +71,7 @@ void PHOSL1phaseCalibDevice::endOfStream(o2::framework::EndOfStreamContext& ec)
   ec.outputs().snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, "PHOS_L1phase", 0}, info);
   // Send summary to QC
   LOG(info) << "Sending histos to QC ";
-  ec.outputs().snapshot(o2::framework::Output{"PHS", "L1PHASEHISTO", 0, o2::framework::Lifetime::Sporadic}, mCalibrator->getQcHistos());
+  ec.outputs().snapshot(o2::framework::Output{"PHS", "L1PHASEHISTO", 0}, mCalibrator->getQcHistos());
 }
 
 o2::framework::DataProcessorSpec o2::phos::getPHOSL1phaseCalibDeviceSpec()
diff --git a/Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx b/Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx
index 292e083d4bb48..bc650dfaa54b8 100644
--- a/Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx
+++ b/Detectors/PHOS/workflow/src/RawToCellConverterSpec.cxx
@@ -115,7 +115,7 @@ void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
       ctx.outputs().snapshot(o2::framework::Output{"PHS", "RAWHWERRORS", 0}, mOutputHWErrors);
       if (mFillChi2) {
         mOutputFitChi.clear();
-        ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLFITQA", 0, o2::framework::Lifetime::QA}, mOutputFitChi);
+        ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLFITQA", 0}, mOutputFitChi);
       }
       return; // empty TF, nothing to process
     }
@@ -294,7 +294,7 @@ void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
   ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLTRIGREC", mflpId}, mOutputTriggerRecords);
   ctx.outputs().snapshot(o2::framework::Output{"PHS", "RAWHWERRORS", 0}, mOutputHWErrors);
   if (mFillChi2) {
-    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLFITQA", 0, o2::framework::Lifetime::QA}, mOutputFitChi);
+    ctx.outputs().snapshot(o2::framework::Output{"PHS", "CELLFITQA", 0}, mOutputFitChi);
   }
 }
 
diff --git a/Detectors/TPC/workflow/include/TPCWorkflow/TPCDistributeSACSpec.h b/Detectors/TPC/workflow/include/TPCWorkflow/TPCDistributeSACSpec.h
index f7c207ba3a27b..ee05a58b5dcbe 100644
--- a/Detectors/TPC/workflow/include/TPCWorkflow/TPCDistributeSACSpec.h
+++ b/Detectors/TPC/workflow/include/TPCWorkflow/TPCDistributeSACSpec.h
@@ -102,7 +102,7 @@ class TPCDistributeSACSpec : public o2::framework::Task
   void sendOutput(o2::framework::ProcessingContext& pc)
   {
     LOGP(info, "Sending SACs on lane: {} for {} TFs", mCurrentOutLane, mProcessedTFs);
-    pc.outputs().snapshot(Output{gDataOriginTPC, getDataDescriptionSACCCDB(), 0, Lifetime::Sporadic}, mCCDBTimeStamp);
+    pc.outputs().snapshot(Output{gDataOriginTPC, getDataDescriptionSACCCDB(), 0}, mCCDBTimeStamp);
     for (unsigned int i = 0; i < o2::tpc::GEMSTACKS; ++i) {
       pc.outputs().snapshot(Output{gDataOriginTPC, mDataDescrOut[mCurrentOutLane], header::DataHeader::SubSpecificationType{i}}, mSACs[i]);
     }
diff --git a/Detectors/TPC/workflow/src/OccupancyFilterSpec.cxx b/Detectors/TPC/workflow/src/OccupancyFilterSpec.cxx
index 9891d80a39503..5ef2bdc7adc51 100644
--- a/Detectors/TPC/workflow/src/OccupancyFilterSpec.cxx
+++ b/Detectors/TPC/workflow/src/OccupancyFilterSpec.cxx
@@ -106,7 +106,7 @@ class OccupancyFilterDevice : public o2::framework::Task
   {
     o2::tpc::TPCSectorHeader header{sector};
     header.activeSectors = (0x1 << sector);
-    output.snapshot(Output{gDataOriginTPC, "FILTERDIG", static_cast<SubSpecificationType>(sector), Lifetime::Sporadic, header}, digits);
+    output.snapshot(Output{gDataOriginTPC, "FILTERDIG", static_cast<SubSpecificationType>(sector), header}, digits);
   }
 };
 
diff --git a/Detectors/TPC/workflow/src/TPCVDriftTglCalibSpec.cxx b/Detectors/TPC/workflow/src/TPCVDriftTglCalibSpec.cxx
index d48dde325c614..ea50a32c6a906 100644
--- a/Detectors/TPC/workflow/src/TPCVDriftTglCalibSpec.cxx
+++ b/Detectors/TPC/workflow/src/TPCVDriftTglCalibSpec.cxx
@@ -104,8 +104,8 @@ void TPCVDriftTglCalibSpec::sendOutput(DataAllocator& output)
     auto image = o2::ccdb::CcdbApi::createObjectImage(&payloadVec[i], &w);
     LOG(info) << "Sending object " << w.getPath() << "/" << w.getFileName() << " of size " << image->size()
               << " bytes, valid for " << w.getStartValidityTimestamp() << " : " << w.getEndValidityTimestamp();
-    output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBPayload, "TPCVDTGL", i, Lifetime::Sporadic}, *image.get()); // vector<char>
-    output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, "TPCVDTGL", i, Lifetime::Sporadic}, w);            // root-serialized
+    output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBPayload, "TPCVDTGL", i}, *image.get()); // vector<char>
+    output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, "TPCVDTGL", i}, w);            // root-serialized
   }
   if (payloadVec.size()) {
     mCalibrator->initOutput(); // reset the outputs once they are already sent
diff --git a/Detectors/ZDC/calib/src/BaselineCalibEPNSpec.cxx b/Detectors/ZDC/calib/src/BaselineCalibEPNSpec.cxx
index 422221157f877..328f824918cd4 100644
--- a/Detectors/ZDC/calib/src/BaselineCalibEPNSpec.cxx
+++ b/Detectors/ZDC/calib/src/BaselineCalibEPNSpec.cxx
@@ -127,7 +127,7 @@ void BaselineCalibEPNSpec::run(ProcessingContext& pc)
     }
     // Send intermediate calibration data
     auto& summary = mWorker.mData.getSummary();
-    o2::framework::Output outputData("ZDC", "BASECALIBDATA", 0, Lifetime::Sporadic);
+    o2::framework::Output outputData("ZDC", "BASECALIBDATA", 0);
     pc.outputs().snapshot(outputData, summary);
     if (pc.transitionState() == TransitionHandlingState::Requested) {
       // End of processing for this run
diff --git a/Detectors/ZDC/calib/src/NoiseCalibEPNSpec.cxx b/Detectors/ZDC/calib/src/NoiseCalibEPNSpec.cxx
index 32fe2ee2eb1fb..186cec68517ea 100644
--- a/Detectors/ZDC/calib/src/NoiseCalibEPNSpec.cxx
+++ b/Detectors/ZDC/calib/src/NoiseCalibEPNSpec.cxx
@@ -113,19 +113,19 @@ void NoiseCalibEPNSpec::run(ProcessingContext& pc)
     if (mVerbosity > DbgMedium && mModTF > 0) {
       LOG(info) << "Send intermediate calibration data mProcessed=" << mProcessed << " >= mModTF=" << mModTF;
     }
-    o2::framework::Output outputData("ZDC", "NOISECALIBDATA", 0, Lifetime::Sporadic);
+    o2::framework::Output outputData("ZDC", "NOISECALIBDATA", 0);
     pc.outputs().snapshot(outputData, summary);
     for (int ih = 0; ih < NChannels; ih++) {
       {
-        o2::framework::Output output("ZDC", "NOISE_1DH", ih, Lifetime::Sporadic);
+        o2::framework::Output output("ZDC", "NOISE_1DH", ih);
         pc.outputs().snapshot(output, mWorker.mH[0][ih]->getBase());
       }
       {
-        o2::framework::Output output("ZDC", "NOISE_1DH_S", ih, Lifetime::Sporadic);
+        o2::framework::Output output("ZDC", "NOISE_1DH_S", ih);
         pc.outputs().snapshot(output, mWorker.mH[1][ih]->getBase());
       }
       {
-        o2::framework::Output output("ZDC", "NOISE_1DH_D", ih, Lifetime::Sporadic);
+        o2::framework::Output output("ZDC", "NOISE_1DH_D", ih);
         pc.outputs().snapshot(output, mWorker.mH[2][ih]->getBase());
       }
     }

From 01b54b1b6334f3ddd758394d93409178ac92e585 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 5 Dec 2023 15:27:13 +0100
Subject: [PATCH 154/726] DPL: drop lifetime member from Output

This is apparently not really used for anything and in general not passed
correctly. Lifetime is a static property of the data flow, so it does not
make sense to have it on a per message basis.
---
 Framework/CCDBSupport/src/CCDBHelpers.cxx     |  4 ++--
 Framework/Core/include/Framework/Output.h     | 19 ++----------------
 Framework/Core/src/CommonServices.cxx         |  2 +-
 Framework/Core/src/DataAllocator.cxx          |  4 ++--
 Framework/Core/test/test_DataAllocator.cxx    | 20 +++++++++----------
 Framework/Core/test/test_ParallelPipeline.cxx |  8 ++++----
 Framework/Core/test/test_ProcessorOptions.cxx |  2 +-
 .../Core/test/test_StaggeringWorkflow.cxx     |  6 +++---
 Framework/Core/test/test_TableBuilder.cxx     |  2 +-
 .../Utils/include/DPLUtils/RootTreeReader.h   |  8 ++++----
 Framework/Utils/src/Utils.cxx                 |  2 +-
 Framework/Utils/test/test_RootTreeReader.cxx  |  5 ++---
 .../test/test_RootTreeWriterWorkflow.cxx      |  2 +-
 13 files changed, 34 insertions(+), 50 deletions(-)

diff --git a/Framework/CCDBSupport/src/CCDBHelpers.cxx b/Framework/CCDBSupport/src/CCDBHelpers.cxx
index 0e067dc2f2d52..e45e1c6996a98 100644
--- a/Framework/CCDBSupport/src/CCDBHelpers.cxx
+++ b/Framework/CCDBSupport/src/CCDBHelpers.cxx
@@ -185,7 +185,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
     LOGP(debug, "Fetching object for route {}", DataSpecUtils::describe(route.matcher));
     objCnt++;
     auto concrete = DataSpecUtils::asConcreteDataMatcher(route.matcher);
-    Output output{concrete.origin, concrete.description, concrete.subSpec, route.matcher.lifetime};
+    Output output{concrete.origin, concrete.description, concrete.subSpec};
     auto&& v = allocator.makeVector<char>(output);
     std::map<std::string, std::string> metadata;
     std::map<std::string, std::string> headers;
@@ -343,7 +343,7 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
             checkValidity = true; // never skip check if the cache is empty
           }
           LOG(debug) << "checkValidity = " << checkValidity << " for TF " << timingInfo.timeslice;
-          Output output{"CTP", "OrbitReset", 0, Lifetime::Condition};
+          Output output{"CTP", "OrbitReset", 0};
           Long64_t newOrbitResetTime = orbitResetTime;
           auto&& v = allocator.makeVector<char>(output);
           const auto& api = helper->getAPI(path);
diff --git a/Framework/Core/include/Framework/Output.h b/Framework/Core/include/Framework/Output.h
index 9f85d6e8edb92..a2abf5f7600a9 100644
--- a/Framework/Core/include/Framework/Output.h
+++ b/Framework/Core/include/Framework/Output.h
@@ -28,7 +28,6 @@ struct Output {
   header::DataOrigin origin;
   header::DataDescription description;
   header::DataHeader::SubSpecificationType subSpec = 0;
-  enum Lifetime lifetime = Lifetime::Timeframe;
   header::Stack metaHeader = {};
 
   Output(header::DataOrigin o, header::DataDescription d) : origin(o), description(d) {}
@@ -38,22 +37,11 @@ struct Output {
   {
   }
 
-  Output(header::DataOrigin o, header::DataDescription d, header::DataHeader::SubSpecificationType s, Lifetime l)
-    : origin(o), description(d), subSpec(s), lifetime(l)
-  {
-  }
-
   Output(header::DataOrigin o, header::DataDescription d, header::DataHeader::SubSpecificationType s, header::Stack&& stack)
     : origin(o), description(d), subSpec(s), metaHeader(std::move(stack))
   {
   }
 
-  Output(header::DataOrigin o, header::DataDescription d, header::DataHeader::SubSpecificationType s, Lifetime l,
-         header::Stack&& stack)
-    : origin(o), description(d), subSpec(s), lifetime(l), metaHeader(std::move(stack))
-  {
-  }
-
   Output(header::DataHeader const& header)
     : origin(header.dataOrigin), description(header.dataDescription), subSpec(header.subSpecification)
   {
@@ -65,7 +53,6 @@ struct Output {
     : origin(rhs.origin),
       description(rhs.description),
       subSpec(rhs.subSpec),
-      lifetime(rhs.lifetime),
       metaHeader(std::move(rhs.metaHeader))
   {
   }
@@ -77,16 +64,14 @@ struct Output {
     origin = rhs.origin;
     description = rhs.description;
     subSpec = rhs.subSpec;
-    lifetime = rhs.lifetime;
     metaHeader = std::move(rhs.metaHeader);
     return *this;
   }
 
   bool operator==(const Output& that) const
   {
-    return origin == that.origin && description == that.description && subSpec == that.subSpec &&
-           lifetime == that.lifetime;
-  };
+    return origin == that.origin && description == that.description && subSpec == that.subSpec;
+  }
 };
 
 } // namespace o2
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 7d502216bc509..05b8fba47e532 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -425,7 +425,7 @@ o2::framework::ServiceSpec CommonServices::ccdbSupportSpec()
           if (concrete.subSpec == 0) {
             continue;
           }
-          auto& stfDist = pc.outputs().make<o2::header::STFHeader>(Output{concrete.origin, concrete.description, concrete.subSpec, output.matcher.lifetime});
+          auto& stfDist = pc.outputs().make<o2::header::STFHeader>(Output{concrete.origin, concrete.description, concrete.subSpec});
           stfDist.id = timingInfo.timeslice;
           stfDist.firstOrbit = timingInfo.firstTForbit;
           stfDist.runNumber = timingInfo.runNumber;
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index 296e094bffe26..1230d2e942f6e 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -294,7 +294,7 @@ Output DataAllocator::getOutputByBind(OutputRef&& ref)
     if (allowedOutputRoutes[ri].matcher.binding.value == ref.label) {
       auto spec = allowedOutputRoutes[ri].matcher;
       auto dataType = DataSpecUtils::asConcreteDataTypeMatcher(spec);
-      return Output{dataType.origin, dataType.description, ref.subSpec, spec.lifetime, std::move(ref.headerStack)};
+      return Output{dataType.origin, dataType.description, ref.subSpec, std::move(ref.headerStack)};
     }
   }
   std::string availableRoutes;
@@ -342,7 +342,7 @@ void DataAllocator::cookDeadBeef(const Output& spec)
   // We get the output route from the original spec, but we send it
   // using the binding of the deadbeef subSpecification.
   RouteIndex routeIndex = matchDataHeader(spec, timingInfo.timeslice);
-  auto deadBeefOutput = Output{spec.origin, spec.description, 0xdeadbeef, Lifetime::Timeframe};
+  auto deadBeefOutput = Output{spec.origin, spec.description, 0xdeadbeef};
   auto headerMessage = headerMessageFromOutput(deadBeefOutput, routeIndex, header::gSerializationMethodNone, 0);
 
   addPartToContext(proxy.createOutputMessage(routeIndex, 0), deadBeefOutput, header::gSerializationMethodNone);
diff --git a/Framework/Core/test/test_DataAllocator.cxx b/Framework/Core/test/test_DataAllocator.cxx
index d880da72d7cb4..894fb5f38368a 100644
--- a/Framework/Core/test/test_DataAllocator.cxx
+++ b/Framework/Core/test/test_DataAllocator.cxx
@@ -45,7 +45,7 @@ using namespace o2::framework;
 // this function is only used to do the static checks for API return types
 void doTypeChecks()
 {
-  const Output output{"TST", "DUMMY", 0, Lifetime::Timeframe};
+  const Output output{"TST", "DUMMY", 0};
   // we require references to objects owned by allocator context
   static_assert(std::is_lvalue_reference<decltype(std::declval<DataAllocator>().make<int>(output))>::value);
   static_assert(std::is_lvalue_reference<decltype(std::declval<DataAllocator>().make<std::string>(output, "test"))>::value);
@@ -83,24 +83,24 @@ DataProcessorSpec getSourceSpec()
     // picked by the framework is no serialization
     test::MetaHeader meta1{42};
     test::MetaHeader meta2{23};
-    pc.outputs().snapshot(Output{"TST", "MESSAGEABLE", 0, Lifetime::Timeframe, {meta1, meta2}}, a);
-    pc.outputs().snapshot(Output{"TST", "MSGBLEROOTSRLZ", 0, Lifetime::Timeframe},
+    pc.outputs().snapshot(Output{"TST", "MESSAGEABLE", 0, {meta1, meta2}}, a);
+    pc.outputs().snapshot(Output{"TST", "MSGBLEROOTSRLZ", 0},
                           o2::framework::ROOTSerialized<decltype(a)>(a));
     // class Polymorphic is not messageable, so the serialization type is deduced
     // from the fact that the type has a dictionary and can be ROOT-serialized.
-    pc.outputs().snapshot(Output{"TST", "ROOTNONTOBJECT", 0, Lifetime::Timeframe}, b);
+    pc.outputs().snapshot(Output{"TST", "ROOTNONTOBJECT", 0}, b);
     // vector of ROOT serializable class
-    pc.outputs().snapshot(Output{"TST", "ROOTVECTOR", 0, Lifetime::Timeframe}, c);
+    pc.outputs().snapshot(Output{"TST", "ROOTVECTOR", 0}, c);
     // deque of simple types
-    pc.outputs().snapshot(Output{"TST", "DEQUE", 0, Lifetime::Timeframe}, testDequePayload);
+    pc.outputs().snapshot(Output{"TST", "DEQUE", 0}, testDequePayload);
     // likewise, passed anonymously with char type and class name
     o2::framework::ROOTSerialized<char, const char> d(*((char*)&c), "vector<o2::test::Polymorphic>");
-    pc.outputs().snapshot(Output{"TST", "ROOTSERLZDVEC", 0, Lifetime::Timeframe}, d);
+    pc.outputs().snapshot(Output{"TST", "ROOTSERLZDVEC", 0}, d);
     // vector of ROOT serializable class wrapped with TClass info as hint
     auto* cl = TClass::GetClass(typeid(decltype(c)));
     ASSERT_ERROR(cl != nullptr);
     o2::framework::ROOTSerialized<char, TClass> e(*((char*)&c), cl);
-    pc.outputs().snapshot(Output{"TST", "ROOTSERLZDVEC2", 0, Lifetime::Timeframe}, e);
+    pc.outputs().snapshot(Output{"TST", "ROOTSERLZDVEC2", 0}, e);
     // test the 'make' methods
     pc.outputs().make<o2::test::TriviallyCopyable>(OutputRef{"makesingle", 0}) = a;
     auto& multi = pc.outputs().make<o2::test::TriviallyCopyable>(OutputRef{"makespan", 0}, 3);
@@ -111,7 +111,7 @@ DataProcessorSpec getSourceSpec()
     // test the adopt method
     auto freefct = [](void* data, void* hint) {}; // simply ignore the cleanup for the test
     static std::string teststring = "adoptchunk";
-    pc.outputs().adoptChunk(Output{"TST", "ADOPTCHUNK", 0, Lifetime::Timeframe}, teststring.data(), teststring.length(), freefct, nullptr);
+    pc.outputs().adoptChunk(Output{"TST", "ADOPTCHUNK", 0}, teststring.data(), teststring.length(), freefct, nullptr);
     // test resizable data chunk, initial size 0 and grow
     auto& growchunk = pc.outputs().newChunk(OutputRef{"growchunk", 0}, 0);
     growchunk.resize(sizeof(o2::test::TriviallyCopyable));
@@ -307,7 +307,7 @@ DataProcessorSpec getSinkSpec()
     ASSERT_ERROR((object12[0] == o2::test::TriviallyCopyable{42, 23, 0xdead}));
     ASSERT_ERROR((object12[1] == o2::test::TriviallyCopyable{10, 20, 0xacdc}));
     // forward the read-only span on a different route
-    pc.outputs().snapshot(Output{"TST", "MSGABLVECTORCPY", 0, Lifetime::Timeframe}, object12);
+    pc.outputs().snapshot(Output{"TST", "MSGABLVECTORCPY", 0}, object12);
 
     LOG(info) << "extracting TNamed object from input13";
     auto object13 = pc.inputs().get<TNamed*>("input13");
diff --git a/Framework/Core/test/test_ParallelPipeline.cxx b/Framework/Core/test/test_ParallelPipeline.cxx
index e13869873fca1..47106855a8f74 100644
--- a/Framework/Core/test/test_ParallelPipeline.cxx
+++ b/Framework/Core/test/test_ParallelPipeline.cxx
@@ -64,7 +64,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
                     << *input.spec << ": " << *((int*)input.payload);
          auto const* dataheader = DataRefUtils::getHeader<o2::header::DataHeader*>(input);
          //auto& data = ctx.outputs().make<int>(OutputRef{"output", dataheader->subSpecification});
-         auto& data = ctx.outputs().make<int>(Output{"TST", "PREPROC", dataheader->subSpecification, Lifetime::Timeframe});
+         auto& data = ctx.outputs().make<int>(Output{"TST", "PREPROC", dataheader->subSpecification});
          ASSERT_ERROR(ctx.inputs().get<int>(input.spec->binding.c_str()) == parallelContext.index1D());
          data = parallelContext.index1D();
        }
@@ -85,10 +85,10 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
          // TODO: there is a bug in the API for using OutputRef, returns an rvalue which can not be bound to
          // lvalue reference
          //auto& data = ctx.outputs().make<int>(OutputRef{"output", dataheader->subSpecification});
-         auto& data = ctx.outputs().make<int>(Output{"TST", "DATA", dataheader->subSpecification, Lifetime::Timeframe});
+         auto& data = ctx.outputs().make<int>(Output{"TST", "DATA", dataheader->subSpecification});
          data = ctx.inputs().get<int>(input.spec->binding.c_str());
          //auto& meta = ctx.outputs().make<int>(OutputRef{"metadt", dataheader->subSpecification});
-         auto& meta = ctx.outputs().make<int>(Output{"TST", "META", dataheader->subSpecification, Lifetime::Timeframe});
+         auto& meta = ctx.outputs().make<int>(Output{"TST", "META", dataheader->subSpecification});
          meta = dataheader->subSpecification;
        }
      }}},
@@ -145,7 +145,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
           if (multiplicities[pipeline] == 0) {
             continue;
           }
-          ctx.outputs().make<int>(Output{"TST", "TRIGGER", subspecs[index], Lifetime::Timeframe}) = pipeline;
+          ctx.outputs().make<int>(Output{"TST", "TRIGGER", subspecs[index]}) = pipeline;
           multiplicities[pipeline++]--;
           if (pipeline >= nPipelines) {
             pipeline = 0;
diff --git a/Framework/Core/test/test_ProcessorOptions.cxx b/Framework/Core/test/test_ProcessorOptions.cxx
index 8de90e0e23441..1751176eb639e 100644
--- a/Framework/Core/test/test_ProcessorOptions.cxx
+++ b/Framework/Core/test/test_ProcessorOptions.cxx
@@ -45,7 +45,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const&)
             }
             // there is nothing to do, simply stop the workflow but we have to send at least one message
             // to make sure that the callback of the consumer is called
-            ctx.outputs().make<int>(Output{"TST", "TEST", 0, Lifetime::Timeframe}) = 42;
+            ctx.outputs().make<int>(Output{"TST", "TEST", 0}) = 42;
             ctx.services().get<ControlService>().endOfStream();
             *isReady = true;
           };
diff --git a/Framework/Core/test/test_StaggeringWorkflow.cxx b/Framework/Core/test/test_StaggeringWorkflow.cxx
index 0e98c4324bc76..edc07d598465b 100644
--- a/Framework/Core/test/test_StaggeringWorkflow.cxx
+++ b/Framework/Core/test/test_StaggeringWorkflow.cxx
@@ -85,9 +85,9 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
       // since the snapshot copy is ready for sending it is scheduled but held back
       // because of the CompletionPolicy trigger matcher. This message will be
       // sent together with the second message.
-      outputs.snapshot(Output{"PROD", "CHANNEL", subspec, Lifetime::Timeframe}, subspec);
+      outputs.snapshot(Output{"PROD", "CHANNEL", subspec}, subspec);
       device.waitFor(100);
-      outputs.snapshot(Output{"PROD", "TRIGGER", subspec, Lifetime::Timeframe}, subspec);
+      outputs.snapshot(Output{"PROD", "TRIGGER", subspec}, subspec);
       device.waitFor(100);
     }
     control.endOfStream();
@@ -106,7 +106,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
       LOG(info) << "processing " << input.spec->binding << " " << data;
       // check if the channel binding starts with 'trigger'
       if (input.spec->binding.find("trigger") == 0) {
-        pc.outputs().make<MyDataType>(Output{"PROC", "CHANNEL", data, Lifetime::Timeframe}) = data;
+        pc.outputs().make<MyDataType>(Output{"PROC", "CHANNEL", data}) = data;
       }
       nActiveInputs++;
     }
diff --git a/Framework/Core/test/test_TableBuilder.cxx b/Framework/Core/test/test_TableBuilder.cxx
index a7b4739054d03..26a0f5a930b73 100644
--- a/Framework/Core/test/test_TableBuilder.cxx
+++ b/Framework/Core/test/test_TableBuilder.cxx
@@ -240,7 +240,7 @@ TEST_CASE("TestSoAIntegration")
 
 TEST_CASE("TestDataAllocatorReturnType")
 {
-  const Output output{"TST", "DUMMY", 0, Lifetime::Timeframe};
+  const Output output{"TST", "DUMMY", 0};
 }
 
 TEST_CASE("TestPodInjestion")
diff --git a/Framework/Utils/include/DPLUtils/RootTreeReader.h b/Framework/Utils/include/DPLUtils/RootTreeReader.h
index 42a8c9d01d0d0..109d79f9fad78 100644
--- a/Framework/Utils/include/DPLUtils/RootTreeReader.h
+++ b/Framework/Utils/include/DPLUtils/RootTreeReader.h
@@ -51,11 +51,11 @@ struct DefaultKey {
   enum Lifetime lifetime = Lifetime::Timeframe;
 
   DefaultKey(const Output& desc)
-    : origin(desc.origin), description(desc.description), subSpec(desc.subSpec), lifetime(desc.lifetime)
+    : origin(desc.origin), description(desc.description), subSpec(desc.subSpec)
   {
   }
 
-  operator Output() const { return Output{origin, description, subSpec, lifetime}; }
+  operator Output() const { return Output{origin, description, subSpec}; }
 };
 } // namespace rtr
 
@@ -302,7 +302,7 @@ class GenericRootTreeReader
       }
 
       auto snapshot = [&context, &stackcreator](const KeyType& key, const auto& object) {
-        context.outputs().snapshot(Output{key.origin, key.description, key.subSpec, key.lifetime, std::move(stackcreator())}, object);
+        context.outputs().snapshot(Output{key.origin, key.description, key.subSpec, std::move(stackcreator())}, object);
       };
 
       char* data = nullptr;
@@ -310,7 +310,7 @@ class GenericRootTreeReader
       mBranch->GetEntry(entry);
 
       // execute hook if it was registered; if this return true do not proceed further
-      if (mPublishHook != nullptr && (*mPublishHook).hook(mName, context, Output{mKey.origin, mKey.description, mKey.subSpec, mKey.lifetime, std::move(stackcreator())}, data)) {
+      if (mPublishHook != nullptr && (*mPublishHook).hook(mName, context, Output{mKey.origin, mKey.description, mKey.subSpec, std::move(stackcreator())}, data)) {
 
       }
       // try to figureout when we need to do something special
diff --git a/Framework/Utils/src/Utils.cxx b/Framework/Utils/src/Utils.cxx
index 3a52c46e074af..1dc0882c2f55d 100644
--- a/Framework/Utils/src/Utils.cxx
+++ b/Framework/Utils/src/Utils.cxx
@@ -28,7 +28,7 @@ namespace workflows
 Output getOutput(const o2f::OutputSpec outputSpec)
 {
   auto concrete = DataSpecUtils::asConcreteDataMatcher(outputSpec);
-  return Output{concrete.origin, concrete.description, concrete.subSpec, outputSpec.lifetime};
+  return Output{concrete.origin, concrete.description, concrete.subSpec};
 }
 
 // This method can convert a vector of OutputSpec into a vector of Output.
diff --git a/Framework/Utils/test/test_RootTreeReader.cxx b/Framework/Utils/test/test_RootTreeReader.cxx
index 0390a29f807b1..bf652fca6f97a 100644
--- a/Framework/Utils/test/test_RootTreeReader.cxx
+++ b/Framework/Utils/test/test_RootTreeReader.cxx
@@ -67,11 +67,10 @@ DataProcessorSpec getSourceSpec()
       testFile->Close();
     }
 
-    constexpr auto persistency = Lifetime::Transient;
     auto reader = std::make_shared<RootTreeReader>("testtree",       // tree name
                                                    fileName.c_str(), // input file name
-                                                   RootTreeReader::BranchDefinition<std::vector<o2::test::TriviallyCopyable>>{Output{"TST", "ARRAYOFMSGBL", 0, persistency}, "msgblarray"},
-                                                   Output{"TST", "ARRAYOFDATA", 0, persistency},
+                                                   RootTreeReader::BranchDefinition<std::vector<o2::test::TriviallyCopyable>>{Output{"TST", "ARRAYOFMSGBL", 0}, "msgblarray"},
+                                                   Output{"TST", "ARRAYOFDATA", 0},
                                                    "dataarray",
                                                    RootTreeReader::PublishingMode::Single);
 
diff --git a/Framework/Utils/test/test_RootTreeWriterWorkflow.cxx b/Framework/Utils/test/test_RootTreeWriterWorkflow.cxx
index 3213c1775c758..9a9b849d8d9d2 100644
--- a/Framework/Utils/test/test_RootTreeWriterWorkflow.cxx
+++ b/Framework/Utils/test/test_RootTreeWriterWorkflow.cxx
@@ -143,7 +143,7 @@ DataProcessorSpec getSourceSpec()
       o2::test::Polymorphic a(*counter);
       pc.outputs().snapshot(OutputRef{"output", 0}, a);
       pc.outputs().snapshot(OutputRef{"output", 1}, a);
-      int& metadata = pc.outputs().make<int>(Output{"TST", "METADATA", 0, Lifetime::Timeframe});
+      int& metadata = pc.outputs().make<int>(Output{"TST", "METADATA", 0});
       metadata = *counter;
       *counter = *counter + 1;
       if (*counter >= sTreeSize) {

From 582c6d50c4d5cbbc79016f486b0f453cf5f6aa39 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 5 Dec 2023 21:49:04 +0100
Subject: [PATCH 155/726] DPL: drop lifetime member from Output

This is apparently not really used for anything and in general not passed
correctly. Lifetime is a static property of the data flow, so it does not
make sense to have it on a per message basis.
---
 Framework/CCDBSupport/src/CCDBHelpers.cxx     |  4 ++--
 Framework/Core/include/Framework/Output.h     | 19 ++----------------
 Framework/Core/src/CommonServices.cxx         |  2 +-
 Framework/Core/src/DataAllocator.cxx          |  4 ++--
 Framework/Core/test/test_DataAllocator.cxx    | 20 +++++++++----------
 Framework/Core/test/test_ParallelPipeline.cxx |  8 ++++----
 Framework/Core/test/test_ProcessorOptions.cxx |  2 +-
 .../Core/test/test_StaggeringWorkflow.cxx     |  6 +++---
 Framework/Core/test/test_TableBuilder.cxx     |  2 +-
 .../Utils/include/DPLUtils/RootTreeReader.h   |  8 ++++----
 Framework/Utils/src/Utils.cxx                 |  2 +-
 Framework/Utils/test/test_RootTreeReader.cxx  |  5 ++---
 .../test/test_RootTreeWriterWorkflow.cxx      |  2 +-
 13 files changed, 34 insertions(+), 50 deletions(-)

diff --git a/Framework/CCDBSupport/src/CCDBHelpers.cxx b/Framework/CCDBSupport/src/CCDBHelpers.cxx
index 0e067dc2f2d52..e45e1c6996a98 100644
--- a/Framework/CCDBSupport/src/CCDBHelpers.cxx
+++ b/Framework/CCDBSupport/src/CCDBHelpers.cxx
@@ -185,7 +185,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
     LOGP(debug, "Fetching object for route {}", DataSpecUtils::describe(route.matcher));
     objCnt++;
     auto concrete = DataSpecUtils::asConcreteDataMatcher(route.matcher);
-    Output output{concrete.origin, concrete.description, concrete.subSpec, route.matcher.lifetime};
+    Output output{concrete.origin, concrete.description, concrete.subSpec};
     auto&& v = allocator.makeVector<char>(output);
     std::map<std::string, std::string> metadata;
     std::map<std::string, std::string> headers;
@@ -343,7 +343,7 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
             checkValidity = true; // never skip check if the cache is empty
           }
           LOG(debug) << "checkValidity = " << checkValidity << " for TF " << timingInfo.timeslice;
-          Output output{"CTP", "OrbitReset", 0, Lifetime::Condition};
+          Output output{"CTP", "OrbitReset", 0};
           Long64_t newOrbitResetTime = orbitResetTime;
           auto&& v = allocator.makeVector<char>(output);
           const auto& api = helper->getAPI(path);
diff --git a/Framework/Core/include/Framework/Output.h b/Framework/Core/include/Framework/Output.h
index 9f85d6e8edb92..a2abf5f7600a9 100644
--- a/Framework/Core/include/Framework/Output.h
+++ b/Framework/Core/include/Framework/Output.h
@@ -28,7 +28,6 @@ struct Output {
   header::DataOrigin origin;
   header::DataDescription description;
   header::DataHeader::SubSpecificationType subSpec = 0;
-  enum Lifetime lifetime = Lifetime::Timeframe;
   header::Stack metaHeader = {};
 
   Output(header::DataOrigin o, header::DataDescription d) : origin(o), description(d) {}
@@ -38,22 +37,11 @@ struct Output {
   {
   }
 
-  Output(header::DataOrigin o, header::DataDescription d, header::DataHeader::SubSpecificationType s, Lifetime l)
-    : origin(o), description(d), subSpec(s), lifetime(l)
-  {
-  }
-
   Output(header::DataOrigin o, header::DataDescription d, header::DataHeader::SubSpecificationType s, header::Stack&& stack)
     : origin(o), description(d), subSpec(s), metaHeader(std::move(stack))
   {
   }
 
-  Output(header::DataOrigin o, header::DataDescription d, header::DataHeader::SubSpecificationType s, Lifetime l,
-         header::Stack&& stack)
-    : origin(o), description(d), subSpec(s), lifetime(l), metaHeader(std::move(stack))
-  {
-  }
-
   Output(header::DataHeader const& header)
     : origin(header.dataOrigin), description(header.dataDescription), subSpec(header.subSpecification)
   {
@@ -65,7 +53,6 @@ struct Output {
     : origin(rhs.origin),
       description(rhs.description),
       subSpec(rhs.subSpec),
-      lifetime(rhs.lifetime),
       metaHeader(std::move(rhs.metaHeader))
   {
   }
@@ -77,16 +64,14 @@ struct Output {
     origin = rhs.origin;
     description = rhs.description;
     subSpec = rhs.subSpec;
-    lifetime = rhs.lifetime;
     metaHeader = std::move(rhs.metaHeader);
     return *this;
   }
 
   bool operator==(const Output& that) const
   {
-    return origin == that.origin && description == that.description && subSpec == that.subSpec &&
-           lifetime == that.lifetime;
-  };
+    return origin == that.origin && description == that.description && subSpec == that.subSpec;
+  }
 };
 
 } // namespace o2
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 7d502216bc509..05b8fba47e532 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -425,7 +425,7 @@ o2::framework::ServiceSpec CommonServices::ccdbSupportSpec()
           if (concrete.subSpec == 0) {
             continue;
           }
-          auto& stfDist = pc.outputs().make<o2::header::STFHeader>(Output{concrete.origin, concrete.description, concrete.subSpec, output.matcher.lifetime});
+          auto& stfDist = pc.outputs().make<o2::header::STFHeader>(Output{concrete.origin, concrete.description, concrete.subSpec});
           stfDist.id = timingInfo.timeslice;
           stfDist.firstOrbit = timingInfo.firstTForbit;
           stfDist.runNumber = timingInfo.runNumber;
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index 296e094bffe26..1230d2e942f6e 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -294,7 +294,7 @@ Output DataAllocator::getOutputByBind(OutputRef&& ref)
     if (allowedOutputRoutes[ri].matcher.binding.value == ref.label) {
       auto spec = allowedOutputRoutes[ri].matcher;
       auto dataType = DataSpecUtils::asConcreteDataTypeMatcher(spec);
-      return Output{dataType.origin, dataType.description, ref.subSpec, spec.lifetime, std::move(ref.headerStack)};
+      return Output{dataType.origin, dataType.description, ref.subSpec, std::move(ref.headerStack)};
     }
   }
   std::string availableRoutes;
@@ -342,7 +342,7 @@ void DataAllocator::cookDeadBeef(const Output& spec)
   // We get the output route from the original spec, but we send it
   // using the binding of the deadbeef subSpecification.
   RouteIndex routeIndex = matchDataHeader(spec, timingInfo.timeslice);
-  auto deadBeefOutput = Output{spec.origin, spec.description, 0xdeadbeef, Lifetime::Timeframe};
+  auto deadBeefOutput = Output{spec.origin, spec.description, 0xdeadbeef};
   auto headerMessage = headerMessageFromOutput(deadBeefOutput, routeIndex, header::gSerializationMethodNone, 0);
 
   addPartToContext(proxy.createOutputMessage(routeIndex, 0), deadBeefOutput, header::gSerializationMethodNone);
diff --git a/Framework/Core/test/test_DataAllocator.cxx b/Framework/Core/test/test_DataAllocator.cxx
index d880da72d7cb4..894fb5f38368a 100644
--- a/Framework/Core/test/test_DataAllocator.cxx
+++ b/Framework/Core/test/test_DataAllocator.cxx
@@ -45,7 +45,7 @@ using namespace o2::framework;
 // this function is only used to do the static checks for API return types
 void doTypeChecks()
 {
-  const Output output{"TST", "DUMMY", 0, Lifetime::Timeframe};
+  const Output output{"TST", "DUMMY", 0};
   // we require references to objects owned by allocator context
   static_assert(std::is_lvalue_reference<decltype(std::declval<DataAllocator>().make<int>(output))>::value);
   static_assert(std::is_lvalue_reference<decltype(std::declval<DataAllocator>().make<std::string>(output, "test"))>::value);
@@ -83,24 +83,24 @@ DataProcessorSpec getSourceSpec()
     // picked by the framework is no serialization
     test::MetaHeader meta1{42};
     test::MetaHeader meta2{23};
-    pc.outputs().snapshot(Output{"TST", "MESSAGEABLE", 0, Lifetime::Timeframe, {meta1, meta2}}, a);
-    pc.outputs().snapshot(Output{"TST", "MSGBLEROOTSRLZ", 0, Lifetime::Timeframe},
+    pc.outputs().snapshot(Output{"TST", "MESSAGEABLE", 0, {meta1, meta2}}, a);
+    pc.outputs().snapshot(Output{"TST", "MSGBLEROOTSRLZ", 0},
                           o2::framework::ROOTSerialized<decltype(a)>(a));
     // class Polymorphic is not messageable, so the serialization type is deduced
     // from the fact that the type has a dictionary and can be ROOT-serialized.
-    pc.outputs().snapshot(Output{"TST", "ROOTNONTOBJECT", 0, Lifetime::Timeframe}, b);
+    pc.outputs().snapshot(Output{"TST", "ROOTNONTOBJECT", 0}, b);
     // vector of ROOT serializable class
-    pc.outputs().snapshot(Output{"TST", "ROOTVECTOR", 0, Lifetime::Timeframe}, c);
+    pc.outputs().snapshot(Output{"TST", "ROOTVECTOR", 0}, c);
     // deque of simple types
-    pc.outputs().snapshot(Output{"TST", "DEQUE", 0, Lifetime::Timeframe}, testDequePayload);
+    pc.outputs().snapshot(Output{"TST", "DEQUE", 0}, testDequePayload);
     // likewise, passed anonymously with char type and class name
     o2::framework::ROOTSerialized<char, const char> d(*((char*)&c), "vector<o2::test::Polymorphic>");
-    pc.outputs().snapshot(Output{"TST", "ROOTSERLZDVEC", 0, Lifetime::Timeframe}, d);
+    pc.outputs().snapshot(Output{"TST", "ROOTSERLZDVEC", 0}, d);
     // vector of ROOT serializable class wrapped with TClass info as hint
     auto* cl = TClass::GetClass(typeid(decltype(c)));
     ASSERT_ERROR(cl != nullptr);
     o2::framework::ROOTSerialized<char, TClass> e(*((char*)&c), cl);
-    pc.outputs().snapshot(Output{"TST", "ROOTSERLZDVEC2", 0, Lifetime::Timeframe}, e);
+    pc.outputs().snapshot(Output{"TST", "ROOTSERLZDVEC2", 0}, e);
     // test the 'make' methods
     pc.outputs().make<o2::test::TriviallyCopyable>(OutputRef{"makesingle", 0}) = a;
     auto& multi = pc.outputs().make<o2::test::TriviallyCopyable>(OutputRef{"makespan", 0}, 3);
@@ -111,7 +111,7 @@ DataProcessorSpec getSourceSpec()
     // test the adopt method
     auto freefct = [](void* data, void* hint) {}; // simply ignore the cleanup for the test
     static std::string teststring = "adoptchunk";
-    pc.outputs().adoptChunk(Output{"TST", "ADOPTCHUNK", 0, Lifetime::Timeframe}, teststring.data(), teststring.length(), freefct, nullptr);
+    pc.outputs().adoptChunk(Output{"TST", "ADOPTCHUNK", 0}, teststring.data(), teststring.length(), freefct, nullptr);
     // test resizable data chunk, initial size 0 and grow
     auto& growchunk = pc.outputs().newChunk(OutputRef{"growchunk", 0}, 0);
     growchunk.resize(sizeof(o2::test::TriviallyCopyable));
@@ -307,7 +307,7 @@ DataProcessorSpec getSinkSpec()
     ASSERT_ERROR((object12[0] == o2::test::TriviallyCopyable{42, 23, 0xdead}));
     ASSERT_ERROR((object12[1] == o2::test::TriviallyCopyable{10, 20, 0xacdc}));
     // forward the read-only span on a different route
-    pc.outputs().snapshot(Output{"TST", "MSGABLVECTORCPY", 0, Lifetime::Timeframe}, object12);
+    pc.outputs().snapshot(Output{"TST", "MSGABLVECTORCPY", 0}, object12);
 
     LOG(info) << "extracting TNamed object from input13";
     auto object13 = pc.inputs().get<TNamed*>("input13");
diff --git a/Framework/Core/test/test_ParallelPipeline.cxx b/Framework/Core/test/test_ParallelPipeline.cxx
index e13869873fca1..47106855a8f74 100644
--- a/Framework/Core/test/test_ParallelPipeline.cxx
+++ b/Framework/Core/test/test_ParallelPipeline.cxx
@@ -64,7 +64,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
                     << *input.spec << ": " << *((int*)input.payload);
          auto const* dataheader = DataRefUtils::getHeader<o2::header::DataHeader*>(input);
          //auto& data = ctx.outputs().make<int>(OutputRef{"output", dataheader->subSpecification});
-         auto& data = ctx.outputs().make<int>(Output{"TST", "PREPROC", dataheader->subSpecification, Lifetime::Timeframe});
+         auto& data = ctx.outputs().make<int>(Output{"TST", "PREPROC", dataheader->subSpecification});
          ASSERT_ERROR(ctx.inputs().get<int>(input.spec->binding.c_str()) == parallelContext.index1D());
          data = parallelContext.index1D();
        }
@@ -85,10 +85,10 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
          // TODO: there is a bug in the API for using OutputRef, returns an rvalue which can not be bound to
          // lvalue reference
          //auto& data = ctx.outputs().make<int>(OutputRef{"output", dataheader->subSpecification});
-         auto& data = ctx.outputs().make<int>(Output{"TST", "DATA", dataheader->subSpecification, Lifetime::Timeframe});
+         auto& data = ctx.outputs().make<int>(Output{"TST", "DATA", dataheader->subSpecification});
          data = ctx.inputs().get<int>(input.spec->binding.c_str());
          //auto& meta = ctx.outputs().make<int>(OutputRef{"metadt", dataheader->subSpecification});
-         auto& meta = ctx.outputs().make<int>(Output{"TST", "META", dataheader->subSpecification, Lifetime::Timeframe});
+         auto& meta = ctx.outputs().make<int>(Output{"TST", "META", dataheader->subSpecification});
          meta = dataheader->subSpecification;
        }
      }}},
@@ -145,7 +145,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
           if (multiplicities[pipeline] == 0) {
             continue;
           }
-          ctx.outputs().make<int>(Output{"TST", "TRIGGER", subspecs[index], Lifetime::Timeframe}) = pipeline;
+          ctx.outputs().make<int>(Output{"TST", "TRIGGER", subspecs[index]}) = pipeline;
           multiplicities[pipeline++]--;
           if (pipeline >= nPipelines) {
             pipeline = 0;
diff --git a/Framework/Core/test/test_ProcessorOptions.cxx b/Framework/Core/test/test_ProcessorOptions.cxx
index 8de90e0e23441..1751176eb639e 100644
--- a/Framework/Core/test/test_ProcessorOptions.cxx
+++ b/Framework/Core/test/test_ProcessorOptions.cxx
@@ -45,7 +45,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const&)
             }
             // there is nothing to do, simply stop the workflow but we have to send at least one message
             // to make sure that the callback of the consumer is called
-            ctx.outputs().make<int>(Output{"TST", "TEST", 0, Lifetime::Timeframe}) = 42;
+            ctx.outputs().make<int>(Output{"TST", "TEST", 0}) = 42;
             ctx.services().get<ControlService>().endOfStream();
             *isReady = true;
           };
diff --git a/Framework/Core/test/test_StaggeringWorkflow.cxx b/Framework/Core/test/test_StaggeringWorkflow.cxx
index 0e98c4324bc76..edc07d598465b 100644
--- a/Framework/Core/test/test_StaggeringWorkflow.cxx
+++ b/Framework/Core/test/test_StaggeringWorkflow.cxx
@@ -85,9 +85,9 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
       // since the snapshot copy is ready for sending it is scheduled but held back
       // because of the CompletionPolicy trigger matcher. This message will be
       // sent together with the second message.
-      outputs.snapshot(Output{"PROD", "CHANNEL", subspec, Lifetime::Timeframe}, subspec);
+      outputs.snapshot(Output{"PROD", "CHANNEL", subspec}, subspec);
       device.waitFor(100);
-      outputs.snapshot(Output{"PROD", "TRIGGER", subspec, Lifetime::Timeframe}, subspec);
+      outputs.snapshot(Output{"PROD", "TRIGGER", subspec}, subspec);
       device.waitFor(100);
     }
     control.endOfStream();
@@ -106,7 +106,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
       LOG(info) << "processing " << input.spec->binding << " " << data;
       // check if the channel binding starts with 'trigger'
       if (input.spec->binding.find("trigger") == 0) {
-        pc.outputs().make<MyDataType>(Output{"PROC", "CHANNEL", data, Lifetime::Timeframe}) = data;
+        pc.outputs().make<MyDataType>(Output{"PROC", "CHANNEL", data}) = data;
       }
       nActiveInputs++;
     }
diff --git a/Framework/Core/test/test_TableBuilder.cxx b/Framework/Core/test/test_TableBuilder.cxx
index a7b4739054d03..26a0f5a930b73 100644
--- a/Framework/Core/test/test_TableBuilder.cxx
+++ b/Framework/Core/test/test_TableBuilder.cxx
@@ -240,7 +240,7 @@ TEST_CASE("TestSoAIntegration")
 
 TEST_CASE("TestDataAllocatorReturnType")
 {
-  const Output output{"TST", "DUMMY", 0, Lifetime::Timeframe};
+  const Output output{"TST", "DUMMY", 0};
 }
 
 TEST_CASE("TestPodInjestion")
diff --git a/Framework/Utils/include/DPLUtils/RootTreeReader.h b/Framework/Utils/include/DPLUtils/RootTreeReader.h
index 42a8c9d01d0d0..109d79f9fad78 100644
--- a/Framework/Utils/include/DPLUtils/RootTreeReader.h
+++ b/Framework/Utils/include/DPLUtils/RootTreeReader.h
@@ -51,11 +51,11 @@ struct DefaultKey {
   enum Lifetime lifetime = Lifetime::Timeframe;
 
   DefaultKey(const Output& desc)
-    : origin(desc.origin), description(desc.description), subSpec(desc.subSpec), lifetime(desc.lifetime)
+    : origin(desc.origin), description(desc.description), subSpec(desc.subSpec)
   {
   }
 
-  operator Output() const { return Output{origin, description, subSpec, lifetime}; }
+  operator Output() const { return Output{origin, description, subSpec}; }
 };
 } // namespace rtr
 
@@ -302,7 +302,7 @@ class GenericRootTreeReader
       }
 
       auto snapshot = [&context, &stackcreator](const KeyType& key, const auto& object) {
-        context.outputs().snapshot(Output{key.origin, key.description, key.subSpec, key.lifetime, std::move(stackcreator())}, object);
+        context.outputs().snapshot(Output{key.origin, key.description, key.subSpec, std::move(stackcreator())}, object);
       };
 
       char* data = nullptr;
@@ -310,7 +310,7 @@ class GenericRootTreeReader
       mBranch->GetEntry(entry);
 
       // execute hook if it was registered; if this return true do not proceed further
-      if (mPublishHook != nullptr && (*mPublishHook).hook(mName, context, Output{mKey.origin, mKey.description, mKey.subSpec, mKey.lifetime, std::move(stackcreator())}, data)) {
+      if (mPublishHook != nullptr && (*mPublishHook).hook(mName, context, Output{mKey.origin, mKey.description, mKey.subSpec, std::move(stackcreator())}, data)) {
 
       }
       // try to figureout when we need to do something special
diff --git a/Framework/Utils/src/Utils.cxx b/Framework/Utils/src/Utils.cxx
index 3a52c46e074af..1dc0882c2f55d 100644
--- a/Framework/Utils/src/Utils.cxx
+++ b/Framework/Utils/src/Utils.cxx
@@ -28,7 +28,7 @@ namespace workflows
 Output getOutput(const o2f::OutputSpec outputSpec)
 {
   auto concrete = DataSpecUtils::asConcreteDataMatcher(outputSpec);
-  return Output{concrete.origin, concrete.description, concrete.subSpec, outputSpec.lifetime};
+  return Output{concrete.origin, concrete.description, concrete.subSpec};
 }
 
 // This method can convert a vector of OutputSpec into a vector of Output.
diff --git a/Framework/Utils/test/test_RootTreeReader.cxx b/Framework/Utils/test/test_RootTreeReader.cxx
index 0390a29f807b1..bf652fca6f97a 100644
--- a/Framework/Utils/test/test_RootTreeReader.cxx
+++ b/Framework/Utils/test/test_RootTreeReader.cxx
@@ -67,11 +67,10 @@ DataProcessorSpec getSourceSpec()
       testFile->Close();
     }
 
-    constexpr auto persistency = Lifetime::Transient;
     auto reader = std::make_shared<RootTreeReader>("testtree",       // tree name
                                                    fileName.c_str(), // input file name
-                                                   RootTreeReader::BranchDefinition<std::vector<o2::test::TriviallyCopyable>>{Output{"TST", "ARRAYOFMSGBL", 0, persistency}, "msgblarray"},
-                                                   Output{"TST", "ARRAYOFDATA", 0, persistency},
+                                                   RootTreeReader::BranchDefinition<std::vector<o2::test::TriviallyCopyable>>{Output{"TST", "ARRAYOFMSGBL", 0}, "msgblarray"},
+                                                   Output{"TST", "ARRAYOFDATA", 0},
                                                    "dataarray",
                                                    RootTreeReader::PublishingMode::Single);
 
diff --git a/Framework/Utils/test/test_RootTreeWriterWorkflow.cxx b/Framework/Utils/test/test_RootTreeWriterWorkflow.cxx
index 3213c1775c758..9a9b849d8d9d2 100644
--- a/Framework/Utils/test/test_RootTreeWriterWorkflow.cxx
+++ b/Framework/Utils/test/test_RootTreeWriterWorkflow.cxx
@@ -143,7 +143,7 @@ DataProcessorSpec getSourceSpec()
       o2::test::Polymorphic a(*counter);
       pc.outputs().snapshot(OutputRef{"output", 0}, a);
       pc.outputs().snapshot(OutputRef{"output", 1}, a);
-      int& metadata = pc.outputs().make<int>(Output{"TST", "METADATA", 0, Lifetime::Timeframe});
+      int& metadata = pc.outputs().make<int>(Output{"TST", "METADATA", 0});
       metadata = *counter;
       *counter = *counter + 1;
       if (*counter >= sTreeSize) {

From 546a0985e104e43a0a2112ce0a009f8b5548b926 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 6 Dec 2023 12:03:33 +0100
Subject: [PATCH 156/726] DPL: improve crash reason message

---
 Framework/Core/src/runDataProcessing.cxx | 25 ++++++++----------------
 1 file changed, 8 insertions(+), 17 deletions(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 54af8e9b4a25e..d42eeafb63006 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -578,31 +578,22 @@ void handle_crash(int sig)
 {
   // dump demangled stack trace
   void* array[1024];
-
   int size = backtrace(array, 1024);
 
   {
-    char const* msg = "*** Program crashed (Segmentation fault, FPE, BUS, ABRT, KILL, Unhandled Exception, ...)\nBacktrace by DPL:\n";
-    auto retVal = write(STDERR_FILENO, msg, strlen(msg));
-    msg = "UNKNOWN SIGNAL\n";
-    if (sig == SIGSEGV) {
-      msg = "SEGMENTATION FAULT\n";
-    } else if (sig == SIGABRT) {
-      msg = "ABRT\n";
-    } else if (sig == SIGBUS) {
-      msg = "BUS ERROR\n";
-    } else if (sig == SIGILL) {
-      msg = "ILLEGAL INSTRUCTION\n";
-    } else if (sig == SIGFPE) {
+    char buffer[1024];
+    char const* msg = "*** Program crashed (%s)\nBacktrace by DPL:\n";
+    snprintf(buffer, 1024, msg, strsignal(sig));
+    if (sig == SIGFPE) {
       if (std::fetestexcept(FE_DIVBYZERO)) {
-        msg = "FLOATING POINT EXCEPTION (DIVISION BY ZERO)\n";
+        snprintf(buffer, 1024, msg, "FLOATING POINT EXCEPTION - DIVISION BY ZERO");
       } else if (std::fetestexcept(FE_INVALID)) {
-        msg = "FLOATING POINT EXCEPTION (INVALID RESULT)\n";
+        snprintf(buffer, 1024, msg, "FLOATING POINT EXCEPTION - INVALID RESULT");
       } else {
-        msg = "FLOATING POINT EXCEPTION (UNKNOWN REASON)\n";
+        snprintf(buffer, 1024, msg, "FLOATING POINT EXCEPTION - UNKNOWN REASON");
       }
     }
-    retVal = write(STDERR_FILENO, msg, strlen(msg));
+    auto retVal = write(STDERR_FILENO, buffer, strlen(buffer));
     (void)retVal;
   }
   demangled_backtrace_symbols(array, size, STDERR_FILENO);

From c1e355cfcfeab45548852b62bfe2f32b805ee775 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 7 Dec 2023 22:08:39 +0100
Subject: [PATCH 157/726] DPL: exit with 128 + signal number when dying due to
 a signal

---
 Framework/Core/src/runDataProcessing.cxx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index d42eeafb63006..21bf8fba49a61 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -874,6 +874,7 @@ bool processSigChild(DeviceInfos& infos, DeviceSpecs& specs)
         } else {
           if (WIFSIGNALED(status)) {
             int exitSignal = WTERMSIG(status);
+            es = exitSignal + 128;
             LOGP(error, "Workflow crashed - pid {} ({}) was killed abnormally with {} and exited with {}", pid, id, strsignal(exitSignal), es);
           } else {
             LOGP(error, "pid {} ({}) crashed with or was killed with exit code {}", pid, id, es);

From 1878f8738fea3ce850090e824d288d5d9036bdb4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 7 Dec 2023 22:08:39 +0100
Subject: [PATCH 158/726] DPL: exit with 128 + signal number when dying due to
 a signal

---
 Framework/Core/src/runDataProcessing.cxx | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index d42eeafb63006..def8a76a12c49 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -874,9 +874,11 @@ bool processSigChild(DeviceInfos& infos, DeviceSpecs& specs)
         } else {
           if (WIFSIGNALED(status)) {
             int exitSignal = WTERMSIG(status);
-            LOGP(error, "Workflow crashed - pid {} ({}) was killed abnormally with {} and exited with {}", pid, id, strsignal(exitSignal), es);
+            es = exitSignal + 128;
+            LOGP(error, "Workflow crashed - PID {} ({}) was killed abnormally with {} and exited code was set to {}.", pid, id, strsignal(exitSignal), es);
           } else {
-            LOGP(error, "pid {} ({}) crashed with or was killed with exit code {}", pid, id, es);
+            es = 128;
+            LOGP(error, "PID {} ({}) did not exit correctly however it's not clear why. Exit code forced to {}.", pid, id, es);
           }
         }
         hasError |= true;

From 9410a658a04f85f840512bffc15e6dfa67eabdd5 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 9 Dec 2023 17:29:50 +0100
Subject: [PATCH 159/726] Fix a few more Output with Lifetime::Timeframe

---
 .../include/CommonDataFormat/FlatHisto1D.h       |  2 +-
 .../include/CommonDataFormat/FlatHisto2D.h       |  2 +-
 .../workflow/include/FT0Workflow/RawReaderFT0.h  | 16 ++++++++--------
 .../FT0/workflow/src/FT0DataDecoderDPLSpec.cxx   |  9 ++++-----
 .../tofworkflow/src/RecoWorkflowSpec.cxx         |  6 +++---
 .../src/TPCResidualReaderSpec.cxx                |  2 +-
 .../PHOS/calib/src/PHOSTurnonCalibDevice.cxx     |  4 ++--
 .../TPC/workflow/src/LinkZSToDigitsSpec.cxx      |  2 +-
 .../ITS3/workflow/src/ClusterReaderSpec.cxx      | 10 +++++-----
 9 files changed, 26 insertions(+), 27 deletions(-)

diff --git a/DataFormats/common/include/CommonDataFormat/FlatHisto1D.h b/DataFormats/common/include/CommonDataFormat/FlatHisto1D.h
index ac91eafc4d147..f8d49042dd8c5 100644
--- a/DataFormats/common/include/CommonDataFormat/FlatHisto1D.h
+++ b/DataFormats/common/include/CommonDataFormat/FlatHisto1D.h
@@ -34,7 +34,7 @@ namespace dataformats
   Fast 1D histo class which can be messages as
   FlatHisto1D<float> histo(nbins, xmin, xmax);
   histo.fill(...);
-  pc.outputs().snapshot(Output{"Origin", "Desc", 0, Lifetime::Timeframe}, histo.getBase());
+  pc.outputs().snapshot(Output{"Origin", "Desc", 0}, histo.getBase());
 
   and received (read only!) as
   const auto hdata = pc.inputs().get<gsl::span<float>>("histodata");
diff --git a/DataFormats/common/include/CommonDataFormat/FlatHisto2D.h b/DataFormats/common/include/CommonDataFormat/FlatHisto2D.h
index 9102d53edeaaa..47793618c0649 100644
--- a/DataFormats/common/include/CommonDataFormat/FlatHisto2D.h
+++ b/DataFormats/common/include/CommonDataFormat/FlatHisto2D.h
@@ -35,7 +35,7 @@ namespace dataformats
   Fast 2D histo class which can be messages as
   FlatHisto2D<float> histo(nbinsX, xmin, xmax, nbinsY, ymin, ymax);
   histo.fill(...);
-  pc.outputs().snapshot(Output{"Origin", "Desc", 0, Lifetime::Timeframe}, histo.getBase());
+  pc.outputs().snapshot(Output{"Origin", "Desc", 0}, histo.getBase());
 
   and received (read only!) as
   const auto hdata = pc.inputs().get<gsl::span<float>>("histodata");
diff --git a/Detectors/FIT/FT0/workflow/include/FT0Workflow/RawReaderFT0.h b/Detectors/FIT/FT0/workflow/include/FT0Workflow/RawReaderFT0.h
index f50b3d224b32c..f7729394db652 100644
--- a/Detectors/FIT/FT0/workflow/include/FT0Workflow/RawReaderFT0.h
+++ b/Detectors/FIT/FT0/workflow/include/FT0Workflow/RawReaderFT0.h
@@ -80,10 +80,10 @@ class RawReaderFT0 : public RawReaderFT0BaseNorm
   }
   void makeSnapshot(o2::framework::ProcessingContext& pc)
   {
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSBC", 0, o2::framework::Lifetime::Timeframe}, mVecDigits);
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSCH", 0, o2::framework::Lifetime::Timeframe}, mVecChannelData);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSBC", 0}, mVecDigits);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSCH", 0}, mVecChannelData);
     if constexpr (sUseTrgInput) {
-      pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "TRIGGERINPUT", 0, o2::framework::Lifetime::Timeframe}, mVecTriggerInput);
+      pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "TRIGGERINPUT", 0}, mVecTriggerInput);
     }
   }
   bool mDumpData;
@@ -136,11 +136,11 @@ class RawReaderFT0ext : public RawReaderFT0BaseExt
   }
   void makeSnapshot(o2::framework::ProcessingContext& pc)
   {
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSBC", 0, o2::framework::Lifetime::Timeframe}, mVecDigits);
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSCH", 0, o2::framework::Lifetime::Timeframe}, mVecChannelData);
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSTRGEXT", 0, o2::framework::Lifetime::Timeframe}, mVecTrgExt);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSBC", 0}, mVecDigits);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSCH", 0}, mVecChannelData);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSTRGEXT", 0}, mVecTrgExt);
     if constexpr (sUseTrgInput) {
-      pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "TRIGGERINPUT", 0, o2::framework::Lifetime::Timeframe}, mVecTriggerInput);
+      pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "TRIGGERINPUT", 0}, mVecTriggerInput);
     }
   }
   bool mDumpData;
@@ -153,4 +153,4 @@ class RawReaderFT0ext : public RawReaderFT0BaseExt
 } // namespace ft0
 } // namespace o2
 
-#endif
\ No newline at end of file
+#endif
diff --git a/Detectors/FIT/FT0/workflow/src/FT0DataDecoderDPLSpec.cxx b/Detectors/FIT/FT0/workflow/src/FT0DataDecoderDPLSpec.cxx
index 945e7ad9d84d1..6d81740dd2173 100644
--- a/Detectors/FIT/FT0/workflow/src/FT0DataDecoderDPLSpec.cxx
+++ b/Detectors/FIT/FT0/workflow/src/FT0DataDecoderDPLSpec.cxx
@@ -28,12 +28,11 @@ namespace ft0
 void FT0DataDecoderDPLSpec::run(ProcessingContext& pc)
 {
   auto t1 = std::chrono::high_resolution_clock::now();
-
   auto dummyOutput = [&pc, this]() {
     this->mVecDigits.resize(0);
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSBC", 0, o2::framework::Lifetime::Timeframe}, mVecDigits);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSBC", 0}, mVecDigits);
     this->mVecChannelData.resize(0);
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSCH", 0, o2::framework::Lifetime::Timeframe}, mVecChannelData);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSCH", 0}, mVecChannelData);
   };
 
   // if we see requested data type input with 0xDEADBEEF subspec and 0 payload this means that the "delayed message"
@@ -529,8 +528,8 @@ void FT0DataDecoderDPLSpec::run(ProcessingContext& pc)
     // Due to empty Digit container this dummy object will never participate in any further tasks.
     mVecChannelData.emplace_back();
   }
-  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSBC", 0, o2::framework::Lifetime::Timeframe}, mVecDigits);
-  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSCH", 0, o2::framework::Lifetime::Timeframe}, mVecChannelData);
+  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSBC", 0}, mVecDigits);
+  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSCH", 0}, mVecChannelData);
   auto t2 = std::chrono::high_resolution_clock::now();
   auto delay = std::chrono::duration_cast<std::chrono::microseconds>(t2 - t1);
   LOG(debug) << "Decoder delay: " << delay.count();
diff --git a/Detectors/GlobalTrackingWorkflow/tofworkflow/src/RecoWorkflowSpec.cxx b/Detectors/GlobalTrackingWorkflow/tofworkflow/src/RecoWorkflowSpec.cxx
index 18b26355ff5ec..ab4f90464b31b 100644
--- a/Detectors/GlobalTrackingWorkflow/tofworkflow/src/RecoWorkflowSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/tofworkflow/src/RecoWorkflowSpec.cxx
@@ -97,11 +97,11 @@ class TOFDPLRecoWorkflowTask
     //           << " DIGITS TO " << mClustersArray.size() << " CLUSTERS";
 
     // send matching-info
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MTC_ITSTPC", 0, Lifetime::Timeframe}, mMatcher.getMatchedTrackVector());
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MTC_ITSTPC", 0}, mMatcher.getMatchedTrackVector());
     if (mUseMC) {
-      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MCMATCHTOF", 0, Lifetime::Timeframe}, mMatcher.getMatchedTOFLabelsVector());
+      pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MCMATCHTOF", 0}, mMatcher.getMatchedTOFLabelsVector());
     }
-    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CALIBDATA", 0, Lifetime::Timeframe}, mMatcher.getCalibVector());
+    pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "CALIBDATA", 0}, mMatcher.getCalibVector());
     mTimer.Stop();
   }
 
diff --git a/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCResidualReaderSpec.cxx b/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCResidualReaderSpec.cxx
index 23a6c8fd06694..b3040d99bc4f2 100644
--- a/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCResidualReaderSpec.cxx
+++ b/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCResidualReaderSpec.cxx
@@ -251,7 +251,7 @@ void TPCResidualReader::run(ProcessingContext& pc)
   mTrackResiduals.closeOutputFile(); // FIXME remove when map output is handled properly
 
   // const auto& voxResArray = mTrackResiduals.getVoxelResults(); // array with one vector of results per sector
-  // pc.outputs().snapshot(Output{"GLO", "VOXELRESULTS", 0, Lifetime::Timeframe}, voxResArray); // send results as one large vector?
+  // pc.outputs().snapshot(Output{"GLO", "VOXELRESULTS", 0}, voxResArray); // send results as one large vector?
 
   pc.services().get<ControlService>().endOfStream();
   pc.services().get<ControlService>().readyToQuit(QuitRequest::Me);
diff --git a/Detectors/PHOS/calib/src/PHOSTurnonCalibDevice.cxx b/Detectors/PHOS/calib/src/PHOSTurnonCalibDevice.cxx
index f4ff1a5c4042b..52ec8cef0b438 100644
--- a/Detectors/PHOS/calib/src/PHOSTurnonCalibDevice.cxx
+++ b/Detectors/PHOS/calib/src/PHOSTurnonCalibDevice.cxx
@@ -80,8 +80,8 @@ void PHOSTurnonCalibDevice::endOfStream(o2::framework::EndOfStreamContext& ec)
     LOG(alarm) << "Incorrect fit results";
   }
   // //Send result to QC
-  // ec.outputs().snapshot(o2::framework::Output{"PHS", "TRIGMAPDIFF", 0, o2::framework::Lifetime::Timeframe}, mTrigMapDiff);
-  // ec.outputs().snapshot(o2::framework::Output{"PHS", "TURNONDIFF", 0, o2::framework::Lifetime::Timeframe}, mTurnOnDiff);
+  // ec.outputs().snapshot(o2::framework::Output{"PHS", "TRIGMAPDIFF", 0}, mTrigMapDiff);
+  // ec.outputs().snapshot(o2::framework::Output{"PHS", "TURNONDIFF", 0}, mTurnOnDiff);
 }
 
 o2::framework::DataProcessorSpec o2::phos::getPHOSTurnonCalibDeviceSpec(bool useCCDB)
diff --git a/Detectors/TPC/workflow/src/LinkZSToDigitsSpec.cxx b/Detectors/TPC/workflow/src/LinkZSToDigitsSpec.cxx
index c42a93f7a7f04..a278809cbeccd 100644
--- a/Detectors/TPC/workflow/src/LinkZSToDigitsSpec.cxx
+++ b/Detectors/TPC/workflow/src/LinkZSToDigitsSpec.cxx
@@ -109,7 +109,7 @@ o2::framework::DataProcessorSpec getLinkZSToDigitsSpec(int channel, const std::s
         o2::tpc::TPCSectorHeader header{sector};
         header.activeSectors = processAttributes->activeSectors;
         // digit for now are transported per sector, not per lane
-        // pc.outputs().snapshot(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(channel), Lifetime::Timeframe, header},
+        // pc.outputs().snapshot(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(channel), header},
         pc.outputs().snapshot(Output{"TPC", "DIGITS", static_cast<SubSpecificationType>(sector), header},
                               const_cast<std::vector<o2::tpc::Digit>&>(digits));
       };
diff --git a/Detectors/Upgrades/ITS3/workflow/src/ClusterReaderSpec.cxx b/Detectors/Upgrades/ITS3/workflow/src/ClusterReaderSpec.cxx
index b50ab65d3720f..093e53c92eaf3 100644
--- a/Detectors/Upgrades/ITS3/workflow/src/ClusterReaderSpec.cxx
+++ b/Detectors/Upgrades/ITS3/workflow/src/ClusterReaderSpec.cxx
@@ -53,14 +53,14 @@ void ClusterReader::run(ProcessingContext& pc)
 
   // This is a very ugly way of providing DataDescription, which anyway does not need to contain detector name.
   // To be fixed once the names-definition class is ready
-  pc.outputs().snapshot(Output{mOrigin, "CLUSTERSROF", 0, Lifetime::Timeframe}, mClusROFRec);
-  pc.outputs().snapshot(Output{mOrigin, "COMPCLUSTERS", 0, Lifetime::Timeframe}, mClusterCompArray);
+  pc.outputs().snapshot(Output{mOrigin, "CLUSTERSROF", 0}, mClusROFRec);
+  pc.outputs().snapshot(Output{mOrigin, "COMPCLUSTERS", 0}, mClusterCompArray);
   if (mUsePatterns) {
-    pc.outputs().snapshot(Output{mOrigin, "PATTERNS", 0, Lifetime::Timeframe}, mPatternsArray);
+    pc.outputs().snapshot(Output{mOrigin, "PATTERNS", 0}, mPatternsArray);
   }
   if (mUseMC) {
-    pc.outputs().snapshot(Output{mOrigin, "CLUSTERSMCTR", 0, Lifetime::Timeframe}, mClusterMCTruth);
-    pc.outputs().snapshot(Output{mOrigin, "CLUSTERSMC2ROF", 0, Lifetime::Timeframe}, mClusMC2ROFs);
+    pc.outputs().snapshot(Output{mOrigin, "CLUSTERSMCTR", 0}, mClusterMCTruth);
+    pc.outputs().snapshot(Output{mOrigin, "CLUSTERSMC2ROF", 0}, mClusMC2ROFs);
   }
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {

From e89c48c00dddeeccc32892887a19e1f7169f46c0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 7 Dec 2023 22:22:43 +0100
Subject: [PATCH 160/726] DPL: drop lifetime member from Output

This is apparently not really used for anything and in general not passed
correctly. Lifetime is a static property of the data flow, so it does not
make sense to have it on a per message basis.
---
 Framework/CCDBSupport/src/CCDBHelpers.cxx     |  4 ++--
 Framework/Core/include/Framework/Output.h     | 19 ++----------------
 Framework/Core/src/CommonServices.cxx         |  2 +-
 Framework/Core/src/DataAllocator.cxx          |  4 ++--
 Framework/Core/test/test_DataAllocator.cxx    | 20 +++++++++----------
 Framework/Core/test/test_ParallelPipeline.cxx |  8 ++++----
 Framework/Core/test/test_ProcessorOptions.cxx |  2 +-
 .../Core/test/test_StaggeringWorkflow.cxx     |  6 +++---
 Framework/Core/test/test_TableBuilder.cxx     |  2 +-
 .../Utils/include/DPLUtils/RootTreeReader.h   |  8 ++++----
 Framework/Utils/src/Utils.cxx                 |  2 +-
 Framework/Utils/test/test_RootTreeReader.cxx  |  5 ++---
 .../test/test_RootTreeWriterWorkflow.cxx      |  2 +-
 13 files changed, 34 insertions(+), 50 deletions(-)

diff --git a/Framework/CCDBSupport/src/CCDBHelpers.cxx b/Framework/CCDBSupport/src/CCDBHelpers.cxx
index 0e067dc2f2d52..e45e1c6996a98 100644
--- a/Framework/CCDBSupport/src/CCDBHelpers.cxx
+++ b/Framework/CCDBSupport/src/CCDBHelpers.cxx
@@ -185,7 +185,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
     LOGP(debug, "Fetching object for route {}", DataSpecUtils::describe(route.matcher));
     objCnt++;
     auto concrete = DataSpecUtils::asConcreteDataMatcher(route.matcher);
-    Output output{concrete.origin, concrete.description, concrete.subSpec, route.matcher.lifetime};
+    Output output{concrete.origin, concrete.description, concrete.subSpec};
     auto&& v = allocator.makeVector<char>(output);
     std::map<std::string, std::string> metadata;
     std::map<std::string, std::string> headers;
@@ -343,7 +343,7 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
             checkValidity = true; // never skip check if the cache is empty
           }
           LOG(debug) << "checkValidity = " << checkValidity << " for TF " << timingInfo.timeslice;
-          Output output{"CTP", "OrbitReset", 0, Lifetime::Condition};
+          Output output{"CTP", "OrbitReset", 0};
           Long64_t newOrbitResetTime = orbitResetTime;
           auto&& v = allocator.makeVector<char>(output);
           const auto& api = helper->getAPI(path);
diff --git a/Framework/Core/include/Framework/Output.h b/Framework/Core/include/Framework/Output.h
index 9f85d6e8edb92..a2abf5f7600a9 100644
--- a/Framework/Core/include/Framework/Output.h
+++ b/Framework/Core/include/Framework/Output.h
@@ -28,7 +28,6 @@ struct Output {
   header::DataOrigin origin;
   header::DataDescription description;
   header::DataHeader::SubSpecificationType subSpec = 0;
-  enum Lifetime lifetime = Lifetime::Timeframe;
   header::Stack metaHeader = {};
 
   Output(header::DataOrigin o, header::DataDescription d) : origin(o), description(d) {}
@@ -38,22 +37,11 @@ struct Output {
   {
   }
 
-  Output(header::DataOrigin o, header::DataDescription d, header::DataHeader::SubSpecificationType s, Lifetime l)
-    : origin(o), description(d), subSpec(s), lifetime(l)
-  {
-  }
-
   Output(header::DataOrigin o, header::DataDescription d, header::DataHeader::SubSpecificationType s, header::Stack&& stack)
     : origin(o), description(d), subSpec(s), metaHeader(std::move(stack))
   {
   }
 
-  Output(header::DataOrigin o, header::DataDescription d, header::DataHeader::SubSpecificationType s, Lifetime l,
-         header::Stack&& stack)
-    : origin(o), description(d), subSpec(s), lifetime(l), metaHeader(std::move(stack))
-  {
-  }
-
   Output(header::DataHeader const& header)
     : origin(header.dataOrigin), description(header.dataDescription), subSpec(header.subSpecification)
   {
@@ -65,7 +53,6 @@ struct Output {
     : origin(rhs.origin),
       description(rhs.description),
       subSpec(rhs.subSpec),
-      lifetime(rhs.lifetime),
       metaHeader(std::move(rhs.metaHeader))
   {
   }
@@ -77,16 +64,14 @@ struct Output {
     origin = rhs.origin;
     description = rhs.description;
     subSpec = rhs.subSpec;
-    lifetime = rhs.lifetime;
     metaHeader = std::move(rhs.metaHeader);
     return *this;
   }
 
   bool operator==(const Output& that) const
   {
-    return origin == that.origin && description == that.description && subSpec == that.subSpec &&
-           lifetime == that.lifetime;
-  };
+    return origin == that.origin && description == that.description && subSpec == that.subSpec;
+  }
 };
 
 } // namespace o2
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 7d502216bc509..05b8fba47e532 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -425,7 +425,7 @@ o2::framework::ServiceSpec CommonServices::ccdbSupportSpec()
           if (concrete.subSpec == 0) {
             continue;
           }
-          auto& stfDist = pc.outputs().make<o2::header::STFHeader>(Output{concrete.origin, concrete.description, concrete.subSpec, output.matcher.lifetime});
+          auto& stfDist = pc.outputs().make<o2::header::STFHeader>(Output{concrete.origin, concrete.description, concrete.subSpec});
           stfDist.id = timingInfo.timeslice;
           stfDist.firstOrbit = timingInfo.firstTForbit;
           stfDist.runNumber = timingInfo.runNumber;
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index 296e094bffe26..1230d2e942f6e 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -294,7 +294,7 @@ Output DataAllocator::getOutputByBind(OutputRef&& ref)
     if (allowedOutputRoutes[ri].matcher.binding.value == ref.label) {
       auto spec = allowedOutputRoutes[ri].matcher;
       auto dataType = DataSpecUtils::asConcreteDataTypeMatcher(spec);
-      return Output{dataType.origin, dataType.description, ref.subSpec, spec.lifetime, std::move(ref.headerStack)};
+      return Output{dataType.origin, dataType.description, ref.subSpec, std::move(ref.headerStack)};
     }
   }
   std::string availableRoutes;
@@ -342,7 +342,7 @@ void DataAllocator::cookDeadBeef(const Output& spec)
   // We get the output route from the original spec, but we send it
   // using the binding of the deadbeef subSpecification.
   RouteIndex routeIndex = matchDataHeader(spec, timingInfo.timeslice);
-  auto deadBeefOutput = Output{spec.origin, spec.description, 0xdeadbeef, Lifetime::Timeframe};
+  auto deadBeefOutput = Output{spec.origin, spec.description, 0xdeadbeef};
   auto headerMessage = headerMessageFromOutput(deadBeefOutput, routeIndex, header::gSerializationMethodNone, 0);
 
   addPartToContext(proxy.createOutputMessage(routeIndex, 0), deadBeefOutput, header::gSerializationMethodNone);
diff --git a/Framework/Core/test/test_DataAllocator.cxx b/Framework/Core/test/test_DataAllocator.cxx
index d880da72d7cb4..894fb5f38368a 100644
--- a/Framework/Core/test/test_DataAllocator.cxx
+++ b/Framework/Core/test/test_DataAllocator.cxx
@@ -45,7 +45,7 @@ using namespace o2::framework;
 // this function is only used to do the static checks for API return types
 void doTypeChecks()
 {
-  const Output output{"TST", "DUMMY", 0, Lifetime::Timeframe};
+  const Output output{"TST", "DUMMY", 0};
   // we require references to objects owned by allocator context
   static_assert(std::is_lvalue_reference<decltype(std::declval<DataAllocator>().make<int>(output))>::value);
   static_assert(std::is_lvalue_reference<decltype(std::declval<DataAllocator>().make<std::string>(output, "test"))>::value);
@@ -83,24 +83,24 @@ DataProcessorSpec getSourceSpec()
     // picked by the framework is no serialization
     test::MetaHeader meta1{42};
     test::MetaHeader meta2{23};
-    pc.outputs().snapshot(Output{"TST", "MESSAGEABLE", 0, Lifetime::Timeframe, {meta1, meta2}}, a);
-    pc.outputs().snapshot(Output{"TST", "MSGBLEROOTSRLZ", 0, Lifetime::Timeframe},
+    pc.outputs().snapshot(Output{"TST", "MESSAGEABLE", 0, {meta1, meta2}}, a);
+    pc.outputs().snapshot(Output{"TST", "MSGBLEROOTSRLZ", 0},
                           o2::framework::ROOTSerialized<decltype(a)>(a));
     // class Polymorphic is not messageable, so the serialization type is deduced
     // from the fact that the type has a dictionary and can be ROOT-serialized.
-    pc.outputs().snapshot(Output{"TST", "ROOTNONTOBJECT", 0, Lifetime::Timeframe}, b);
+    pc.outputs().snapshot(Output{"TST", "ROOTNONTOBJECT", 0}, b);
     // vector of ROOT serializable class
-    pc.outputs().snapshot(Output{"TST", "ROOTVECTOR", 0, Lifetime::Timeframe}, c);
+    pc.outputs().snapshot(Output{"TST", "ROOTVECTOR", 0}, c);
     // deque of simple types
-    pc.outputs().snapshot(Output{"TST", "DEQUE", 0, Lifetime::Timeframe}, testDequePayload);
+    pc.outputs().snapshot(Output{"TST", "DEQUE", 0}, testDequePayload);
     // likewise, passed anonymously with char type and class name
     o2::framework::ROOTSerialized<char, const char> d(*((char*)&c), "vector<o2::test::Polymorphic>");
-    pc.outputs().snapshot(Output{"TST", "ROOTSERLZDVEC", 0, Lifetime::Timeframe}, d);
+    pc.outputs().snapshot(Output{"TST", "ROOTSERLZDVEC", 0}, d);
     // vector of ROOT serializable class wrapped with TClass info as hint
     auto* cl = TClass::GetClass(typeid(decltype(c)));
     ASSERT_ERROR(cl != nullptr);
     o2::framework::ROOTSerialized<char, TClass> e(*((char*)&c), cl);
-    pc.outputs().snapshot(Output{"TST", "ROOTSERLZDVEC2", 0, Lifetime::Timeframe}, e);
+    pc.outputs().snapshot(Output{"TST", "ROOTSERLZDVEC2", 0}, e);
     // test the 'make' methods
     pc.outputs().make<o2::test::TriviallyCopyable>(OutputRef{"makesingle", 0}) = a;
     auto& multi = pc.outputs().make<o2::test::TriviallyCopyable>(OutputRef{"makespan", 0}, 3);
@@ -111,7 +111,7 @@ DataProcessorSpec getSourceSpec()
     // test the adopt method
     auto freefct = [](void* data, void* hint) {}; // simply ignore the cleanup for the test
     static std::string teststring = "adoptchunk";
-    pc.outputs().adoptChunk(Output{"TST", "ADOPTCHUNK", 0, Lifetime::Timeframe}, teststring.data(), teststring.length(), freefct, nullptr);
+    pc.outputs().adoptChunk(Output{"TST", "ADOPTCHUNK", 0}, teststring.data(), teststring.length(), freefct, nullptr);
     // test resizable data chunk, initial size 0 and grow
     auto& growchunk = pc.outputs().newChunk(OutputRef{"growchunk", 0}, 0);
     growchunk.resize(sizeof(o2::test::TriviallyCopyable));
@@ -307,7 +307,7 @@ DataProcessorSpec getSinkSpec()
     ASSERT_ERROR((object12[0] == o2::test::TriviallyCopyable{42, 23, 0xdead}));
     ASSERT_ERROR((object12[1] == o2::test::TriviallyCopyable{10, 20, 0xacdc}));
     // forward the read-only span on a different route
-    pc.outputs().snapshot(Output{"TST", "MSGABLVECTORCPY", 0, Lifetime::Timeframe}, object12);
+    pc.outputs().snapshot(Output{"TST", "MSGABLVECTORCPY", 0}, object12);
 
     LOG(info) << "extracting TNamed object from input13";
     auto object13 = pc.inputs().get<TNamed*>("input13");
diff --git a/Framework/Core/test/test_ParallelPipeline.cxx b/Framework/Core/test/test_ParallelPipeline.cxx
index e13869873fca1..47106855a8f74 100644
--- a/Framework/Core/test/test_ParallelPipeline.cxx
+++ b/Framework/Core/test/test_ParallelPipeline.cxx
@@ -64,7 +64,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
                     << *input.spec << ": " << *((int*)input.payload);
          auto const* dataheader = DataRefUtils::getHeader<o2::header::DataHeader*>(input);
          //auto& data = ctx.outputs().make<int>(OutputRef{"output", dataheader->subSpecification});
-         auto& data = ctx.outputs().make<int>(Output{"TST", "PREPROC", dataheader->subSpecification, Lifetime::Timeframe});
+         auto& data = ctx.outputs().make<int>(Output{"TST", "PREPROC", dataheader->subSpecification});
          ASSERT_ERROR(ctx.inputs().get<int>(input.spec->binding.c_str()) == parallelContext.index1D());
          data = parallelContext.index1D();
        }
@@ -85,10 +85,10 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
          // TODO: there is a bug in the API for using OutputRef, returns an rvalue which can not be bound to
          // lvalue reference
          //auto& data = ctx.outputs().make<int>(OutputRef{"output", dataheader->subSpecification});
-         auto& data = ctx.outputs().make<int>(Output{"TST", "DATA", dataheader->subSpecification, Lifetime::Timeframe});
+         auto& data = ctx.outputs().make<int>(Output{"TST", "DATA", dataheader->subSpecification});
          data = ctx.inputs().get<int>(input.spec->binding.c_str());
          //auto& meta = ctx.outputs().make<int>(OutputRef{"metadt", dataheader->subSpecification});
-         auto& meta = ctx.outputs().make<int>(Output{"TST", "META", dataheader->subSpecification, Lifetime::Timeframe});
+         auto& meta = ctx.outputs().make<int>(Output{"TST", "META", dataheader->subSpecification});
          meta = dataheader->subSpecification;
        }
      }}},
@@ -145,7 +145,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
           if (multiplicities[pipeline] == 0) {
             continue;
           }
-          ctx.outputs().make<int>(Output{"TST", "TRIGGER", subspecs[index], Lifetime::Timeframe}) = pipeline;
+          ctx.outputs().make<int>(Output{"TST", "TRIGGER", subspecs[index]}) = pipeline;
           multiplicities[pipeline++]--;
           if (pipeline >= nPipelines) {
             pipeline = 0;
diff --git a/Framework/Core/test/test_ProcessorOptions.cxx b/Framework/Core/test/test_ProcessorOptions.cxx
index 8de90e0e23441..1751176eb639e 100644
--- a/Framework/Core/test/test_ProcessorOptions.cxx
+++ b/Framework/Core/test/test_ProcessorOptions.cxx
@@ -45,7 +45,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const&)
             }
             // there is nothing to do, simply stop the workflow but we have to send at least one message
             // to make sure that the callback of the consumer is called
-            ctx.outputs().make<int>(Output{"TST", "TEST", 0, Lifetime::Timeframe}) = 42;
+            ctx.outputs().make<int>(Output{"TST", "TEST", 0}) = 42;
             ctx.services().get<ControlService>().endOfStream();
             *isReady = true;
           };
diff --git a/Framework/Core/test/test_StaggeringWorkflow.cxx b/Framework/Core/test/test_StaggeringWorkflow.cxx
index 0e98c4324bc76..edc07d598465b 100644
--- a/Framework/Core/test/test_StaggeringWorkflow.cxx
+++ b/Framework/Core/test/test_StaggeringWorkflow.cxx
@@ -85,9 +85,9 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
       // since the snapshot copy is ready for sending it is scheduled but held back
       // because of the CompletionPolicy trigger matcher. This message will be
       // sent together with the second message.
-      outputs.snapshot(Output{"PROD", "CHANNEL", subspec, Lifetime::Timeframe}, subspec);
+      outputs.snapshot(Output{"PROD", "CHANNEL", subspec}, subspec);
       device.waitFor(100);
-      outputs.snapshot(Output{"PROD", "TRIGGER", subspec, Lifetime::Timeframe}, subspec);
+      outputs.snapshot(Output{"PROD", "TRIGGER", subspec}, subspec);
       device.waitFor(100);
     }
     control.endOfStream();
@@ -106,7 +106,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
       LOG(info) << "processing " << input.spec->binding << " " << data;
       // check if the channel binding starts with 'trigger'
       if (input.spec->binding.find("trigger") == 0) {
-        pc.outputs().make<MyDataType>(Output{"PROC", "CHANNEL", data, Lifetime::Timeframe}) = data;
+        pc.outputs().make<MyDataType>(Output{"PROC", "CHANNEL", data}) = data;
       }
       nActiveInputs++;
     }
diff --git a/Framework/Core/test/test_TableBuilder.cxx b/Framework/Core/test/test_TableBuilder.cxx
index a7b4739054d03..26a0f5a930b73 100644
--- a/Framework/Core/test/test_TableBuilder.cxx
+++ b/Framework/Core/test/test_TableBuilder.cxx
@@ -240,7 +240,7 @@ TEST_CASE("TestSoAIntegration")
 
 TEST_CASE("TestDataAllocatorReturnType")
 {
-  const Output output{"TST", "DUMMY", 0, Lifetime::Timeframe};
+  const Output output{"TST", "DUMMY", 0};
 }
 
 TEST_CASE("TestPodInjestion")
diff --git a/Framework/Utils/include/DPLUtils/RootTreeReader.h b/Framework/Utils/include/DPLUtils/RootTreeReader.h
index 42a8c9d01d0d0..109d79f9fad78 100644
--- a/Framework/Utils/include/DPLUtils/RootTreeReader.h
+++ b/Framework/Utils/include/DPLUtils/RootTreeReader.h
@@ -51,11 +51,11 @@ struct DefaultKey {
   enum Lifetime lifetime = Lifetime::Timeframe;
 
   DefaultKey(const Output& desc)
-    : origin(desc.origin), description(desc.description), subSpec(desc.subSpec), lifetime(desc.lifetime)
+    : origin(desc.origin), description(desc.description), subSpec(desc.subSpec)
   {
   }
 
-  operator Output() const { return Output{origin, description, subSpec, lifetime}; }
+  operator Output() const { return Output{origin, description, subSpec}; }
 };
 } // namespace rtr
 
@@ -302,7 +302,7 @@ class GenericRootTreeReader
       }
 
       auto snapshot = [&context, &stackcreator](const KeyType& key, const auto& object) {
-        context.outputs().snapshot(Output{key.origin, key.description, key.subSpec, key.lifetime, std::move(stackcreator())}, object);
+        context.outputs().snapshot(Output{key.origin, key.description, key.subSpec, std::move(stackcreator())}, object);
       };
 
       char* data = nullptr;
@@ -310,7 +310,7 @@ class GenericRootTreeReader
       mBranch->GetEntry(entry);
 
       // execute hook if it was registered; if this return true do not proceed further
-      if (mPublishHook != nullptr && (*mPublishHook).hook(mName, context, Output{mKey.origin, mKey.description, mKey.subSpec, mKey.lifetime, std::move(stackcreator())}, data)) {
+      if (mPublishHook != nullptr && (*mPublishHook).hook(mName, context, Output{mKey.origin, mKey.description, mKey.subSpec, std::move(stackcreator())}, data)) {
 
       }
       // try to figureout when we need to do something special
diff --git a/Framework/Utils/src/Utils.cxx b/Framework/Utils/src/Utils.cxx
index 3a52c46e074af..1dc0882c2f55d 100644
--- a/Framework/Utils/src/Utils.cxx
+++ b/Framework/Utils/src/Utils.cxx
@@ -28,7 +28,7 @@ namespace workflows
 Output getOutput(const o2f::OutputSpec outputSpec)
 {
   auto concrete = DataSpecUtils::asConcreteDataMatcher(outputSpec);
-  return Output{concrete.origin, concrete.description, concrete.subSpec, outputSpec.lifetime};
+  return Output{concrete.origin, concrete.description, concrete.subSpec};
 }
 
 // This method can convert a vector of OutputSpec into a vector of Output.
diff --git a/Framework/Utils/test/test_RootTreeReader.cxx b/Framework/Utils/test/test_RootTreeReader.cxx
index 0390a29f807b1..bf652fca6f97a 100644
--- a/Framework/Utils/test/test_RootTreeReader.cxx
+++ b/Framework/Utils/test/test_RootTreeReader.cxx
@@ -67,11 +67,10 @@ DataProcessorSpec getSourceSpec()
       testFile->Close();
     }
 
-    constexpr auto persistency = Lifetime::Transient;
     auto reader = std::make_shared<RootTreeReader>("testtree",       // tree name
                                                    fileName.c_str(), // input file name
-                                                   RootTreeReader::BranchDefinition<std::vector<o2::test::TriviallyCopyable>>{Output{"TST", "ARRAYOFMSGBL", 0, persistency}, "msgblarray"},
-                                                   Output{"TST", "ARRAYOFDATA", 0, persistency},
+                                                   RootTreeReader::BranchDefinition<std::vector<o2::test::TriviallyCopyable>>{Output{"TST", "ARRAYOFMSGBL", 0}, "msgblarray"},
+                                                   Output{"TST", "ARRAYOFDATA", 0},
                                                    "dataarray",
                                                    RootTreeReader::PublishingMode::Single);
 
diff --git a/Framework/Utils/test/test_RootTreeWriterWorkflow.cxx b/Framework/Utils/test/test_RootTreeWriterWorkflow.cxx
index 3213c1775c758..9a9b849d8d9d2 100644
--- a/Framework/Utils/test/test_RootTreeWriterWorkflow.cxx
+++ b/Framework/Utils/test/test_RootTreeWriterWorkflow.cxx
@@ -143,7 +143,7 @@ DataProcessorSpec getSourceSpec()
       o2::test::Polymorphic a(*counter);
       pc.outputs().snapshot(OutputRef{"output", 0}, a);
       pc.outputs().snapshot(OutputRef{"output", 1}, a);
-      int& metadata = pc.outputs().make<int>(Output{"TST", "METADATA", 0, Lifetime::Timeframe});
+      int& metadata = pc.outputs().make<int>(Output{"TST", "METADATA", 0});
       metadata = *counter;
       *counter = *counter + 1;
       if (*counter >= sTreeSize) {

From 97b5ea02a641dbe1cb0af59b6fd8180cf76d63ff Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 11 Dec 2023 08:45:47 +0100
Subject: [PATCH 161/726] Fix error message

---
 Detectors/Upgrades/ITS3/macros/test/CreateDictionariesITS3.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Detectors/Upgrades/ITS3/macros/test/CreateDictionariesITS3.C b/Detectors/Upgrades/ITS3/macros/test/CreateDictionariesITS3.C
index 1c53390946c01..b466ebf222878 100644
--- a/Detectors/Upgrades/ITS3/macros/test/CreateDictionariesITS3.C
+++ b/Detectors/Upgrades/ITS3/macros/test/CreateDictionariesITS3.C
@@ -168,7 +168,7 @@ void CreateDictionariesITS3(bool saveDeltas = false,
   }
   clusTree->GetEntry(0);
   if (clusTree->GetEntries() > 1 && !hitfile.empty()) {
-    LOGP(error, "Hits are provided but the cluster tree containes {} entries, looks like real data");
+    LOGP(error, "Hits are provided but the cluster tree containes {} entries, looks like real data", clusTree->GetEntries());
     return;
   }
 

From 3ec19bc77c2403b3014446ab9921bc648d18b2c1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 11 Dec 2023 23:05:54 +0100
Subject: [PATCH 162/726] DPL: add test to verify query builder with
 Lifetime::timeframe

---
 .../Core/test/test_DataDescriptorMatcher.cxx  | 25 ++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/test/test_DataDescriptorMatcher.cxx b/Framework/Core/test/test_DataDescriptorMatcher.cxx
index 8b74f7d9f6b6b..a76fddb925817 100644
--- a/Framework/Core/test/test_DataDescriptorMatcher.cxx
+++ b/Framework/Core/test/test_DataDescriptorMatcher.cxx
@@ -660,7 +660,7 @@ TEST_CASE("DataQuery")
 
   auto result5 = DataDescriptorQueryBuilder::parse("x:TST/A1/0?lifetime=sporadic&ccdb-path=GLO/Config/GRPECS&key3=value3");
   REQUIRE(result5.size() == 1);
-  result5[0].lifetime = Lifetime::Sporadic;
+  REQUIRE(result5[0].lifetime == Lifetime::Sporadic);
   REQUIRE(result5[0].metadata.size() == 3);
   REQUIRE(result5[0].metadata[0].name == "lifetime");
   REQUIRE(result5[0].metadata[0].defaultValue.get<std::string>() == "sporadic");
@@ -668,6 +668,29 @@ TEST_CASE("DataQuery")
   REQUIRE(result5[0].metadata[1].defaultValue.get<std::string>() == "GLO/Config/GRPECS");
   REQUIRE(result5[0].metadata[2].name == "key3");
   REQUIRE(result5[0].metadata[2].defaultValue.get<std::string>() == "value3");
+
+}
+
+TEST_CASE("DataQueryLifetime") {
+  auto result0 = DataDescriptorQueryBuilder::parse("x:TST/A1?lifetime=timeframe");
+  REQUIRE(result0.size() == 1);
+  REQUIRE(result0[0].lifetime == Lifetime::Timeframe);
+  auto result1 = DataDescriptorQueryBuilder::parse("x:TST/A1?lifetime=sporadic");
+  REQUIRE(result1.size() == 1);
+  REQUIRE(result1[0].lifetime == Lifetime::Sporadic);
+  auto result2 = DataDescriptorQueryBuilder::parse("x:TST/A1?lifetime=condition");
+  REQUIRE(result2.size() == 1);
+  REQUIRE(result2[0].lifetime == Lifetime::Condition);
+
+  auto result3 = DataDescriptorQueryBuilder::parse("x:TST/A1/1?lifetime=timeframe");
+  REQUIRE(result3.size() == 1);
+  REQUIRE(result3[0].lifetime == Lifetime::Timeframe);
+  auto result4 = DataDescriptorQueryBuilder::parse("x:TST/A1/2?lifetime=sporadic");
+  REQUIRE(result4.size() == 1);
+  REQUIRE(result4[0].lifetime == Lifetime::Sporadic);
+  auto result5 = DataDescriptorQueryBuilder::parse("x:TST/A1/3?lifetime=condition");
+  REQUIRE(result5.size() == 1);
+  REQUIRE(result5[0].lifetime == Lifetime::Condition);
 }
 
 // Make sure that 10 and 1 subspect are matched differently

From ced2ae84ecabcbfbf77a43da72aa0ede70ea2b98 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 11 Dec 2023 23:40:05 +0100
Subject: [PATCH 163/726] DPL: add test to verify query builder with
 Lifetime::timeframe

---
 .../Core/test/test_DataDescriptorMatcher.cxx  | 25 ++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/test/test_DataDescriptorMatcher.cxx b/Framework/Core/test/test_DataDescriptorMatcher.cxx
index 8b74f7d9f6b6b..e372ed06a8cb7 100644
--- a/Framework/Core/test/test_DataDescriptorMatcher.cxx
+++ b/Framework/Core/test/test_DataDescriptorMatcher.cxx
@@ -660,7 +660,7 @@ TEST_CASE("DataQuery")
 
   auto result5 = DataDescriptorQueryBuilder::parse("x:TST/A1/0?lifetime=sporadic&ccdb-path=GLO/Config/GRPECS&key3=value3");
   REQUIRE(result5.size() == 1);
-  result5[0].lifetime = Lifetime::Sporadic;
+  REQUIRE(result5[0].lifetime == Lifetime::Sporadic);
   REQUIRE(result5[0].metadata.size() == 3);
   REQUIRE(result5[0].metadata[0].name == "lifetime");
   REQUIRE(result5[0].metadata[0].defaultValue.get<std::string>() == "sporadic");
@@ -670,6 +670,29 @@ TEST_CASE("DataQuery")
   REQUIRE(result5[0].metadata[2].defaultValue.get<std::string>() == "value3");
 }
 
+TEST_CASE("DataQueryLifetime")
+{
+  auto result0 = DataDescriptorQueryBuilder::parse("x:TST/A1?lifetime=timeframe");
+  REQUIRE(result0.size() == 1);
+  REQUIRE(result0[0].lifetime == Lifetime::Timeframe);
+  auto result1 = DataDescriptorQueryBuilder::parse("x:TST/A1?lifetime=sporadic");
+  REQUIRE(result1.size() == 1);
+  REQUIRE(result1[0].lifetime == Lifetime::Sporadic);
+  auto result2 = DataDescriptorQueryBuilder::parse("x:TST/A1?lifetime=condition");
+  REQUIRE(result2.size() == 1);
+  REQUIRE(result2[0].lifetime == Lifetime::Condition);
+
+  auto result3 = DataDescriptorQueryBuilder::parse("x:TST/A1/1?lifetime=timeframe");
+  REQUIRE(result3.size() == 1);
+  REQUIRE(result3[0].lifetime == Lifetime::Timeframe);
+  auto result4 = DataDescriptorQueryBuilder::parse("x:TST/A1/2?lifetime=sporadic");
+  REQUIRE(result4.size() == 1);
+  REQUIRE(result4[0].lifetime == Lifetime::Sporadic);
+  auto result5 = DataDescriptorQueryBuilder::parse("x:TST/A1/3?lifetime=condition");
+  REQUIRE(result5.size() == 1);
+  REQUIRE(result5[0].lifetime == Lifetime::Condition);
+}
+
 // Make sure that 10 and 1 subspect are matched differently
 
 TEST_CASE("MatchSubspec")

From 7700c44dbca21d6e5ae89ddb3cc2ec84522c3ec0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 12 Dec 2023 13:06:16 +0100
Subject: [PATCH 164/726] Fix sign in AltroDecoder

A 1 bit signed bitfield is -1, not 1.
---
 Detectors/PHOS/reconstruction/src/AltroDecoder.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Detectors/PHOS/reconstruction/src/AltroDecoder.cxx b/Detectors/PHOS/reconstruction/src/AltroDecoder.cxx
index a1e205cbd2cbc..7202b27325c0e 100644
--- a/Detectors/PHOS/reconstruction/src/AltroDecoder.cxx
+++ b/Detectors/PHOS/reconstruction/src/AltroDecoder.cxx
@@ -284,7 +284,7 @@ void AltroDecoder::readTRUDigits(short absId, int payloadSize)
     currentsample += bunchlength + 2;
   }
   truDigitPack dp = {0};
-  dp.mHeader = 1;
+  dp.mHeader = -1;
   dp.mAmp = maxAmp;
   dp.mTime = timeBin;
   int chId = (absId - Mapping::NCHANNELS - 1) % 224;

From acfe390e6b8458ad29fc13da5332775fe7f64e37 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 12 Dec 2023 13:26:27 +0100
Subject: [PATCH 165/726] DPL: improve test to check for broken case in QC

---
 Framework/Core/test/test_DataDescriptorMatcher.cxx | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/test/test_DataDescriptorMatcher.cxx b/Framework/Core/test/test_DataDescriptorMatcher.cxx
index e372ed06a8cb7..9f83c2b5b5ac7 100644
--- a/Framework/Core/test/test_DataDescriptorMatcher.cxx
+++ b/Framework/Core/test/test_DataDescriptorMatcher.cxx
@@ -12,6 +12,7 @@
 #include "Framework/DataDescriptorMatcher.h"
 #include "Framework/DataDescriptorQueryBuilder.h"
 #include "Framework/InputSpec.h"
+#include "Framework/DataSpecUtils.h"
 #include "Headers/Stack.h"
 
 #include <catch_amalgamated.hpp>
@@ -688,8 +689,11 @@ TEST_CASE("DataQueryLifetime")
   auto result4 = DataDescriptorQueryBuilder::parse("x:TST/A1/2?lifetime=sporadic");
   REQUIRE(result4.size() == 1);
   REQUIRE(result4[0].lifetime == Lifetime::Sporadic);
-  auto result5 = DataDescriptorQueryBuilder::parse("x:TST/A1/3?lifetime=condition");
+  auto result5 = DataDescriptorQueryBuilder::parse("noiseMap:TRD/NOISEMAP?lifetime=condition&ccdb-path=TRD/Calib/NoiseMapMCM");
   REQUIRE(result5.size() == 1);
+  auto concrete = DataSpecUtils::asConcreteDataTypeMatcher(result5[0]);
+  REQUIRE(concrete.origin.as<std::string>() == "TRD");
+  REQUIRE(concrete.description.as<std::string>() == "NOISEMAP");
   REQUIRE(result5[0].lifetime == Lifetime::Condition);
 }
 

From 022b9cc2f60c778c47cb24d0489423c4af12ee8b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 12 Dec 2023 15:35:24 +0100
Subject: [PATCH 166/726] DPL: get rid of simplified CompletionPolicy

The full blown version has been there for a while now, and in any case
it's required if one wants to access the oldest possible timeframe.
---
 .../TPCSectorCompletionPolicy.h                 |  2 +-
 .../Core/include/Framework/CompletionPolicy.h   |  9 ++-------
 Framework/Core/src/CompletionPolicyHelpers.cxx  | 17 +++++++----------
 Framework/Core/src/DataRelayer.cxx              | 13 +++++--------
 Framework/Core/test/test_CompletionPolicy.cxx   |  9 +++++++--
 Framework/Core/test/test_StaggeringWorkflow.cxx |  2 +-
 6 files changed, 23 insertions(+), 29 deletions(-)

diff --git a/Detectors/TPC/workflow/readers/include/TPCReaderWorkflow/TPCSectorCompletionPolicy.h b/Detectors/TPC/workflow/readers/include/TPCReaderWorkflow/TPCSectorCompletionPolicy.h
index 33abded35624a..9f2a8b31dbdba 100644
--- a/Detectors/TPC/workflow/readers/include/TPCReaderWorkflow/TPCSectorCompletionPolicy.h
+++ b/Detectors/TPC/workflow/readers/include/TPCReaderWorkflow/TPCSectorCompletionPolicy.h
@@ -91,7 +91,7 @@ class TPCSectorCompletionPolicy
       return std::regex_match(device.name.begin(), device.name.end(), std::regex(expression.c_str()));
     };
 
-    auto callback = [bRequireAll = mRequireAll, inputMatchers = mInputMatchers, externalInputMatchers = mExternalInputMatchers, pTpcSectorMask = mTpcSectorMask, orderCheck = mOrderCheck](framework::InputSpan const& inputs) -> framework::CompletionPolicy::CompletionOp {
+    auto callback = [bRequireAll = mRequireAll, inputMatchers = mInputMatchers, externalInputMatchers = mExternalInputMatchers, pTpcSectorMask = mTpcSectorMask, orderCheck = mOrderCheck](framework::InputSpan const& inputs, auto const&, auto&) -> framework::CompletionPolicy::CompletionOp {
       unsigned long tpcSectorMask = pTpcSectorMask ? *pTpcSectorMask : 0xFFFFFFFFF;
       std::bitset<NSectors> validSectors = 0;
       bool haveMatchedInput = false;
diff --git a/Framework/Core/include/Framework/CompletionPolicy.h b/Framework/Core/include/Framework/CompletionPolicy.h
index eda45bd315471..55d3014166956 100644
--- a/Framework/Core/include/Framework/CompletionPolicy.h
+++ b/Framework/Core/include/Framework/CompletionPolicy.h
@@ -64,26 +64,21 @@ struct CompletionPolicy {
 
   using Matcher = std::function<bool(DeviceSpec const& device)>;
   using InputSetElement = DataRef;
-  using Callback = std::function<CompletionOp(InputSpan const&)>;
   using CallbackFull = std::function<CompletionOp(InputSpan const&, std::vector<InputSpec> const&, ServiceRegistryRef&)>;
   using CallbackConfigureRelayer = std::function<void(DataRelayer&)>;
 
   /// Constructor
   CompletionPolicy()
-    : name{}, matcher{}, callback{} {}
+    : name{}, matcher{}, callbackFull{} {}
   /// Constructor for emplace_back
-  CompletionPolicy(std::string _name, Matcher _matcher, Callback _callback, bool _balanceChannels = true)
-    : name(std::move(_name)), matcher(std::move(_matcher)), callback(std::move(_callback)), callbackFull{nullptr}, balanceChannels{_balanceChannels} {}
   CompletionPolicy(std::string _name, Matcher _matcher, CallbackFull _callback, bool _balanceChannels = true)
-    : name(std::move(_name)), matcher(std::move(_matcher)), callback(nullptr), callbackFull{std::move(_callback)}, balanceChannels{_balanceChannels} {}
+    : name(std::move(_name)), matcher(std::move(_matcher)), callbackFull{std::move(_callback)}, balanceChannels{_balanceChannels} {}
 
   /// Name of the policy itself.
   std::string name = "";
   /// Callback to be used to understand if the policy should apply
   /// to the given device.
   Matcher matcher = nullptr;
-  /// Actual policy which decides what to do with a partial InputRecord.
-  Callback callback = nullptr;
   /// Actual policy which decides what to do with a partial InputRecord, extended version
   CallbackFull callbackFull = nullptr;
   /// A callback which allows you to configure the behavior of the data relayer associated
diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx
index 34fc002428e12..5c38446fedaee 100644
--- a/Framework/Core/src/CompletionPolicyHelpers.cxx
+++ b/Framework/Core/src/CompletionPolicyHelpers.cxx
@@ -21,9 +21,6 @@
 #include <cassert>
 #include <regex>
 
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wpedantic"
-
 namespace o2::framework
 {
 
@@ -35,7 +32,7 @@ CompletionPolicy CompletionPolicyHelpers::defineByNameOrigin(std::string const&
 
   auto originReceived = std::make_shared<std::vector<uint64_t>>();
 
-  auto callback = [originReceived, origin, op](InputSpan const& inputRefs) -> CompletionPolicy::CompletionOp {
+  auto callback = [originReceived, origin, op](InputSpan const& inputRefs, std::vector<InputSpec> const&, ServiceRegistryRef&) -> CompletionPolicy::CompletionOp {
     // update list of the start times of inputs with origin @origin
     for (auto& ref : inputRefs) {
       if (ref.header != nullptr) {
@@ -77,7 +74,7 @@ CompletionPolicy CompletionPolicyHelpers::defineByName(std::string const& name,
   auto matcher = [name](DeviceSpec const& device) -> bool {
     return std::regex_match(device.name.begin(), device.name.end(), std::regex(name));
   };
-  auto callback = [op](InputSpan const&) -> CompletionPolicy::CompletionOp {
+  auto callback = [op](InputSpan const&, std::vector<InputSpec> const& specs, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     return op;
   };
   switch (op) {
@@ -108,7 +105,8 @@ CompletionPolicy CompletionPolicyHelpers::defineByName(std::string const& name,
 
 CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, CompletionPolicy::Matcher matcher)
 {
-  auto callback = [](InputSpan const& inputs) -> CompletionPolicy::CompletionOp {
+  auto callback = [](InputSpan const& inputs, std::vector<InputSpec> const& specs, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
+    assert(inputs.size() == spec.size());
     for (auto& input : inputs) {
       if (input.header == nullptr) {
         return CompletionPolicy::CompletionOp::Wait;
@@ -123,7 +121,7 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAllOrdered(const char* name
 {
   auto callbackFull = [](InputSpan const& inputs, std::vector<InputSpec> const&, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     auto& decongestionService = ref.get<DecongestionService>();
-    decongestionService.orderedCompletionPolicyActive = 1;
+    decongestionService.orderedCompletionPolicyActive = true;
     for (auto& input : inputs) {
       if (input.header == nullptr) {
         return CompletionPolicy::CompletionOp::Wait;
@@ -199,7 +197,7 @@ CompletionPolicy CompletionPolicyHelpers::consumeExistingWhenAny(const char* nam
 
 CompletionPolicy CompletionPolicyHelpers::consumeWhenAny(const char* name, CompletionPolicy::Matcher matcher)
 {
-  auto callback = [](InputSpan const& inputs) -> CompletionPolicy::CompletionOp {
+  auto callback = [](InputSpan const& inputs, std::vector<InputSpec> const&, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     for (auto& input : inputs) {
       if (input.header != nullptr) {
         return CompletionPolicy::CompletionOp::Consume;
@@ -289,7 +287,7 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAnyWithAllConditions(std::s
 
 CompletionPolicy CompletionPolicyHelpers::processWhenAny(const char* name, CompletionPolicy::Matcher matcher)
 {
-  auto callback = [](InputSpan const& inputs) -> CompletionPolicy::CompletionOp {
+  auto callback = [](InputSpan const& inputs, std::vector<InputSpec> const&, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     size_t present = 0;
     for (auto& input : inputs) {
       if (input.header != nullptr) {
@@ -307,4 +305,3 @@ CompletionPolicy CompletionPolicyHelpers::processWhenAny(const char* name, Compl
 }
 
 } // namespace o2::framework
-#pragma GCC diagnostic pop
diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index e9591a9cc76ea..b3d20e19852fa 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -675,6 +675,9 @@ void DataRelayer::getReadyToProcess(std::vector<DataRelayer::RecordAction>& comp
       notDirty++;
       continue;
     }
+    if (!mCompletionPolicy.callbackFull) {
+      throw runtime_error_f("Completion police %s has no callback set", mCompletionPolicy.name.c_str());
+    }
     auto partial = getPartialRecord(li);
     // TODO: get the data ref from message model
     auto getter = [&partial](size_t idx, size_t part) {
@@ -692,14 +695,8 @@ void DataRelayer::getReadyToProcess(std::vector<DataRelayer::RecordAction>& comp
       return partial[idx].size();
     };
     InputSpan span{getter, nPartsGetter, static_cast<size_t>(partial.size())};
-    CompletionPolicy::CompletionOp action;
-    if (mCompletionPolicy.callback) {
-      action = mCompletionPolicy.callback(span);
-    } else if (mCompletionPolicy.callbackFull) {
-      action = mCompletionPolicy.callbackFull(span, mInputs, mContext);
-    } else {
-      throw runtime_error_f("Completion police %s has no callback set", mCompletionPolicy.name.c_str());
-    }
+    CompletionPolicy::CompletionOp action = mCompletionPolicy.callbackFull(span, mInputs, mContext);
+
     auto& variables = mTimesliceIndex.getVariablesForSlot(slot);
     auto timeslice = std::get_if<uint64_t>(&variables.get(0));
     switch (action) {
diff --git a/Framework/Core/test/test_CompletionPolicy.cxx b/Framework/Core/test/test_CompletionPolicy.cxx
index 3bb2dc12caff0..059f20b352b3d 100644
--- a/Framework/Core/test/test_CompletionPolicy.cxx
+++ b/Framework/Core/test/test_CompletionPolicy.cxx
@@ -12,6 +12,7 @@
 #include <catch_amalgamated.hpp>
 #include "Framework/CompletionPolicy.h"
 #include "Framework/CompletionPolicyHelpers.h"
+#include "Framework/ServiceRegistry.h"
 #include "Headers/DataHeader.h"
 #include "Headers/NameHeader.h"
 #include "Framework/CompletionPolicy.h"
@@ -39,7 +40,9 @@ TEST_CASE("TestCompletionPolicy_callback")
     return true;
   };
 
-  auto callback = [&stack](InputSpan const& inputRefs) {
+  ServiceRegistry services;
+
+  auto callback = [&stack](InputSpan const& inputRefs, std::vector<InputSpec> const&, ServiceRegistryRef&) {
     for (auto const& ref : inputRefs) {
       auto const* header = CompletionPolicyHelpers::getHeader<o2::header::DataHeader>(ref);
       REQUIRE(header == reinterpret_cast<o2::header::DataHeader*>(stack.data()));
@@ -53,7 +56,9 @@ TEST_CASE("TestCompletionPolicy_callback")
     {"test", matcher, callback}};
   CompletionPolicy::InputSetElement ref{nullptr, reinterpret_cast<const char*>(stack.data()), nullptr};
   InputSpan const& inputs{[&ref](size_t) { return ref; }, 1};
+  std::vector<InputSpec> specs;
+  ServiceRegistryRef servicesRef{services};
   for (auto& policy : policies) {
-    policy.callback(inputs);
+    policy.callbackFull(inputs, specs, servicesRef);
   }
 }
diff --git a/Framework/Core/test/test_StaggeringWorkflow.cxx b/Framework/Core/test/test_StaggeringWorkflow.cxx
index edc07d598465b..590ce83cef467 100644
--- a/Framework/Core/test/test_StaggeringWorkflow.cxx
+++ b/Framework/Core/test/test_StaggeringWorkflow.cxx
@@ -53,7 +53,7 @@ void customize(std::vector<o2::framework::CompletionPolicy>& policies)
                         // search for spec names starting with "processor"
                         return spec.name.find("processor") == 0;
                       },
-                      [](auto const&) { return o2::framework::CompletionPolicy::CompletionOp::Consume; }});
+                      [](auto const&, auto const&, auto &) { return o2::framework::CompletionPolicy::CompletionOp::Consume; }});
 }
 
 #include "Framework/runDataProcessing.h"

From bdecacebf99acbac801cdbc35e45973a9f32b600 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 12 Dec 2023 15:35:24 +0100
Subject: [PATCH 167/726] DPL: wait as long as possible for Sporadic inputs

Right now if we have the standard consumeWhenAll policy and we have a sporadic
input, it will wait indefinitely until all the inputs arrive or it will drop
timeframes without that sporadic input.

This changes the behavior and waits only until the oldest possible timeframe
does not allow the Sporadic input to be there. At which point, it schedules the
processing in any case, under the assumption that a task declaring a sporadic
input knows what to do in case it's not there.
---
 .../Core/src/CompletionPolicyHelpers.cxx      | 25 ++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx
index 5c38446fedaee..769692c1d6637 100644
--- a/Framework/Core/src/CompletionPolicyHelpers.cxx
+++ b/Framework/Core/src/CompletionPolicyHelpers.cxx
@@ -15,6 +15,7 @@
 #include "Framework/DeviceSpec.h"
 #include "Framework/CompilerBuiltins.h"
 #include "Framework/Logger.h"
+#include "Framework/TimesliceIndex.h"
 #include "Framework/TimingInfo.h"
 #include "DecongestionService.h"
 
@@ -107,10 +108,32 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
 {
   auto callback = [](InputSpan const& inputs, std::vector<InputSpec> const& specs, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     assert(inputs.size() == spec.size());
+
+    size_t si = 0;
+    bool missingSporadic = false;
+    size_t currentTimeslice = -1;
     for (auto& input : inputs) {
-      if (input.header == nullptr) {
+      assert(si < specs.size());
+      auto& spec = specs[si++];
+      if (input.header == nullptr && spec.lifetime != Lifetime::Sporadic) {
         return CompletionPolicy::CompletionOp::Wait;
       }
+      if (input.header == nullptr && spec.lifetime == Lifetime::Sporadic) {
+        missingSporadic |= true;
+      }
+      if (input.header != nullptr) {
+        auto const* dph = framework::DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(input);
+        if (dph && !TimingInfo::timesliceIsTimer(dph->startTime)) {
+          currentTimeslice = dph->startTime;
+          break;
+        }
+      }
+    }
+    // If some sporadic inputs are missing, we wait for them util we are sure they will not come,
+    // i.e. until the oldest possible timeslice is beyond the timeslice of the input.
+    auto& timesliceIndex = ref.get<TimesliceIndex>();
+    if (missingSporadic && currentTimeslice >= timesliceIndex.getOldestPossibleInput().timeslice.value) {
+      return CompletionPolicy::CompletionOp::Wait;
     }
     return CompletionPolicy::CompletionOp::Consume;
   };

From 11347973a5173489725d27b02e9a32ad0299c55d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 12 Dec 2023 23:21:04 +0100
Subject: [PATCH 168/726] Fix missing argument in logging statement

---
 Detectors/ITSMFT/ITS/macros/test/CreateDictionaries.C | 2 +-
 Detectors/ITSMFT/MFT/macros/test/CreateDictionaries.C | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Detectors/ITSMFT/ITS/macros/test/CreateDictionaries.C b/Detectors/ITSMFT/ITS/macros/test/CreateDictionaries.C
index f0a2ef0de6398..3f906ebc04e17 100644
--- a/Detectors/ITSMFT/ITS/macros/test/CreateDictionaries.C
+++ b/Detectors/ITSMFT/ITS/macros/test/CreateDictionaries.C
@@ -140,7 +140,7 @@ void CreateDictionaries(bool saveDeltas = false,
   }
   clusTree->GetEntry(0);
   if (clusTree->GetEntries() > 1 && !hitfile.empty()) {
-    LOGP(error, "Hits are provided but the cluster tree containes {} entries, looks like real data");
+    LOGP(error, "Hits are provided but the cluster tree containes {} entries, looks like real data", clusTree->GetEntries());
     return;
   }
 
diff --git a/Detectors/ITSMFT/MFT/macros/test/CreateDictionaries.C b/Detectors/ITSMFT/MFT/macros/test/CreateDictionaries.C
index 3d6fb3a7ef59f..b610156f0cc4c 100644
--- a/Detectors/ITSMFT/MFT/macros/test/CreateDictionaries.C
+++ b/Detectors/ITSMFT/MFT/macros/test/CreateDictionaries.C
@@ -140,7 +140,7 @@ void CreateDictionaries(bool saveDeltas = false,
   }
   clusTree->GetEntry(0);
   if (clusTree->GetEntries() > 1 && !hitfile.empty()) {
-    LOGP(error, "Hits are provided but the cluster tree containes {} entries, looks like real data");
+    LOGP(error, "Hits are provided but the cluster tree containes {} entries, looks like real data", clusTree->GetEntries());
     return;
   }
 

From 1937f15524250f78b59e2ce1e66ede378648d298 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 13 Dec 2023 09:32:09 +0100
Subject: [PATCH 169/726] Add missing header

---
 .../ITSMFT/common/include/DataFormatsITSMFT/CompCluster.h        | 1 +
 1 file changed, 1 insertion(+)

diff --git a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/CompCluster.h b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/CompCluster.h
index 772a9ae12a81a..0cd5d0e11f19f 100644
--- a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/CompCluster.h
+++ b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/CompCluster.h
@@ -15,6 +15,7 @@
 #define ALICEO2_ITSMFT_COMPCLUSTER_H
 
 #include <Rtypes.h>
+#include <iosfwd>
 
 namespace o2
 {

From ac663ddbfab4ea63ca551243b48ed88d4eb58177 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 13 Dec 2023 11:26:50 +0100
Subject: [PATCH 170/726] Fix brown paperbag indexing

---
 .../GlobalTracking/src/MatchITSTPCQC.cxx      | 52 +++++++++----------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/Detectors/GlobalTracking/src/MatchITSTPCQC.cxx b/Detectors/GlobalTracking/src/MatchITSTPCQC.cxx
index be787d14196d9..ade9e4b4ec6ff 100644
--- a/Detectors/GlobalTracking/src/MatchITSTPCQC.cxx
+++ b/Detectors/GlobalTracking/src/MatchITSTPCQC.cxx
@@ -811,48 +811,48 @@ void MatchITSTPCQC::finalize()
   // first we use denominators and nominators to set the TEfficiency; later they are scaled
 
   // some checks
-  for (int i = 0; i < matchType::SIZE; ++i) {
-    for (int i = 0; i < mPtDen[i]->GetNbinsX(); ++i) {
+  for (int ti = 0; ti < matchType::SIZE; ++ti) {
+    for (int i = 0; i < mPtDen[ti]->GetNbinsX(); ++i) {
       if (mPtDen[i]->GetBinContent(i + 1) < mPtNum[i]->GetBinContent(i + 1)) {
-        LOG(error) << title[i] << ": bin " << i + 1 << " in [" << mPtNum[i]->GetBinLowEdge(i + 1) << " , " << mPtNum[i]->GetBinLowEdge(i + 1) + mPtNum[i]->GetBinWidth(i + 1) << "]: mPtDen[i] = " << mPtDen[i]->GetBinContent(i + 1) << ", mPtNum[i] = " << mPtNum[i]->GetBinContent(i + 1);
+        LOG(error) << title[ti] << ": bin " << i + 1 << " in [" << mPtNum[ti]->GetBinLowEdge(i + 1) << " , " << mPtNum[ti]->GetBinLowEdge(i + 1) + mPtNum[ti]->GetBinWidth(i + 1) << "]: mPtDen[i] = " << mPtDen[ti]->GetBinContent(i + 1) << ", mPtNum[i] = " << mPtNum[ti]->GetBinContent(i + 1);
       }
     }
-    for (int i = 0; i < mPtDen_noEta0[i]->GetNbinsX(); ++i) {
-      if (mPtDen_noEta0[i]->GetBinContent(i + 1) < mPtNum_noEta0[i]->GetBinContent(i + 1)) {
-        LOG(error) << title[i] << ": bin " << i + 1 << " in [" << mPtNum_noEta0[i]->GetBinLowEdge(i + 1) << " , " << mPtNum_noEta0[i]->GetBinLowEdge(i + 1) + mPtNum_noEta0[i]->GetBinWidth(i + 1) << "]: mPtDen_noEta0[i] = " << mPtDen_noEta0[i]->GetBinContent(i + 1) << ", mPtNum_noEta0[i] = " << mPtNum_noEta0[i]->GetBinContent(i + 1);
+    for (int i = 0; i < mPtDen_noEta0[ti]->GetNbinsX(); ++i) {
+      if (mPtDen_noEta0[ti]->GetBinContent(i + 1) < mPtNum_noEta0[ti]->GetBinContent(i + 1)) {
+        LOG(error) << title[ti] << ": bin " << i + 1 << " in [" << mPtNum_noEta0[ti]->GetBinLowEdge(i + 1) << " , " << mPtNum_noEta0[ti]->GetBinLowEdge(i + 1) + mPtNum_noEta0[ti]->GetBinWidth(i + 1) << "]: mPtDen_noEta0[i] = " << mPtDen_noEta0[ti]->GetBinContent(i + 1) << ", mPtNum_noEta0[i] = " << mPtNum_noEta0[ti]->GetBinContent(i + 1);
       }
     }
-    for (int i = 0; i < mPhiDen[i]->GetNbinsX(); ++i) {
-      if (mPhiDen[i]->GetBinContent(i + 1) < mPhiNum[i]->GetBinContent(i + 1)) {
-        LOG(error) << title[i] << ": bin " << i + 1 << " in [" << mPhiNum[i]->GetBinLowEdge(i + 1) << " , " << mPhiNum[i]->GetBinLowEdge(i + 1) + mPhiNum[i]->GetBinWidth(i + 1) << "]: mPhiDen[i] = " << mPhiDen[i]->GetBinContent(i + 1) << ", mPhiNum[i] = " << mPhiNum[i]->GetBinContent(i + 1);
+    for (int i = 0; i < mPhiDen[ti]->GetNbinsX(); ++i) {
+      if (mPhiDen[ti]->GetBinContent(i + 1) < mPhiNum[ti]->GetBinContent(i + 1)) {
+        LOG(error) << title[ti] << ": bin " << i + 1 << " in [" << mPhiNum[ti]->GetBinLowEdge(i + 1) << " , " << mPhiNum[ti]->GetBinLowEdge(i + 1) + mPhiNum[ti]->GetBinWidth(i + 1) << "]: mPhiDen[i] = " << mPhiDen[ti]->GetBinContent(i + 1) << ", mPhiNum[i] = " << mPhiNum[ti]->GetBinContent(i + 1);
       }
     }
-    for (int i = 0; i < mEtaDen[i]->GetNbinsX(); ++i) {
-      if (mEtaDen[i]->GetBinContent(i + 1) < mEtaNum[i]->GetBinContent(i + 1)) {
-        LOG(error) << title[i] << ": bin " << i + 1 << " in [" << mEtaNum[i]->GetBinLowEdge(i + 1) << " , " << mEtaNum[i]->GetBinLowEdge(i + 1) + mEtaNum[i]->GetBinWidth(i + 1) << "]: mEtaDen[i] = " << mEtaDen[i]->GetBinContent(i + 1) << ", mEtaNum[i] = " << mEtaNum[i]->GetBinContent(i + 1);
+    for (int i = 0; i < mEtaDen[ti]->GetNbinsX(); ++i) {
+      if (mEtaDen[ti]->GetBinContent(i + 1) < mEtaNum[ti]->GetBinContent(i + 1)) {
+        LOG(error) << title[ti] << ": bin " << i + 1 << " in [" << mEtaNum[ti]->GetBinLowEdge(i + 1) << " , " << mEtaNum[ti]->GetBinLowEdge(i + 1) + mEtaNum[ti]->GetBinWidth(i + 1) << "]: mEtaDen[i] = " << mEtaDen[ti]->GetBinContent(i + 1) << ", mEtaNum[i] = " << mEtaNum[ti]->GetBinContent(i + 1);
       }
     }
 
     // filling the efficiency
-    setEfficiency(mFractionITSTPCmatch[i], mPtNum[i], mPtDen[i]);
-    setEfficiency(mFractionITSTPCmatch_noEta0[i], mPtNum_noEta0[i], mPtDen_noEta0[i]);
-    setEfficiency(mFractionITSTPCmatchPhi[i], mPhiNum[i], mPhiDen[i]);
-    setEfficiency(mFractionITSTPCmatchEta[i], mEtaNum[i], mEtaDen[i]);
-    setEfficiency(mFractionITSTPCmatchPhiVsPt[i], mPhiVsPtNum[i], mPhiVsPtDen[i], true);
-    setEfficiency(mFractionITSTPCmatchEtaVsPt[i], mEtaVsPtNum[i], mEtaVsPtDen[i], true);
-    setEfficiency(mFractionITSTPCmatch1OverPt[i], m1OverPtNum[i], m1OverPtDen[i]);
+    setEfficiency(mFractionITSTPCmatch[ti], mPtNum[ti], mPtDen[ti]);
+    setEfficiency(mFractionITSTPCmatch_noEta0[ti], mPtNum_noEta0[ti], mPtDen_noEta0[ti]);
+    setEfficiency(mFractionITSTPCmatchPhi[ti], mPhiNum[ti], mPhiDen[ti]);
+    setEfficiency(mFractionITSTPCmatchEta[ti], mEtaNum[ti], mEtaDen[ti]);
+    setEfficiency(mFractionITSTPCmatchPhiVsPt[ti], mPhiVsPtNum[ti], mPhiVsPtDen[ti], true);
+    setEfficiency(mFractionITSTPCmatchEtaVsPt[ti], mEtaVsPtNum[ti], mEtaVsPtDen[ti], true);
+    setEfficiency(mFractionITSTPCmatch1OverPt[ti], m1OverPtNum[ti], m1OverPtDen[ti]);
     if (mUseTrkPID) { // Vs Tracking PID hypothesis
       for (int j = 0; j < o2::track::PID::NIDs; ++j) {
-        setEfficiency(mFractionITSTPCmatchPtVsTrkPID[i][j], mPtNumVsTrkPID[i][j], mPtDenVsTrkPID[i][j]);
-        setEfficiency(mFractionITSTPCmatchPhiVsTrkPID[i][j], mPhiNumVsTrkPID[i][j], mPhiDenVsTrkPID[i][j]);
-        setEfficiency(mFractionITSTPCmatchEtaVsTrkPID[i][j], mEtaNumVsTrkPID[i][j], mEtaDenVsTrkPID[i][j]);
+        setEfficiency(mFractionITSTPCmatchPtVsTrkPID[ti][j], mPtNumVsTrkPID[ti][j], mPtDenVsTrkPID[ti][j]);
+        setEfficiency(mFractionITSTPCmatchPhiVsTrkPID[ti][j], mPhiNumVsTrkPID[ti][j], mPhiDenVsTrkPID[ti][j]);
+        setEfficiency(mFractionITSTPCmatchEtaVsTrkPID[ti][j], mEtaNumVsTrkPID[ti][j], mEtaDenVsTrkPID[ti][j]);
       }
     }
     if (mUseMC) {
-      setEfficiency(mFractionITSTPCmatchPhysPrim[i], mPtPhysPrimNum[i], mPtPhysPrimDen[i]);
-      setEfficiency(mFractionITSTPCmatchPhiPhysPrim[i], mPhiPhysPrimNum[i], mPhiPhysPrimDen[i]);
-      setEfficiency(mFractionITSTPCmatchEtaPhysPrim[i], mEtaPhysPrimNum[i], mEtaPhysPrimDen[i]);
-      setEfficiency(mFractionITSTPCmatchPhysPrim1OverPt[i], m1OverPtPhysPrimNum[i], m1OverPtPhysPrimDen[i]);
+      setEfficiency(mFractionITSTPCmatchPhysPrim[ti], mPtPhysPrimNum[ti], mPtPhysPrimDen[ti]);
+      setEfficiency(mFractionITSTPCmatchPhiPhysPrim[ti], mPhiPhysPrimNum[ti], mPhiPhysPrimDen[ti]);
+      setEfficiency(mFractionITSTPCmatchEtaPhysPrim[ti], mEtaPhysPrimNum[ti], mEtaPhysPrimDen[ti]);
+      setEfficiency(mFractionITSTPCmatchPhysPrim1OverPt[ti], m1OverPtPhysPrimNum[ti], m1OverPtPhysPrimDen[ti]);
     }
   }
   /*

From e88ed40161f331a31e47cdd0afdc64162ec28587 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 13 Dec 2023 11:30:38 +0100
Subject: [PATCH 171/726] Fix brown paperbag indexing

---
 .../GlobalTracking/src/MatchITSTPCQC.cxx      | 54 +++++++++----------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/Detectors/GlobalTracking/src/MatchITSTPCQC.cxx b/Detectors/GlobalTracking/src/MatchITSTPCQC.cxx
index be787d14196d9..fea0a2b2af3a3 100644
--- a/Detectors/GlobalTracking/src/MatchITSTPCQC.cxx
+++ b/Detectors/GlobalTracking/src/MatchITSTPCQC.cxx
@@ -811,48 +811,48 @@ void MatchITSTPCQC::finalize()
   // first we use denominators and nominators to set the TEfficiency; later they are scaled
 
   // some checks
-  for (int i = 0; i < matchType::SIZE; ++i) {
-    for (int i = 0; i < mPtDen[i]->GetNbinsX(); ++i) {
-      if (mPtDen[i]->GetBinContent(i + 1) < mPtNum[i]->GetBinContent(i + 1)) {
-        LOG(error) << title[i] << ": bin " << i + 1 << " in [" << mPtNum[i]->GetBinLowEdge(i + 1) << " , " << mPtNum[i]->GetBinLowEdge(i + 1) + mPtNum[i]->GetBinWidth(i + 1) << "]: mPtDen[i] = " << mPtDen[i]->GetBinContent(i + 1) << ", mPtNum[i] = " << mPtNum[i]->GetBinContent(i + 1);
+  for (int ti = 0; ti < matchType::SIZE; ++ti) {
+    for (int i = 0; i < mPtDen[ti]->GetNbinsX(); ++i) {
+      if (mPtDen[ti]->GetBinContent(i + 1) < mPtNum[ti]->GetBinContent(i + 1)) {
+        LOG(error) << title[ti] << ": bin " << i + 1 << " in [" << mPtNum[ti]->GetBinLowEdge(i + 1) << " , " << mPtNum[ti]->GetBinLowEdge(i + 1) + mPtNum[ti]->GetBinWidth(i + 1) << "]: mPtDen[i] = " << mPtDen[ti]->GetBinContent(i + 1) << ", mPtNum[i] = " << mPtNum[ti]->GetBinContent(i + 1);
       }
     }
-    for (int i = 0; i < mPtDen_noEta0[i]->GetNbinsX(); ++i) {
-      if (mPtDen_noEta0[i]->GetBinContent(i + 1) < mPtNum_noEta0[i]->GetBinContent(i + 1)) {
-        LOG(error) << title[i] << ": bin " << i + 1 << " in [" << mPtNum_noEta0[i]->GetBinLowEdge(i + 1) << " , " << mPtNum_noEta0[i]->GetBinLowEdge(i + 1) + mPtNum_noEta0[i]->GetBinWidth(i + 1) << "]: mPtDen_noEta0[i] = " << mPtDen_noEta0[i]->GetBinContent(i + 1) << ", mPtNum_noEta0[i] = " << mPtNum_noEta0[i]->GetBinContent(i + 1);
+    for (int i = 0; i < mPtDen_noEta0[ti]->GetNbinsX(); ++i) {
+      if (mPtDen_noEta0[ti]->GetBinContent(i + 1) < mPtNum_noEta0[ti]->GetBinContent(i + 1)) {
+        LOG(error) << title[ti] << ": bin " << i + 1 << " in [" << mPtNum_noEta0[ti]->GetBinLowEdge(i + 1) << " , " << mPtNum_noEta0[ti]->GetBinLowEdge(i + 1) + mPtNum_noEta0[ti]->GetBinWidth(i + 1) << "]: mPtDen_noEta0[i] = " << mPtDen_noEta0[ti]->GetBinContent(i + 1) << ", mPtNum_noEta0[i] = " << mPtNum_noEta0[ti]->GetBinContent(i + 1);
       }
     }
-    for (int i = 0; i < mPhiDen[i]->GetNbinsX(); ++i) {
-      if (mPhiDen[i]->GetBinContent(i + 1) < mPhiNum[i]->GetBinContent(i + 1)) {
-        LOG(error) << title[i] << ": bin " << i + 1 << " in [" << mPhiNum[i]->GetBinLowEdge(i + 1) << " , " << mPhiNum[i]->GetBinLowEdge(i + 1) + mPhiNum[i]->GetBinWidth(i + 1) << "]: mPhiDen[i] = " << mPhiDen[i]->GetBinContent(i + 1) << ", mPhiNum[i] = " << mPhiNum[i]->GetBinContent(i + 1);
+    for (int i = 0; i < mPhiDen[ti]->GetNbinsX(); ++i) {
+      if (mPhiDen[ti]->GetBinContent(i + 1) < mPhiNum[ti]->GetBinContent(i + 1)) {
+        LOG(error) << title[ti] << ": bin " << i + 1 << " in [" << mPhiNum[ti]->GetBinLowEdge(i + 1) << " , " << mPhiNum[ti]->GetBinLowEdge(i + 1) + mPhiNum[ti]->GetBinWidth(i + 1) << "]: mPhiDen[i] = " << mPhiDen[ti]->GetBinContent(i + 1) << ", mPhiNum[i] = " << mPhiNum[ti]->GetBinContent(i + 1);
       }
     }
-    for (int i = 0; i < mEtaDen[i]->GetNbinsX(); ++i) {
-      if (mEtaDen[i]->GetBinContent(i + 1) < mEtaNum[i]->GetBinContent(i + 1)) {
-        LOG(error) << title[i] << ": bin " << i + 1 << " in [" << mEtaNum[i]->GetBinLowEdge(i + 1) << " , " << mEtaNum[i]->GetBinLowEdge(i + 1) + mEtaNum[i]->GetBinWidth(i + 1) << "]: mEtaDen[i] = " << mEtaDen[i]->GetBinContent(i + 1) << ", mEtaNum[i] = " << mEtaNum[i]->GetBinContent(i + 1);
+    for (int i = 0; i < mEtaDen[ti]->GetNbinsX(); ++i) {
+      if (mEtaDen[ti]->GetBinContent(i + 1) < mEtaNum[ti]->GetBinContent(i + 1)) {
+        LOG(error) << title[ti] << ": bin " << i + 1 << " in [" << mEtaNum[ti]->GetBinLowEdge(i + 1) << " , " << mEtaNum[ti]->GetBinLowEdge(i + 1) + mEtaNum[ti]->GetBinWidth(i + 1) << "]: mEtaDen[i] = " << mEtaDen[ti]->GetBinContent(i + 1) << ", mEtaNum[i] = " << mEtaNum[ti]->GetBinContent(i + 1);
       }
     }
 
     // filling the efficiency
-    setEfficiency(mFractionITSTPCmatch[i], mPtNum[i], mPtDen[i]);
-    setEfficiency(mFractionITSTPCmatch_noEta0[i], mPtNum_noEta0[i], mPtDen_noEta0[i]);
-    setEfficiency(mFractionITSTPCmatchPhi[i], mPhiNum[i], mPhiDen[i]);
-    setEfficiency(mFractionITSTPCmatchEta[i], mEtaNum[i], mEtaDen[i]);
-    setEfficiency(mFractionITSTPCmatchPhiVsPt[i], mPhiVsPtNum[i], mPhiVsPtDen[i], true);
-    setEfficiency(mFractionITSTPCmatchEtaVsPt[i], mEtaVsPtNum[i], mEtaVsPtDen[i], true);
-    setEfficiency(mFractionITSTPCmatch1OverPt[i], m1OverPtNum[i], m1OverPtDen[i]);
+    setEfficiency(mFractionITSTPCmatch[ti], mPtNum[ti], mPtDen[ti]);
+    setEfficiency(mFractionITSTPCmatch_noEta0[ti], mPtNum_noEta0[ti], mPtDen_noEta0[ti]);
+    setEfficiency(mFractionITSTPCmatchPhi[ti], mPhiNum[ti], mPhiDen[ti]);
+    setEfficiency(mFractionITSTPCmatchEta[ti], mEtaNum[ti], mEtaDen[ti]);
+    setEfficiency(mFractionITSTPCmatchPhiVsPt[ti], mPhiVsPtNum[ti], mPhiVsPtDen[ti], true);
+    setEfficiency(mFractionITSTPCmatchEtaVsPt[ti], mEtaVsPtNum[ti], mEtaVsPtDen[ti], true);
+    setEfficiency(mFractionITSTPCmatch1OverPt[ti], m1OverPtNum[ti], m1OverPtDen[ti]);
     if (mUseTrkPID) { // Vs Tracking PID hypothesis
       for (int j = 0; j < o2::track::PID::NIDs; ++j) {
-        setEfficiency(mFractionITSTPCmatchPtVsTrkPID[i][j], mPtNumVsTrkPID[i][j], mPtDenVsTrkPID[i][j]);
-        setEfficiency(mFractionITSTPCmatchPhiVsTrkPID[i][j], mPhiNumVsTrkPID[i][j], mPhiDenVsTrkPID[i][j]);
-        setEfficiency(mFractionITSTPCmatchEtaVsTrkPID[i][j], mEtaNumVsTrkPID[i][j], mEtaDenVsTrkPID[i][j]);
+        setEfficiency(mFractionITSTPCmatchPtVsTrkPID[ti][j], mPtNumVsTrkPID[ti][j], mPtDenVsTrkPID[ti][j]);
+        setEfficiency(mFractionITSTPCmatchPhiVsTrkPID[ti][j], mPhiNumVsTrkPID[ti][j], mPhiDenVsTrkPID[ti][j]);
+        setEfficiency(mFractionITSTPCmatchEtaVsTrkPID[ti][j], mEtaNumVsTrkPID[ti][j], mEtaDenVsTrkPID[ti][j]);
       }
     }
     if (mUseMC) {
-      setEfficiency(mFractionITSTPCmatchPhysPrim[i], mPtPhysPrimNum[i], mPtPhysPrimDen[i]);
-      setEfficiency(mFractionITSTPCmatchPhiPhysPrim[i], mPhiPhysPrimNum[i], mPhiPhysPrimDen[i]);
-      setEfficiency(mFractionITSTPCmatchEtaPhysPrim[i], mEtaPhysPrimNum[i], mEtaPhysPrimDen[i]);
-      setEfficiency(mFractionITSTPCmatchPhysPrim1OverPt[i], m1OverPtPhysPrimNum[i], m1OverPtPhysPrimDen[i]);
+      setEfficiency(mFractionITSTPCmatchPhysPrim[ti], mPtPhysPrimNum[ti], mPtPhysPrimDen[ti]);
+      setEfficiency(mFractionITSTPCmatchPhiPhysPrim[ti], mPhiPhysPrimNum[ti], mPhiPhysPrimDen[ti]);
+      setEfficiency(mFractionITSTPCmatchEtaPhysPrim[ti], mEtaPhysPrimNum[ti], mEtaPhysPrimDen[ti]);
+      setEfficiency(mFractionITSTPCmatchPhysPrim1OverPt[ti], m1OverPtPhysPrimNum[ti], m1OverPtPhysPrimDen[ti]);
     }
   }
   /*

From 0953e70583a59468ad741b08a174338f5badbf9b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 13 Dec 2023 21:45:27 +0100
Subject: [PATCH 172/726] DPL: get rid of simplified CompletionPolicy

The full blown version has been there for a while now, and in any case
it's required if one wants to access the oldest possible timeframe.
---
 .../TPCSectorCompletionPolicy.h                 |  2 +-
 .../Core/include/Framework/CompletionPolicy.h   |  9 ++-------
 Framework/Core/src/CompletionPolicyHelpers.cxx  | 17 +++++++----------
 Framework/Core/src/DataRelayer.cxx              | 13 +++++--------
 Framework/Core/test/test_CompletionPolicy.cxx   |  9 +++++++--
 Framework/Core/test/test_StaggeringWorkflow.cxx |  2 +-
 6 files changed, 23 insertions(+), 29 deletions(-)

diff --git a/Detectors/TPC/workflow/readers/include/TPCReaderWorkflow/TPCSectorCompletionPolicy.h b/Detectors/TPC/workflow/readers/include/TPCReaderWorkflow/TPCSectorCompletionPolicy.h
index 33abded35624a..9f2a8b31dbdba 100644
--- a/Detectors/TPC/workflow/readers/include/TPCReaderWorkflow/TPCSectorCompletionPolicy.h
+++ b/Detectors/TPC/workflow/readers/include/TPCReaderWorkflow/TPCSectorCompletionPolicy.h
@@ -91,7 +91,7 @@ class TPCSectorCompletionPolicy
       return std::regex_match(device.name.begin(), device.name.end(), std::regex(expression.c_str()));
     };
 
-    auto callback = [bRequireAll = mRequireAll, inputMatchers = mInputMatchers, externalInputMatchers = mExternalInputMatchers, pTpcSectorMask = mTpcSectorMask, orderCheck = mOrderCheck](framework::InputSpan const& inputs) -> framework::CompletionPolicy::CompletionOp {
+    auto callback = [bRequireAll = mRequireAll, inputMatchers = mInputMatchers, externalInputMatchers = mExternalInputMatchers, pTpcSectorMask = mTpcSectorMask, orderCheck = mOrderCheck](framework::InputSpan const& inputs, auto const&, auto&) -> framework::CompletionPolicy::CompletionOp {
       unsigned long tpcSectorMask = pTpcSectorMask ? *pTpcSectorMask : 0xFFFFFFFFF;
       std::bitset<NSectors> validSectors = 0;
       bool haveMatchedInput = false;
diff --git a/Framework/Core/include/Framework/CompletionPolicy.h b/Framework/Core/include/Framework/CompletionPolicy.h
index eda45bd315471..55d3014166956 100644
--- a/Framework/Core/include/Framework/CompletionPolicy.h
+++ b/Framework/Core/include/Framework/CompletionPolicy.h
@@ -64,26 +64,21 @@ struct CompletionPolicy {
 
   using Matcher = std::function<bool(DeviceSpec const& device)>;
   using InputSetElement = DataRef;
-  using Callback = std::function<CompletionOp(InputSpan const&)>;
   using CallbackFull = std::function<CompletionOp(InputSpan const&, std::vector<InputSpec> const&, ServiceRegistryRef&)>;
   using CallbackConfigureRelayer = std::function<void(DataRelayer&)>;
 
   /// Constructor
   CompletionPolicy()
-    : name{}, matcher{}, callback{} {}
+    : name{}, matcher{}, callbackFull{} {}
   /// Constructor for emplace_back
-  CompletionPolicy(std::string _name, Matcher _matcher, Callback _callback, bool _balanceChannels = true)
-    : name(std::move(_name)), matcher(std::move(_matcher)), callback(std::move(_callback)), callbackFull{nullptr}, balanceChannels{_balanceChannels} {}
   CompletionPolicy(std::string _name, Matcher _matcher, CallbackFull _callback, bool _balanceChannels = true)
-    : name(std::move(_name)), matcher(std::move(_matcher)), callback(nullptr), callbackFull{std::move(_callback)}, balanceChannels{_balanceChannels} {}
+    : name(std::move(_name)), matcher(std::move(_matcher)), callbackFull{std::move(_callback)}, balanceChannels{_balanceChannels} {}
 
   /// Name of the policy itself.
   std::string name = "";
   /// Callback to be used to understand if the policy should apply
   /// to the given device.
   Matcher matcher = nullptr;
-  /// Actual policy which decides what to do with a partial InputRecord.
-  Callback callback = nullptr;
   /// Actual policy which decides what to do with a partial InputRecord, extended version
   CallbackFull callbackFull = nullptr;
   /// A callback which allows you to configure the behavior of the data relayer associated
diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx
index 34fc002428e12..b3e0621bf984e 100644
--- a/Framework/Core/src/CompletionPolicyHelpers.cxx
+++ b/Framework/Core/src/CompletionPolicyHelpers.cxx
@@ -21,9 +21,6 @@
 #include <cassert>
 #include <regex>
 
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wpedantic"
-
 namespace o2::framework
 {
 
@@ -35,7 +32,7 @@ CompletionPolicy CompletionPolicyHelpers::defineByNameOrigin(std::string const&
 
   auto originReceived = std::make_shared<std::vector<uint64_t>>();
 
-  auto callback = [originReceived, origin, op](InputSpan const& inputRefs) -> CompletionPolicy::CompletionOp {
+  auto callback = [originReceived, origin, op](InputSpan const& inputRefs, std::vector<InputSpec> const&, ServiceRegistryRef&) -> CompletionPolicy::CompletionOp {
     // update list of the start times of inputs with origin @origin
     for (auto& ref : inputRefs) {
       if (ref.header != nullptr) {
@@ -77,7 +74,7 @@ CompletionPolicy CompletionPolicyHelpers::defineByName(std::string const& name,
   auto matcher = [name](DeviceSpec const& device) -> bool {
     return std::regex_match(device.name.begin(), device.name.end(), std::regex(name));
   };
-  auto callback = [op](InputSpan const&) -> CompletionPolicy::CompletionOp {
+  auto callback = [op](InputSpan const&, std::vector<InputSpec> const& specs, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     return op;
   };
   switch (op) {
@@ -108,7 +105,8 @@ CompletionPolicy CompletionPolicyHelpers::defineByName(std::string const& name,
 
 CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, CompletionPolicy::Matcher matcher)
 {
-  auto callback = [](InputSpan const& inputs) -> CompletionPolicy::CompletionOp {
+  auto callback = [](InputSpan const& inputs, std::vector<InputSpec> const& specs, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
+    assert(inputs.size() == specs.size());
     for (auto& input : inputs) {
       if (input.header == nullptr) {
         return CompletionPolicy::CompletionOp::Wait;
@@ -123,7 +121,7 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAllOrdered(const char* name
 {
   auto callbackFull = [](InputSpan const& inputs, std::vector<InputSpec> const&, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     auto& decongestionService = ref.get<DecongestionService>();
-    decongestionService.orderedCompletionPolicyActive = 1;
+    decongestionService.orderedCompletionPolicyActive = true;
     for (auto& input : inputs) {
       if (input.header == nullptr) {
         return CompletionPolicy::CompletionOp::Wait;
@@ -199,7 +197,7 @@ CompletionPolicy CompletionPolicyHelpers::consumeExistingWhenAny(const char* nam
 
 CompletionPolicy CompletionPolicyHelpers::consumeWhenAny(const char* name, CompletionPolicy::Matcher matcher)
 {
-  auto callback = [](InputSpan const& inputs) -> CompletionPolicy::CompletionOp {
+  auto callback = [](InputSpan const& inputs, std::vector<InputSpec> const&, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     for (auto& input : inputs) {
       if (input.header != nullptr) {
         return CompletionPolicy::CompletionOp::Consume;
@@ -289,7 +287,7 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAnyWithAllConditions(std::s
 
 CompletionPolicy CompletionPolicyHelpers::processWhenAny(const char* name, CompletionPolicy::Matcher matcher)
 {
-  auto callback = [](InputSpan const& inputs) -> CompletionPolicy::CompletionOp {
+  auto callback = [](InputSpan const& inputs, std::vector<InputSpec> const&, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     size_t present = 0;
     for (auto& input : inputs) {
       if (input.header != nullptr) {
@@ -307,4 +305,3 @@ CompletionPolicy CompletionPolicyHelpers::processWhenAny(const char* name, Compl
 }
 
 } // namespace o2::framework
-#pragma GCC diagnostic pop
diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index e9591a9cc76ea..b3d20e19852fa 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -675,6 +675,9 @@ void DataRelayer::getReadyToProcess(std::vector<DataRelayer::RecordAction>& comp
       notDirty++;
       continue;
     }
+    if (!mCompletionPolicy.callbackFull) {
+      throw runtime_error_f("Completion police %s has no callback set", mCompletionPolicy.name.c_str());
+    }
     auto partial = getPartialRecord(li);
     // TODO: get the data ref from message model
     auto getter = [&partial](size_t idx, size_t part) {
@@ -692,14 +695,8 @@ void DataRelayer::getReadyToProcess(std::vector<DataRelayer::RecordAction>& comp
       return partial[idx].size();
     };
     InputSpan span{getter, nPartsGetter, static_cast<size_t>(partial.size())};
-    CompletionPolicy::CompletionOp action;
-    if (mCompletionPolicy.callback) {
-      action = mCompletionPolicy.callback(span);
-    } else if (mCompletionPolicy.callbackFull) {
-      action = mCompletionPolicy.callbackFull(span, mInputs, mContext);
-    } else {
-      throw runtime_error_f("Completion police %s has no callback set", mCompletionPolicy.name.c_str());
-    }
+    CompletionPolicy::CompletionOp action = mCompletionPolicy.callbackFull(span, mInputs, mContext);
+
     auto& variables = mTimesliceIndex.getVariablesForSlot(slot);
     auto timeslice = std::get_if<uint64_t>(&variables.get(0));
     switch (action) {
diff --git a/Framework/Core/test/test_CompletionPolicy.cxx b/Framework/Core/test/test_CompletionPolicy.cxx
index 3bb2dc12caff0..059f20b352b3d 100644
--- a/Framework/Core/test/test_CompletionPolicy.cxx
+++ b/Framework/Core/test/test_CompletionPolicy.cxx
@@ -12,6 +12,7 @@
 #include <catch_amalgamated.hpp>
 #include "Framework/CompletionPolicy.h"
 #include "Framework/CompletionPolicyHelpers.h"
+#include "Framework/ServiceRegistry.h"
 #include "Headers/DataHeader.h"
 #include "Headers/NameHeader.h"
 #include "Framework/CompletionPolicy.h"
@@ -39,7 +40,9 @@ TEST_CASE("TestCompletionPolicy_callback")
     return true;
   };
 
-  auto callback = [&stack](InputSpan const& inputRefs) {
+  ServiceRegistry services;
+
+  auto callback = [&stack](InputSpan const& inputRefs, std::vector<InputSpec> const&, ServiceRegistryRef&) {
     for (auto const& ref : inputRefs) {
       auto const* header = CompletionPolicyHelpers::getHeader<o2::header::DataHeader>(ref);
       REQUIRE(header == reinterpret_cast<o2::header::DataHeader*>(stack.data()));
@@ -53,7 +56,9 @@ TEST_CASE("TestCompletionPolicy_callback")
     {"test", matcher, callback}};
   CompletionPolicy::InputSetElement ref{nullptr, reinterpret_cast<const char*>(stack.data()), nullptr};
   InputSpan const& inputs{[&ref](size_t) { return ref; }, 1};
+  std::vector<InputSpec> specs;
+  ServiceRegistryRef servicesRef{services};
   for (auto& policy : policies) {
-    policy.callback(inputs);
+    policy.callbackFull(inputs, specs, servicesRef);
   }
 }
diff --git a/Framework/Core/test/test_StaggeringWorkflow.cxx b/Framework/Core/test/test_StaggeringWorkflow.cxx
index edc07d598465b..590ce83cef467 100644
--- a/Framework/Core/test/test_StaggeringWorkflow.cxx
+++ b/Framework/Core/test/test_StaggeringWorkflow.cxx
@@ -53,7 +53,7 @@ void customize(std::vector<o2::framework::CompletionPolicy>& policies)
                         // search for spec names starting with "processor"
                         return spec.name.find("processor") == 0;
                       },
-                      [](auto const&) { return o2::framework::CompletionPolicy::CompletionOp::Consume; }});
+                      [](auto const&, auto const&, auto &) { return o2::framework::CompletionPolicy::CompletionOp::Consume; }});
 }
 
 #include "Framework/runDataProcessing.h"

From 8642aeeac6d26535a84e2dd101a0bd148e5f417c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 13 Dec 2023 21:45:27 +0100
Subject: [PATCH 173/726] DPL: wait as long as possible for Sporadic inputs

Right now if we have the standard consumeWhenAll policy and we have a sporadic
input, it will wait indefinitely until all the inputs arrive or it will drop
timeframes without that sporadic input.

This changes the behavior and waits only until the oldest possible timeframe
does not allow the Sporadic input to be there. At which point, it schedules the
processing in any case, under the assumption that a task declaring a sporadic
input knows what to do in case it's not there.
---
 .../Core/src/CompletionPolicyHelpers.cxx      | 25 ++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx
index b3e0621bf984e..ed18d89eea013 100644
--- a/Framework/Core/src/CompletionPolicyHelpers.cxx
+++ b/Framework/Core/src/CompletionPolicyHelpers.cxx
@@ -15,6 +15,7 @@
 #include "Framework/DeviceSpec.h"
 #include "Framework/CompilerBuiltins.h"
 #include "Framework/Logger.h"
+#include "Framework/TimesliceIndex.h"
 #include "Framework/TimingInfo.h"
 #include "DecongestionService.h"
 
@@ -107,10 +108,32 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
 {
   auto callback = [](InputSpan const& inputs, std::vector<InputSpec> const& specs, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     assert(inputs.size() == specs.size());
+
+    size_t si = 0;
+    bool missingSporadic = false;
+    size_t currentTimeslice = -1;
     for (auto& input : inputs) {
-      if (input.header == nullptr) {
+      assert(si < specs.size());
+      auto& spec = specs[si++];
+      if (input.header == nullptr && spec.lifetime != Lifetime::Sporadic) {
         return CompletionPolicy::CompletionOp::Wait;
       }
+      if (input.header == nullptr && spec.lifetime == Lifetime::Sporadic) {
+        missingSporadic |= true;
+      }
+      if (input.header != nullptr) {
+        auto const* dph = framework::DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(input);
+        if (dph && !TimingInfo::timesliceIsTimer(dph->startTime)) {
+          currentTimeslice = dph->startTime;
+          break;
+        }
+      }
+    }
+    // If some sporadic inputs are missing, we wait for them util we are sure they will not come,
+    // i.e. until the oldest possible timeslice is beyond the timeslice of the input.
+    auto& timesliceIndex = ref.get<TimesliceIndex>();
+    if (missingSporadic && currentTimeslice >= timesliceIndex.getOldestPossibleInput().timeslice.value) {
+      return CompletionPolicy::CompletionOp::Wait;
     }
     return CompletionPolicy::CompletionOp::Consume;
   };

From a1e0e758fa1e163c7301f981f55c103eb6393939 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sun, 31 Dec 2023 10:44:50 +0100
Subject: [PATCH 174/726] DPL: ignore SIGUSR1 until we are ready to handle it

This should fix a race condition when the parent decides to quit too
quickly.
---
 Framework/Core/src/runDataProcessing.cxx | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 66679d83dc4ec..e9d1f3d4ee314 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -646,6 +646,12 @@ void spawnDevice(uv_loop_t* loop,
   if (id == 0) {
     // We allow being debugged and do not terminate on SIGTRAP
     signal(SIGTRAP, SIG_IGN);
+    // We immediately ignore SIGUSR1 and SIGUSR2 so that we do not
+    // get killed by the parent trying to force stepping children.
+    // We will re-enable them later on, when it is actually safe to
+    // do so.
+    signal(SIGUSR1, SIG_IGN);
+    signal(SIGUSR2, SIG_IGN);
 
     // This is the child.
     // For stdout / stderr, we close the read part of the pipe, the

From 5ee4098581dbac30c31d9a76e9dd38861684053f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 11:24:22 +0100
Subject: [PATCH 175/726] DPL: get rid of warning

---
 Framework/Core/src/FairMQDeviceProxy.cxx | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Framework/Core/src/FairMQDeviceProxy.cxx b/Framework/Core/src/FairMQDeviceProxy.cxx
index a83057e9578be..06a404360d741 100644
--- a/Framework/Core/src/FairMQDeviceProxy.cxx
+++ b/Framework/Core/src/FairMQDeviceProxy.cxx
@@ -320,10 +320,7 @@ void FairMQDeviceProxy::bind(std::vector<OutputRoute> const& outputs, std::vecto
       mInputRoutes.emplace_back(RouteState{channelIndex, false});
       ri++;
     }
-    for (auto& route : mInputRoutes) {
-      assert(route.channel.value != -1);
-      assert(route.channel.value < mInputChannels.size());
-    }
+    assert(std::all_of(mInputRoutes.begin(), mInputRoutes.end(), [s=mInputChannels.size()](InputRoute const&route) {return route.channel.value != -1 && route.channel.value < s;}));
     LOGP(detail, "Total input channels found {}, total routes {}", mInputChannels.size(), mInputRoutes.size());
     assert(mInputRoutes.size() == inputs.size());
   }

From 2f1c99314b819dc2a288f89a2e3d9539ac83b3c8 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:29:24 +0100
Subject: [PATCH 176/726] Fix CheckClusterSize.C to work with C++20

---
 Detectors/Upgrades/ITS3/macros/test/CheckClusterSize.C | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Detectors/Upgrades/ITS3/macros/test/CheckClusterSize.C b/Detectors/Upgrades/ITS3/macros/test/CheckClusterSize.C
index cd69cb9c2cf3c..c433ab855b70d 100755
--- a/Detectors/Upgrades/ITS3/macros/test/CheckClusterSize.C
+++ b/Detectors/Upgrades/ITS3/macros/test/CheckClusterSize.C
@@ -43,8 +43,9 @@
 #include "SimulationDataFormat/MCCompLabel.h"
 #include "SimulationDataFormat/MCEventHeader.h"
 #include "SimulationDataFormat/MCTrack.h"
-#include "SimulationDataFormat/MCTruthContainer.h"
 #endif
+#define ENABLE_UPGRADES
+#include "SimulationDataFormat/MCTruthContainer.h"
 
 static constexpr int nLayers = 4; // 3 Layers + 1 combined outer layer
 

From 865a73033cdde0a20b7a61fda20fd98122d20d2f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:29:41 +0100
Subject: [PATCH 177/726] Get readZDCDigits.C to work with C++20

---
 Detectors/ZDC/macro/readZDCDigits.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Detectors/ZDC/macro/readZDCDigits.C b/Detectors/ZDC/macro/readZDCDigits.C
index b5baa77d0ef99..d6987ecc98994 100644
--- a/Detectors/ZDC/macro/readZDCDigits.C
+++ b/Detectors/ZDC/macro/readZDCDigits.C
@@ -5,13 +5,13 @@
 #include <TStopwatch.h>
 #include <memory>
 #include <iostream>
-#include "DataFormatsZDC/BCData.h"
 #include "DataFormatsZDC/ChannelData.h"
 #include "SimulationDataFormat/MCTruthContainer.h"
 #include "SimulationDataFormat/MCCompLabel.h"
 
 #endif
 
+#include "DataFormatsZDC/BCData.h"
 #include "DataFormatsZDC/MCLabel.h"
 #include "Framework/Logger.h"
 

From cc70cb60b0db5998c114e73dd17b546f6271275a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:29:41 +0100
Subject: [PATCH 178/726] Fix run_digi2rawVarPage_its.C to work with C++20

---
 Detectors/ITSMFT/ITS/macros/test/run_digi2rawVarPage_its.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Detectors/ITSMFT/ITS/macros/test/run_digi2rawVarPage_its.C b/Detectors/ITSMFT/ITS/macros/test/run_digi2rawVarPage_its.C
index 253726f9c90fe..1c682fc25fff1 100644
--- a/Detectors/ITSMFT/ITS/macros/test/run_digi2rawVarPage_its.C
+++ b/Detectors/ITSMFT/ITS/macros/test/run_digi2rawVarPage_its.C
@@ -12,9 +12,9 @@
 #include "DataFormatsITSMFT/ROFRecord.h"
 #include "DataFormatsParameters/GRPObject.h"
 #include "DataFormatsITSMFT/Digit.h"
-#include "ITSMFTSimulation/MC2RawEncoder.h"
 #endif
 
+#include "ITSMFTSimulation/MC2RawEncoder.h"
 // demo macro the MC->raw conversion with new (variable page size) format
 void setupLinks(o2::itsmft::MC2RawEncoder<o2::itsmft::ChipMappingITS>& m2r, const std::string& outPrefix);
 

From e0a802c422ef02ba198e454f5333deeaaca1da70 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:29:41 +0100
Subject: [PATCH 179/726] Fix findTOFclusterFromLabel.C for C++20

---
 Detectors/TOF/prototyping/findTOFclusterFromLabel.C | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Detectors/TOF/prototyping/findTOFclusterFromLabel.C b/Detectors/TOF/prototyping/findTOFclusterFromLabel.C
index 6dc6299f3d46b..67ed487ecebc1 100644
--- a/Detectors/TOF/prototyping/findTOFclusterFromLabel.C
+++ b/Detectors/TOF/prototyping/findTOFclusterFromLabel.C
@@ -1,16 +1,16 @@
 #if !defined(__CLING__) || defined(__ROOTCLING__)
 #include "TFile.h"
 #include "TTree.h"
-#include "DataFormatsTOF/Cluster.h"
-#include "SimulationDataFormat/MCTruthContainer.h"
 #include "SimulationDataFormat/MCCompLabel.h"
 #include "SimulationDataFormat/MCTrack.h"
-#include "DataFormatsTPC/TrackTPC.h"
 #include "ReconstructionDataFormats/TrackTPCITS.h"
 #include "CommonDataFormat/EvIndex.h"
 #include "TOFBase/Geo.h"
+#include "DataFormatsTPC/TrackTPC.h"
 #include "TOFBase/Digit.h"
 #endif
+#include "SimulationDataFormat/MCTruthContainer.h"
+#include "DataFormatsTOF/Cluster.h"
 
 void findTOFclusterFromLabel(int trackID, int eventID = 0, int sourceID = 0)
 {

From f915223c762b5e4d9151891c90c649d4f6e964dd Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:30:53 +0100
Subject: [PATCH 180/726] Fix CheckTracks.C for C++20

---
 Detectors/ITSMFT/ITS/macros/test/CheckTracks.C | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Detectors/ITSMFT/ITS/macros/test/CheckTracks.C b/Detectors/ITSMFT/ITS/macros/test/CheckTracks.C
index 79b2f52ad29df..11b03531d4af3 100644
--- a/Detectors/ITSMFT/ITS/macros/test/CheckTracks.C
+++ b/Detectors/ITSMFT/ITS/macros/test/CheckTracks.C
@@ -15,16 +15,16 @@
 
 #include "TGeoGlobalMagField.h"
 #include "Field/MagneticField.h"
-#include "DetectorsBase/Propagator.h"
 #include "ITSBase/GeometryTGeo.h"
-#include "SimulationDataFormat/TrackReference.h"
+#include "DataFormatsITS/TrackITS.h"
 #include "SimulationDataFormat/MCTrack.h"
 #include "SimulationDataFormat/MCCompLabel.h"
-#include "SimulationDataFormat/MCTruthContainer.h"
-#include "DataFormatsITSMFT/CompCluster.h"
-#include "DataFormatsITS/TrackITS.h"
+#include "SimulationDataFormat/TrackReference.h"
+#include "DetectorsBase/Propagator.h"
 
 #endif
+#include "DataFormatsITSMFT/CompCluster.h"
+#include "SimulationDataFormat/MCTruthContainer.h"
 
 using namespace std;
 

From 402b1eaf54aaa12711b90f8538df8118e33755ee Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:31:14 +0100
Subject: [PATCH 181/726] Fix CheckClusters.C for C++20

---
 Detectors/ITSMFT/ITS/macros/test/CheckClusters.C | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/Detectors/ITSMFT/ITS/macros/test/CheckClusters.C b/Detectors/ITSMFT/ITS/macros/test/CheckClusters.C
index 3f17e42b3a712..d4707bd683a30 100644
--- a/Detectors/ITSMFT/ITS/macros/test/CheckClusters.C
+++ b/Detectors/ITSMFT/ITS/macros/test/CheckClusters.C
@@ -1,6 +1,18 @@
 /// \file CheckClusters.C
 /// \brief Simple macro to check ITSU clusters
 
+#include <TCanvas.h>
+#include <TFile.h>
+#include <TH2F.h>
+#include <TNtuple.h>
+#include <TString.h>
+#include <TTree.h>
+
+#include "ITSMFTBase/SegmentationAlpide.h"
+#include "ITSBase/GeometryTGeo.h"
+#include "DataFormatsITSMFT/CompCluster.h"
+#include "DataFormatsITSMFT/TopologyDictionary.h"
+#include "ITSMFTSimulation/Hit.h"
 #if !defined(__CLING__) || defined(__ROOTCLING__)
 #include <TCanvas.h>
 #include <TFile.h>

From 065aeb939f41ccc61c91369ce3996a93eae316dd Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:37:00 +0100
Subject: [PATCH 182/726] Fix CheckVertices.C for C++20

---
 Detectors/ITSMFT/ITS/macros/test/CheckVertices.C | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Detectors/ITSMFT/ITS/macros/test/CheckVertices.C b/Detectors/ITSMFT/ITS/macros/test/CheckVertices.C
index f348272425b77..2da3ed9b97a28 100644
--- a/Detectors/ITSMFT/ITS/macros/test/CheckVertices.C
+++ b/Detectors/ITSMFT/ITS/macros/test/CheckVertices.C
@@ -29,6 +29,7 @@
 #include "DataFormatsITSMFT/ROFRecord.h"
 #include "DataFormatsITS/TrackITS.h"
 #endif
+#include "DataFormatsITSMFT/CompCluster.h"
 
 o2::MCCompLabel getMainLabel(std::vector<o2::MCCompLabel>& labs);
 
@@ -335,4 +336,4 @@ void CheckVertices(const int dumprof = -1, std::string path = "tf1/", std::strin
   LOGP(info, "Summary:");
   LOGP(info, "Found {} vertices in {} usable out of {} simulated", nvt, nevts, simVerts.size());
   LOGP(info, "Average good vertexing efficiency: {}%", (addeff / (float)nroffilled) * 100);
-}
\ No newline at end of file
+}

From 9621b319d0fd99a98075bafd9c7aacec11b809f2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:31:14 +0100
Subject: [PATCH 183/726] Fix convertTreeTo02object.C for C++20

---
 Detectors/TOF/prototyping/convertTreeTo02object.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Detectors/TOF/prototyping/convertTreeTo02object.C b/Detectors/TOF/prototyping/convertTreeTo02object.C
index 060072ef21e6a..2e46e9842bf33 100644
--- a/Detectors/TOF/prototyping/convertTreeTo02object.C
+++ b/Detectors/TOF/prototyping/convertTreeTo02object.C
@@ -3,8 +3,8 @@
 #include "TTree.h"
 #include "TLeaf.h"
 #include <stdio.h>
-#include "DataFormatsTOF/CalibInfoTOF.h"
 #endif
+#include "DataFormatsTOF/CalibInfoTOF.h"
 
 void convertTreeTo02object()
 {

From adc9d104ce7bebacb6f87a9ec25d5f22c6720e02 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:31:14 +0100
Subject: [PATCH 184/726] Fix CheckHits.C for C++20

---
 Detectors/Upgrades/ITS3/macros/test/CheckHits.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Detectors/Upgrades/ITS3/macros/test/CheckHits.C b/Detectors/Upgrades/ITS3/macros/test/CheckHits.C
index bb1a8073fc646..5c0f945898471 100644
--- a/Detectors/Upgrades/ITS3/macros/test/CheckHits.C
+++ b/Detectors/Upgrades/ITS3/macros/test/CheckHits.C
@@ -31,12 +31,12 @@
 #include <utility>
 #include <vector>
 
+#endif
 #define ENABLE_UPGRADES
 #include "CommonConstants/MathConstants.h"
 #include "ITS3Base/SegmentationSuperAlpide.h"
 #include "ITSMFTSimulation/Hit.h"
 #include "SimulationDataFormat/MCTrack.h"
-#endif
 
 using o2::itsmft::Hit;
 

From b74c34495bd145e454e0455e0cab0e2714b3dc6f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:31:14 +0100
Subject: [PATCH 185/726] Fix CheckDuplicates.C for C++20

---
 Detectors/ITSMFT/ITS/macros/test/CheckDuplicates.C | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/Detectors/ITSMFT/ITS/macros/test/CheckDuplicates.C b/Detectors/ITSMFT/ITS/macros/test/CheckDuplicates.C
index 2736a79beb528..e36f249ef70dc 100644
--- a/Detectors/ITSMFT/ITS/macros/test/CheckDuplicates.C
+++ b/Detectors/ITSMFT/ITS/macros/test/CheckDuplicates.C
@@ -23,11 +23,10 @@
 #include <TStyle.h>
 #include <TGaxis.h>
 
-#include "DataFormatsITSMFT/ROFRecord.h"
-#include "DataFormatsITS/TrackITS.h"
-#include "Framework/Logger.h"
 
+#include "DataFormatsITS/TrackITS.h"
 #endif
+#include "DataFormatsITSMFT/ROFRecord.h"
 
 void CheckDuplicates(TString tracfile = "./o2trac_its.root", TString output = ".", TString rootSave = "", int rofStart = 0, int rofEnd = -1, int windowSize = 2, bool includeSame = 1)
 {

From d50929303da5930023ad16168a5c30db888910d5 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:31:14 +0100
Subject: [PATCH 186/726] Fix CheckTracksITS3.C for C++20

---
 .../ITS3/macros/test/CheckTracksITS3.C        | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/Detectors/Upgrades/ITS3/macros/test/CheckTracksITS3.C b/Detectors/Upgrades/ITS3/macros/test/CheckTracksITS3.C
index 8b0383246e7fc..c04dae6309d7e 100644
--- a/Detectors/Upgrades/ITS3/macros/test/CheckTracksITS3.C
+++ b/Detectors/Upgrades/ITS3/macros/test/CheckTracksITS3.C
@@ -12,7 +12,17 @@
 /// \file CheckTracksITS3.C
 /// \brief Simple macro to check ITS3 tracks
 
+#include <iostream>
 #if !defined(__CLING__) || defined(__ROOTCLING__)
+#include <TROOT.h>
+
+#include "DataFormatsITS/TrackITS.h"
+#include "TGeoGlobalMagField.h"
+#include "Field/MagneticField.h"
+#include "DetectorsBase/Propagator.h"
+#include "ITSBase/GeometryTGeo.h"
+#include "DataFormatsITS3/CompCluster.h"
+#endif
 #include <array>
 #include <cmath>
 #include <iostream>
@@ -26,23 +36,13 @@
 #include <THStack.h>
 #include <TLegend.h>
 #include <TPad.h>
-#include <TROOT.h>
-
-#include "TGeoGlobalMagField.h"
-#include "Field/MagneticField.h"
-#include "DataFormatsITS3/CompCluster.h"
 #include "DataFormatsITSMFT/CompCluster.h"
-#include "DataFormatsITS/TrackITS.h"
-#include "DetectorsBase/Propagator.h"
-#include "ITSBase/GeometryTGeo.h"
 #include "SimulationDataFormat/TrackReference.h"
 #include "SimulationDataFormat/MCTrack.h"
 #include "SimulationDataFormat/MCCompLabel.h"
 #include "SimulationDataFormat/MCTruthContainer.h"
 #include "SimulationDataFormat/MCEventHeader.h"
 
-#endif
-
 using namespace std;
 
 struct ParticleInfo {

From a7db50078b637bad188eed12b300383fb0d37b8f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:31:14 +0100
Subject: [PATCH 187/726] Fix misAlign.C to work with C++20

---
 Detectors/MUON/MCH/Geometry/Test/misAlign.C | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/Detectors/MUON/MCH/Geometry/Test/misAlign.C b/Detectors/MUON/MCH/Geometry/Test/misAlign.C
index 38a4a7c32eaeb..c49fe5717a36b 100644
--- a/Detectors/MUON/MCH/Geometry/Test/misAlign.C
+++ b/Detectors/MUON/MCH/Geometry/Test/misAlign.C
@@ -1,21 +1,14 @@
+#include "DetectorsBase/MaterialManager.h"
+#include "DetectorsCommonDataFormats/AlignParam.h"
 #if !defined(__CLING__) || defined(__ROOTCLING__)
 
 //#include "MCHGeometryTest/Helpers.h"
-#include "DetectorsBase/GeometryManager.h"
-#include "DetectorsBase/MaterialManager.h"
-#include "DetectorsCommonDataFormats/AlignParam.h"
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "DetectorsCommonDataFormats/DetectorNameConf.h"
 
 #include "MCHGeometryMisAligner/MisAligner.h"
 #include "MCHGeometryTransformer/Transformations.h"
 
-#include "MCHGeometryTest/Helpers.h"
-#include "MCHGeometryCreator/Geometry.h"
-#include "CCDB/CcdbApi.h"
-
-#include "MathUtils/Cartesian.h"
-#include "Math/GenVector/Cartesian3D.h"
 #include "TGLRnrCtx.h"
 #include "TGLViewer.h"
 #include "TGeoManager.h"
@@ -26,7 +19,13 @@
 
 #include <iostream>
 #include <fmt/format.h>
+#include "MCHGeometryTest/Helpers.h"
+#include "MCHGeometryCreator/Geometry.h"
+#include "CCDB/CcdbApi.h"
 
+#include "MathUtils/Cartesian.h"
+#include "Math/GenVector/Cartesian3D.h"
+#include "DetectorsBase/GeometryManager.h"
 #endif
 
 // void misAlign()

From fe746e2806acae6e75370067b536769fa16880f0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:39:19 +0100
Subject: [PATCH 188/726] Fix CheckTracksCA.C to work with C++20

---
 Detectors/ITSMFT/ITS/macros/test/CheckTracksCA.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Detectors/ITSMFT/ITS/macros/test/CheckTracksCA.C b/Detectors/ITSMFT/ITS/macros/test/CheckTracksCA.C
index f1a3d2af4e469..7312a6845f1a3 100644
--- a/Detectors/ITSMFT/ITS/macros/test/CheckTracksCA.C
+++ b/Detectors/ITSMFT/ITS/macros/test/CheckTracksCA.C
@@ -21,10 +21,10 @@
 #include "SimulationDataFormat/MCTrack.h"
 #include "SimulationDataFormat/MCCompLabel.h"
 #include "SimulationDataFormat/MCTruthContainer.h"
-#include "DataFormatsITSMFT/CompCluster.h"
 #include "DataFormatsITS/TrackITS.h"
 
 #endif
+#include "DataFormatsITSMFT/CompCluster.h"
 
 using namespace std;
 

From 695718eab76b4eb7cf6f4284837c2b86cdd72cc8 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:37:04 +0100
Subject: [PATCH 189/726] Fix toyCluster.C to work with C++20

---
 Detectors/TPC/simulation/macro/toyCluster.C | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Detectors/TPC/simulation/macro/toyCluster.C b/Detectors/TPC/simulation/macro/toyCluster.C
index 2087a7773fe22..b7c5f74633cfc 100644
--- a/Detectors/TPC/simulation/macro/toyCluster.C
+++ b/Detectors/TPC/simulation/macro/toyCluster.C
@@ -59,6 +59,7 @@
 #include "TPCSimulation/GEMAmplification.h"
 #endif
 #include <boost/property_tree/ptree.hpp>
+#include "DataFormatsTPC/Helpers.h"
 
 using namespace o2::tpc;
 void fillTPCHits(const float theta, const float phi, const float dedx, std::vector<HitGroup>& hitGroupSector, std::pair<GlobalPosition3D, GlobalPosition3D>& trackInfo);

From d70a4c956b66f3a94079dd5d3e445353d76b0e56 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 13:16:55 +0100
Subject: [PATCH 190/726] Fix macros to work with C++20

---
 macro/run_clus_tpc.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macro/run_clus_tpc.C b/macro/run_clus_tpc.C
index ce703e65ae999..950eb45f68851 100644
--- a/macro/run_clus_tpc.C
+++ b/macro/run_clus_tpc.C
@@ -14,8 +14,8 @@
 #include "FairParRootFileIo.h"
 #include <FairRootFileSink.h>
 
-#include "TPCReconstruction/ClustererTask.h"
 #endif
+#include "TPCReconstruction/ClustererTask.h"
 
 void run_clus_tpc(std::string outputfile = "o2clus_tpc.root", std::string inputfile = "o2dig.root",
                   std::string paramfile = "o2sim_par.root", bool isContinuous = true, unsigned threads = 0);

From e83336c1a32c547d627c1b50956e6ffb3331d068 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 13:24:09 +0100
Subject: [PATCH 191/726] Fix macros to compile with C++20

---
 Framework/CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 Framework/CHANGELOG.md

diff --git a/Framework/CHANGELOG.md b/Framework/CHANGELOG.md
new file mode 100644
index 0000000000000..2973174f81d28
--- /dev/null
+++ b/Framework/CHANGELOG.md
@@ -0,0 +1,4 @@
+* Improved C++20 support. Most of the macros which were failing when C++20
+  support is enabled now seem to work fine. The issue seems to be related to
+  some forward declaration logic which seems to be not working correctly in
+  ROOT 6.30.01. The issue is discussed in <https://github.com/root-project/root/issues/14230> and it seems to be not trivial to fix with the current ROOT version.

From 65ff7892821b932a2a4864b19be3c0096e13734f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:39:18 +0100
Subject: [PATCH 192/726] Fix CheckDuplicates.C for C++20

---
 Detectors/ITSMFT/ITS/macros/test/CheckDuplicates.C | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Detectors/ITSMFT/ITS/macros/test/CheckDuplicates.C b/Detectors/ITSMFT/ITS/macros/test/CheckDuplicates.C
index 2736a79beb528..abe707225ab30 100644
--- a/Detectors/ITSMFT/ITS/macros/test/CheckDuplicates.C
+++ b/Detectors/ITSMFT/ITS/macros/test/CheckDuplicates.C
@@ -23,11 +23,9 @@
 #include <TStyle.h>
 #include <TGaxis.h>
 
-#include "DataFormatsITSMFT/ROFRecord.h"
 #include "DataFormatsITS/TrackITS.h"
-#include "Framework/Logger.h"
-
 #endif
+#include "DataFormatsITSMFT/ROFRecord.h"
 
 void CheckDuplicates(TString tracfile = "./o2trac_its.root", TString output = ".", TString rootSave = "", int rofStart = 0, int rofEnd = -1, int windowSize = 2, bool includeSame = 1)
 {

From 48a887cd562980018f5329a86bdfdc1c76b443ee Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:39:19 +0100
Subject: [PATCH 193/726] Fix misAlign.C to work with C++20

---
 Detectors/MUON/MCH/Geometry/Test/misAlign.C | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/Detectors/MUON/MCH/Geometry/Test/misAlign.C b/Detectors/MUON/MCH/Geometry/Test/misAlign.C
index 38a4a7c32eaeb..c49fe5717a36b 100644
--- a/Detectors/MUON/MCH/Geometry/Test/misAlign.C
+++ b/Detectors/MUON/MCH/Geometry/Test/misAlign.C
@@ -1,21 +1,14 @@
+#include "DetectorsBase/MaterialManager.h"
+#include "DetectorsCommonDataFormats/AlignParam.h"
 #if !defined(__CLING__) || defined(__ROOTCLING__)
 
 //#include "MCHGeometryTest/Helpers.h"
-#include "DetectorsBase/GeometryManager.h"
-#include "DetectorsBase/MaterialManager.h"
-#include "DetectorsCommonDataFormats/AlignParam.h"
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "DetectorsCommonDataFormats/DetectorNameConf.h"
 
 #include "MCHGeometryMisAligner/MisAligner.h"
 #include "MCHGeometryTransformer/Transformations.h"
 
-#include "MCHGeometryTest/Helpers.h"
-#include "MCHGeometryCreator/Geometry.h"
-#include "CCDB/CcdbApi.h"
-
-#include "MathUtils/Cartesian.h"
-#include "Math/GenVector/Cartesian3D.h"
 #include "TGLRnrCtx.h"
 #include "TGLViewer.h"
 #include "TGeoManager.h"
@@ -26,7 +19,13 @@
 
 #include <iostream>
 #include <fmt/format.h>
+#include "MCHGeometryTest/Helpers.h"
+#include "MCHGeometryCreator/Geometry.h"
+#include "CCDB/CcdbApi.h"
 
+#include "MathUtils/Cartesian.h"
+#include "Math/GenVector/Cartesian3D.h"
+#include "DetectorsBase/GeometryManager.h"
 #endif
 
 // void misAlign()

From 902ceb512df89a817a439edda65e6c5a3aeb9440 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:39:20 +0100
Subject: [PATCH 194/726] Fix CheckTracksCA.C to work with C++20

---
 Detectors/ITSMFT/ITS/macros/test/CheckTracksCA.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Detectors/ITSMFT/ITS/macros/test/CheckTracksCA.C b/Detectors/ITSMFT/ITS/macros/test/CheckTracksCA.C
index f1a3d2af4e469..7312a6845f1a3 100644
--- a/Detectors/ITSMFT/ITS/macros/test/CheckTracksCA.C
+++ b/Detectors/ITSMFT/ITS/macros/test/CheckTracksCA.C
@@ -21,10 +21,10 @@
 #include "SimulationDataFormat/MCTrack.h"
 #include "SimulationDataFormat/MCCompLabel.h"
 #include "SimulationDataFormat/MCTruthContainer.h"
-#include "DataFormatsITSMFT/CompCluster.h"
 #include "DataFormatsITS/TrackITS.h"
 
 #endif
+#include "DataFormatsITSMFT/CompCluster.h"
 
 using namespace std;
 

From 0394453dbf8d15484f5d92d3f6676484134b0c5a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 12:39:20 +0100
Subject: [PATCH 195/726] Fix toyCluster.C to work with C++20

---
 Detectors/TPC/simulation/macro/toyCluster.C | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Detectors/TPC/simulation/macro/toyCluster.C b/Detectors/TPC/simulation/macro/toyCluster.C
index 2087a7773fe22..b7c5f74633cfc 100644
--- a/Detectors/TPC/simulation/macro/toyCluster.C
+++ b/Detectors/TPC/simulation/macro/toyCluster.C
@@ -59,6 +59,7 @@
 #include "TPCSimulation/GEMAmplification.h"
 #endif
 #include <boost/property_tree/ptree.hpp>
+#include "DataFormatsTPC/Helpers.h"
 
 using namespace o2::tpc;
 void fillTPCHits(const float theta, const float phi, const float dedx, std::vector<HitGroup>& hitGroupSector, std::pair<GlobalPosition3D, GlobalPosition3D>& trackInfo);

From 41e1fcd045cf0313af4925e49d10a159a3b40c98 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 13:28:05 +0100
Subject: [PATCH 196/726] Fix run_clus_tpc to work with C++20

---
 macro/run_clus_tpc.C | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macro/run_clus_tpc.C b/macro/run_clus_tpc.C
index ce703e65ae999..950eb45f68851 100644
--- a/macro/run_clus_tpc.C
+++ b/macro/run_clus_tpc.C
@@ -14,8 +14,8 @@
 #include "FairParRootFileIo.h"
 #include <FairRootFileSink.h>
 
-#include "TPCReconstruction/ClustererTask.h"
 #endif
+#include "TPCReconstruction/ClustererTask.h"
 
 void run_clus_tpc(std::string outputfile = "o2clus_tpc.root", std::string inputfile = "o2dig.root",
                   std::string paramfile = "o2sim_par.root", bool isContinuous = true, unsigned threads = 0);

From e41c4dad9bdd5e37f1bdcc91403ac759f6313eaa Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 13:28:13 +0100
Subject: [PATCH 197/726] Fix macros to compile with C++20

---
 Framework/CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 Framework/CHANGELOG.md

diff --git a/Framework/CHANGELOG.md b/Framework/CHANGELOG.md
new file mode 100644
index 0000000000000..2973174f81d28
--- /dev/null
+++ b/Framework/CHANGELOG.md
@@ -0,0 +1,4 @@
+* Improved C++20 support. Most of the macros which were failing when C++20
+  support is enabled now seem to work fine. The issue seems to be related to
+  some forward declaration logic which seems to be not working correctly in
+  ROOT 6.30.01. The issue is discussed in <https://github.com/root-project/root/issues/14230> and it seems to be not trivial to fix with the current ROOT version.

From b10d39b0634af85e70e6e77b87b747ce720ca445 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 9 Jan 2024 13:46:44 +0100
Subject: [PATCH 198/726] Drop --old option for real

---
 Detectors/TOF/compression/src/tof-compressor.cxx | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/Detectors/TOF/compression/src/tof-compressor.cxx b/Detectors/TOF/compression/src/tof-compressor.cxx
index 3bb0899f5e91c..dde9cc3cc6c23 100644
--- a/Detectors/TOF/compression/src/tof-compressor.cxx
+++ b/Detectors/TOF/compression/src/tof-compressor.cxx
@@ -44,7 +44,6 @@ void customize(std::vector<ConfigParamSpec>& workflowOptions)
   workflowOptions.push_back(paranoid);
   workflowOptions.push_back(ignoreStf);
   workflowOptions.push_back(payloadlim);
-  workflowOptions.emplace_back(ConfigParamSpec{"old", VariantType::Bool, false, {"use the non-DPL version of the compressor"}});
   workflowOptions.push_back(ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}});
 }
 
@@ -60,10 +59,6 @@ WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
   auto verbose = cfgc.options().get<bool>("tof-compressor-verbose");
   auto paranoid = cfgc.options().get<bool>("tof-compressor-paranoid");
   auto ignoreStf = cfgc.options().get<bool>("ignore-dist-stf");
-  auto old = cfgc.options().get<bool>("old");
-  if (old) {
-    LOGP(error, "--old option is not actually supported, using the new version");
-  }
   auto payloadLim = cfgc.options().get<long>("payload-limit");
 
   std::vector<OutputSpec> outputs;

From 91f782e229996d55cb7343b2cd0e695cd612df40 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 10 Jan 2024 11:41:03 +0100
Subject: [PATCH 199/726] DPL: fix warning and simplify assert.

---
 Framework/Core/include/Framework/MessageContext.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Framework/Core/include/Framework/MessageContext.h b/Framework/Core/include/Framework/MessageContext.h
index 4f6674576e8ef..c3b0e1f822b81 100644
--- a/Framework/Core/include/Framework/MessageContext.h
+++ b/Framework/Core/include/Framework/MessageContext.h
@@ -498,9 +498,7 @@ class MessageContext
   void clear()
   {
     // Verify that everything has been sent on clear.
-    for (auto& m : mMessages) {
-      assert(m->empty());
-    }
+    assert(std::all_of(mMessages.begin(), mMessages.end(), [](auto& m) { return m->empty(); }));
     mMessages.clear();
   }
 

From 42a900a173bab9553508242361dea19030544f84 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 10 Jan 2024 11:43:57 +0100
Subject: [PATCH 200/726] DPL: move MessageContext::schedule out of line

---
 .../Core/include/Framework/MessageContext.h   | 33 +------------------
 Framework/Core/src/MessageContext.cxx         | 33 +++++++++++++++++++
 2 files changed, 34 insertions(+), 32 deletions(-)

diff --git a/Framework/Core/include/Framework/MessageContext.h b/Framework/Core/include/Framework/MessageContext.h
index c3b0e1f822b81..c1f2c695ff58a 100644
--- a/Framework/Core/include/Framework/MessageContext.h
+++ b/Framework/Core/include/Framework/MessageContext.h
@@ -444,38 +444,7 @@ class MessageContext
   /// Schedule a context object for sending.
   /// The object is considered complete at this point and is sent directly through the dispatcher callback
   /// of the context if initialized.
-  void schedule(Messages::value_type&& message)
-  {
-    auto const* header = message->header();
-    if (header == nullptr) {
-      throw std::logic_error("No valid header message found");
-    }
-    mScheduledMessages.emplace_back(std::move(message));
-    if (mDispatchControl.dispatch != nullptr) {
-      // send all scheduled messages if there is no trigger callback or its result is true
-      if (mDispatchControl.trigger == nullptr || mDispatchControl.trigger(*header)) {
-        std::vector<fair::mq::Parts> outputsPerChannel;
-        outputsPerChannel.resize(mProxy.getNumOutputChannels());
-        for (auto& message : mScheduledMessages) {
-          fair::mq::Parts parts = message->finalize();
-          assert(message->empty());
-          assert(parts.Size() == 2);
-          for (auto& part : parts) {
-            outputsPerChannel[mProxy.getOutputChannelIndex(message->route()).value].AddPart(std::move(part));
-          }
-        }
-        for (int ci = 0; ci < mProxy.getNumOutputChannels(); ++ci) {
-          auto& parts = outputsPerChannel[ci];
-          if (parts.Size() == 0) {
-            continue;
-          }
-          mDispatchControl.dispatch(std::move(parts), ChannelIndex{ci}, DefaultChannelIndex);
-        }
-        mDidDispatch = mScheduledMessages.empty() == false;
-        mScheduledMessages.clear();
-      }
-    }
-  }
+  void schedule(Messages::value_type&& message);
 
   Messages getMessagesForSending()
   {
diff --git a/Framework/Core/src/MessageContext.cxx b/Framework/Core/src/MessageContext.cxx
index 74a7a838aa78e..a7365594785c7 100644
--- a/Framework/Core/src/MessageContext.cxx
+++ b/Framework/Core/src/MessageContext.cxx
@@ -123,4 +123,37 @@ void MessageContext::pruneFromCache(int64_t id)
   mMessageCache.erase(id);
 }
 
+void MessageContext::schedule(Messages::value_type&& message)
+{
+  auto const* header = message->header();
+  if (header == nullptr) {
+    throw std::logic_error("No valid header message found");
+  }
+  mScheduledMessages.emplace_back(std::move(message));
+  if (mDispatchControl.dispatch != nullptr) {
+    // send all scheduled messages if there is no trigger callback or its result is true
+    if (mDispatchControl.trigger == nullptr || mDispatchControl.trigger(*header)) {
+      std::vector<fair::mq::Parts> outputsPerChannel;
+      outputsPerChannel.resize(mProxy.getNumOutputChannels());
+      for (auto& message : mScheduledMessages) {
+        fair::mq::Parts parts = message->finalize();
+        assert(message->empty());
+        assert(parts.Size() == 2);
+        for (auto& part : parts) {
+          outputsPerChannel[mProxy.getOutputChannelIndex(message->route()).value].AddPart(std::move(part));
+        }
+      }
+      for (int ci = 0; ci < mProxy.getNumOutputChannels(); ++ci) {
+        auto& parts = outputsPerChannel[ci];
+        if (parts.Size() == 0) {
+          continue;
+        }
+        mDispatchControl.dispatch(std::move(parts), ChannelIndex{ci}, DefaultChannelIndex);
+      }
+      mDidDispatch = mScheduledMessages.empty() == false;
+      mScheduledMessages.clear();
+    }
+  }
+}
+
 } // namespace o2::framework

From 51820539a485c939a0a1c6c490549c8b06167d37 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 10 Jan 2024 11:41:03 +0100
Subject: [PATCH 201/726] DPL: drop duplicate include statement

---
 Framework/Core/test/test_TMessageSerializer.cxx | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Framework/Core/test/test_TMessageSerializer.cxx b/Framework/Core/test/test_TMessageSerializer.cxx
index b54d3cf1a7da5..bc5f817400a44 100644
--- a/Framework/Core/test/test_TMessageSerializer.cxx
+++ b/Framework/Core/test/test_TMessageSerializer.cxx
@@ -13,7 +13,6 @@
 #include "Framework/RuntimeError.h"
 #include "TestClasses.h"
 #include <catch_amalgamated.hpp>
-#include <catch_amalgamated.hpp>
 #include <utility>
 
 using namespace o2::framework;

From 3882264a0b149759338ba9e39a5bf5b5bfee9c03 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 10 Jan 2024 11:43:58 +0100
Subject: [PATCH 202/726] DPL: detect when a Lifetime::Timeframe output is
 missing

---
 .../Core/include/Framework/StreamContext.h    |  7 +++
 Framework/Core/src/CommonServices.cxx         | 27 ++++++++
 Framework/Core/src/DataAllocator.cxx          |  3 +
 Framework/TestWorkflows/CMakeLists.txt        |  3 +
 .../test/test_DetectMissingTimeframe.cxx      | 62 +++++++++++++++++++
 5 files changed, 102 insertions(+)
 create mode 100644 Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx

diff --git a/Framework/Core/include/Framework/StreamContext.h b/Framework/Core/include/Framework/StreamContext.h
index 0ab353e147276..198802179a419 100644
--- a/Framework/Core/include/Framework/StreamContext.h
+++ b/Framework/Core/include/Framework/StreamContext.h
@@ -63,6 +63,13 @@ struct StreamContext {
   // Notice that in such a case all the services will be created upfront, so
   // the callback will be called for all of them.
   std::vector<ServiceStartStreamHandle> preStartStreamHandles;
+
+  // Information on wether or not all the required routes have been created.
+  // This is used to check if the LifetimeTimeframe routes were all created
+  // for a given iteration.
+  // This is in the stream context to allow tracking data creation on a per thread
+  // basis.
+  std::vector<bool> routeCreated;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 05b8fba47e532..b5351c8822cb1 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -162,6 +162,33 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
     .uniqueId = simpleServiceId<StreamContext>(),
     .init = simpleServiceInit<StreamContext, StreamContext, ServiceKind::Stream>(),
     .configure = noConfiguration(),
+    .preProcessing = [](ProcessingContext& context, void* service) {
+      auto* stream = (StreamContext*)service;
+      auto& routes = context.services().get<DeviceSpec const>().outputs;
+      // Notice I need to do this here, because different invocation for 
+      // the same stream might be referring to different data processors.
+      // We should probably have a context which is per stream of a specific
+      // data processor.
+      stream->routeCreated.resize(routes.size());
+      // Reset the routeCreated at every processing step
+      std::fill(stream->routeCreated.begin(), stream->routeCreated.end(), false); },
+    .postProcessing = [](ProcessingContext& processingContext, void* service) {
+      auto* stream = (StreamContext*)service;
+      auto& routes = processingContext.services().get<DeviceSpec const>().outputs;
+      auto& timeslice = processingContext.services().get<TimingInfo>().timeslice;
+      for (size_t ri = 0; ri < routes.size(); ++ri) {
+        if (stream->routeCreated[ri] == true) {
+          continue;
+        }
+        auto &route = routes[ri];
+        auto &matcher = route.matcher;
+        if ((timeslice % route.maxTimeslices) != route.timeslice) {
+          continue;
+        }
+        if (matcher.lifetime == Lifetime::Timeframe) {
+          LOGP(error, "Expected Lifetime::Timeframe data {} was not created for timeslice {} and might result in dropped timeframes", DataSpecUtils::describe(matcher), timeslice);
+        }
+      } },
     .kind = ServiceKind::Stream};
 }
 
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index 1230d2e942f6e..9ce72aed8c1cf 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -19,6 +19,7 @@
 #include "Framework/FairMQResizableBuffer.h"
 #include "Framework/DataProcessingContext.h"
 #include "Framework/DeviceSpec.h"
+#include "Framework/StreamContext.h"
 #include "Headers/Stack.h"
 
 #include <fairmq/Device.h>
@@ -47,10 +48,12 @@ DataAllocator::DataAllocator(ServiceRegistryRef contextRegistry)
 RouteIndex DataAllocator::matchDataHeader(const Output& spec, size_t timeslice)
 {
   auto& allowedOutputRoutes = mRegistry.get<DeviceSpec const>().outputs;
+  auto& stream = mRegistry.get<o2::framework::StreamContext>();
   // FIXME: we should take timeframeId into account as well.
   for (auto ri = 0; ri < allowedOutputRoutes.size(); ++ri) {
     auto& route = allowedOutputRoutes[ri];
     if (DataSpecUtils::match(route.matcher, spec.origin, spec.description, spec.subSpec) && ((timeslice % route.maxTimeslices) == route.timeslice)) {
+      stream.routeCreated[ri] = true;
       return RouteIndex{ri};
     }
   }
diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index 7a396c0f2c3a9..64e9dddbb020e 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -12,6 +12,9 @@
 o2_add_dpl_workflow(dummy-workflow
                   SOURCES src/o2DummyWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
+o2_add_dpl_workflow(detect-missing-timeframe
+                  SOURCES test/test_DetectMissingTimeframe.cxx
+                  COMPONENT_NAME TestWorkflows)
 
 o2_add_dpl_workflow(o2rootmessage-workflow
                   SOURCES "src/test_o2RootMessageWorkflow.cxx"
diff --git a/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx b/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
new file mode 100644
index 0000000000000..3374cd9ad6cf2
--- /dev/null
+++ b/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
@@ -0,0 +1,62 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/DataTakingContext.h"
+#include "Framework/CompletionPolicyHelpers.h"
+#include "Framework/DeviceSpec.h"
+#include "Framework/RawDeviceService.h"
+#include "Framework/ControlService.h"
+#include "Framework/Configurable.h"
+#include "Framework/RunningWorkflowInfo.h"
+#include "Framework/RateLimiter.h"
+#include <fairmq/Device.h>
+
+#include <iostream>
+#include <chrono>
+#include <thread>
+#include <vector>
+
+using namespace o2::framework;
+
+#include "Framework/runDataProcessing.h"
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& specs)
+{
+  DataProcessorSpec a{
+    .name = "A",
+    .outputs = {OutputSpec{{"a1"}, "TST", "A1"},
+                OutputSpec{{"a2"}, "TST", "A2"}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, RawDeviceService& device, DataTakingContext& context, ProcessingContext& pcx) {
+        outputs.make<int>(OutputRef{"a1"}, 1);
+        static int i = 0;
+        outputs.make<int>(OutputRef{"a1"}, 1);
+        if (i++ % 2 == 0) {
+          outputs.make<int>(OutputRef{"a2"}, 1);
+        }
+      })},
+  };
+  DataProcessorSpec d{
+    .name = "D",
+    .inputs = {InputSpec{"a1", "TST", "A1"},
+               InputSpec{"a2", "TST", "A2"}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](InputRecord& inputs) {
+        auto ref = inputs.get("a1");
+        auto header = o2::header::get<const DataProcessingHeader*>(ref.header);
+        LOG(info) << "Start time: " << header->startTime;
+      })},
+  };
+
+  return workflow::concat(WorkflowSpec{a},
+                          WorkflowSpec{d});
+}

From 8d12ff88993c9fb217f6dd00b4c1658b8779f6be Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 10 Jan 2024 11:49:51 +0100
Subject: [PATCH 203/726] DPL: detect when a Lifetime::Timeframe output is
 missing

---
 .../Core/include/Framework/StreamContext.h    |  7 +++
 Framework/Core/src/CommonServices.cxx         | 27 ++++++++
 Framework/Core/src/DataAllocator.cxx          |  3 +
 Framework/TestWorkflows/CMakeLists.txt        |  3 +
 .../test/test_DetectMissingTimeframe.cxx      | 62 +++++++++++++++++++
 5 files changed, 102 insertions(+)
 create mode 100644 Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx

diff --git a/Framework/Core/include/Framework/StreamContext.h b/Framework/Core/include/Framework/StreamContext.h
index 0ab353e147276..198802179a419 100644
--- a/Framework/Core/include/Framework/StreamContext.h
+++ b/Framework/Core/include/Framework/StreamContext.h
@@ -63,6 +63,13 @@ struct StreamContext {
   // Notice that in such a case all the services will be created upfront, so
   // the callback will be called for all of them.
   std::vector<ServiceStartStreamHandle> preStartStreamHandles;
+
+  // Information on wether or not all the required routes have been created.
+  // This is used to check if the LifetimeTimeframe routes were all created
+  // for a given iteration.
+  // This is in the stream context to allow tracking data creation on a per thread
+  // basis.
+  std::vector<bool> routeCreated;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 05b8fba47e532..1a79cc8afacba 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -162,6 +162,33 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
     .uniqueId = simpleServiceId<StreamContext>(),
     .init = simpleServiceInit<StreamContext, StreamContext, ServiceKind::Stream>(),
     .configure = noConfiguration(),
+    .preProcessing = [](ProcessingContext& context, void* service) {
+      auto* stream = (StreamContext*)service;
+      auto& routes = context.services().get<DeviceSpec const>().outputs;
+      // Notice I need to do this here, because different invocation for
+      // the same stream might be referring to different data processors.
+      // We should probably have a context which is per stream of a specific
+      // data processor.
+      stream->routeCreated.resize(routes.size());
+      // Reset the routeCreated at every processing step
+      std::fill(stream->routeCreated.begin(), stream->routeCreated.end(), false); },
+    .postProcessing = [](ProcessingContext& processingContext, void* service) {
+      auto* stream = (StreamContext*)service;
+      auto& routes = processingContext.services().get<DeviceSpec const>().outputs;
+      auto& timeslice = processingContext.services().get<TimingInfo>().timeslice;
+      for (size_t ri = 0; ri < routes.size(); ++ri) {
+        if (stream->routeCreated[ri] == true) {
+          continue;
+        }
+        auto &route = routes[ri];
+        auto &matcher = route.matcher;
+        if ((timeslice % route.maxTimeslices) != route.timeslice) {
+          continue;
+        }
+        if (matcher.lifetime == Lifetime::Timeframe) {
+          LOGP(error, "Expected Lifetime::Timeframe data {} was not created for timeslice {} and might result in dropped timeframes", DataSpecUtils::describe(matcher), timeslice);
+        }
+      } },
     .kind = ServiceKind::Stream};
 }
 
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index 1230d2e942f6e..9ce72aed8c1cf 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -19,6 +19,7 @@
 #include "Framework/FairMQResizableBuffer.h"
 #include "Framework/DataProcessingContext.h"
 #include "Framework/DeviceSpec.h"
+#include "Framework/StreamContext.h"
 #include "Headers/Stack.h"
 
 #include <fairmq/Device.h>
@@ -47,10 +48,12 @@ DataAllocator::DataAllocator(ServiceRegistryRef contextRegistry)
 RouteIndex DataAllocator::matchDataHeader(const Output& spec, size_t timeslice)
 {
   auto& allowedOutputRoutes = mRegistry.get<DeviceSpec const>().outputs;
+  auto& stream = mRegistry.get<o2::framework::StreamContext>();
   // FIXME: we should take timeframeId into account as well.
   for (auto ri = 0; ri < allowedOutputRoutes.size(); ++ri) {
     auto& route = allowedOutputRoutes[ri];
     if (DataSpecUtils::match(route.matcher, spec.origin, spec.description, spec.subSpec) && ((timeslice % route.maxTimeslices) == route.timeslice)) {
+      stream.routeCreated[ri] = true;
       return RouteIndex{ri};
     }
   }
diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index 7a396c0f2c3a9..64e9dddbb020e 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -12,6 +12,9 @@
 o2_add_dpl_workflow(dummy-workflow
                   SOURCES src/o2DummyWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
+o2_add_dpl_workflow(detect-missing-timeframe
+                  SOURCES test/test_DetectMissingTimeframe.cxx
+                  COMPONENT_NAME TestWorkflows)
 
 o2_add_dpl_workflow(o2rootmessage-workflow
                   SOURCES "src/test_o2RootMessageWorkflow.cxx"
diff --git a/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx b/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
new file mode 100644
index 0000000000000..3374cd9ad6cf2
--- /dev/null
+++ b/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
@@ -0,0 +1,62 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/DataTakingContext.h"
+#include "Framework/CompletionPolicyHelpers.h"
+#include "Framework/DeviceSpec.h"
+#include "Framework/RawDeviceService.h"
+#include "Framework/ControlService.h"
+#include "Framework/Configurable.h"
+#include "Framework/RunningWorkflowInfo.h"
+#include "Framework/RateLimiter.h"
+#include <fairmq/Device.h>
+
+#include <iostream>
+#include <chrono>
+#include <thread>
+#include <vector>
+
+using namespace o2::framework;
+
+#include "Framework/runDataProcessing.h"
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& specs)
+{
+  DataProcessorSpec a{
+    .name = "A",
+    .outputs = {OutputSpec{{"a1"}, "TST", "A1"},
+                OutputSpec{{"a2"}, "TST", "A2"}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, RawDeviceService& device, DataTakingContext& context, ProcessingContext& pcx) {
+        outputs.make<int>(OutputRef{"a1"}, 1);
+        static int i = 0;
+        outputs.make<int>(OutputRef{"a1"}, 1);
+        if (i++ % 2 == 0) {
+          outputs.make<int>(OutputRef{"a2"}, 1);
+        }
+      })},
+  };
+  DataProcessorSpec d{
+    .name = "D",
+    .inputs = {InputSpec{"a1", "TST", "A1"},
+               InputSpec{"a2", "TST", "A2"}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](InputRecord& inputs) {
+        auto ref = inputs.get("a1");
+        auto header = o2::header::get<const DataProcessingHeader*>(ref.header);
+        LOG(info) << "Start time: " << header->startTime;
+      })},
+  };
+
+  return workflow::concat(WorkflowSpec{a},
+                          WorkflowSpec{d});
+}

From b41d10b74e22566e2ae77e0a4f3e08a439531b87 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 11 Jan 2024 13:48:25 +0100
Subject: [PATCH 204/726] DPL: remove need for special engineering type

We can simply use Instruments one and convert them to something sensible when
using the FairLogger implementation.
---
 Framework/Core/src/runDataProcessing.cxx      |  4 +-
 .../Foundation/include/Framework/Signpost.h   | 48 ++++++++++++++-----
 Framework/Foundation/test/test_Signpost.cxx   |  2 +-
 .../Foundation/test/test_SignpostLogger.cxx   |  2 +-
 4 files changed, 40 insertions(+), 16 deletions(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index e9d1f3d4ee314..325417235cd83 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -800,7 +800,7 @@ void processChildrenOutput(DriverInfo& driverInfo,
     }
 
     O2_SIGNPOST_ID_FROM_POINTER(sid, driver, &info);
-    O2_SIGNPOST_START(driver, sid, "bytes_processed", "bytes processed by " O2_ENG_TYPE(pid, "d"), info.pid);
+    O2_SIGNPOST_START(driver, sid, "bytes_processed", "bytes processed by %{xcode:pid}d", info.pid);
 
     std::string_view s = info.unprinted;
     size_t pos = 0;
@@ -848,7 +848,7 @@ void processChildrenOutput(DriverInfo& driverInfo,
     size_t oldSize = info.unprinted.size();
     info.unprinted = std::string(s);
     int64_t bytesProcessed = oldSize - info.unprinted.size();
-    O2_SIGNPOST_END(driver, sid, "bytes_processed", "bytes processed by " O2_ENG_TYPE(network - size - in - bytes, PRIi64), bytesProcessed);
+    O2_SIGNPOST_END(driver, sid, "bytes_processed", "bytes processed by %{xcode:network-size-in-bytes}" PRIi64, bytesProcessed);
   }
 }
 
diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index 44421687a75c9..f286e43d16afc 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -12,6 +12,7 @@
 #define O2_FRAMEWORK_SIGNPOST_H_
 
 #include <atomic>
+#include <array>
 
 struct o2_log_handle_t {
   char const* name = nullptr;
@@ -19,6 +20,31 @@ struct o2_log_handle_t {
   o2_log_handle_t* next = nullptr;
 };
 
+// Helper function which replaces engineering types with a printf
+// compatible format string.
+template <auto N>
+constexpr auto remove_engineering_type(char const (&src)[N]) {
+  std::array<char, N> res = {};
+  // do whatever string manipulation you want in res.
+  char *t = res.data();
+  for (int i = 0; i < N; ++i) {
+    if (src[i] == '%' && src[i+1] == '{')
+    {
+      *t++ = src[i];
+      while (src[i] != '}' && src[i] != 0) {
+        ++i;
+      }
+      if (src[i] == 0) {
+        *t = 0;
+        return res;
+      }
+    } else {
+      *t++ = src[i];
+    }
+  }
+  return res;
+}
+
 // Loggers registry is actually a feature available to all platforms
 // We use this to register the loggers and to walk over them.
 // So that also on mac we can have a list of all the registered loggers.
@@ -71,10 +97,10 @@ void* _o2_log_create(char const* name, char const* category);
 #define O2_LOG_DEBUG(log, ...) os_log_debug(private_o2_log_##log, __VA_ARGS__)
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer) os_signpost_id_t name = os_signpost_id_make_with_pointer(private_o2_log_##log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log) os_signpost_id_t name = os_signpost_id_generate(private_o2_log_##log)
-#define O2_SIGNPOST_EVENT_EMIT(log, id, name, ...) os_signpost_event_emit(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_SIGNPOST_START(log, id, name, ...) os_signpost_interval_begin(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_SIGNPOST_END(log, id, name, ...) os_signpost_interval_end(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_ENG_TYPE(x, what) "%{xcode:" #x "}" what
+// FIXME: use __VA_OPT__ when available in C++20
+#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) os_signpost_event_emit(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_START(log, id, name, format, ...) os_signpost_interval_begin(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_END(log, id, name, format, ...) os_signpost_interval_end(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
 
 #ifdef O2_SIGNPOST_IMPLEMENTATION
 /// We use a wrapper so that we can keep track of the logs.
@@ -465,10 +491,9 @@ void _o2_log_set_stacktrace(_o2_log_t* log, int stacktrace)
 #define O2_LOG_DEBUG(log, ...) O2_LOG_MACRO(__VA_ARGS__)
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer) _o2_signpost_id_t name = _o2_signpost_id_make_with_pointer(private_o2_log_##log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log) _o2_signpost_id_t name = _o2_signpost_id_generate_local(private_o2_log_##log)
-#define O2_SIGNPOST_EVENT_EMIT(log, id, name, ...) _o2_signpost_event_emit(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_SIGNPOST_START(log, id, name, ...) _o2_signpost_interval_begin(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_SIGNPOST_END(log, id, name, ...) _o2_signpost_interval_end(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_ENG_TYPE(x, what) "%" what
+#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
+#define O2_SIGNPOST_START(log, id, name, format, ...) _o2_signpost_interval_begin(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
+#define O2_SIGNPOST_END(log, id, name, format, ...) _o2_signpost_interval_end(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
 #else // This is the release implementation, it does nothing.
 #define O2_DECLARE_DYNAMIC_LOG(x)
 #define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(x)
@@ -478,10 +503,9 @@ void _o2_log_set_stacktrace(_o2_log_t* log, int stacktrace)
 #define O2_LOG_DEBUG(log, ...)
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log)
-#define O2_SIGNPOST_EVENT_EMIT(log, id, name, ...)
-#define O2_SIGNPOST_START(log, id, name, ...)
-#define O2_SIGNPOST_END(log, id, name, ...)
-#define O2_ENG_TYPE(x)
+#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...)
+#define O2_SIGNPOST_START(log, id, name, format, ...)
+#define O2_SIGNPOST_END(log, id, name, format, ...)
 #endif
 
 #endif // O2_FRAMEWORK_SIGNPOST_H_
diff --git a/Framework/Foundation/test/test_Signpost.cxx b/Framework/Foundation/test/test_Signpost.cxx
index faf5a259fc6a6..24b6afaec5c3d 100644
--- a/Framework/Foundation/test/test_Signpost.cxx
+++ b/Framework/Foundation/test/test_Signpost.cxx
@@ -38,7 +38,7 @@ int main(int argc, char** argv)
 
   // This has an engineering type, which we will not use on Linux / FairLogger
   O2_SIGNPOST_ID_FROM_POINTER(id4, test_Signpost, &id3);
-  O2_SIGNPOST_START(test_Signpost, id4, "Test category", "A signpost with an engineering type formatter " O2_ENG_TYPE(size - in - bytes, "d"), 1);
+  O2_SIGNPOST_START(test_Signpost, id4, "Test category", "A signpost with an engineering type formatter %{size-in-bytes}d", 1);
   O2_SIGNPOST_END(test_Signpost, id4, "Test category", "A signpost interval from a pointer");
 
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
diff --git a/Framework/Foundation/test/test_SignpostLogger.cxx b/Framework/Foundation/test/test_SignpostLogger.cxx
index 4ce862dfb4c74..74da35abf7c70 100644
--- a/Framework/Foundation/test/test_SignpostLogger.cxx
+++ b/Framework/Foundation/test/test_SignpostLogger.cxx
@@ -46,7 +46,7 @@ int main(int argc, char** argv)
 
   // This has an engineering type, which we will not use on Linux / FairLogger
   O2_SIGNPOST_ID_FROM_POINTER(id4, test_Signpost, &id3);
-  O2_SIGNPOST_START(test_Signpost, id4, "Test category", "A signpost with an engineering type formatter " O2_ENG_TYPE(size - in - bytes, "d"), 1);
+  O2_SIGNPOST_START(test_Signpost, id4, "Test category", "A signpost with an engineering type formatter %{size-in-bytes}d", 1);
   O2_SIGNPOST_END(test_Signpost, id4, "Test category", "A signpost interval from a pointer");
 
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");

From 3e520dfee34d48a356e06e64bb0a3a6079ee8efd Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 11 Jan 2024 19:54:46 +0100
Subject: [PATCH 205/726] DPL: remove need for special engineering type

We can simply use Instruments one and convert them to something sensible when
using the FairLogger implementation.
---
 Framework/Core/src/runDataProcessing.cxx      |  4 +-
 .../Foundation/include/Framework/Signpost.h   | 48 ++++++++++++++-----
 Framework/Foundation/test/test_Signpost.cxx   |  2 +-
 .../Foundation/test/test_SignpostLogger.cxx   |  2 +-
 4 files changed, 40 insertions(+), 16 deletions(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index e9d1f3d4ee314..325417235cd83 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -800,7 +800,7 @@ void processChildrenOutput(DriverInfo& driverInfo,
     }
 
     O2_SIGNPOST_ID_FROM_POINTER(sid, driver, &info);
-    O2_SIGNPOST_START(driver, sid, "bytes_processed", "bytes processed by " O2_ENG_TYPE(pid, "d"), info.pid);
+    O2_SIGNPOST_START(driver, sid, "bytes_processed", "bytes processed by %{xcode:pid}d", info.pid);
 
     std::string_view s = info.unprinted;
     size_t pos = 0;
@@ -848,7 +848,7 @@ void processChildrenOutput(DriverInfo& driverInfo,
     size_t oldSize = info.unprinted.size();
     info.unprinted = std::string(s);
     int64_t bytesProcessed = oldSize - info.unprinted.size();
-    O2_SIGNPOST_END(driver, sid, "bytes_processed", "bytes processed by " O2_ENG_TYPE(network - size - in - bytes, PRIi64), bytesProcessed);
+    O2_SIGNPOST_END(driver, sid, "bytes_processed", "bytes processed by %{xcode:network-size-in-bytes}" PRIi64, bytesProcessed);
   }
 }
 
diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index 44421687a75c9..f286e43d16afc 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -12,6 +12,7 @@
 #define O2_FRAMEWORK_SIGNPOST_H_
 
 #include <atomic>
+#include <array>
 
 struct o2_log_handle_t {
   char const* name = nullptr;
@@ -19,6 +20,31 @@ struct o2_log_handle_t {
   o2_log_handle_t* next = nullptr;
 };
 
+// Helper function which replaces engineering types with a printf
+// compatible format string.
+template <auto N>
+constexpr auto remove_engineering_type(char const (&src)[N]) {
+  std::array<char, N> res = {};
+  // do whatever string manipulation you want in res.
+  char *t = res.data();
+  for (int i = 0; i < N; ++i) {
+    if (src[i] == '%' && src[i+1] == '{')
+    {
+      *t++ = src[i];
+      while (src[i] != '}' && src[i] != 0) {
+        ++i;
+      }
+      if (src[i] == 0) {
+        *t = 0;
+        return res;
+      }
+    } else {
+      *t++ = src[i];
+    }
+  }
+  return res;
+}
+
 // Loggers registry is actually a feature available to all platforms
 // We use this to register the loggers and to walk over them.
 // So that also on mac we can have a list of all the registered loggers.
@@ -71,10 +97,10 @@ void* _o2_log_create(char const* name, char const* category);
 #define O2_LOG_DEBUG(log, ...) os_log_debug(private_o2_log_##log, __VA_ARGS__)
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer) os_signpost_id_t name = os_signpost_id_make_with_pointer(private_o2_log_##log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log) os_signpost_id_t name = os_signpost_id_generate(private_o2_log_##log)
-#define O2_SIGNPOST_EVENT_EMIT(log, id, name, ...) os_signpost_event_emit(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_SIGNPOST_START(log, id, name, ...) os_signpost_interval_begin(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_SIGNPOST_END(log, id, name, ...) os_signpost_interval_end(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_ENG_TYPE(x, what) "%{xcode:" #x "}" what
+// FIXME: use __VA_OPT__ when available in C++20
+#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) os_signpost_event_emit(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_START(log, id, name, format, ...) os_signpost_interval_begin(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_END(log, id, name, format, ...) os_signpost_interval_end(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
 
 #ifdef O2_SIGNPOST_IMPLEMENTATION
 /// We use a wrapper so that we can keep track of the logs.
@@ -465,10 +491,9 @@ void _o2_log_set_stacktrace(_o2_log_t* log, int stacktrace)
 #define O2_LOG_DEBUG(log, ...) O2_LOG_MACRO(__VA_ARGS__)
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer) _o2_signpost_id_t name = _o2_signpost_id_make_with_pointer(private_o2_log_##log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log) _o2_signpost_id_t name = _o2_signpost_id_generate_local(private_o2_log_##log)
-#define O2_SIGNPOST_EVENT_EMIT(log, id, name, ...) _o2_signpost_event_emit(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_SIGNPOST_START(log, id, name, ...) _o2_signpost_interval_begin(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_SIGNPOST_END(log, id, name, ...) _o2_signpost_interval_end(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_ENG_TYPE(x, what) "%" what
+#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
+#define O2_SIGNPOST_START(log, id, name, format, ...) _o2_signpost_interval_begin(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
+#define O2_SIGNPOST_END(log, id, name, format, ...) _o2_signpost_interval_end(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
 #else // This is the release implementation, it does nothing.
 #define O2_DECLARE_DYNAMIC_LOG(x)
 #define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(x)
@@ -478,10 +503,9 @@ void _o2_log_set_stacktrace(_o2_log_t* log, int stacktrace)
 #define O2_LOG_DEBUG(log, ...)
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log)
-#define O2_SIGNPOST_EVENT_EMIT(log, id, name, ...)
-#define O2_SIGNPOST_START(log, id, name, ...)
-#define O2_SIGNPOST_END(log, id, name, ...)
-#define O2_ENG_TYPE(x)
+#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...)
+#define O2_SIGNPOST_START(log, id, name, format, ...)
+#define O2_SIGNPOST_END(log, id, name, format, ...)
 #endif
 
 #endif // O2_FRAMEWORK_SIGNPOST_H_
diff --git a/Framework/Foundation/test/test_Signpost.cxx b/Framework/Foundation/test/test_Signpost.cxx
index faf5a259fc6a6..24b6afaec5c3d 100644
--- a/Framework/Foundation/test/test_Signpost.cxx
+++ b/Framework/Foundation/test/test_Signpost.cxx
@@ -38,7 +38,7 @@ int main(int argc, char** argv)
 
   // This has an engineering type, which we will not use on Linux / FairLogger
   O2_SIGNPOST_ID_FROM_POINTER(id4, test_Signpost, &id3);
-  O2_SIGNPOST_START(test_Signpost, id4, "Test category", "A signpost with an engineering type formatter " O2_ENG_TYPE(size - in - bytes, "d"), 1);
+  O2_SIGNPOST_START(test_Signpost, id4, "Test category", "A signpost with an engineering type formatter %{size-in-bytes}d", 1);
   O2_SIGNPOST_END(test_Signpost, id4, "Test category", "A signpost interval from a pointer");
 
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
diff --git a/Framework/Foundation/test/test_SignpostLogger.cxx b/Framework/Foundation/test/test_SignpostLogger.cxx
index 4ce862dfb4c74..74da35abf7c70 100644
--- a/Framework/Foundation/test/test_SignpostLogger.cxx
+++ b/Framework/Foundation/test/test_SignpostLogger.cxx
@@ -46,7 +46,7 @@ int main(int argc, char** argv)
 
   // This has an engineering type, which we will not use on Linux / FairLogger
   O2_SIGNPOST_ID_FROM_POINTER(id4, test_Signpost, &id3);
-  O2_SIGNPOST_START(test_Signpost, id4, "Test category", "A signpost with an engineering type formatter " O2_ENG_TYPE(size - in - bytes, "d"), 1);
+  O2_SIGNPOST_START(test_Signpost, id4, "Test category", "A signpost with an engineering type formatter %{size-in-bytes}d", 1);
   O2_SIGNPOST_END(test_Signpost, id4, "Test category", "A signpost interval from a pointer");
 
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");

From 8e52e59a168bc106a3a8e15bd28ac34d7df07da1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 11 Jan 2024 20:14:16 +0100
Subject: [PATCH 206/726] DPL: detect when a Lifetime::Timeframe output is
 missing

---
 .../Core/include/Framework/MessageContext.h   |  1 +
 .../Core/include/Framework/StreamContext.h    |  7 +++
 Framework/Core/src/CommonServices.cxx         | 42 +++++++++++++
 Framework/Core/src/DataAllocator.cxx          |  3 +
 Framework/TestWorkflows/CMakeLists.txt        |  3 +
 .../test/test_DetectMissingTimeframe.cxx      | 62 +++++++++++++++++++
 6 files changed, 118 insertions(+)
 create mode 100644 Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx

diff --git a/Framework/Core/include/Framework/MessageContext.h b/Framework/Core/include/Framework/MessageContext.h
index c1f2c695ff58a..9e7268f614fda 100644
--- a/Framework/Core/include/Framework/MessageContext.h
+++ b/Framework/Core/include/Framework/MessageContext.h
@@ -496,6 +496,7 @@ class MessageContext
   o2::header::Stack* findMessageHeaderStack(const Output& spec);
   int countDeviceOutputs(bool excludeDPLOrigin = false);
   void fakeDispatch() { mDidDispatch = true; }
+  bool didDispatch() { return mDidDispatch; }
   o2::framework::DataProcessingHeader* findMessageDataProcessingHeader(const Output& spec);
   std::pair<o2::header::DataHeader*, o2::framework::DataProcessingHeader*> findMessageHeaders(const Output& spec);
 
diff --git a/Framework/Core/include/Framework/StreamContext.h b/Framework/Core/include/Framework/StreamContext.h
index 0ab353e147276..198802179a419 100644
--- a/Framework/Core/include/Framework/StreamContext.h
+++ b/Framework/Core/include/Framework/StreamContext.h
@@ -63,6 +63,13 @@ struct StreamContext {
   // Notice that in such a case all the services will be created upfront, so
   // the callback will be called for all of them.
   std::vector<ServiceStartStreamHandle> preStartStreamHandles;
+
+  // Information on wether or not all the required routes have been created.
+  // This is used to check if the LifetimeTimeframe routes were all created
+  // for a given iteration.
+  // This is in the stream context to allow tracking data creation on a per thread
+  // basis.
+  std::vector<bool> routeCreated;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 05b8fba47e532..4656ddeb12e63 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -162,6 +162,48 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
     .uniqueId = simpleServiceId<StreamContext>(),
     .init = simpleServiceInit<StreamContext, StreamContext, ServiceKind::Stream>(),
     .configure = noConfiguration(),
+    .preProcessing = [](ProcessingContext& context, void* service) {
+      auto* stream = (StreamContext*)service;
+      auto& routes = context.services().get<DeviceSpec const>().outputs;
+      // Notice I need to do this here, because different invocation for
+      // the same stream might be referring to different data processors.
+      // We should probably have a context which is per stream of a specific
+      // data processor.
+      stream->routeCreated.resize(routes.size());
+      // Reset the routeCreated at every processing step
+      std::fill(stream->routeCreated.begin(), stream->routeCreated.end(), false); },
+    .postProcessing = [](ProcessingContext& processingContext, void* service) {
+      auto* stream = (StreamContext*)service;
+      auto& routes = processingContext.services().get<DeviceSpec const>().outputs;
+      auto& timeslice = processingContext.services().get<TimingInfo>().timeslice;
+      auto& messageContext = processingContext.services().get<MessageContext>();
+      // Check if we never created any data for this timeslice
+      // if we did not, but we still have didDispatched set to true
+      // it means it was created out of band.
+      bool didCreate = false;
+      for (size_t ri = 0; ri < routes.size(); ++ri) {
+        if (stream->routeCreated[ri] == true) {
+          didCreate = true;
+          break;
+        }
+      }
+      if (didCreate == false && messageContext.didDispatch() == true) {
+        LOGP(debug, "Data created out of band");
+        return;
+      }
+      for (size_t ri = 0; ri < routes.size(); ++ri) {
+        if (stream->routeCreated[ri] == true) {
+          continue;
+        }
+        auto &route = routes[ri];
+        auto &matcher = route.matcher;
+        if ((timeslice % route.maxTimeslices) != route.timeslice) {
+          continue;
+        }
+        if (matcher.lifetime == Lifetime::Timeframe) {
+          LOGP(error, "Expected Lifetime::Timeframe data {} was not created for timeslice {} and might result in dropped timeframes", DataSpecUtils::describe(matcher), timeslice);
+        }
+      } },
     .kind = ServiceKind::Stream};
 }
 
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index 1230d2e942f6e..9ce72aed8c1cf 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -19,6 +19,7 @@
 #include "Framework/FairMQResizableBuffer.h"
 #include "Framework/DataProcessingContext.h"
 #include "Framework/DeviceSpec.h"
+#include "Framework/StreamContext.h"
 #include "Headers/Stack.h"
 
 #include <fairmq/Device.h>
@@ -47,10 +48,12 @@ DataAllocator::DataAllocator(ServiceRegistryRef contextRegistry)
 RouteIndex DataAllocator::matchDataHeader(const Output& spec, size_t timeslice)
 {
   auto& allowedOutputRoutes = mRegistry.get<DeviceSpec const>().outputs;
+  auto& stream = mRegistry.get<o2::framework::StreamContext>();
   // FIXME: we should take timeframeId into account as well.
   for (auto ri = 0; ri < allowedOutputRoutes.size(); ++ri) {
     auto& route = allowedOutputRoutes[ri];
     if (DataSpecUtils::match(route.matcher, spec.origin, spec.description, spec.subSpec) && ((timeslice % route.maxTimeslices) == route.timeslice)) {
+      stream.routeCreated[ri] = true;
       return RouteIndex{ri};
     }
   }
diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index 7a396c0f2c3a9..64e9dddbb020e 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -12,6 +12,9 @@
 o2_add_dpl_workflow(dummy-workflow
                   SOURCES src/o2DummyWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
+o2_add_dpl_workflow(detect-missing-timeframe
+                  SOURCES test/test_DetectMissingTimeframe.cxx
+                  COMPONENT_NAME TestWorkflows)
 
 o2_add_dpl_workflow(o2rootmessage-workflow
                   SOURCES "src/test_o2RootMessageWorkflow.cxx"
diff --git a/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx b/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
new file mode 100644
index 0000000000000..3374cd9ad6cf2
--- /dev/null
+++ b/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
@@ -0,0 +1,62 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/DataTakingContext.h"
+#include "Framework/CompletionPolicyHelpers.h"
+#include "Framework/DeviceSpec.h"
+#include "Framework/RawDeviceService.h"
+#include "Framework/ControlService.h"
+#include "Framework/Configurable.h"
+#include "Framework/RunningWorkflowInfo.h"
+#include "Framework/RateLimiter.h"
+#include <fairmq/Device.h>
+
+#include <iostream>
+#include <chrono>
+#include <thread>
+#include <vector>
+
+using namespace o2::framework;
+
+#include "Framework/runDataProcessing.h"
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& specs)
+{
+  DataProcessorSpec a{
+    .name = "A",
+    .outputs = {OutputSpec{{"a1"}, "TST", "A1"},
+                OutputSpec{{"a2"}, "TST", "A2"}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, RawDeviceService& device, DataTakingContext& context, ProcessingContext& pcx) {
+        outputs.make<int>(OutputRef{"a1"}, 1);
+        static int i = 0;
+        outputs.make<int>(OutputRef{"a1"}, 1);
+        if (i++ % 2 == 0) {
+          outputs.make<int>(OutputRef{"a2"}, 1);
+        }
+      })},
+  };
+  DataProcessorSpec d{
+    .name = "D",
+    .inputs = {InputSpec{"a1", "TST", "A1"},
+               InputSpec{"a2", "TST", "A2"}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](InputRecord& inputs) {
+        auto ref = inputs.get("a1");
+        auto header = o2::header::get<const DataProcessingHeader*>(ref.header);
+        LOG(info) << "Start time: " << header->startTime;
+      })},
+  };
+
+  return workflow::concat(WorkflowSpec{a},
+                          WorkflowSpec{d});
+}

From 232bc84bb69b23651c7bf56003a694bb3f4c6377 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 12 Jan 2024 06:52:11 +0100
Subject: [PATCH 207/726] DPL: remove need for special engineering type

We can simply use Instruments one and convert them to something sensible when
using the FairLogger implementation.
---
 Framework/Core/src/runDataProcessing.cxx      |  4 +-
 .../Foundation/include/Framework/Signpost.h   | 48 ++++++++++++++-----
 Framework/Foundation/test/test_Signpost.cxx   |  2 +-
 .../Foundation/test/test_SignpostLogger.cxx   |  2 +-
 4 files changed, 40 insertions(+), 16 deletions(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index e9d1f3d4ee314..325417235cd83 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -800,7 +800,7 @@ void processChildrenOutput(DriverInfo& driverInfo,
     }
 
     O2_SIGNPOST_ID_FROM_POINTER(sid, driver, &info);
-    O2_SIGNPOST_START(driver, sid, "bytes_processed", "bytes processed by " O2_ENG_TYPE(pid, "d"), info.pid);
+    O2_SIGNPOST_START(driver, sid, "bytes_processed", "bytes processed by %{xcode:pid}d", info.pid);
 
     std::string_view s = info.unprinted;
     size_t pos = 0;
@@ -848,7 +848,7 @@ void processChildrenOutput(DriverInfo& driverInfo,
     size_t oldSize = info.unprinted.size();
     info.unprinted = std::string(s);
     int64_t bytesProcessed = oldSize - info.unprinted.size();
-    O2_SIGNPOST_END(driver, sid, "bytes_processed", "bytes processed by " O2_ENG_TYPE(network - size - in - bytes, PRIi64), bytesProcessed);
+    O2_SIGNPOST_END(driver, sid, "bytes_processed", "bytes processed by %{xcode:network-size-in-bytes}" PRIi64, bytesProcessed);
   }
 }
 
diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index 44421687a75c9..eca9e26f77469 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -12,6 +12,7 @@
 #define O2_FRAMEWORK_SIGNPOST_H_
 
 #include <atomic>
+#include <array>
 
 struct o2_log_handle_t {
   char const* name = nullptr;
@@ -19,6 +20,31 @@ struct o2_log_handle_t {
   o2_log_handle_t* next = nullptr;
 };
 
+// Helper function which replaces engineering types with a printf
+// compatible format string.
+template <auto N>
+constexpr auto remove_engineering_type(char const (&src)[N])
+{
+  std::array<char, N> res = {};
+  // do whatever string manipulation you want in res.
+  char* t = res.data();
+  for (int i = 0; i < N; ++i) {
+    if (src[i] == '%' && src[i + 1] == '{') {
+      *t++ = src[i];
+      while (src[i] != '}' && src[i] != 0) {
+        ++i;
+      }
+      if (src[i] == 0) {
+        *t = 0;
+        return res;
+      }
+    } else {
+      *t++ = src[i];
+    }
+  }
+  return res;
+}
+
 // Loggers registry is actually a feature available to all platforms
 // We use this to register the loggers and to walk over them.
 // So that also on mac we can have a list of all the registered loggers.
@@ -71,10 +97,10 @@ void* _o2_log_create(char const* name, char const* category);
 #define O2_LOG_DEBUG(log, ...) os_log_debug(private_o2_log_##log, __VA_ARGS__)
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer) os_signpost_id_t name = os_signpost_id_make_with_pointer(private_o2_log_##log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log) os_signpost_id_t name = os_signpost_id_generate(private_o2_log_##log)
-#define O2_SIGNPOST_EVENT_EMIT(log, id, name, ...) os_signpost_event_emit(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_SIGNPOST_START(log, id, name, ...) os_signpost_interval_begin(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_SIGNPOST_END(log, id, name, ...) os_signpost_interval_end(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_ENG_TYPE(x, what) "%{xcode:" #x "}" what
+// FIXME: use __VA_OPT__ when available in C++20
+#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) os_signpost_event_emit(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_START(log, id, name, format, ...) os_signpost_interval_begin(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_END(log, id, name, format, ...) os_signpost_interval_end(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
 
 #ifdef O2_SIGNPOST_IMPLEMENTATION
 /// We use a wrapper so that we can keep track of the logs.
@@ -465,10 +491,9 @@ void _o2_log_set_stacktrace(_o2_log_t* log, int stacktrace)
 #define O2_LOG_DEBUG(log, ...) O2_LOG_MACRO(__VA_ARGS__)
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer) _o2_signpost_id_t name = _o2_signpost_id_make_with_pointer(private_o2_log_##log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log) _o2_signpost_id_t name = _o2_signpost_id_generate_local(private_o2_log_##log)
-#define O2_SIGNPOST_EVENT_EMIT(log, id, name, ...) _o2_signpost_event_emit(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_SIGNPOST_START(log, id, name, ...) _o2_signpost_interval_begin(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_SIGNPOST_END(log, id, name, ...) _o2_signpost_interval_end(private_o2_log_##log, id, name, __VA_ARGS__)
-#define O2_ENG_TYPE(x, what) "%" what
+#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
+#define O2_SIGNPOST_START(log, id, name, format, ...) _o2_signpost_interval_begin(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
+#define O2_SIGNPOST_END(log, id, name, format, ...) _o2_signpost_interval_end(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
 #else // This is the release implementation, it does nothing.
 #define O2_DECLARE_DYNAMIC_LOG(x)
 #define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(x)
@@ -478,10 +503,9 @@ void _o2_log_set_stacktrace(_o2_log_t* log, int stacktrace)
 #define O2_LOG_DEBUG(log, ...)
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log)
-#define O2_SIGNPOST_EVENT_EMIT(log, id, name, ...)
-#define O2_SIGNPOST_START(log, id, name, ...)
-#define O2_SIGNPOST_END(log, id, name, ...)
-#define O2_ENG_TYPE(x)
+#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...)
+#define O2_SIGNPOST_START(log, id, name, format, ...)
+#define O2_SIGNPOST_END(log, id, name, format, ...)
 #endif
 
 #endif // O2_FRAMEWORK_SIGNPOST_H_
diff --git a/Framework/Foundation/test/test_Signpost.cxx b/Framework/Foundation/test/test_Signpost.cxx
index faf5a259fc6a6..24b6afaec5c3d 100644
--- a/Framework/Foundation/test/test_Signpost.cxx
+++ b/Framework/Foundation/test/test_Signpost.cxx
@@ -38,7 +38,7 @@ int main(int argc, char** argv)
 
   // This has an engineering type, which we will not use on Linux / FairLogger
   O2_SIGNPOST_ID_FROM_POINTER(id4, test_Signpost, &id3);
-  O2_SIGNPOST_START(test_Signpost, id4, "Test category", "A signpost with an engineering type formatter " O2_ENG_TYPE(size - in - bytes, "d"), 1);
+  O2_SIGNPOST_START(test_Signpost, id4, "Test category", "A signpost with an engineering type formatter %{size-in-bytes}d", 1);
   O2_SIGNPOST_END(test_Signpost, id4, "Test category", "A signpost interval from a pointer");
 
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
diff --git a/Framework/Foundation/test/test_SignpostLogger.cxx b/Framework/Foundation/test/test_SignpostLogger.cxx
index 4ce862dfb4c74..74da35abf7c70 100644
--- a/Framework/Foundation/test/test_SignpostLogger.cxx
+++ b/Framework/Foundation/test/test_SignpostLogger.cxx
@@ -46,7 +46,7 @@ int main(int argc, char** argv)
 
   // This has an engineering type, which we will not use on Linux / FairLogger
   O2_SIGNPOST_ID_FROM_POINTER(id4, test_Signpost, &id3);
-  O2_SIGNPOST_START(test_Signpost, id4, "Test category", "A signpost with an engineering type formatter " O2_ENG_TYPE(size - in - bytes, "d"), 1);
+  O2_SIGNPOST_START(test_Signpost, id4, "Test category", "A signpost with an engineering type formatter %{size-in-bytes}d", 1);
   O2_SIGNPOST_END(test_Signpost, id4, "Test category", "A signpost interval from a pointer");
 
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");

From 44d49baba95acd91b6226d1fcd43d494f0eff9c8 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 12 Jan 2024 12:37:11 +0100
Subject: [PATCH 208/726] DPL: wait as long as possible for Sporadic inputs

Right now if we have the standard consumeWhenAll policy and we have a sporadic
input, it will wait indefinitely until all the inputs arrive or it will drop
timeframes without that sporadic input.

This changes the behavior and waits only until the oldest possible timeframe
does not allow the Sporadic input to be there. At which point, it schedules the
processing in any case, under the assumption that a task declaring a sporadic
input knows what to do in case it's not there.
---
 .../Core/src/CompletionPolicyHelpers.cxx      | 27 +++++-
 Framework/TestWorkflows/CMakeLists.txt        |  5 ++
 .../test/test_WaitUntilPossible.cxx           | 82 +++++++++++++++++++
 3 files changed, 113 insertions(+), 1 deletion(-)
 create mode 100644 Framework/TestWorkflows/test/test_WaitUntilPossible.cxx

diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx
index b3e0621bf984e..a24c7342bd6fb 100644
--- a/Framework/Core/src/CompletionPolicyHelpers.cxx
+++ b/Framework/Core/src/CompletionPolicyHelpers.cxx
@@ -15,6 +15,7 @@
 #include "Framework/DeviceSpec.h"
 #include "Framework/CompilerBuiltins.h"
 #include "Framework/Logger.h"
+#include "Framework/TimesliceIndex.h"
 #include "Framework/TimingInfo.h"
 #include "DecongestionService.h"
 
@@ -107,10 +108,34 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
 {
   auto callback = [](InputSpan const& inputs, std::vector<InputSpec> const& specs, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     assert(inputs.size() == specs.size());
+
+    size_t si = 0;
+    bool missingSporadic = false;
+    bool missingTimeframe = false;
+    size_t currentTimeslice = -1;
     for (auto& input : inputs) {
-      if (input.header == nullptr) {
+      assert(si < specs.size());
+      auto& spec = specs[si++];
+      if (input.header == nullptr && spec.lifetime != Lifetime::Sporadic) {
         return CompletionPolicy::CompletionOp::Wait;
       }
+      if (input.header == nullptr && spec.lifetime == Lifetime::Sporadic) {
+        missingSporadic |= true;
+      }
+      if (input.header != nullptr && currentTimeslice == -1) {
+        auto const* dph = framework::DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(input);
+        if (dph && !TimingInfo::timesliceIsTimer(dph->startTime)) {
+          currentTimeslice = dph->startTime;
+        }
+      }
+    }
+    // If some sporadic inputs are missing, we wait for them util we are sure they will not come,
+    // i.e. until the oldest possible timeslice is beyond the timeslice of the input.
+    auto& timesliceIndex = ref.get<TimesliceIndex>();
+    auto oldestPossibleTimeslice = timesliceIndex.getOldestPossibleInput().timeslice.value;
+
+    if (missingSporadic && currentTimeslice >= oldestPossibleTimeslice) {
+      return CompletionPolicy::CompletionOp::Retry;
     }
     return CompletionPolicy::CompletionOp::Consume;
   };
diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index 64e9dddbb020e..1aa20b4efdd3b 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -12,9 +12,14 @@
 o2_add_dpl_workflow(dummy-workflow
                   SOURCES src/o2DummyWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(detect-missing-timeframe
                   SOURCES test/test_DetectMissingTimeframe.cxx
                   COMPONENT_NAME TestWorkflows)
+                
+o2_add_dpl_workflow(wait-until-possible
+                  SOURCES test/test_WaitUntilPossible.cxx
+                  COMPONENT_NAME TestWorkflows)
 
 o2_add_dpl_workflow(o2rootmessage-workflow
                   SOURCES "src/test_o2RootMessageWorkflow.cxx"
diff --git a/Framework/TestWorkflows/test/test_WaitUntilPossible.cxx b/Framework/TestWorkflows/test/test_WaitUntilPossible.cxx
new file mode 100644
index 0000000000000..d9ba315ef02b8
--- /dev/null
+++ b/Framework/TestWorkflows/test/test_WaitUntilPossible.cxx
@@ -0,0 +1,82 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/DataTakingContext.h"
+#include "Framework/CompletionPolicyHelpers.h"
+#include "Framework/DeviceSpec.h"
+#include "Framework/RawDeviceService.h"
+#include "Framework/ControlService.h"
+#include "Framework/Configurable.h"
+#include "Framework/RunningWorkflowInfo.h"
+#include "Framework/RateLimiter.h"
+#include <fairmq/Device.h>
+
+#include <iostream>
+#include <chrono>
+#include <thread>
+#include <vector>
+
+using namespace o2::framework;
+
+#include "Framework/runDataProcessing.h"
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& specs)
+{
+  DataProcessorSpec a{
+    .name = "A",
+    .outputs = {OutputSpec{{"data"}, "TST", "A1", 0}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, RawDeviceService& device, DataTakingContext& context, ProcessingContext& pcx) {
+        LOG(info) << "Data TST/A1/0 created";
+        outputs.make<int>(OutputRef{"data"}, 1);
+      })},
+  };
+  DataProcessorSpec b{
+    .name = "B",
+    .outputs = {OutputSpec{{"sporadic"}, "TST", "B1", 0, Lifetime::Sporadic}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, RawDeviceService& device, DataTakingContext& context, ProcessingContext& pcx) {
+        // This will always be late, however since the oldest possible timeframe
+        // will be used to decide the scheduling, it will not be dropped.
+        sleep(1);
+        // We also create it only every second time, so that we can check that
+        // the sporadic output is not mandatory.
+        static int i = 0;
+        if (i++ % 2 == 0) {
+          LOG(info) << "Data TST/B1/0 created";
+          outputs.make<int>(OutputRef{"sporadic"}, 1);
+        }
+      })},
+  };
+  DataProcessorSpec d{
+    .name = "D",
+    .inputs = {InputSpec{"a1", "TST", "A1", 0, Lifetime::Timeframe},
+               InputSpec{"b1", "TST", "B1", 0, Lifetime::Sporadic}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](InputRecord& inputs) {
+        auto refA = inputs.get("a1");
+        auto headerA = o2::header::get<const DataProcessingHeader*>(refA.header);
+        LOG(info) << "Start time: " << headerA->startTime;
+        auto refB = inputs.get("b1");
+        if (!refB.header) {
+          LOG(info) << "No sporadic input for start time " << headerA->startTime;
+          return;
+        }
+        auto headerB = o2::header::get<const DataProcessingHeader*>(refB.header);
+        LOG(info) << "Start time: " << headerB->startTime;
+      })},
+  };
+
+  return workflow::concat(WorkflowSpec{a},
+                          WorkflowSpec{b},
+                          WorkflowSpec{d});
+}

From a59ea0ead75c70e355ac5b4e7eea23a86b4dcdd8 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 12 Jan 2024 12:43:10 +0100
Subject: [PATCH 209/726] DPL: wait as long as possible for Sporadic inputs

Right now if we have the standard consumeWhenAll policy and we have a sporadic
input, it will wait indefinitely until all the inputs arrive or it will drop
timeframes without that sporadic input.

This changes the behavior and waits only until the oldest possible timeframe
does not allow the Sporadic input to be there. At which point, it schedules the
processing in any case, under the assumption that a task declaring a sporadic
input knows what to do in case it's not there.
---
 .../Core/src/CompletionPolicyHelpers.cxx      | 26 +++++-
 Framework/TestWorkflows/CMakeLists.txt        |  5 ++
 .../test/test_WaitUntilPossible.cxx           | 82 +++++++++++++++++++
 3 files changed, 112 insertions(+), 1 deletion(-)
 create mode 100644 Framework/TestWorkflows/test/test_WaitUntilPossible.cxx

diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx
index b3e0621bf984e..61aba49be14d1 100644
--- a/Framework/Core/src/CompletionPolicyHelpers.cxx
+++ b/Framework/Core/src/CompletionPolicyHelpers.cxx
@@ -15,6 +15,7 @@
 #include "Framework/DeviceSpec.h"
 #include "Framework/CompilerBuiltins.h"
 #include "Framework/Logger.h"
+#include "Framework/TimesliceIndex.h"
 #include "Framework/TimingInfo.h"
 #include "DecongestionService.h"
 
@@ -107,10 +108,33 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
 {
   auto callback = [](InputSpan const& inputs, std::vector<InputSpec> const& specs, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     assert(inputs.size() == specs.size());
+
+    size_t si = 0;
+    bool missingSporadic = false;
+    size_t currentTimeslice = -1;
     for (auto& input : inputs) {
-      if (input.header == nullptr) {
+      assert(si < specs.size());
+      auto& spec = specs[si++];
+      if (input.header == nullptr && spec.lifetime != Lifetime::Sporadic) {
         return CompletionPolicy::CompletionOp::Wait;
       }
+      if (input.header == nullptr && spec.lifetime == Lifetime::Sporadic) {
+        missingSporadic |= true;
+      }
+      if (input.header != nullptr && currentTimeslice == -1) {
+        auto const* dph = framework::DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(input);
+        if (dph && !TimingInfo::timesliceIsTimer(dph->startTime)) {
+          currentTimeslice = dph->startTime;
+        }
+      }
+    }
+    // If some sporadic inputs are missing, we wait for them util we are sure they will not come,
+    // i.e. until the oldest possible timeslice is beyond the timeslice of the input.
+    auto& timesliceIndex = ref.get<TimesliceIndex>();
+    auto oldestPossibleTimeslice = timesliceIndex.getOldestPossibleInput().timeslice.value;
+
+    if (missingSporadic && currentTimeslice >= oldestPossibleTimeslice) {
+      return CompletionPolicy::CompletionOp::Retry;
     }
     return CompletionPolicy::CompletionOp::Consume;
   };
diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index 64e9dddbb020e..1aa20b4efdd3b 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -12,9 +12,14 @@
 o2_add_dpl_workflow(dummy-workflow
                   SOURCES src/o2DummyWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(detect-missing-timeframe
                   SOURCES test/test_DetectMissingTimeframe.cxx
                   COMPONENT_NAME TestWorkflows)
+                
+o2_add_dpl_workflow(wait-until-possible
+                  SOURCES test/test_WaitUntilPossible.cxx
+                  COMPONENT_NAME TestWorkflows)
 
 o2_add_dpl_workflow(o2rootmessage-workflow
                   SOURCES "src/test_o2RootMessageWorkflow.cxx"
diff --git a/Framework/TestWorkflows/test/test_WaitUntilPossible.cxx b/Framework/TestWorkflows/test/test_WaitUntilPossible.cxx
new file mode 100644
index 0000000000000..d9ba315ef02b8
--- /dev/null
+++ b/Framework/TestWorkflows/test/test_WaitUntilPossible.cxx
@@ -0,0 +1,82 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/DataTakingContext.h"
+#include "Framework/CompletionPolicyHelpers.h"
+#include "Framework/DeviceSpec.h"
+#include "Framework/RawDeviceService.h"
+#include "Framework/ControlService.h"
+#include "Framework/Configurable.h"
+#include "Framework/RunningWorkflowInfo.h"
+#include "Framework/RateLimiter.h"
+#include <fairmq/Device.h>
+
+#include <iostream>
+#include <chrono>
+#include <thread>
+#include <vector>
+
+using namespace o2::framework;
+
+#include "Framework/runDataProcessing.h"
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& specs)
+{
+  DataProcessorSpec a{
+    .name = "A",
+    .outputs = {OutputSpec{{"data"}, "TST", "A1", 0}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, RawDeviceService& device, DataTakingContext& context, ProcessingContext& pcx) {
+        LOG(info) << "Data TST/A1/0 created";
+        outputs.make<int>(OutputRef{"data"}, 1);
+      })},
+  };
+  DataProcessorSpec b{
+    .name = "B",
+    .outputs = {OutputSpec{{"sporadic"}, "TST", "B1", 0, Lifetime::Sporadic}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, RawDeviceService& device, DataTakingContext& context, ProcessingContext& pcx) {
+        // This will always be late, however since the oldest possible timeframe
+        // will be used to decide the scheduling, it will not be dropped.
+        sleep(1);
+        // We also create it only every second time, so that we can check that
+        // the sporadic output is not mandatory.
+        static int i = 0;
+        if (i++ % 2 == 0) {
+          LOG(info) << "Data TST/B1/0 created";
+          outputs.make<int>(OutputRef{"sporadic"}, 1);
+        }
+      })},
+  };
+  DataProcessorSpec d{
+    .name = "D",
+    .inputs = {InputSpec{"a1", "TST", "A1", 0, Lifetime::Timeframe},
+               InputSpec{"b1", "TST", "B1", 0, Lifetime::Sporadic}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](InputRecord& inputs) {
+        auto refA = inputs.get("a1");
+        auto headerA = o2::header::get<const DataProcessingHeader*>(refA.header);
+        LOG(info) << "Start time: " << headerA->startTime;
+        auto refB = inputs.get("b1");
+        if (!refB.header) {
+          LOG(info) << "No sporadic input for start time " << headerA->startTime;
+          return;
+        }
+        auto headerB = o2::header::get<const DataProcessingHeader*>(refB.header);
+        LOG(info) << "Start time: " << headerB->startTime;
+      })},
+  };
+
+  return workflow::concat(WorkflowSpec{a},
+                          WorkflowSpec{b},
+                          WorkflowSpec{d});
+}

From d4f5f6e452c57121534614d56038d25e4377e3ff Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 12 Jan 2024 14:21:30 +0100
Subject: [PATCH 210/726] DPL: wait as long as possible for Sporadic inputs

Right now if we have the standard consumeWhenAll policy and we have a sporadic
input, it will wait indefinitely until all the inputs arrive or it will drop
timeframes without that sporadic input.

This changes the behavior and waits only until the oldest possible timeframe
does not allow the Sporadic input to be there. At which point, it schedules the
processing in any case, under the assumption that a task declaring a sporadic
input knows what to do in case it's not there.
---
 .../Core/src/CompletionPolicyHelpers.cxx      | 26 +++++-
 Framework/Core/test/test_DataRelayer.cxx      | 11 +++
 Framework/TestWorkflows/CMakeLists.txt        |  5 ++
 .../test/test_WaitUntilPossible.cxx           | 82 +++++++++++++++++++
 4 files changed, 123 insertions(+), 1 deletion(-)
 create mode 100644 Framework/TestWorkflows/test/test_WaitUntilPossible.cxx

diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx
index b3e0621bf984e..61aba49be14d1 100644
--- a/Framework/Core/src/CompletionPolicyHelpers.cxx
+++ b/Framework/Core/src/CompletionPolicyHelpers.cxx
@@ -15,6 +15,7 @@
 #include "Framework/DeviceSpec.h"
 #include "Framework/CompilerBuiltins.h"
 #include "Framework/Logger.h"
+#include "Framework/TimesliceIndex.h"
 #include "Framework/TimingInfo.h"
 #include "DecongestionService.h"
 
@@ -107,10 +108,33 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
 {
   auto callback = [](InputSpan const& inputs, std::vector<InputSpec> const& specs, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     assert(inputs.size() == specs.size());
+
+    size_t si = 0;
+    bool missingSporadic = false;
+    size_t currentTimeslice = -1;
     for (auto& input : inputs) {
-      if (input.header == nullptr) {
+      assert(si < specs.size());
+      auto& spec = specs[si++];
+      if (input.header == nullptr && spec.lifetime != Lifetime::Sporadic) {
         return CompletionPolicy::CompletionOp::Wait;
       }
+      if (input.header == nullptr && spec.lifetime == Lifetime::Sporadic) {
+        missingSporadic |= true;
+      }
+      if (input.header != nullptr && currentTimeslice == -1) {
+        auto const* dph = framework::DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(input);
+        if (dph && !TimingInfo::timesliceIsTimer(dph->startTime)) {
+          currentTimeslice = dph->startTime;
+        }
+      }
+    }
+    // If some sporadic inputs are missing, we wait for them util we are sure they will not come,
+    // i.e. until the oldest possible timeslice is beyond the timeslice of the input.
+    auto& timesliceIndex = ref.get<TimesliceIndex>();
+    auto oldestPossibleTimeslice = timesliceIndex.getOldestPossibleInput().timeslice.value;
+
+    if (missingSporadic && currentTimeslice >= oldestPossibleTimeslice) {
+      return CompletionPolicy::CompletionOp::Retry;
     }
     return CompletionPolicy::CompletionOp::Consume;
   };
diff --git a/Framework/Core/test/test_DataRelayer.cxx b/Framework/Core/test/test_DataRelayer.cxx
index 788417778642b..bd1ead40c9773 100644
--- a/Framework/Core/test/test_DataRelayer.cxx
+++ b/Framework/Core/test/test_DataRelayer.cxx
@@ -76,6 +76,7 @@ TEST_CASE("DataRelayer")
     std::vector<ForwardRoute> forwards;
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::consumeWhenAny();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -124,6 +125,7 @@ TEST_CASE("DataRelayer")
     std::vector<ForwardRoute> forwards;
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::consumeWhenAny();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -184,6 +186,7 @@ TEST_CASE("DataRelayer")
 
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::consumeWhenAll();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -263,6 +266,7 @@ TEST_CASE("DataRelayer")
 
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::consumeWhenAll();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -346,6 +350,7 @@ TEST_CASE("DataRelayer")
     auto policy = CompletionPolicyHelpers::consumeWhenAll();
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
     DataRelayer relayer(policy, inputs, index, {registry});
     // Only two messages to fill the cache.
     relayer.setPipelineLength(2);
@@ -420,6 +425,7 @@ TEST_CASE("DataRelayer")
     std::vector<ForwardRoute> forwards;
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::processWhenAny();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -490,6 +496,7 @@ TEST_CASE("DataRelayer")
     std::vector<ForwardRoute> forwards;
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::processWhenAny();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -547,6 +554,7 @@ TEST_CASE("DataRelayer")
     std::vector<ForwardRoute> forwards;
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::processWhenAny();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -605,6 +613,7 @@ TEST_CASE("DataRelayer")
     std::vector<ForwardRoute> forwards;
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::processWhenAny();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -670,6 +679,7 @@ TEST_CASE("DataRelayer")
     std::vector<ForwardRoute> forwards;
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::consumeWhenAny();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -722,6 +732,7 @@ TEST_CASE("DataRelayer")
     std::vector<ForwardRoute> forwards;
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::consumeWhenAny();
     DataRelayer relayer(policy, inputs, index, {registry});
diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index 64e9dddbb020e..1aa20b4efdd3b 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -12,9 +12,14 @@
 o2_add_dpl_workflow(dummy-workflow
                   SOURCES src/o2DummyWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(detect-missing-timeframe
                   SOURCES test/test_DetectMissingTimeframe.cxx
                   COMPONENT_NAME TestWorkflows)
+                
+o2_add_dpl_workflow(wait-until-possible
+                  SOURCES test/test_WaitUntilPossible.cxx
+                  COMPONENT_NAME TestWorkflows)
 
 o2_add_dpl_workflow(o2rootmessage-workflow
                   SOURCES "src/test_o2RootMessageWorkflow.cxx"
diff --git a/Framework/TestWorkflows/test/test_WaitUntilPossible.cxx b/Framework/TestWorkflows/test/test_WaitUntilPossible.cxx
new file mode 100644
index 0000000000000..d9ba315ef02b8
--- /dev/null
+++ b/Framework/TestWorkflows/test/test_WaitUntilPossible.cxx
@@ -0,0 +1,82 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/DataTakingContext.h"
+#include "Framework/CompletionPolicyHelpers.h"
+#include "Framework/DeviceSpec.h"
+#include "Framework/RawDeviceService.h"
+#include "Framework/ControlService.h"
+#include "Framework/Configurable.h"
+#include "Framework/RunningWorkflowInfo.h"
+#include "Framework/RateLimiter.h"
+#include <fairmq/Device.h>
+
+#include <iostream>
+#include <chrono>
+#include <thread>
+#include <vector>
+
+using namespace o2::framework;
+
+#include "Framework/runDataProcessing.h"
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& specs)
+{
+  DataProcessorSpec a{
+    .name = "A",
+    .outputs = {OutputSpec{{"data"}, "TST", "A1", 0}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, RawDeviceService& device, DataTakingContext& context, ProcessingContext& pcx) {
+        LOG(info) << "Data TST/A1/0 created";
+        outputs.make<int>(OutputRef{"data"}, 1);
+      })},
+  };
+  DataProcessorSpec b{
+    .name = "B",
+    .outputs = {OutputSpec{{"sporadic"}, "TST", "B1", 0, Lifetime::Sporadic}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, RawDeviceService& device, DataTakingContext& context, ProcessingContext& pcx) {
+        // This will always be late, however since the oldest possible timeframe
+        // will be used to decide the scheduling, it will not be dropped.
+        sleep(1);
+        // We also create it only every second time, so that we can check that
+        // the sporadic output is not mandatory.
+        static int i = 0;
+        if (i++ % 2 == 0) {
+          LOG(info) << "Data TST/B1/0 created";
+          outputs.make<int>(OutputRef{"sporadic"}, 1);
+        }
+      })},
+  };
+  DataProcessorSpec d{
+    .name = "D",
+    .inputs = {InputSpec{"a1", "TST", "A1", 0, Lifetime::Timeframe},
+               InputSpec{"b1", "TST", "B1", 0, Lifetime::Sporadic}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](InputRecord& inputs) {
+        auto refA = inputs.get("a1");
+        auto headerA = o2::header::get<const DataProcessingHeader*>(refA.header);
+        LOG(info) << "Start time: " << headerA->startTime;
+        auto refB = inputs.get("b1");
+        if (!refB.header) {
+          LOG(info) << "No sporadic input for start time " << headerA->startTime;
+          return;
+        }
+        auto headerB = o2::header::get<const DataProcessingHeader*>(refB.header);
+        LOG(info) << "Start time: " << headerB->startTime;
+      })},
+  };
+
+  return workflow::concat(WorkflowSpec{a},
+                          WorkflowSpec{b},
+                          WorkflowSpec{d});
+}

From 799976301a2984351675b5da7973fa9f2dcc4423 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 12 Jan 2024 14:54:00 +0100
Subject: [PATCH 211/726] DPL: wait as long as possible for Sporadic inputs

Right now if we have the standard consumeWhenAll policy and we have a sporadic
input, it will wait indefinitely until all the inputs arrive or it will drop
timeframes without that sporadic input.

This changes the behavior and waits only until the oldest possible timeframe
does not allow the Sporadic input to be there. At which point, it schedules the
processing in any case, under the assumption that a task declaring a sporadic
input knows what to do in case it's not there.
---
 .../Core/src/CompletionPolicyHelpers.cxx      | 26 +++++-
 Framework/Core/test/test_DataRelayer.cxx      | 11 +++
 Framework/TestWorkflows/CMakeLists.txt        |  5 ++
 .../test/test_WaitUntilPossible.cxx           | 82 +++++++++++++++++++
 4 files changed, 123 insertions(+), 1 deletion(-)
 create mode 100644 Framework/TestWorkflows/test/test_WaitUntilPossible.cxx

diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx
index b3e0621bf984e..8c4b38ba3355c 100644
--- a/Framework/Core/src/CompletionPolicyHelpers.cxx
+++ b/Framework/Core/src/CompletionPolicyHelpers.cxx
@@ -15,6 +15,7 @@
 #include "Framework/DeviceSpec.h"
 #include "Framework/CompilerBuiltins.h"
 #include "Framework/Logger.h"
+#include "Framework/TimesliceIndex.h"
 #include "Framework/TimingInfo.h"
 #include "DecongestionService.h"
 
@@ -107,10 +108,33 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
 {
   auto callback = [](InputSpan const& inputs, std::vector<InputSpec> const& specs, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     assert(inputs.size() == specs.size());
+
+    size_t si = 0;
+    bool missingSporadic = false;
+    size_t currentTimeslice = -1;
     for (auto& input : inputs) {
-      if (input.header == nullptr) {
+      assert(si < specs.size());
+      auto& spec = specs[si++];
+      if (input.header == nullptr && spec.lifetime != Lifetime::Sporadic) {
         return CompletionPolicy::CompletionOp::Wait;
       }
+      if (input.header == nullptr && spec.lifetime == Lifetime::Sporadic) {
+        missingSporadic = true;
+      }
+      if (input.header != nullptr && currentTimeslice == -1) {
+        auto const* dph = framework::DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(input);
+        if (dph && !TimingInfo::timesliceIsTimer(dph->startTime)) {
+          currentTimeslice = dph->startTime;
+        }
+      }
+    }
+    // If some sporadic inputs are missing, we wait for them util we are sure they will not come,
+    // i.e. until the oldest possible timeslice is beyond the timeslice of the input.
+    auto& timesliceIndex = ref.get<TimesliceIndex>();
+    auto oldestPossibleTimeslice = timesliceIndex.getOldestPossibleInput().timeslice.value;
+
+    if (missingSporadic && currentTimeslice >= oldestPossibleTimeslice) {
+      return CompletionPolicy::CompletionOp::Retry;
     }
     return CompletionPolicy::CompletionOp::Consume;
   };
diff --git a/Framework/Core/test/test_DataRelayer.cxx b/Framework/Core/test/test_DataRelayer.cxx
index 788417778642b..bd1ead40c9773 100644
--- a/Framework/Core/test/test_DataRelayer.cxx
+++ b/Framework/Core/test/test_DataRelayer.cxx
@@ -76,6 +76,7 @@ TEST_CASE("DataRelayer")
     std::vector<ForwardRoute> forwards;
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::consumeWhenAny();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -124,6 +125,7 @@ TEST_CASE("DataRelayer")
     std::vector<ForwardRoute> forwards;
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::consumeWhenAny();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -184,6 +186,7 @@ TEST_CASE("DataRelayer")
 
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::consumeWhenAll();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -263,6 +266,7 @@ TEST_CASE("DataRelayer")
 
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::consumeWhenAll();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -346,6 +350,7 @@ TEST_CASE("DataRelayer")
     auto policy = CompletionPolicyHelpers::consumeWhenAll();
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
     DataRelayer relayer(policy, inputs, index, {registry});
     // Only two messages to fill the cache.
     relayer.setPipelineLength(2);
@@ -420,6 +425,7 @@ TEST_CASE("DataRelayer")
     std::vector<ForwardRoute> forwards;
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::processWhenAny();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -490,6 +496,7 @@ TEST_CASE("DataRelayer")
     std::vector<ForwardRoute> forwards;
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::processWhenAny();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -547,6 +554,7 @@ TEST_CASE("DataRelayer")
     std::vector<ForwardRoute> forwards;
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::processWhenAny();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -605,6 +613,7 @@ TEST_CASE("DataRelayer")
     std::vector<ForwardRoute> forwards;
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::processWhenAny();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -670,6 +679,7 @@ TEST_CASE("DataRelayer")
     std::vector<ForwardRoute> forwards;
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::consumeWhenAny();
     DataRelayer relayer(policy, inputs, index, {registry});
@@ -722,6 +732,7 @@ TEST_CASE("DataRelayer")
     std::vector<ForwardRoute> forwards;
     std::vector<InputChannelInfo> infos{1};
     TimesliceIndex index{1, infos};
+    ref.registerService(ServiceRegistryHelpers::handleForService<TimesliceIndex>(&index));
 
     auto policy = CompletionPolicyHelpers::consumeWhenAny();
     DataRelayer relayer(policy, inputs, index, {registry});
diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index 64e9dddbb020e..784e07c803d76 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -12,10 +12,15 @@
 o2_add_dpl_workflow(dummy-workflow
                   SOURCES src/o2DummyWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(detect-missing-timeframe
                   SOURCES test/test_DetectMissingTimeframe.cxx
                   COMPONENT_NAME TestWorkflows)
 
+o2_add_dpl_workflow(wait-until-possible
+                  SOURCES test/test_WaitUntilPossible.cxx
+                  COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(o2rootmessage-workflow
                   SOURCES "src/test_o2RootMessageWorkflow.cxx"
                   COMPONENT_NAME TestWorkflows)
diff --git a/Framework/TestWorkflows/test/test_WaitUntilPossible.cxx b/Framework/TestWorkflows/test/test_WaitUntilPossible.cxx
new file mode 100644
index 0000000000000..d9ba315ef02b8
--- /dev/null
+++ b/Framework/TestWorkflows/test/test_WaitUntilPossible.cxx
@@ -0,0 +1,82 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/DataTakingContext.h"
+#include "Framework/CompletionPolicyHelpers.h"
+#include "Framework/DeviceSpec.h"
+#include "Framework/RawDeviceService.h"
+#include "Framework/ControlService.h"
+#include "Framework/Configurable.h"
+#include "Framework/RunningWorkflowInfo.h"
+#include "Framework/RateLimiter.h"
+#include <fairmq/Device.h>
+
+#include <iostream>
+#include <chrono>
+#include <thread>
+#include <vector>
+
+using namespace o2::framework;
+
+#include "Framework/runDataProcessing.h"
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& specs)
+{
+  DataProcessorSpec a{
+    .name = "A",
+    .outputs = {OutputSpec{{"data"}, "TST", "A1", 0}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, RawDeviceService& device, DataTakingContext& context, ProcessingContext& pcx) {
+        LOG(info) << "Data TST/A1/0 created";
+        outputs.make<int>(OutputRef{"data"}, 1);
+      })},
+  };
+  DataProcessorSpec b{
+    .name = "B",
+    .outputs = {OutputSpec{{"sporadic"}, "TST", "B1", 0, Lifetime::Sporadic}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, RawDeviceService& device, DataTakingContext& context, ProcessingContext& pcx) {
+        // This will always be late, however since the oldest possible timeframe
+        // will be used to decide the scheduling, it will not be dropped.
+        sleep(1);
+        // We also create it only every second time, so that we can check that
+        // the sporadic output is not mandatory.
+        static int i = 0;
+        if (i++ % 2 == 0) {
+          LOG(info) << "Data TST/B1/0 created";
+          outputs.make<int>(OutputRef{"sporadic"}, 1);
+        }
+      })},
+  };
+  DataProcessorSpec d{
+    .name = "D",
+    .inputs = {InputSpec{"a1", "TST", "A1", 0, Lifetime::Timeframe},
+               InputSpec{"b1", "TST", "B1", 0, Lifetime::Sporadic}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](InputRecord& inputs) {
+        auto refA = inputs.get("a1");
+        auto headerA = o2::header::get<const DataProcessingHeader*>(refA.header);
+        LOG(info) << "Start time: " << headerA->startTime;
+        auto refB = inputs.get("b1");
+        if (!refB.header) {
+          LOG(info) << "No sporadic input for start time " << headerA->startTime;
+          return;
+        }
+        auto headerB = o2::header::get<const DataProcessingHeader*>(refB.header);
+        LOG(info) << "Start time: " << headerB->startTime;
+      })},
+  };
+
+  return workflow::concat(WorkflowSpec{a},
+                          WorkflowSpec{b},
+                          WorkflowSpec{d});
+}

From 50b71eb7083d016ab32addf592cfbb7724566f96 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 12 Jan 2024 16:25:30 +0100
Subject: [PATCH 212/726] Drop unused output

The newly introduced check now complains about this output which is
actually simply never used.
---
 Detectors/PHOS/calib/src/PHOSRunbyrunCalibDevice.cxx | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Detectors/PHOS/calib/src/PHOSRunbyrunCalibDevice.cxx b/Detectors/PHOS/calib/src/PHOSRunbyrunCalibDevice.cxx
index ed88d40daf049..cf767cb76c7ad 100644
--- a/Detectors/PHOS/calib/src/PHOSRunbyrunCalibDevice.cxx
+++ b/Detectors/PHOS/calib/src/PHOSRunbyrunCalibDevice.cxx
@@ -133,7 +133,6 @@ o2::framework::DataProcessorSpec o2::phos::getPHOSRunbyrunCalibDeviceSpec(bool u
 {
 
   std::vector<OutputSpec> outputs;
-  outputs.emplace_back("PHS", "RUNBYRUNHISTOS", 0, o2::framework::Lifetime::Timeframe);
   outputs.emplace_back(o2::calibration::Utils::gDataOriginCDBPayload, "PHOS_Runbyrun", 0, Lifetime::Sporadic);
   outputs.emplace_back(o2::calibration::Utils::gDataOriginCDBWrapper, "PHOS_Runbyrun", 0, Lifetime::Sporadic);
 

From d99c82ba49c2a2caaf0a1e33cb3a4bb6b80148e1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 15 Jan 2024 13:02:30 +0100
Subject: [PATCH 213/726] DPL: drop unused o2_signpost_id_make_with_pointer
 method

Probably old remaining from a previous implementation.
---
 Framework/Foundation/include/Framework/Signpost.h | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index eca9e26f77469..64d92d67acbf7 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -218,7 +218,6 @@ bool _o2_lock_free_stack_push(_o2_lock_free_stack& stack, const int& value, bool
 bool _o2_lock_free_stack_pop(_o2_lock_free_stack& stack, int& value, bool spin = false);
 //_o2_signpost_id_t _o2_signpost_id_generate_local(_o2_log_t* log);
 //_o2_signpost_id_t _o2_signpost_id_make_with_pointer(_o2_log_t* log, void* pointer);
-_o2_signpost_index_t o2_signpost_id_make_with_pointer(_o2_log_t* log, void* pointer);
 void* _o2_log_create(char const* name, int stacktrace);
 void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...);
 void _o2_signpost_interval_begin(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...);
@@ -246,14 +245,6 @@ inline _o2_signpost_id_t _o2_signpost_id_make_with_pointer(_o2_log_t* log, void*
   return uniqueId;
 }
 
-inline _o2_signpost_index_t o2_signpost_id_make_with_pointer(_o2_log_t* log, void* pointer)
-{
-  _o2_signpost_index_t signpost_index;
-  _o2_lock_free_stack_pop(log->slots, signpost_index, true);
-  log->ids[signpost_index].id = (int64_t)pointer;
-  return signpost_index;
-}
-
 // Implementation start here. Include this file with O2_SIGNPOST_IMPLEMENTATION defined in one file of your
 // project.
 #ifdef O2_SIGNPOST_IMPLEMENTATION

From 2a74f97657e41bf1b4295670b758c244631a2a29 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 09:22:29 +0100
Subject: [PATCH 214/726] DPL: rework signposts support

The logger based Signposts are now behaving the same on macOS and linux when
running normally.

The difference between the two implementations will now be that if Instruments
enables some OSLog namespace, we stop printing the messages and use the os_signpost
API instead.
---
 .../Foundation/include/Framework/Signpost.h   | 191 +++++++++---------
 Framework/Foundation/test/test_Signpost.cxx   |   2 +-
 .../Foundation/test/test_SignpostLogger.cxx   |   2 +-
 3 files changed, 94 insertions(+), 101 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index 64d92d67acbf7..d2bdec0ec8246 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -13,6 +13,9 @@
 
 #include <atomic>
 #include <array>
+#ifdef __APPLE__
+#include <os/log.h>
+#endif
 
 struct o2_log_handle_t {
   char const* name = nullptr;
@@ -22,6 +25,7 @@ struct o2_log_handle_t {
 
 // Helper function which replaces engineering types with a printf
 // compatible format string.
+// FIXME: make this consteval when available in C++20
 template <auto N>
 constexpr auto remove_engineering_type(char const (&src)[N])
 {
@@ -82,78 +86,37 @@ o2_log_handle_t* o2_walk_logs(bool (*callback)(char const* name, void* log, void
 }
 #endif
 
-#if !defined(O2_FORCE_LOGGER_SIGNPOST) && defined(__APPLE__) && (!defined(NDEBUG) || defined(O2_FORCE_SIGNPOSTS))
+#if defined(__APPLE__)
 #include <os/log.h>
 #include <os/signpost.h>
 #include <cstring>
-void* _o2_log_create(char const* name, char const* category);
-#define O2_DECLARE_DYNAMIC_LOG(x) static os_log_t private_o2_log_##x = (os_log_t)_o2_log_create("ch.cern.aliceo2." #x, OS_LOG_CATEGORY_DYNAMIC_TRACING)
-#define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(x) static os_log_t private_o2_log_##x = (os_log_t)_o2_log_create("ch.cern.aliceo2." #x, OS_LOG_CATEGORY_DYNAMIC_STACK_TRACING)
-// This is a no-op on macOS using the os_signpost API because only external instruments can enable/disable dynamic signposts
-#define O2_LOG_ENABLE_DYNAMIC(log)
-// This is a no-op on macOS using the os_signpost API because only external instruments can enable/disable dynamic signposts
-#define O2_LOG_ENABLE_STACKTRACE(log)
-#define O2_DECLARE_LOG(x, category) static os_log_t private_o2_log_##x = (os_log_t)_o2_log_create("ch.cern.aliceo2." #x, #category)
-#define O2_LOG_DEBUG(log, ...) os_log_debug(private_o2_log_##log, __VA_ARGS__)
-#define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer) os_signpost_id_t name = os_signpost_id_make_with_pointer(private_o2_log_##log, pointer)
-#define O2_SIGNPOST_ID_GENERATE(name, log) os_signpost_id_t name = os_signpost_id_generate(private_o2_log_##log)
+#define O2_LOG_DEBUG_MAC(log, ...) os_log_debug(private_o2_log_##log, __VA_ARGS__)
 // FIXME: use __VA_OPT__ when available in C++20
-#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) os_signpost_event_emit(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
-#define O2_SIGNPOST_START(log, id, name, format, ...) os_signpost_interval_begin(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
-#define O2_SIGNPOST_END(log, id, name, format, ...) os_signpost_interval_end(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
-
-#ifdef O2_SIGNPOST_IMPLEMENTATION
-/// We use a wrapper so that we can keep track of the logs.
-void* _o2_log_create(char const* name, char const* category)
-{
-  // iterate over the list of logs and check if we already have
-  // one with the same name.
-  auto findLogByName = [](char const* name, void* log, void* context) -> bool {
-    char const* currentName = (char const*)context;
-    if (strcmp(name, currentName) == 0) {
-      return false;
-    }
-    return true;
-  };
-
-  o2_log_handle_t* handle = o2_walk_logs(findLogByName, (void*)name);
-
-  // If we found one, return it.
-  if (handle) {
-    return handle->log;
-  }
-  // Otherwise, create a new one and add it to the end of the list.
-  os_log_t log = os_log_create(name, category);
-  o2_log_handle_t* newHandle = new o2_log_handle_t();
-  newHandle->log = log;
-  newHandle->name = strdup(name);
-  newHandle->next = o2_get_logs_tail().load();
-  // Until I manage to replace the log I have in next, keep trying.
-  // Notice this does not protect against two threads trying to insert
-  // a log with the same name. I should probably do a sorted insert for that.
-  while (!o2_get_logs_tail().compare_exchange_weak(newHandle->next, newHandle,
-                                                   std::memory_order_release,
-                                                   std::memory_order_relaxed)) {
-    newHandle->next = o2_get_logs_tail();
-  }
-
-  return log;
-}
+#define O2_SIGNPOST_EVENT_EMIT_MAC(log, id, name, format, ...) os_signpost_event_emit(private_o2_log_##log->os_log, (uint64_t)id.value, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_START_MAC(log, id, name, format, ...) os_signpost_interval_begin(private_o2_log_##log->os_log, (uint64_t)id.value, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_END_MAC(log, id, name, format, ...) os_signpost_interval_end(private_o2_log_##log->os_log, (uint64_t)id.value, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_ENABLED_MAC(log) os_signpost_enabled(private_o2_log_##log->os_log)
+#else
+// These are no-ops on linux.
+#define O2_DECLARE_LOG_MAC(x, category)
+#define O2_LOG_DEBUG_MAC(log, ...)
+#define O2_SIGNPOST_EVENT_EMIT_MAC(log, id, name, format, ...)
+#define O2_SIGNPOST_START_MAC(log, id, name, format, ...)
+#define O2_SIGNPOST_END_MAC(log, id, name, format, ...)
+#define O2_SIGNPOST_ENABLED_MAC(log) false
+#endif // __APPLE__
+
+// Unless we are on apple we enable checking for signposts only if in debug mode or if we force them.
+#if defined(__APPLE__) || defined(O2_FORCE_SIGNPOSTS) || !defined(NDEBUG)
+#define O2_LOG_ENABLED(log) private_o2_log_##log->stacktrace
+#else
+#define O2_LOG_ENABLED(log) false
 #endif
 
-#elif !defined(NDEBUG) || defined(O2_FORCE_LOGGER_SIGNPOST) || defined(O2_FORCE_SIGNPOSTS)
-
-#ifndef O2_LOG_MACRO
-#if __has_include("Framework/Logger.h")
+#if !defined(O2_LOG_MACRO) && __has_include("Framework/Logger.h")
 #include "Framework/Logger.h"
-// If NDEBUG is not defined, we use the logger to print out the signposts at the debug level.
-#if !defined(NDEBUG)
-#define O2_LOG_MACRO(...) LOGF(debug, __VA_ARGS__)
-#elif defined(O2_FORCE_LOGGER_SIGNPOST) || defined(O2_FORCE_SIGNPOSTS)
-// If we force the presence of the logger, we use it to print out the signposts at the detail level, which is not optimized out.
 #define O2_LOG_MACRO(...) LOGF(info, __VA_ARGS__)
-#endif
-#else
+#elif !defined(O2_LOG_MACRO)
 // If we do not have the fairlogger, we simply print out the signposts to the console.
 // This is useful for things like the tests, which this way do not need to depend on the FairLogger.
 #define O2_LOG_MACRO(...) \
@@ -161,8 +124,9 @@ void* _o2_log_create(char const* name, char const* category)
     printf(__VA_ARGS__);  \
     printf("\n");         \
   } while (0)
-#endif
-#endif
+#else
+#define O2_LOG_MACRO(...)
+#endif // O2_LOG_MACRO
 
 // This is the linux implementation, it is not as nice as the apple one and simply prints out
 // the signpost information to the log.
@@ -181,7 +145,7 @@ struct _o2_lock_free_stack {
 // A log is simply an inbox which keeps track of the available id, so that we can print out different signposts
 // with different indentation levels.
 // supports up to 1024 paralle signposts before it spinlocks.
-typedef int _o2_signpost_index_t;
+using _o2_signpost_index_t = int;
 
 struct _o2_activity_t {
   // How much the activity is indented in the output log.
@@ -191,10 +155,13 @@ struct _o2_activity_t {
 
 struct _o2_signpost_id_t {
   // The id of the activity.
-  int64_t id = -1;
+  int64_t value = -1;
 };
 
 struct _o2_log_t {
+#ifdef __APPLE__
+  os_log_t os_log = nullptr;
+#endif
   // A circular buffer of available slots. Each unique interval pulls an id from this buffer.
   _o2_lock_free_stack slots;
   // Up to 256 activities can be active at the same time.
@@ -211,7 +178,10 @@ struct _o2_log_t {
   // 0 means the log is disabled.
   // 1 means only the current signpost is printed.
   // >1 means the current signpost and n levels of the stacktrace are printed.
-  std::atomic<int> stacktrace = 1;
+  std::atomic<int> stacktrace = 0;
+
+  // Default stacktrace level for the log, when enabled.
+  int defaultStacktrace = 1;
 };
 
 bool _o2_lock_free_stack_push(_o2_lock_free_stack& stack, const int& value, bool spin = false);
@@ -310,7 +280,7 @@ bool _o2_lock_free_stack_pop(_o2_lock_free_stack& stack, int& value, bool spin)
   }
 }
 
-void* _o2_log_create(char const* name, int stacktrace)
+void* _o2_log_create(char const* name, int defaultStacktrace)
 {
   // iterate over the list of logs and check if we already have
   // one with the same name.
@@ -328,7 +298,7 @@ void* _o2_log_create(char const* name, int stacktrace)
     return handle->log;
   }
   // Otherwise, create a new one and add it to the end of the list.
-  _o2_log_t* log = new _o2_log_t();
+  auto* log = new _o2_log_t();
   // Write the initial 256 ids to the inbox, in reverse, so that the
   // linear search below is just for an handful of elements.
   int n = _o2_lock_free_stack::N;
@@ -336,9 +306,18 @@ void* _o2_log_create(char const* name, int stacktrace)
     _o2_signpost_index_t signpost_index{n - 1 - i};
     _o2_lock_free_stack_push(log->slots, signpost_index, true);
   }
-  log->stacktrace = stacktrace;
-  o2_log_handle_t* newHandle = new o2_log_handle_t();
+  log->defaultStacktrace = defaultStacktrace;
+  auto* newHandle = new o2_log_handle_t();
   newHandle->log = log;
+#ifdef __APPLE__
+  // On macOS, we use the os_signpost API so that when we are
+  // using instruments we can see the messages there.
+  if (defaultStacktrace > 1) {
+    log->os_log = os_log_create(name, OS_LOG_CATEGORY_DYNAMIC_STACK_TRACING);
+  } else {
+    log->os_log = os_log_create(name, OS_LOG_CATEGORY_DYNAMIC_TRACING);
+  }
+#endif
   newHandle->name = strdup(name);
   newHandle->next = o2_get_logs_tail().load();
   // Until I manage to replace the log I have in next, keep trying.
@@ -357,10 +336,6 @@ void* _o2_log_create(char const* name, int stacktrace)
 // If the slot is empty, it will return the id and increment the indentation level.
 void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...)
 {
-  // Nothing to be done
-  if (log->stacktrace == 0) {
-    return;
-  }
   va_list args;
   va_start(args, format);
 
@@ -368,10 +343,10 @@ void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* n
   int leading = 0;
 
   // This is the equivalent of exclusive
-  if (id.id != 0) {
+  if (id.value != 0) {
     int i = 0;
     for (i = 0; i < log->ids.size(); ++i) {
-      if (log->ids[i].id == id.id) {
+      if (log->ids[i].value == id.value) {
         break;
       }
     }
@@ -385,7 +360,7 @@ void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* n
   }
 
   char prebuffer[4096];
-  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16s*>%*c", id.id, name, leading, ' ');
+  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16s*>%*c", id.value, name, leading, ' ');
   vsnprintf(prebuffer + s, 4096 - s, format, args);
   va_end(args);
   O2_LOG_MACRO("%s", prebuffer);
@@ -395,22 +370,19 @@ void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* n
 // If the slot is empty, it will return the id and increment the indentation level.
 void _o2_signpost_interval_begin(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...)
 {
-  if (log->stacktrace == 0) {
-    return;
-  }
   va_list args;
   va_start(args, format);
   // This is a unique slot for this interval.
   _o2_signpost_index_t signpost_index;
   _o2_lock_free_stack_pop(log->slots, signpost_index, true);
   // Put the id in the slot, to close things or to attach signposts to a given interval
-  log->ids[signpost_index].id = id.id;
+  log->ids[signpost_index].value = id.value;
   auto* activity = &log->activities[signpost_index];
   activity->indentation = log->current_indentation++;
   activity->name = name;
   int leading = activity->indentation * 2;
   char prebuffer[4096];
-  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16sS>%*c", id.id, name, leading, ' ');
+  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16sS>%*c", id.value, name, leading, ' ');
   vsnprintf(prebuffer + s, 4096 - s, format, args);
   va_end(args);
   O2_LOG_MACRO("%s", prebuffer);
@@ -424,7 +396,7 @@ void _o2_signpost_interval_end_v(_o2_log_t* log, _o2_signpost_id_t id, char cons
   // Find the index of the activity
   int i = 0;
   for (i = 0; i < log->ids.size(); ++i) {
-    if (log->ids[i].id == id.id) {
+    if (log->ids[i].value == id.value) {
       break;
     }
   }
@@ -439,13 +411,13 @@ void _o2_signpost_interval_end_v(_o2_log_t* log, _o2_signpost_id_t id, char cons
   _o2_activity_t* activity = &log->activities[i];
   int leading = activity->indentation * 2;
   char prebuffer[4096];
-  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16sE>%*c", id.id, name, leading, ' ');
+  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16sE>%*c", id.value, name, leading, ' ');
   vsnprintf(prebuffer + s, 4096 - s, format, args);
   O2_LOG_MACRO("%s", prebuffer);
   // Clear the slot
   activity->indentation = -1;
   activity->name = nullptr;
-  log->ids[i].id = -1;
+  log->ids[i].value = -1;
   // Put back the slot
   log->current_indentation--;
   _o2_signpost_index_t signpost_index{i};
@@ -468,29 +440,50 @@ void _o2_log_set_stacktrace(_o2_log_t* log, int stacktrace)
 }
 #endif // O2_SIGNPOST_IMPLEMENTATION
 
-/// Dynamic logs need to be enabled via the O2_LOG_ENABLE_DYNAMIC macro. Notice this will only work
+#if defined(__APPLE__) || defined(O2_FORCE_SIGNPOSTS) || !defined(NDEBUG)
+/// Dynamic logs need to be enabled via the O2_LOG_ENABLE macro. Notice this will only work
 /// for the logger based logging, since the Apple version needs instruments to enable them.
-#define O2_DECLARE_DYNAMIC_LOG(name) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 0)
+#define O2_DECLARE_DYNAMIC_LOG(name) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 1)
 /// For the moment we do not support logs with a stacktrace.
-#define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(name) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 0)
+#define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(name) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 64)
 #define O2_DECLARE_LOG(name, category) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 1)
-#define O2_LOG_ENABLE_DYNAMIC(log) _o2_log_set_stacktrace(private_o2_log_##log, 1)
-// We print out only the first 64 frames.
-#define O2_LOG_ENABLE_STACKTRACE(log) _o2_log_set_stacktrace(private_o2_log_##log, 64)
+// When we enable the log, we set the stacktrace to the default value.
+#define O2_LOG_ENABLE(log) _o2_log_set_stacktrace(private_o2_log_##log, private_o2_log_##log->defaultStacktrace)
+#define O2_LOG_DISABLE(log) _o2_log_set_stacktrace(private_o2_log_##log, 0)
 // For the moment we simply use LOG DEBUG. We should have proper activities so that we can
 // turn on and off the printing.
 #define O2_LOG_DEBUG(log, ...) O2_LOG_MACRO(__VA_ARGS__)
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer) _o2_signpost_id_t name = _o2_signpost_id_make_with_pointer(private_o2_log_##log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log) _o2_signpost_id_t name = _o2_signpost_id_generate_local(private_o2_log_##log)
-#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
-#define O2_SIGNPOST_START(log, id, name, format, ...) _o2_signpost_interval_begin(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
-#define O2_SIGNPOST_END(log, id, name, format, ...) _o2_signpost_interval_end(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
+// In case Instruments is attached, we switch to the Apple signpost API otherwise, both one
+// mac and on linux we use our own implementation, using the logger. We can use the same ids because
+// they are compatible between the two implementations, we also use remove_engineering_type to remove
+// the engineering types from the format string, so that we can use the same format string for both.
+#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) __extension__({                                          \
+  if (O2_SIGNPOST_ENABLED_MAC(log)) {                                                                               \
+    O2_SIGNPOST_EVENT_EMIT_MAC(log, id, name, format, ##__VA_ARGS__);                                               \
+  } else if (private_o2_log_##log->stacktrace) {                                                                    \
+    _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
+  }                                                                                                                 \
+})
+#define O2_SIGNPOST_START(log, id, name, format, ...)                                                                   \
+  if (O2_SIGNPOST_ENABLED_MAC(log)) {                                                                                   \
+    O2_SIGNPOST_START_MAC(log, id, name, format, ##__VA_ARGS__);                                                        \
+  } else if (private_o2_log_##log->stacktrace) {                                                                        \
+    _o2_signpost_interval_begin(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
+  }
+#define O2_SIGNPOST_END(log, id, name, format, ...)                                                                   \
+  if (O2_SIGNPOST_ENABLED_MAC(log)) {                                                                                 \
+    O2_SIGNPOST_END_MAC(log, id, name, format, ##__VA_ARGS__);                                                        \
+  } else if (private_o2_log_##log->stacktrace) {                                                                      \
+    _o2_signpost_interval_end(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
+  }
 #else // This is the release implementation, it does nothing.
 #define O2_DECLARE_DYNAMIC_LOG(x)
 #define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(x)
 #define O2_DECLARE_LOG(x, category)
-#define O2_LOG_ENABLE_DYNAMIC(log)
-#define O2_LOG_ENABLE_STACKTRACE(log)
+#define O2_LOG_ENABLE(log)
+#define O2_LOG_DISABLE(log)
 #define O2_LOG_DEBUG(log, ...)
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log)
diff --git a/Framework/Foundation/test/test_Signpost.cxx b/Framework/Foundation/test/test_Signpost.cxx
index 24b6afaec5c3d..f9b8d4ec0e13a 100644
--- a/Framework/Foundation/test/test_Signpost.cxx
+++ b/Framework/Foundation/test/test_Signpost.cxx
@@ -43,7 +43,7 @@ int main(int argc, char** argv)
 
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
   O2_SIGNPOST_END(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
-  O2_LOG_ENABLE_DYNAMIC(test_SignpostDynamic);
+  O2_LOG_ENABLE(test_SignpostDynamic);
 #ifdef __APPLE__
   // On Apple there is no way to turn on signposts in the logger, so we do not display this message
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will see, because we turned them on");
diff --git a/Framework/Foundation/test/test_SignpostLogger.cxx b/Framework/Foundation/test/test_SignpostLogger.cxx
index 74da35abf7c70..ed917ea7cadf5 100644
--- a/Framework/Foundation/test/test_SignpostLogger.cxx
+++ b/Framework/Foundation/test/test_SignpostLogger.cxx
@@ -51,7 +51,7 @@ int main(int argc, char** argv)
 
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
   O2_SIGNPOST_END(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
-  O2_LOG_ENABLE_DYNAMIC(test_SignpostDynamic);
+  O2_LOG_ENABLE(test_SignpostDynamic);
 #ifdef __APPLE__
   // On Apple there is no way to turn on signposts in the logger, so we do not display this message
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will see, because we turned them on");

From 9f4370f658edb8c2b2a6418ea76891559dc4add1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 09:24:03 +0100
Subject: [PATCH 215/726] DPL: add proper tracing for DataProcessingDevice
 socket callbacks

---
 Framework/Core/src/DataProcessingDevice.cxx | 40 +++++++++++++--------
 1 file changed, 26 insertions(+), 14 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index b03904c5b3776..366bfafd7d801 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -35,6 +35,9 @@
 #include "Framework/TMessageSerializer.h"
 #include "Framework/InputRecord.h"
 #include "Framework/InputSpan.h"
+#if defined(__APPLE__) || defined(NDEBUG)
+#define O2_SIGNPOST_IMPLEMENTATION
+#endif
 #include "Framework/Signpost.h"
 #include "Framework/TimingHelpers.h"
 #include "Framework/SourceInfoHeader.h"
@@ -80,6 +83,8 @@
 #include <sstream>
 #include <boost/property_tree/json_parser.hpp>
 
+O2_DECLARE_DYNAMIC_LOG(device);
+
 using namespace o2::framework;
 using ConfigurationInterface = o2::configuration::ConfigurationInterface;
 using DataHeader = o2::header::DataHeader;
@@ -274,21 +279,22 @@ struct PollerContext {
 void on_socket_polled(uv_poll_t* poller, int status, int events)
 {
   auto* context = (PollerContext*)poller->data;
+  assert(context);
+  O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
   context->state->loopReason |= DeviceState::DATA_SOCKET_POLLED;
   switch (events) {
     case UV_READABLE: {
-      ZoneScopedN("socket readable event");
-      LOG(debug) << "socket polled UV_READABLE: " << context->name;
+      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "Data pending on socket for channel %{public}s", context->name);
       context->state->loopReason |= DeviceState::DATA_INCOMING;
     } break;
     case UV_WRITABLE: {
-      ZoneScopedN("socket writeable");
+      O2_SIGNPOST_END(device, sid, "socket_state", "Socket connected for channel %{public}s", context->name);
       if (context->read) {
-        LOG(debug) << "socket polled UV_CONNECT" << context->name;
+        O2_SIGNPOST_START(device, sid, "socket_state", "Socket connected for read in context %{public}s", context->name);
         uv_poll_start(poller, UV_READABLE | UV_DISCONNECT | UV_PRIORITIZED, &on_socket_polled);
         context->state->loopReason |= DeviceState::DATA_CONNECTED;
       } else {
-        LOG(debug) << "socket polled UV_WRITABLE" << context->name;
+        O2_SIGNPOST_START(device, sid, "socket_state", "Socket connected for write for channel %{public}s", context->name);
         context->state->loopReason |= DeviceState::DATA_OUTGOING;
         // If the socket is writable, fairmq will handle the rest, so we can stop polling and
         // just wait for the disconnect.
@@ -297,12 +303,10 @@ void on_socket_polled(uv_poll_t* poller, int status, int events)
       context->pollerState = PollerContext::PollerState::Connected;
     } break;
     case UV_DISCONNECT: {
-      ZoneScopedN("socket disconnect");
-      LOG(debug) << "socket polled UV_DISCONNECT";
+      O2_SIGNPOST_END(device, sid, "socket_state", "Socket disconnected in context %{public}s", context->name);
     } break;
     case UV_PRIORITIZED: {
-      ZoneScopedN("socket prioritized");
-      LOG(debug) << "socket polled UV_PRIORITIZED";
+      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "Data pending on socket for context %{public}s", context->name);
     } break;
   }
   // We do nothing, all the logic for now stays in DataProcessingDevice::doRun()
@@ -873,7 +877,9 @@ void DataProcessingDevice::startPollers()
   auto& deviceContext = ref.get<DeviceContext>();
   auto& state = ref.get<DeviceState>();
 
-  for (auto& poller : state.activeInputPollers) {
+  for (auto* poller : state.activeInputPollers) {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
+    O2_SIGNPOST_START(device, sid, "socket_state", "Input socket waiting for connection.");
     uv_poll_start(poller, UV_WRITABLE, &on_socket_polled);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Disconnected;
   }
@@ -881,7 +887,9 @@ void DataProcessingDevice::startPollers()
     uv_poll_start(poller, UV_WRITABLE, &on_out_of_band_polled);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Disconnected;
   }
-  for (auto& poller : state.activeOutputPollers) {
+  for (auto* poller : state.activeOutputPollers) {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
+    O2_SIGNPOST_START(device, sid, "socket_state", "Output socket waiting for connection.");
     uv_poll_start(poller, UV_WRITABLE, &on_socket_polled);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Disconnected;
   }
@@ -897,17 +905,21 @@ void DataProcessingDevice::stopPollers()
   auto& deviceContext = ref.get<DeviceContext>();
   auto& state = ref.get<DeviceState>();
   LOGP(detail, "Stopping {} input pollers", state.activeInputPollers.size());
-  for (auto& poller : state.activeInputPollers) {
+  for (auto* poller : state.activeInputPollers) {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
+    O2_SIGNPOST_END(device, sid, "socket_state", "Output socket closed.");
     uv_poll_stop(poller);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Stopped;
   }
   LOGP(detail, "Stopping {} out of band pollers", state.activeOutOfBandPollers.size());
-  for (auto& poller : state.activeOutOfBandPollers) {
+  for (auto* poller : state.activeOutOfBandPollers) {
     uv_poll_stop(poller);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Stopped;
   }
   LOGP(detail, "Stopping {} output pollers", state.activeOutOfBandPollers.size());
-  for (auto& poller : state.activeOutputPollers) {
+  for (auto* poller : state.activeOutputPollers) {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
+    O2_SIGNPOST_END(device, sid, "socket_state", "Output socket closed.");
     uv_poll_stop(poller);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Stopped;
   }

From 0766b8af23b5a64627f8b7dd56ee767130071f4f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 09:24:03 +0100
Subject: [PATCH 216/726] DPL use Signposts to debug consumeWhenAll

---
 Framework/Core/src/CompletionPolicyHelpers.cxx | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx
index 8c4b38ba3355c..a6240b1165b81 100644
--- a/Framework/Core/src/CompletionPolicyHelpers.cxx
+++ b/Framework/Core/src/CompletionPolicyHelpers.cxx
@@ -18,10 +18,13 @@
 #include "Framework/TimesliceIndex.h"
 #include "Framework/TimingInfo.h"
 #include "DecongestionService.h"
+#include "Framework/Signpost.h"
 
 #include <cassert>
 #include <regex>
 
+O2_DECLARE_DYNAMIC_LOG(completion);
+
 namespace o2::framework
 {
 
@@ -108,6 +111,8 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
 {
   auto callback = [](InputSpan const& inputs, std::vector<InputSpec> const& specs, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     assert(inputs.size() == specs.size());
+    O2_SIGNPOST_ID_GENERATE(sid, completion);
+    O2_SIGNPOST_START(completion, sid, "consumeWhenAll", "Completion policy invoked");
 
     size_t si = 0;
     bool missingSporadic = false;
@@ -116,15 +121,18 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
       assert(si < specs.size());
       auto& spec = specs[si++];
       if (input.header == nullptr && spec.lifetime != Lifetime::Sporadic) {
+        O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s due to missing input %lu", "Wait", si);
         return CompletionPolicy::CompletionOp::Wait;
       }
       if (input.header == nullptr && spec.lifetime == Lifetime::Sporadic) {
+        O2_SIGNPOST_EVENT_EMIT(completion, sid, "consumeWhenAll", "Missing sporadic found for route index %lu", si);
         missingSporadic = true;
       }
       if (input.header != nullptr && currentTimeslice == -1) {
         auto const* dph = framework::DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(input);
         if (dph && !TimingInfo::timesliceIsTimer(dph->startTime)) {
           currentTimeslice = dph->startTime;
+          O2_SIGNPOST_EVENT_EMIT(completion, sid, "consumeWhenAll", "currentTimeslice %lu from route index %lu", currentTimeslice, si);
         }
       }
     }
@@ -134,8 +142,10 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
     auto oldestPossibleTimeslice = timesliceIndex.getOldestPossibleInput().timeslice.value;
 
     if (missingSporadic && currentTimeslice >= oldestPossibleTimeslice) {
+      O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s for timeslice %lu > oldestPossibleTimeslice %lu", "Retry", currentTimeslice, oldestPossibleTimeslice);
       return CompletionPolicy::CompletionOp::Retry;
     }
+    O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s for timeslice %lu <= oldestPossibleTimeslice %lu", "Consume", currentTimeslice, oldestPossibleTimeslice);
     return CompletionPolicy::CompletionOp::Consume;
   };
   return CompletionPolicy{name, matcher, callback};

From a9f53d8aee1c66fa8af02efbfd633efab5081e5d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 09:24:03 +0100
Subject: [PATCH 217/726] Ability to turn on / off the logger from the GUI.

---
 .../Core/include/Framework/DeviceControl.h    |  2 +
 .../Core/include/Framework/DeviceState.h      |  9 ++++
 Framework/Core/src/WSDriverClient.cxx         | 41 +++++++++++++++++++
 .../src/FrameworkGUIDeviceInspector.cxx       | 11 +++++
 .../test/test_DetectMissingTimeframe.cxx      |  1 +
 5 files changed, 64 insertions(+)

diff --git a/Framework/Core/include/Framework/DeviceControl.h b/Framework/Core/include/Framework/DeviceControl.h
index 589de3937ee55..ce946e8e77fbf 100644
--- a/Framework/Core/include/Framework/DeviceControl.h
+++ b/Framework/Core/include/Framework/DeviceControl.h
@@ -48,6 +48,8 @@ struct DeviceControl {
   DeviceController* controller = nullptr;
   /// What kind of events should run with the TRACE level
   int tracingFlags = 0;
+  /// What kind of log streams should be enabled
+  int logStreams = 0;
   /// An incremental number to identify the device state
   int requestedState = 0;
 };
diff --git a/Framework/Core/include/Framework/DeviceState.h b/Framework/Core/include/Framework/DeviceState.h
index 3be0302fd4113..4fa72a84cad71 100644
--- a/Framework/Core/include/Framework/DeviceState.h
+++ b/Framework/Core/include/Framework/DeviceState.h
@@ -57,6 +57,13 @@ struct DeviceState {
     DATA_CONNECTED = 1 << 19,     // Data channel connected
   };
 
+  enum LogStreams : int {
+    NO_LOG = 0,
+    DEVICE_LOG = 1 << 0,             // Log for Data Processing Device activities.
+    COMPLETION_LOG = 1 << 1,         // Log for the completion policy of the device.
+    MONITORING_SERVICE_LOG = 1 << 2, // Log for the monitoring service flushing.
+  };
+
   std::vector<InputChannelInfo> inputChannelInfos;
   StreamingState streaming = StreamingState::Streaming;
   bool quitRequested = false;
@@ -93,6 +100,8 @@ struct DeviceState {
   int loopReason = 0;
   /// Bitmask of LoopReason to trace
   int tracingFlags = 0;
+  /// Bitmask of log streams which are available
+  int logStreams = 0;
   /// Stack of the severity, so that we can display only
   /// the bits we are interested in.
   std::vector<int> severityStack;
diff --git a/Framework/Core/src/WSDriverClient.cxx b/Framework/Core/src/WSDriverClient.cxx
index d4ed77b9a004e..234b98b10259d 100644
--- a/Framework/Core/src/WSDriverClient.cxx
+++ b/Framework/Core/src/WSDriverClient.cxx
@@ -16,10 +16,15 @@
 #include "Framework/DeviceSpec.h"
 #include "DriverClientContext.h"
 #include "DPLWebSocket.h"
+#include "Framework/Signpost.h"
 #include <uv.h>
 #include <string_view>
 #include <charconv>
 
+O2_DECLARE_DYNAMIC_LOG(device);
+O2_DECLARE_DYNAMIC_LOG(completion);
+O2_DECLARE_DYNAMIC_LOG(monitoring_service);
+
 namespace o2::framework
 {
 
@@ -152,6 +157,42 @@ void on_connect(uv_connect_t* connection, int status)
     state.tracingFlags = tracingFlags;
   });
 
+  client->observe("/log-streams", [ref = context->ref](std::string_view cmd) {
+    auto& state = ref.get<DeviceState>();
+    static constexpr int prefixSize = std::string_view{"/log-streams "}.size();
+    if (prefixSize > cmd.size()) {
+      LOG(error) << "Malformed log-streams request";
+      return;
+    }
+    cmd.remove_prefix(prefixSize);
+    int logStreams = 0;
+
+    auto error = std::from_chars(cmd.data(), cmd.data() + cmd.size(), logStreams);
+    if (error.ec != std::errc()) {
+      LOG(error) << "Malformed log-streams mask";
+      return;
+    }
+    LOGP(info, "Logstreams flags set to {}", logStreams);
+    state.logStreams = logStreams;
+    if ((state.logStreams & DeviceState::LogStreams::DEVICE_LOG) != 0) {
+      O2_LOG_ENABLE(device);
+    } else {
+      O2_LOG_DISABLE(device);
+    }
+    if ((state.logStreams & DeviceState::LogStreams::COMPLETION_LOG) != 0) {
+      O2_LOG_ENABLE(completion);
+    } else {
+      O2_LOG_DISABLE(completion);
+    }
+
+    if ((state.logStreams & DeviceState::LogStreams::MONITORING_SERVICE_LOG) != 0) {
+      O2_LOG_ENABLE(monitoring_service);
+    } else {
+      O2_LOG_DISABLE(monitoring_service);
+    }
+
+  });
+
   // Client will be filled in the line after. I can probably have a single
   // client per device.
   auto dplClient = std::make_unique<WSDPLClient>();
diff --git a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
index a82753eb5af1f..fe93ca6d0f07f 100644
--- a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
@@ -400,6 +400,17 @@ void displayDeviceInspector(DeviceSpec const& spec,
     }
   }
 
+  bool logsChanged = false;
+  if (ImGui::CollapsingHeader("Signposts", ImGuiTreeNodeFlags_DefaultOpen)) {
+    logsChanged = ImGui::CheckboxFlags("Device", &control.logStreams, DeviceState::LogStreams::DEVICE_LOG);
+    logsChanged = ImGui::CheckboxFlags("Completion", &control.logStreams, DeviceState::LogStreams::COMPLETION_LOG);
+    logsChanged = ImGui::CheckboxFlags("Monitoring", &control.logStreams, DeviceState::LogStreams::MONITORING_SERVICE_LOG);
+    if (logsChanged && control.controller) {
+      std::string cmd = fmt::format("/log-streams {}", control.logStreams);
+      control.controller->write(cmd.c_str(), cmd.size());
+    }
+  }
+
   bool flagsChanged = false;
   if (ImGui::CollapsingHeader("Event loop tracing", ImGuiTreeNodeFlags_DefaultOpen)) {
     flagsChanged |= ImGui::CheckboxFlags("METRICS_MUST_FLUSH", &control.tracingFlags, DeviceState::LoopReason::METRICS_MUST_FLUSH);
diff --git a/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx b/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
index 3374cd9ad6cf2..7ef1370f040b4 100644
--- a/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
+++ b/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
@@ -43,6 +43,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& specs)
         if (i++ % 2 == 0) {
           outputs.make<int>(OutputRef{"a2"}, 1);
         }
+        sleep(1);
       })},
   };
   DataProcessorSpec d{

From f977cf7469d7d1923ffcd37a3193698cdff8be2b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 11:20:50 +0100
Subject: [PATCH 218/726] DPL: do not validate lifetime when consumer uses
 consumeWhenAny

---
 Framework/Core/src/DeviceSpecHelpers.cxx | 16 +++++++++++++---
 Framework/Core/src/WorkflowHelpers.cxx   | 22 ++++++++++++++++++----
 Framework/Core/src/WorkflowHelpers.h     |  6 ++++++
 3 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 6c2063cb59311..359b1a343cb7e 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1074,8 +1074,6 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
 
   WorkflowHelpers::constructGraph(workflow, logicalEdges, outputs, availableForwardsInfo);
 
-  WorkflowHelpers::validateEdges(workflow, logicalEdges, outputs);
-
   // We need to instanciate one device per (me, timeIndex) in the
   // DeviceConnectionEdge. For each device we need one new binding
   // server per (me, other) -> port Moreover for each (me, other,
@@ -1122,10 +1120,13 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
                        inActions, workflow, availableForwardsInfo, channelPolicies, channelPrefix, defaultOffer, overrideServices);
   // We apply the completion policies here since this is where we have all the
   // devices resolved.
-  for (auto& device : devices) {
+  std::map<std::string, DataProcessorPoliciesInfo> policies;
+  for (DeviceSpec& device : devices) {
     bool hasPolicy = false;
+    policies[device.name].completionPolicyName = "unknown";
     for (auto& policy : completionPolicies) {
       if (policy.matcher(device) == true) {
+        policies[policy.name].completionPolicyName = policy.name;
         device.completionPolicy = policy;
         hasPolicy = true;
         break;
@@ -1158,6 +1159,15 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
       throw runtime_error_f("Unable to find a resource policy for %s", device.id.c_str());
     }
   }
+  // Iterate of the workflow and create a consistent vector of DataProcessorPoliciesInfo
+  std::vector<DataProcessorPoliciesInfo> policiesVector;
+  for (size_t wi = 0; wi < workflow.size(); ++wi) {
+    auto& processor = workflow[wi];
+    auto& info = policies[processor.name];
+    policiesVector.push_back(info);
+  }
+
+  WorkflowHelpers::validateEdges(workflow, policiesVector, logicalEdges, outputs);
 
   for (auto& device : devices) {
     device.resourceMonitoringInterval = resourcesMonitoringInterval;
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 360480f80d4ef..6b4ad35f826be 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -1214,8 +1214,14 @@ std::vector<InputSpec> WorkflowHelpers::computeDanglingOutputs(WorkflowSpec cons
   return results;
 }
 
-bool validateLifetime(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
+bool validateLifetime(std::ostream& errors,
+                      DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorPoliciesInfo const& producerPolicies,
+                      DataProcessorSpec const& consumer, InputSpec const& input, DataProcessorPoliciesInfo const& consumerPolicies)
 {
+  // In case the completion policy is consume-any, we do not need to check anything.
+  if (consumerPolicies.completionPolicyName == "consume-any") {
+    return true;
+  }
   if (input.lifetime == Lifetime::Timeframe && output.lifetime == Lifetime::Sporadic) {
     errors << fmt::format("Input {} of {} has lifetime Timeframe, but output {} of {} has lifetime Sporadic\n",
                           DataSpecUtils::describe(input).c_str(), consumer.name,
@@ -1225,7 +1231,9 @@ bool validateLifetime(std::ostream& errors, DataProcessorSpec const& producer, O
   return true;
 }
 
-bool validateExpendable(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
+bool validateExpendable(std::ostream& errors,
+                        DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorPoliciesInfo const& producerPolicies,
+                        DataProcessorSpec const& consumer, InputSpec const& input, DataProcessorPoliciesInfo const& consumerPolicies)
 {
   auto isExpendable = [](DataProcessorLabel const& label) {
     return label.value == "expendable";
@@ -1244,8 +1252,12 @@ bool validateExpendable(std::ostream& errors, DataProcessorSpec const& producer,
   return true;
 }
 
-using Validator = std::function<bool(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)>;
+using Validator = std::function<bool(std::ostream& errors,
+                                     DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorPoliciesInfo const& producerPolicies,
+                                     DataProcessorSpec const& consumer, InputSpec const& input, DataProcessorPoliciesInfo const& consumerPolicies)>;
+
 void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
+                                    std::vector<DataProcessorPoliciesInfo> const& policies,
                                     std::vector<DeviceConnectionEdge> const& edges,
                                     std::vector<OutputSpec> const& outputs)
 {
@@ -1262,10 +1274,12 @@ void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
   for (auto& edge : edges) {
     DataProcessorSpec const& producer = workflow[edge.producer];
     DataProcessorSpec const& consumer = workflow[edge.consumer];
+    DataProcessorPoliciesInfo const& producerPolicies = policies[edge.producer];
+    DataProcessorPoliciesInfo const& consumerPolicies = policies[edge.consumer];
     OutputSpec const& output = outputs[edge.outputGlobalIndex];
     InputSpec const& input = consumer.inputs[edge.consumerInputIndex];
     for (auto& validator : defaultValidators) {
-      hasErrors |= !validator(errors, producer, output, consumer, input);
+      hasErrors |= !validator(errors, producer, output, producerPolicies, consumer, input, consumerPolicies);
     }
   }
   if (hasErrors) {
diff --git a/Framework/Core/src/WorkflowHelpers.h b/Framework/Core/src/WorkflowHelpers.h
index 0a3cd408b3424..055cae1d6011b 100644
--- a/Framework/Core/src/WorkflowHelpers.h
+++ b/Framework/Core/src/WorkflowHelpers.h
@@ -141,6 +141,11 @@ struct TopoIndexInfo {
   friend std::ostream& operator<<(std::ostream& out, TopoIndexInfo const& info);
 };
 
+// Information about the policies which were derived for a given data processor.
+struct DataProcessorPoliciesInfo {
+  std::string completionPolicyName;
+};
+
 struct OutputObj {
   InputSpec spec;
   bool isdangling;
@@ -234,6 +239,7 @@ struct WorkflowHelpers {
   /// For example we should make sure that Lifetime::Timeframe inputs of
   /// one node is not connected to an Output of Lifetime::Sporadic of another node.
   static void validateEdges(WorkflowSpec const& workflow,
+                            std::vector<DataProcessorPoliciesInfo> const& policiesInfos,
                             std::vector<DeviceConnectionEdge> const& edges,
                             std::vector<OutputSpec> const& outputs);
 };

From 6ca6b75d10920b3b6299974fa6604643ceba2222 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 15 Jan 2024 13:02:30 +0100
Subject: [PATCH 219/726] DPL: drop unused o2_signpost_id_make_with_pointer
 method

Probably old remaining from a previous implementation.
---
 Framework/Foundation/include/Framework/Signpost.h | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index eca9e26f77469..64d92d67acbf7 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -218,7 +218,6 @@ bool _o2_lock_free_stack_push(_o2_lock_free_stack& stack, const int& value, bool
 bool _o2_lock_free_stack_pop(_o2_lock_free_stack& stack, int& value, bool spin = false);
 //_o2_signpost_id_t _o2_signpost_id_generate_local(_o2_log_t* log);
 //_o2_signpost_id_t _o2_signpost_id_make_with_pointer(_o2_log_t* log, void* pointer);
-_o2_signpost_index_t o2_signpost_id_make_with_pointer(_o2_log_t* log, void* pointer);
 void* _o2_log_create(char const* name, int stacktrace);
 void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...);
 void _o2_signpost_interval_begin(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...);
@@ -246,14 +245,6 @@ inline _o2_signpost_id_t _o2_signpost_id_make_with_pointer(_o2_log_t* log, void*
   return uniqueId;
 }
 
-inline _o2_signpost_index_t o2_signpost_id_make_with_pointer(_o2_log_t* log, void* pointer)
-{
-  _o2_signpost_index_t signpost_index;
-  _o2_lock_free_stack_pop(log->slots, signpost_index, true);
-  log->ids[signpost_index].id = (int64_t)pointer;
-  return signpost_index;
-}
-
 // Implementation start here. Include this file with O2_SIGNPOST_IMPLEMENTATION defined in one file of your
 // project.
 #ifdef O2_SIGNPOST_IMPLEMENTATION

From 87974b029e76c38f30b82238f03d48ae7732b9ad Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 11:21:05 +0100
Subject: [PATCH 220/726] DPL: rework signposts support

The logger based Signposts are now behaving the same on macOS and linux when
running normally.

The difference between the two implementations will now be that if Instruments
enables some OSLog namespace, we stop printing the messages and use the os_signpost
API instead.
---
 .../Foundation/include/Framework/Signpost.h   | 191 +++++++++---------
 Framework/Foundation/test/test_Signpost.cxx   |   2 +-
 .../Foundation/test/test_SignpostLogger.cxx   |   2 +-
 3 files changed, 94 insertions(+), 101 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index 64d92d67acbf7..d2bdec0ec8246 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -13,6 +13,9 @@
 
 #include <atomic>
 #include <array>
+#ifdef __APPLE__
+#include <os/log.h>
+#endif
 
 struct o2_log_handle_t {
   char const* name = nullptr;
@@ -22,6 +25,7 @@ struct o2_log_handle_t {
 
 // Helper function which replaces engineering types with a printf
 // compatible format string.
+// FIXME: make this consteval when available in C++20
 template <auto N>
 constexpr auto remove_engineering_type(char const (&src)[N])
 {
@@ -82,78 +86,37 @@ o2_log_handle_t* o2_walk_logs(bool (*callback)(char const* name, void* log, void
 }
 #endif
 
-#if !defined(O2_FORCE_LOGGER_SIGNPOST) && defined(__APPLE__) && (!defined(NDEBUG) || defined(O2_FORCE_SIGNPOSTS))
+#if defined(__APPLE__)
 #include <os/log.h>
 #include <os/signpost.h>
 #include <cstring>
-void* _o2_log_create(char const* name, char const* category);
-#define O2_DECLARE_DYNAMIC_LOG(x) static os_log_t private_o2_log_##x = (os_log_t)_o2_log_create("ch.cern.aliceo2." #x, OS_LOG_CATEGORY_DYNAMIC_TRACING)
-#define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(x) static os_log_t private_o2_log_##x = (os_log_t)_o2_log_create("ch.cern.aliceo2." #x, OS_LOG_CATEGORY_DYNAMIC_STACK_TRACING)
-// This is a no-op on macOS using the os_signpost API because only external instruments can enable/disable dynamic signposts
-#define O2_LOG_ENABLE_DYNAMIC(log)
-// This is a no-op on macOS using the os_signpost API because only external instruments can enable/disable dynamic signposts
-#define O2_LOG_ENABLE_STACKTRACE(log)
-#define O2_DECLARE_LOG(x, category) static os_log_t private_o2_log_##x = (os_log_t)_o2_log_create("ch.cern.aliceo2." #x, #category)
-#define O2_LOG_DEBUG(log, ...) os_log_debug(private_o2_log_##log, __VA_ARGS__)
-#define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer) os_signpost_id_t name = os_signpost_id_make_with_pointer(private_o2_log_##log, pointer)
-#define O2_SIGNPOST_ID_GENERATE(name, log) os_signpost_id_t name = os_signpost_id_generate(private_o2_log_##log)
+#define O2_LOG_DEBUG_MAC(log, ...) os_log_debug(private_o2_log_##log, __VA_ARGS__)
 // FIXME: use __VA_OPT__ when available in C++20
-#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) os_signpost_event_emit(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
-#define O2_SIGNPOST_START(log, id, name, format, ...) os_signpost_interval_begin(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
-#define O2_SIGNPOST_END(log, id, name, format, ...) os_signpost_interval_end(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
-
-#ifdef O2_SIGNPOST_IMPLEMENTATION
-/// We use a wrapper so that we can keep track of the logs.
-void* _o2_log_create(char const* name, char const* category)
-{
-  // iterate over the list of logs and check if we already have
-  // one with the same name.
-  auto findLogByName = [](char const* name, void* log, void* context) -> bool {
-    char const* currentName = (char const*)context;
-    if (strcmp(name, currentName) == 0) {
-      return false;
-    }
-    return true;
-  };
-
-  o2_log_handle_t* handle = o2_walk_logs(findLogByName, (void*)name);
-
-  // If we found one, return it.
-  if (handle) {
-    return handle->log;
-  }
-  // Otherwise, create a new one and add it to the end of the list.
-  os_log_t log = os_log_create(name, category);
-  o2_log_handle_t* newHandle = new o2_log_handle_t();
-  newHandle->log = log;
-  newHandle->name = strdup(name);
-  newHandle->next = o2_get_logs_tail().load();
-  // Until I manage to replace the log I have in next, keep trying.
-  // Notice this does not protect against two threads trying to insert
-  // a log with the same name. I should probably do a sorted insert for that.
-  while (!o2_get_logs_tail().compare_exchange_weak(newHandle->next, newHandle,
-                                                   std::memory_order_release,
-                                                   std::memory_order_relaxed)) {
-    newHandle->next = o2_get_logs_tail();
-  }
-
-  return log;
-}
+#define O2_SIGNPOST_EVENT_EMIT_MAC(log, id, name, format, ...) os_signpost_event_emit(private_o2_log_##log->os_log, (uint64_t)id.value, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_START_MAC(log, id, name, format, ...) os_signpost_interval_begin(private_o2_log_##log->os_log, (uint64_t)id.value, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_END_MAC(log, id, name, format, ...) os_signpost_interval_end(private_o2_log_##log->os_log, (uint64_t)id.value, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_ENABLED_MAC(log) os_signpost_enabled(private_o2_log_##log->os_log)
+#else
+// These are no-ops on linux.
+#define O2_DECLARE_LOG_MAC(x, category)
+#define O2_LOG_DEBUG_MAC(log, ...)
+#define O2_SIGNPOST_EVENT_EMIT_MAC(log, id, name, format, ...)
+#define O2_SIGNPOST_START_MAC(log, id, name, format, ...)
+#define O2_SIGNPOST_END_MAC(log, id, name, format, ...)
+#define O2_SIGNPOST_ENABLED_MAC(log) false
+#endif // __APPLE__
+
+// Unless we are on apple we enable checking for signposts only if in debug mode or if we force them.
+#if defined(__APPLE__) || defined(O2_FORCE_SIGNPOSTS) || !defined(NDEBUG)
+#define O2_LOG_ENABLED(log) private_o2_log_##log->stacktrace
+#else
+#define O2_LOG_ENABLED(log) false
 #endif
 
-#elif !defined(NDEBUG) || defined(O2_FORCE_LOGGER_SIGNPOST) || defined(O2_FORCE_SIGNPOSTS)
-
-#ifndef O2_LOG_MACRO
-#if __has_include("Framework/Logger.h")
+#if !defined(O2_LOG_MACRO) && __has_include("Framework/Logger.h")
 #include "Framework/Logger.h"
-// If NDEBUG is not defined, we use the logger to print out the signposts at the debug level.
-#if !defined(NDEBUG)
-#define O2_LOG_MACRO(...) LOGF(debug, __VA_ARGS__)
-#elif defined(O2_FORCE_LOGGER_SIGNPOST) || defined(O2_FORCE_SIGNPOSTS)
-// If we force the presence of the logger, we use it to print out the signposts at the detail level, which is not optimized out.
 #define O2_LOG_MACRO(...) LOGF(info, __VA_ARGS__)
-#endif
-#else
+#elif !defined(O2_LOG_MACRO)
 // If we do not have the fairlogger, we simply print out the signposts to the console.
 // This is useful for things like the tests, which this way do not need to depend on the FairLogger.
 #define O2_LOG_MACRO(...) \
@@ -161,8 +124,9 @@ void* _o2_log_create(char const* name, char const* category)
     printf(__VA_ARGS__);  \
     printf("\n");         \
   } while (0)
-#endif
-#endif
+#else
+#define O2_LOG_MACRO(...)
+#endif // O2_LOG_MACRO
 
 // This is the linux implementation, it is not as nice as the apple one and simply prints out
 // the signpost information to the log.
@@ -181,7 +145,7 @@ struct _o2_lock_free_stack {
 // A log is simply an inbox which keeps track of the available id, so that we can print out different signposts
 // with different indentation levels.
 // supports up to 1024 paralle signposts before it spinlocks.
-typedef int _o2_signpost_index_t;
+using _o2_signpost_index_t = int;
 
 struct _o2_activity_t {
   // How much the activity is indented in the output log.
@@ -191,10 +155,13 @@ struct _o2_activity_t {
 
 struct _o2_signpost_id_t {
   // The id of the activity.
-  int64_t id = -1;
+  int64_t value = -1;
 };
 
 struct _o2_log_t {
+#ifdef __APPLE__
+  os_log_t os_log = nullptr;
+#endif
   // A circular buffer of available slots. Each unique interval pulls an id from this buffer.
   _o2_lock_free_stack slots;
   // Up to 256 activities can be active at the same time.
@@ -211,7 +178,10 @@ struct _o2_log_t {
   // 0 means the log is disabled.
   // 1 means only the current signpost is printed.
   // >1 means the current signpost and n levels of the stacktrace are printed.
-  std::atomic<int> stacktrace = 1;
+  std::atomic<int> stacktrace = 0;
+
+  // Default stacktrace level for the log, when enabled.
+  int defaultStacktrace = 1;
 };
 
 bool _o2_lock_free_stack_push(_o2_lock_free_stack& stack, const int& value, bool spin = false);
@@ -310,7 +280,7 @@ bool _o2_lock_free_stack_pop(_o2_lock_free_stack& stack, int& value, bool spin)
   }
 }
 
-void* _o2_log_create(char const* name, int stacktrace)
+void* _o2_log_create(char const* name, int defaultStacktrace)
 {
   // iterate over the list of logs and check if we already have
   // one with the same name.
@@ -328,7 +298,7 @@ void* _o2_log_create(char const* name, int stacktrace)
     return handle->log;
   }
   // Otherwise, create a new one and add it to the end of the list.
-  _o2_log_t* log = new _o2_log_t();
+  auto* log = new _o2_log_t();
   // Write the initial 256 ids to the inbox, in reverse, so that the
   // linear search below is just for an handful of elements.
   int n = _o2_lock_free_stack::N;
@@ -336,9 +306,18 @@ void* _o2_log_create(char const* name, int stacktrace)
     _o2_signpost_index_t signpost_index{n - 1 - i};
     _o2_lock_free_stack_push(log->slots, signpost_index, true);
   }
-  log->stacktrace = stacktrace;
-  o2_log_handle_t* newHandle = new o2_log_handle_t();
+  log->defaultStacktrace = defaultStacktrace;
+  auto* newHandle = new o2_log_handle_t();
   newHandle->log = log;
+#ifdef __APPLE__
+  // On macOS, we use the os_signpost API so that when we are
+  // using instruments we can see the messages there.
+  if (defaultStacktrace > 1) {
+    log->os_log = os_log_create(name, OS_LOG_CATEGORY_DYNAMIC_STACK_TRACING);
+  } else {
+    log->os_log = os_log_create(name, OS_LOG_CATEGORY_DYNAMIC_TRACING);
+  }
+#endif
   newHandle->name = strdup(name);
   newHandle->next = o2_get_logs_tail().load();
   // Until I manage to replace the log I have in next, keep trying.
@@ -357,10 +336,6 @@ void* _o2_log_create(char const* name, int stacktrace)
 // If the slot is empty, it will return the id and increment the indentation level.
 void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...)
 {
-  // Nothing to be done
-  if (log->stacktrace == 0) {
-    return;
-  }
   va_list args;
   va_start(args, format);
 
@@ -368,10 +343,10 @@ void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* n
   int leading = 0;
 
   // This is the equivalent of exclusive
-  if (id.id != 0) {
+  if (id.value != 0) {
     int i = 0;
     for (i = 0; i < log->ids.size(); ++i) {
-      if (log->ids[i].id == id.id) {
+      if (log->ids[i].value == id.value) {
         break;
       }
     }
@@ -385,7 +360,7 @@ void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* n
   }
 
   char prebuffer[4096];
-  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16s*>%*c", id.id, name, leading, ' ');
+  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16s*>%*c", id.value, name, leading, ' ');
   vsnprintf(prebuffer + s, 4096 - s, format, args);
   va_end(args);
   O2_LOG_MACRO("%s", prebuffer);
@@ -395,22 +370,19 @@ void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* n
 // If the slot is empty, it will return the id and increment the indentation level.
 void _o2_signpost_interval_begin(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...)
 {
-  if (log->stacktrace == 0) {
-    return;
-  }
   va_list args;
   va_start(args, format);
   // This is a unique slot for this interval.
   _o2_signpost_index_t signpost_index;
   _o2_lock_free_stack_pop(log->slots, signpost_index, true);
   // Put the id in the slot, to close things or to attach signposts to a given interval
-  log->ids[signpost_index].id = id.id;
+  log->ids[signpost_index].value = id.value;
   auto* activity = &log->activities[signpost_index];
   activity->indentation = log->current_indentation++;
   activity->name = name;
   int leading = activity->indentation * 2;
   char prebuffer[4096];
-  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16sS>%*c", id.id, name, leading, ' ');
+  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16sS>%*c", id.value, name, leading, ' ');
   vsnprintf(prebuffer + s, 4096 - s, format, args);
   va_end(args);
   O2_LOG_MACRO("%s", prebuffer);
@@ -424,7 +396,7 @@ void _o2_signpost_interval_end_v(_o2_log_t* log, _o2_signpost_id_t id, char cons
   // Find the index of the activity
   int i = 0;
   for (i = 0; i < log->ids.size(); ++i) {
-    if (log->ids[i].id == id.id) {
+    if (log->ids[i].value == id.value) {
       break;
     }
   }
@@ -439,13 +411,13 @@ void _o2_signpost_interval_end_v(_o2_log_t* log, _o2_signpost_id_t id, char cons
   _o2_activity_t* activity = &log->activities[i];
   int leading = activity->indentation * 2;
   char prebuffer[4096];
-  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16sE>%*c", id.id, name, leading, ' ');
+  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16sE>%*c", id.value, name, leading, ' ');
   vsnprintf(prebuffer + s, 4096 - s, format, args);
   O2_LOG_MACRO("%s", prebuffer);
   // Clear the slot
   activity->indentation = -1;
   activity->name = nullptr;
-  log->ids[i].id = -1;
+  log->ids[i].value = -1;
   // Put back the slot
   log->current_indentation--;
   _o2_signpost_index_t signpost_index{i};
@@ -468,29 +440,50 @@ void _o2_log_set_stacktrace(_o2_log_t* log, int stacktrace)
 }
 #endif // O2_SIGNPOST_IMPLEMENTATION
 
-/// Dynamic logs need to be enabled via the O2_LOG_ENABLE_DYNAMIC macro. Notice this will only work
+#if defined(__APPLE__) || defined(O2_FORCE_SIGNPOSTS) || !defined(NDEBUG)
+/// Dynamic logs need to be enabled via the O2_LOG_ENABLE macro. Notice this will only work
 /// for the logger based logging, since the Apple version needs instruments to enable them.
-#define O2_DECLARE_DYNAMIC_LOG(name) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 0)
+#define O2_DECLARE_DYNAMIC_LOG(name) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 1)
 /// For the moment we do not support logs with a stacktrace.
-#define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(name) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 0)
+#define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(name) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 64)
 #define O2_DECLARE_LOG(name, category) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 1)
-#define O2_LOG_ENABLE_DYNAMIC(log) _o2_log_set_stacktrace(private_o2_log_##log, 1)
-// We print out only the first 64 frames.
-#define O2_LOG_ENABLE_STACKTRACE(log) _o2_log_set_stacktrace(private_o2_log_##log, 64)
+// When we enable the log, we set the stacktrace to the default value.
+#define O2_LOG_ENABLE(log) _o2_log_set_stacktrace(private_o2_log_##log, private_o2_log_##log->defaultStacktrace)
+#define O2_LOG_DISABLE(log) _o2_log_set_stacktrace(private_o2_log_##log, 0)
 // For the moment we simply use LOG DEBUG. We should have proper activities so that we can
 // turn on and off the printing.
 #define O2_LOG_DEBUG(log, ...) O2_LOG_MACRO(__VA_ARGS__)
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer) _o2_signpost_id_t name = _o2_signpost_id_make_with_pointer(private_o2_log_##log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log) _o2_signpost_id_t name = _o2_signpost_id_generate_local(private_o2_log_##log)
-#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
-#define O2_SIGNPOST_START(log, id, name, format, ...) _o2_signpost_interval_begin(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
-#define O2_SIGNPOST_END(log, id, name, format, ...) _o2_signpost_interval_end(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
+// In case Instruments is attached, we switch to the Apple signpost API otherwise, both one
+// mac and on linux we use our own implementation, using the logger. We can use the same ids because
+// they are compatible between the two implementations, we also use remove_engineering_type to remove
+// the engineering types from the format string, so that we can use the same format string for both.
+#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) __extension__({                                          \
+  if (O2_SIGNPOST_ENABLED_MAC(log)) {                                                                               \
+    O2_SIGNPOST_EVENT_EMIT_MAC(log, id, name, format, ##__VA_ARGS__);                                               \
+  } else if (private_o2_log_##log->stacktrace) {                                                                    \
+    _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
+  }                                                                                                                 \
+})
+#define O2_SIGNPOST_START(log, id, name, format, ...)                                                                   \
+  if (O2_SIGNPOST_ENABLED_MAC(log)) {                                                                                   \
+    O2_SIGNPOST_START_MAC(log, id, name, format, ##__VA_ARGS__);                                                        \
+  } else if (private_o2_log_##log->stacktrace) {                                                                        \
+    _o2_signpost_interval_begin(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
+  }
+#define O2_SIGNPOST_END(log, id, name, format, ...)                                                                   \
+  if (O2_SIGNPOST_ENABLED_MAC(log)) {                                                                                 \
+    O2_SIGNPOST_END_MAC(log, id, name, format, ##__VA_ARGS__);                                                        \
+  } else if (private_o2_log_##log->stacktrace) {                                                                      \
+    _o2_signpost_interval_end(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
+  }
 #else // This is the release implementation, it does nothing.
 #define O2_DECLARE_DYNAMIC_LOG(x)
 #define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(x)
 #define O2_DECLARE_LOG(x, category)
-#define O2_LOG_ENABLE_DYNAMIC(log)
-#define O2_LOG_ENABLE_STACKTRACE(log)
+#define O2_LOG_ENABLE(log)
+#define O2_LOG_DISABLE(log)
 #define O2_LOG_DEBUG(log, ...)
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log)
diff --git a/Framework/Foundation/test/test_Signpost.cxx b/Framework/Foundation/test/test_Signpost.cxx
index 24b6afaec5c3d..f9b8d4ec0e13a 100644
--- a/Framework/Foundation/test/test_Signpost.cxx
+++ b/Framework/Foundation/test/test_Signpost.cxx
@@ -43,7 +43,7 @@ int main(int argc, char** argv)
 
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
   O2_SIGNPOST_END(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
-  O2_LOG_ENABLE_DYNAMIC(test_SignpostDynamic);
+  O2_LOG_ENABLE(test_SignpostDynamic);
 #ifdef __APPLE__
   // On Apple there is no way to turn on signposts in the logger, so we do not display this message
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will see, because we turned them on");
diff --git a/Framework/Foundation/test/test_SignpostLogger.cxx b/Framework/Foundation/test/test_SignpostLogger.cxx
index 74da35abf7c70..ed917ea7cadf5 100644
--- a/Framework/Foundation/test/test_SignpostLogger.cxx
+++ b/Framework/Foundation/test/test_SignpostLogger.cxx
@@ -51,7 +51,7 @@ int main(int argc, char** argv)
 
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
   O2_SIGNPOST_END(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
-  O2_LOG_ENABLE_DYNAMIC(test_SignpostDynamic);
+  O2_LOG_ENABLE(test_SignpostDynamic);
 #ifdef __APPLE__
   // On Apple there is no way to turn on signposts in the logger, so we do not display this message
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will see, because we turned them on");

From a7d3034558e369aa0ce539d779260b01de23cfc7 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 09:24:03 +0100
Subject: [PATCH 221/726] DPL: add proper tracing for DataProcessingDevice
 socket callbacks

---
 Framework/Core/src/DataProcessingDevice.cxx | 40 +++++++++++++--------
 1 file changed, 26 insertions(+), 14 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index b03904c5b3776..366bfafd7d801 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -35,6 +35,9 @@
 #include "Framework/TMessageSerializer.h"
 #include "Framework/InputRecord.h"
 #include "Framework/InputSpan.h"
+#if defined(__APPLE__) || defined(NDEBUG)
+#define O2_SIGNPOST_IMPLEMENTATION
+#endif
 #include "Framework/Signpost.h"
 #include "Framework/TimingHelpers.h"
 #include "Framework/SourceInfoHeader.h"
@@ -80,6 +83,8 @@
 #include <sstream>
 #include <boost/property_tree/json_parser.hpp>
 
+O2_DECLARE_DYNAMIC_LOG(device);
+
 using namespace o2::framework;
 using ConfigurationInterface = o2::configuration::ConfigurationInterface;
 using DataHeader = o2::header::DataHeader;
@@ -274,21 +279,22 @@ struct PollerContext {
 void on_socket_polled(uv_poll_t* poller, int status, int events)
 {
   auto* context = (PollerContext*)poller->data;
+  assert(context);
+  O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
   context->state->loopReason |= DeviceState::DATA_SOCKET_POLLED;
   switch (events) {
     case UV_READABLE: {
-      ZoneScopedN("socket readable event");
-      LOG(debug) << "socket polled UV_READABLE: " << context->name;
+      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "Data pending on socket for channel %{public}s", context->name);
       context->state->loopReason |= DeviceState::DATA_INCOMING;
     } break;
     case UV_WRITABLE: {
-      ZoneScopedN("socket writeable");
+      O2_SIGNPOST_END(device, sid, "socket_state", "Socket connected for channel %{public}s", context->name);
       if (context->read) {
-        LOG(debug) << "socket polled UV_CONNECT" << context->name;
+        O2_SIGNPOST_START(device, sid, "socket_state", "Socket connected for read in context %{public}s", context->name);
         uv_poll_start(poller, UV_READABLE | UV_DISCONNECT | UV_PRIORITIZED, &on_socket_polled);
         context->state->loopReason |= DeviceState::DATA_CONNECTED;
       } else {
-        LOG(debug) << "socket polled UV_WRITABLE" << context->name;
+        O2_SIGNPOST_START(device, sid, "socket_state", "Socket connected for write for channel %{public}s", context->name);
         context->state->loopReason |= DeviceState::DATA_OUTGOING;
         // If the socket is writable, fairmq will handle the rest, so we can stop polling and
         // just wait for the disconnect.
@@ -297,12 +303,10 @@ void on_socket_polled(uv_poll_t* poller, int status, int events)
       context->pollerState = PollerContext::PollerState::Connected;
     } break;
     case UV_DISCONNECT: {
-      ZoneScopedN("socket disconnect");
-      LOG(debug) << "socket polled UV_DISCONNECT";
+      O2_SIGNPOST_END(device, sid, "socket_state", "Socket disconnected in context %{public}s", context->name);
     } break;
     case UV_PRIORITIZED: {
-      ZoneScopedN("socket prioritized");
-      LOG(debug) << "socket polled UV_PRIORITIZED";
+      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "Data pending on socket for context %{public}s", context->name);
     } break;
   }
   // We do nothing, all the logic for now stays in DataProcessingDevice::doRun()
@@ -873,7 +877,9 @@ void DataProcessingDevice::startPollers()
   auto& deviceContext = ref.get<DeviceContext>();
   auto& state = ref.get<DeviceState>();
 
-  for (auto& poller : state.activeInputPollers) {
+  for (auto* poller : state.activeInputPollers) {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
+    O2_SIGNPOST_START(device, sid, "socket_state", "Input socket waiting for connection.");
     uv_poll_start(poller, UV_WRITABLE, &on_socket_polled);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Disconnected;
   }
@@ -881,7 +887,9 @@ void DataProcessingDevice::startPollers()
     uv_poll_start(poller, UV_WRITABLE, &on_out_of_band_polled);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Disconnected;
   }
-  for (auto& poller : state.activeOutputPollers) {
+  for (auto* poller : state.activeOutputPollers) {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
+    O2_SIGNPOST_START(device, sid, "socket_state", "Output socket waiting for connection.");
     uv_poll_start(poller, UV_WRITABLE, &on_socket_polled);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Disconnected;
   }
@@ -897,17 +905,21 @@ void DataProcessingDevice::stopPollers()
   auto& deviceContext = ref.get<DeviceContext>();
   auto& state = ref.get<DeviceState>();
   LOGP(detail, "Stopping {} input pollers", state.activeInputPollers.size());
-  for (auto& poller : state.activeInputPollers) {
+  for (auto* poller : state.activeInputPollers) {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
+    O2_SIGNPOST_END(device, sid, "socket_state", "Output socket closed.");
     uv_poll_stop(poller);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Stopped;
   }
   LOGP(detail, "Stopping {} out of band pollers", state.activeOutOfBandPollers.size());
-  for (auto& poller : state.activeOutOfBandPollers) {
+  for (auto* poller : state.activeOutOfBandPollers) {
     uv_poll_stop(poller);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Stopped;
   }
   LOGP(detail, "Stopping {} output pollers", state.activeOutOfBandPollers.size());
-  for (auto& poller : state.activeOutputPollers) {
+  for (auto* poller : state.activeOutputPollers) {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
+    O2_SIGNPOST_END(device, sid, "socket_state", "Output socket closed.");
     uv_poll_stop(poller);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Stopped;
   }

From 4df6e4a6d17cbd9241dc7ff2ad3ed5346456bff3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 09:24:03 +0100
Subject: [PATCH 222/726] DPL use Signposts to debug consumeWhenAll

---
 Framework/Core/src/CompletionPolicyHelpers.cxx | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx
index 8c4b38ba3355c..a6240b1165b81 100644
--- a/Framework/Core/src/CompletionPolicyHelpers.cxx
+++ b/Framework/Core/src/CompletionPolicyHelpers.cxx
@@ -18,10 +18,13 @@
 #include "Framework/TimesliceIndex.h"
 #include "Framework/TimingInfo.h"
 #include "DecongestionService.h"
+#include "Framework/Signpost.h"
 
 #include <cassert>
 #include <regex>
 
+O2_DECLARE_DYNAMIC_LOG(completion);
+
 namespace o2::framework
 {
 
@@ -108,6 +111,8 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
 {
   auto callback = [](InputSpan const& inputs, std::vector<InputSpec> const& specs, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     assert(inputs.size() == specs.size());
+    O2_SIGNPOST_ID_GENERATE(sid, completion);
+    O2_SIGNPOST_START(completion, sid, "consumeWhenAll", "Completion policy invoked");
 
     size_t si = 0;
     bool missingSporadic = false;
@@ -116,15 +121,18 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
       assert(si < specs.size());
       auto& spec = specs[si++];
       if (input.header == nullptr && spec.lifetime != Lifetime::Sporadic) {
+        O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s due to missing input %lu", "Wait", si);
         return CompletionPolicy::CompletionOp::Wait;
       }
       if (input.header == nullptr && spec.lifetime == Lifetime::Sporadic) {
+        O2_SIGNPOST_EVENT_EMIT(completion, sid, "consumeWhenAll", "Missing sporadic found for route index %lu", si);
         missingSporadic = true;
       }
       if (input.header != nullptr && currentTimeslice == -1) {
         auto const* dph = framework::DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(input);
         if (dph && !TimingInfo::timesliceIsTimer(dph->startTime)) {
           currentTimeslice = dph->startTime;
+          O2_SIGNPOST_EVENT_EMIT(completion, sid, "consumeWhenAll", "currentTimeslice %lu from route index %lu", currentTimeslice, si);
         }
       }
     }
@@ -134,8 +142,10 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
     auto oldestPossibleTimeslice = timesliceIndex.getOldestPossibleInput().timeslice.value;
 
     if (missingSporadic && currentTimeslice >= oldestPossibleTimeslice) {
+      O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s for timeslice %lu > oldestPossibleTimeslice %lu", "Retry", currentTimeslice, oldestPossibleTimeslice);
       return CompletionPolicy::CompletionOp::Retry;
     }
+    O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s for timeslice %lu <= oldestPossibleTimeslice %lu", "Consume", currentTimeslice, oldestPossibleTimeslice);
     return CompletionPolicy::CompletionOp::Consume;
   };
   return CompletionPolicy{name, matcher, callback};

From d566fd1bc0000efb68deb110b3c61a4f54f25fb5 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 09:41:37 +0100
Subject: [PATCH 223/726] Ability to turn on / off the logger from the GUI.

---
 .../Core/include/Framework/DeviceControl.h    |  2 +
 .../Core/include/Framework/DeviceState.h      |  9 +++++
 Framework/Core/src/WSDriverClient.cxx         | 40 +++++++++++++++++++
 .../src/FrameworkGUIDeviceInspector.cxx       | 11 +++++
 .../test/test_DetectMissingTimeframe.cxx      |  1 +
 5 files changed, 63 insertions(+)

diff --git a/Framework/Core/include/Framework/DeviceControl.h b/Framework/Core/include/Framework/DeviceControl.h
index 589de3937ee55..ce946e8e77fbf 100644
--- a/Framework/Core/include/Framework/DeviceControl.h
+++ b/Framework/Core/include/Framework/DeviceControl.h
@@ -48,6 +48,8 @@ struct DeviceControl {
   DeviceController* controller = nullptr;
   /// What kind of events should run with the TRACE level
   int tracingFlags = 0;
+  /// What kind of log streams should be enabled
+  int logStreams = 0;
   /// An incremental number to identify the device state
   int requestedState = 0;
 };
diff --git a/Framework/Core/include/Framework/DeviceState.h b/Framework/Core/include/Framework/DeviceState.h
index 3be0302fd4113..4fa72a84cad71 100644
--- a/Framework/Core/include/Framework/DeviceState.h
+++ b/Framework/Core/include/Framework/DeviceState.h
@@ -57,6 +57,13 @@ struct DeviceState {
     DATA_CONNECTED = 1 << 19,     // Data channel connected
   };
 
+  enum LogStreams : int {
+    NO_LOG = 0,
+    DEVICE_LOG = 1 << 0,             // Log for Data Processing Device activities.
+    COMPLETION_LOG = 1 << 1,         // Log for the completion policy of the device.
+    MONITORING_SERVICE_LOG = 1 << 2, // Log for the monitoring service flushing.
+  };
+
   std::vector<InputChannelInfo> inputChannelInfos;
   StreamingState streaming = StreamingState::Streaming;
   bool quitRequested = false;
@@ -93,6 +100,8 @@ struct DeviceState {
   int loopReason = 0;
   /// Bitmask of LoopReason to trace
   int tracingFlags = 0;
+  /// Bitmask of log streams which are available
+  int logStreams = 0;
   /// Stack of the severity, so that we can display only
   /// the bits we are interested in.
   std::vector<int> severityStack;
diff --git a/Framework/Core/src/WSDriverClient.cxx b/Framework/Core/src/WSDriverClient.cxx
index d4ed77b9a004e..ac2e3db41fcef 100644
--- a/Framework/Core/src/WSDriverClient.cxx
+++ b/Framework/Core/src/WSDriverClient.cxx
@@ -16,10 +16,15 @@
 #include "Framework/DeviceSpec.h"
 #include "DriverClientContext.h"
 #include "DPLWebSocket.h"
+#include "Framework/Signpost.h"
 #include <uv.h>
 #include <string_view>
 #include <charconv>
 
+O2_DECLARE_DYNAMIC_LOG(device);
+O2_DECLARE_DYNAMIC_LOG(completion);
+O2_DECLARE_DYNAMIC_LOG(monitoring_service);
+
 namespace o2::framework
 {
 
@@ -152,6 +157,41 @@ void on_connect(uv_connect_t* connection, int status)
     state.tracingFlags = tracingFlags;
   });
 
+  client->observe("/log-streams", [ref = context->ref](std::string_view cmd) {
+    auto& state = ref.get<DeviceState>();
+    static constexpr int prefixSize = std::string_view{"/log-streams "}.size();
+    if (prefixSize > cmd.size()) {
+      LOG(error) << "Malformed log-streams request";
+      return;
+    }
+    cmd.remove_prefix(prefixSize);
+    int logStreams = 0;
+
+    auto error = std::from_chars(cmd.data(), cmd.data() + cmd.size(), logStreams);
+    if (error.ec != std::errc()) {
+      LOG(error) << "Malformed log-streams mask";
+      return;
+    }
+    LOGP(info, "Logstreams flags set to {}", logStreams);
+    state.logStreams = logStreams;
+    if ((state.logStreams & DeviceState::LogStreams::DEVICE_LOG) != 0) {
+      O2_LOG_ENABLE(device);
+    } else {
+      O2_LOG_DISABLE(device);
+    }
+    if ((state.logStreams & DeviceState::LogStreams::COMPLETION_LOG) != 0) {
+      O2_LOG_ENABLE(completion);
+    } else {
+      O2_LOG_DISABLE(completion);
+    }
+
+    if ((state.logStreams & DeviceState::LogStreams::MONITORING_SERVICE_LOG) != 0) {
+      O2_LOG_ENABLE(monitoring_service);
+    } else {
+      O2_LOG_DISABLE(monitoring_service);
+    }
+  });
+
   // Client will be filled in the line after. I can probably have a single
   // client per device.
   auto dplClient = std::make_unique<WSDPLClient>();
diff --git a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
index a82753eb5af1f..fe93ca6d0f07f 100644
--- a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
@@ -400,6 +400,17 @@ void displayDeviceInspector(DeviceSpec const& spec,
     }
   }
 
+  bool logsChanged = false;
+  if (ImGui::CollapsingHeader("Signposts", ImGuiTreeNodeFlags_DefaultOpen)) {
+    logsChanged = ImGui::CheckboxFlags("Device", &control.logStreams, DeviceState::LogStreams::DEVICE_LOG);
+    logsChanged = ImGui::CheckboxFlags("Completion", &control.logStreams, DeviceState::LogStreams::COMPLETION_LOG);
+    logsChanged = ImGui::CheckboxFlags("Monitoring", &control.logStreams, DeviceState::LogStreams::MONITORING_SERVICE_LOG);
+    if (logsChanged && control.controller) {
+      std::string cmd = fmt::format("/log-streams {}", control.logStreams);
+      control.controller->write(cmd.c_str(), cmd.size());
+    }
+  }
+
   bool flagsChanged = false;
   if (ImGui::CollapsingHeader("Event loop tracing", ImGuiTreeNodeFlags_DefaultOpen)) {
     flagsChanged |= ImGui::CheckboxFlags("METRICS_MUST_FLUSH", &control.tracingFlags, DeviceState::LoopReason::METRICS_MUST_FLUSH);
diff --git a/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx b/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
index 3374cd9ad6cf2..7ef1370f040b4 100644
--- a/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
+++ b/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
@@ -43,6 +43,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& specs)
         if (i++ % 2 == 0) {
           outputs.make<int>(OutputRef{"a2"}, 1);
         }
+        sleep(1);
       })},
   };
   DataProcessorSpec d{

From 34d79bf5747e7f47eef5453c147f162870720041 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 10:17:25 +0100
Subject: [PATCH 224/726] DPL GUI: add GUI to enable / disable driver signposts

---
 Framework/GUISupport/src/FrameworkGUIDebugger.cxx | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
index 35e0bc563a22e..005f7082d77b4 100644
--- a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
@@ -23,6 +23,7 @@
 #include "FrameworkGUIDataRelayerUsage.h"
 #include "PaletteHelpers.h"
 #include "FrameworkGUIState.h"
+#include "Framework/Signpost.h"
 #include <DebugGUI/icons_font_awesome.h>
 
 #include <fmt/format.h>
@@ -34,6 +35,7 @@
 #include <cinttypes>
 #include <numeric>
 
+O2_DECLARE_DYNAMIC_LOG(driver);
 // Make sure we can use aggregated initialisers.
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wpedantic"
@@ -506,7 +508,7 @@ void displayDeviceMetrics(const char* label,
     case MetricsDisplayStyle::Lines: {
       auto xAxisFlags = ImPlotAxisFlags_None;
       auto yAxisFlags = ImPlotAxisFlags_LockMin;
-      //ImPlot::FitNextPlotAxes(true, true, true, true);
+      // ImPlot::FitNextPlotAxes(true, true, true, true);
       if (ImPlot::BeginPlot("##Some plot", {-1, -1}, axisFlags)) {
         ImPlot::SetupAxes("time", "value", xAxisFlags, yAxisFlags);
         ImPlot::SetupAxisFormat(ImAxis_Y1, formatSI, nullptr);
@@ -1093,6 +1095,17 @@ void displayDriverInfo(DriverInfo const& driverInfo, DriverControl& driverContro
   }
 #endif
 
+  static bool selectedSignpost = false;
+  if (ImGui::CollapsingHeader("Signposts", ImGuiTreeNodeFlags_DefaultOpen)) {
+    if (ImGui::Checkbox("Driver", &selectedSignpost)) {
+      if (selectedSignpost) {
+        O2_LOG_ENABLE(driver);
+      } else {
+        O2_LOG_DISABLE(driver);
+      }
+    }
+  }
+
   for (size_t i = 0; i < driverInfo.states.size(); ++i) {
     ImGui::Text("#%lu: %s", i, DriverInfoHelper::stateToString(driverInfo.states[i]));
   }

From 942a18c7312363a4bed12a85ebce1c804c3cee17 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 15 Jan 2024 13:02:30 +0100
Subject: [PATCH 225/726] DPL: drop unused o2_signpost_id_make_with_pointer
 method

Probably old remaining from a previous implementation.
---
 Framework/Foundation/include/Framework/Signpost.h | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index eca9e26f77469..64d92d67acbf7 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -218,7 +218,6 @@ bool _o2_lock_free_stack_push(_o2_lock_free_stack& stack, const int& value, bool
 bool _o2_lock_free_stack_pop(_o2_lock_free_stack& stack, int& value, bool spin = false);
 //_o2_signpost_id_t _o2_signpost_id_generate_local(_o2_log_t* log);
 //_o2_signpost_id_t _o2_signpost_id_make_with_pointer(_o2_log_t* log, void* pointer);
-_o2_signpost_index_t o2_signpost_id_make_with_pointer(_o2_log_t* log, void* pointer);
 void* _o2_log_create(char const* name, int stacktrace);
 void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...);
 void _o2_signpost_interval_begin(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...);
@@ -246,14 +245,6 @@ inline _o2_signpost_id_t _o2_signpost_id_make_with_pointer(_o2_log_t* log, void*
   return uniqueId;
 }
 
-inline _o2_signpost_index_t o2_signpost_id_make_with_pointer(_o2_log_t* log, void* pointer)
-{
-  _o2_signpost_index_t signpost_index;
-  _o2_lock_free_stack_pop(log->slots, signpost_index, true);
-  log->ids[signpost_index].id = (int64_t)pointer;
-  return signpost_index;
-}
-
 // Implementation start here. Include this file with O2_SIGNPOST_IMPLEMENTATION defined in one file of your
 // project.
 #ifdef O2_SIGNPOST_IMPLEMENTATION

From 7eb936ce0b25f3ee218b3c159d27e87d743b7002 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 11:21:05 +0100
Subject: [PATCH 226/726] DPL: rework signposts support

The logger based Signposts are now behaving the same on macOS and linux when
running normally.

The difference between the two implementations will now be that if Instruments
enables some OSLog namespace, we stop printing the messages and use the os_signpost
API instead.
---
 .../Foundation/include/Framework/Signpost.h   | 191 +++++++++---------
 Framework/Foundation/test/test_Signpost.cxx   |   2 +-
 .../Foundation/test/test_SignpostLogger.cxx   |   2 +-
 3 files changed, 94 insertions(+), 101 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index 64d92d67acbf7..d2bdec0ec8246 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -13,6 +13,9 @@
 
 #include <atomic>
 #include <array>
+#ifdef __APPLE__
+#include <os/log.h>
+#endif
 
 struct o2_log_handle_t {
   char const* name = nullptr;
@@ -22,6 +25,7 @@ struct o2_log_handle_t {
 
 // Helper function which replaces engineering types with a printf
 // compatible format string.
+// FIXME: make this consteval when available in C++20
 template <auto N>
 constexpr auto remove_engineering_type(char const (&src)[N])
 {
@@ -82,78 +86,37 @@ o2_log_handle_t* o2_walk_logs(bool (*callback)(char const* name, void* log, void
 }
 #endif
 
-#if !defined(O2_FORCE_LOGGER_SIGNPOST) && defined(__APPLE__) && (!defined(NDEBUG) || defined(O2_FORCE_SIGNPOSTS))
+#if defined(__APPLE__)
 #include <os/log.h>
 #include <os/signpost.h>
 #include <cstring>
-void* _o2_log_create(char const* name, char const* category);
-#define O2_DECLARE_DYNAMIC_LOG(x) static os_log_t private_o2_log_##x = (os_log_t)_o2_log_create("ch.cern.aliceo2." #x, OS_LOG_CATEGORY_DYNAMIC_TRACING)
-#define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(x) static os_log_t private_o2_log_##x = (os_log_t)_o2_log_create("ch.cern.aliceo2." #x, OS_LOG_CATEGORY_DYNAMIC_STACK_TRACING)
-// This is a no-op on macOS using the os_signpost API because only external instruments can enable/disable dynamic signposts
-#define O2_LOG_ENABLE_DYNAMIC(log)
-// This is a no-op on macOS using the os_signpost API because only external instruments can enable/disable dynamic signposts
-#define O2_LOG_ENABLE_STACKTRACE(log)
-#define O2_DECLARE_LOG(x, category) static os_log_t private_o2_log_##x = (os_log_t)_o2_log_create("ch.cern.aliceo2." #x, #category)
-#define O2_LOG_DEBUG(log, ...) os_log_debug(private_o2_log_##log, __VA_ARGS__)
-#define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer) os_signpost_id_t name = os_signpost_id_make_with_pointer(private_o2_log_##log, pointer)
-#define O2_SIGNPOST_ID_GENERATE(name, log) os_signpost_id_t name = os_signpost_id_generate(private_o2_log_##log)
+#define O2_LOG_DEBUG_MAC(log, ...) os_log_debug(private_o2_log_##log, __VA_ARGS__)
 // FIXME: use __VA_OPT__ when available in C++20
-#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) os_signpost_event_emit(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
-#define O2_SIGNPOST_START(log, id, name, format, ...) os_signpost_interval_begin(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
-#define O2_SIGNPOST_END(log, id, name, format, ...) os_signpost_interval_end(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
-
-#ifdef O2_SIGNPOST_IMPLEMENTATION
-/// We use a wrapper so that we can keep track of the logs.
-void* _o2_log_create(char const* name, char const* category)
-{
-  // iterate over the list of logs and check if we already have
-  // one with the same name.
-  auto findLogByName = [](char const* name, void* log, void* context) -> bool {
-    char const* currentName = (char const*)context;
-    if (strcmp(name, currentName) == 0) {
-      return false;
-    }
-    return true;
-  };
-
-  o2_log_handle_t* handle = o2_walk_logs(findLogByName, (void*)name);
-
-  // If we found one, return it.
-  if (handle) {
-    return handle->log;
-  }
-  // Otherwise, create a new one and add it to the end of the list.
-  os_log_t log = os_log_create(name, category);
-  o2_log_handle_t* newHandle = new o2_log_handle_t();
-  newHandle->log = log;
-  newHandle->name = strdup(name);
-  newHandle->next = o2_get_logs_tail().load();
-  // Until I manage to replace the log I have in next, keep trying.
-  // Notice this does not protect against two threads trying to insert
-  // a log with the same name. I should probably do a sorted insert for that.
-  while (!o2_get_logs_tail().compare_exchange_weak(newHandle->next, newHandle,
-                                                   std::memory_order_release,
-                                                   std::memory_order_relaxed)) {
-    newHandle->next = o2_get_logs_tail();
-  }
-
-  return log;
-}
+#define O2_SIGNPOST_EVENT_EMIT_MAC(log, id, name, format, ...) os_signpost_event_emit(private_o2_log_##log->os_log, (uint64_t)id.value, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_START_MAC(log, id, name, format, ...) os_signpost_interval_begin(private_o2_log_##log->os_log, (uint64_t)id.value, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_END_MAC(log, id, name, format, ...) os_signpost_interval_end(private_o2_log_##log->os_log, (uint64_t)id.value, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_ENABLED_MAC(log) os_signpost_enabled(private_o2_log_##log->os_log)
+#else
+// These are no-ops on linux.
+#define O2_DECLARE_LOG_MAC(x, category)
+#define O2_LOG_DEBUG_MAC(log, ...)
+#define O2_SIGNPOST_EVENT_EMIT_MAC(log, id, name, format, ...)
+#define O2_SIGNPOST_START_MAC(log, id, name, format, ...)
+#define O2_SIGNPOST_END_MAC(log, id, name, format, ...)
+#define O2_SIGNPOST_ENABLED_MAC(log) false
+#endif // __APPLE__
+
+// Unless we are on apple we enable checking for signposts only if in debug mode or if we force them.
+#if defined(__APPLE__) || defined(O2_FORCE_SIGNPOSTS) || !defined(NDEBUG)
+#define O2_LOG_ENABLED(log) private_o2_log_##log->stacktrace
+#else
+#define O2_LOG_ENABLED(log) false
 #endif
 
-#elif !defined(NDEBUG) || defined(O2_FORCE_LOGGER_SIGNPOST) || defined(O2_FORCE_SIGNPOSTS)
-
-#ifndef O2_LOG_MACRO
-#if __has_include("Framework/Logger.h")
+#if !defined(O2_LOG_MACRO) && __has_include("Framework/Logger.h")
 #include "Framework/Logger.h"
-// If NDEBUG is not defined, we use the logger to print out the signposts at the debug level.
-#if !defined(NDEBUG)
-#define O2_LOG_MACRO(...) LOGF(debug, __VA_ARGS__)
-#elif defined(O2_FORCE_LOGGER_SIGNPOST) || defined(O2_FORCE_SIGNPOSTS)
-// If we force the presence of the logger, we use it to print out the signposts at the detail level, which is not optimized out.
 #define O2_LOG_MACRO(...) LOGF(info, __VA_ARGS__)
-#endif
-#else
+#elif !defined(O2_LOG_MACRO)
 // If we do not have the fairlogger, we simply print out the signposts to the console.
 // This is useful for things like the tests, which this way do not need to depend on the FairLogger.
 #define O2_LOG_MACRO(...) \
@@ -161,8 +124,9 @@ void* _o2_log_create(char const* name, char const* category)
     printf(__VA_ARGS__);  \
     printf("\n");         \
   } while (0)
-#endif
-#endif
+#else
+#define O2_LOG_MACRO(...)
+#endif // O2_LOG_MACRO
 
 // This is the linux implementation, it is not as nice as the apple one and simply prints out
 // the signpost information to the log.
@@ -181,7 +145,7 @@ struct _o2_lock_free_stack {
 // A log is simply an inbox which keeps track of the available id, so that we can print out different signposts
 // with different indentation levels.
 // supports up to 1024 paralle signposts before it spinlocks.
-typedef int _o2_signpost_index_t;
+using _o2_signpost_index_t = int;
 
 struct _o2_activity_t {
   // How much the activity is indented in the output log.
@@ -191,10 +155,13 @@ struct _o2_activity_t {
 
 struct _o2_signpost_id_t {
   // The id of the activity.
-  int64_t id = -1;
+  int64_t value = -1;
 };
 
 struct _o2_log_t {
+#ifdef __APPLE__
+  os_log_t os_log = nullptr;
+#endif
   // A circular buffer of available slots. Each unique interval pulls an id from this buffer.
   _o2_lock_free_stack slots;
   // Up to 256 activities can be active at the same time.
@@ -211,7 +178,10 @@ struct _o2_log_t {
   // 0 means the log is disabled.
   // 1 means only the current signpost is printed.
   // >1 means the current signpost and n levels of the stacktrace are printed.
-  std::atomic<int> stacktrace = 1;
+  std::atomic<int> stacktrace = 0;
+
+  // Default stacktrace level for the log, when enabled.
+  int defaultStacktrace = 1;
 };
 
 bool _o2_lock_free_stack_push(_o2_lock_free_stack& stack, const int& value, bool spin = false);
@@ -310,7 +280,7 @@ bool _o2_lock_free_stack_pop(_o2_lock_free_stack& stack, int& value, bool spin)
   }
 }
 
-void* _o2_log_create(char const* name, int stacktrace)
+void* _o2_log_create(char const* name, int defaultStacktrace)
 {
   // iterate over the list of logs and check if we already have
   // one with the same name.
@@ -328,7 +298,7 @@ void* _o2_log_create(char const* name, int stacktrace)
     return handle->log;
   }
   // Otherwise, create a new one and add it to the end of the list.
-  _o2_log_t* log = new _o2_log_t();
+  auto* log = new _o2_log_t();
   // Write the initial 256 ids to the inbox, in reverse, so that the
   // linear search below is just for an handful of elements.
   int n = _o2_lock_free_stack::N;
@@ -336,9 +306,18 @@ void* _o2_log_create(char const* name, int stacktrace)
     _o2_signpost_index_t signpost_index{n - 1 - i};
     _o2_lock_free_stack_push(log->slots, signpost_index, true);
   }
-  log->stacktrace = stacktrace;
-  o2_log_handle_t* newHandle = new o2_log_handle_t();
+  log->defaultStacktrace = defaultStacktrace;
+  auto* newHandle = new o2_log_handle_t();
   newHandle->log = log;
+#ifdef __APPLE__
+  // On macOS, we use the os_signpost API so that when we are
+  // using instruments we can see the messages there.
+  if (defaultStacktrace > 1) {
+    log->os_log = os_log_create(name, OS_LOG_CATEGORY_DYNAMIC_STACK_TRACING);
+  } else {
+    log->os_log = os_log_create(name, OS_LOG_CATEGORY_DYNAMIC_TRACING);
+  }
+#endif
   newHandle->name = strdup(name);
   newHandle->next = o2_get_logs_tail().load();
   // Until I manage to replace the log I have in next, keep trying.
@@ -357,10 +336,6 @@ void* _o2_log_create(char const* name, int stacktrace)
 // If the slot is empty, it will return the id and increment the indentation level.
 void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...)
 {
-  // Nothing to be done
-  if (log->stacktrace == 0) {
-    return;
-  }
   va_list args;
   va_start(args, format);
 
@@ -368,10 +343,10 @@ void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* n
   int leading = 0;
 
   // This is the equivalent of exclusive
-  if (id.id != 0) {
+  if (id.value != 0) {
     int i = 0;
     for (i = 0; i < log->ids.size(); ++i) {
-      if (log->ids[i].id == id.id) {
+      if (log->ids[i].value == id.value) {
         break;
       }
     }
@@ -385,7 +360,7 @@ void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* n
   }
 
   char prebuffer[4096];
-  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16s*>%*c", id.id, name, leading, ' ');
+  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16s*>%*c", id.value, name, leading, ' ');
   vsnprintf(prebuffer + s, 4096 - s, format, args);
   va_end(args);
   O2_LOG_MACRO("%s", prebuffer);
@@ -395,22 +370,19 @@ void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* n
 // If the slot is empty, it will return the id and increment the indentation level.
 void _o2_signpost_interval_begin(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...)
 {
-  if (log->stacktrace == 0) {
-    return;
-  }
   va_list args;
   va_start(args, format);
   // This is a unique slot for this interval.
   _o2_signpost_index_t signpost_index;
   _o2_lock_free_stack_pop(log->slots, signpost_index, true);
   // Put the id in the slot, to close things or to attach signposts to a given interval
-  log->ids[signpost_index].id = id.id;
+  log->ids[signpost_index].value = id.value;
   auto* activity = &log->activities[signpost_index];
   activity->indentation = log->current_indentation++;
   activity->name = name;
   int leading = activity->indentation * 2;
   char prebuffer[4096];
-  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16sS>%*c", id.id, name, leading, ' ');
+  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16sS>%*c", id.value, name, leading, ' ');
   vsnprintf(prebuffer + s, 4096 - s, format, args);
   va_end(args);
   O2_LOG_MACRO("%s", prebuffer);
@@ -424,7 +396,7 @@ void _o2_signpost_interval_end_v(_o2_log_t* log, _o2_signpost_id_t id, char cons
   // Find the index of the activity
   int i = 0;
   for (i = 0; i < log->ids.size(); ++i) {
-    if (log->ids[i].id == id.id) {
+    if (log->ids[i].value == id.value) {
       break;
     }
   }
@@ -439,13 +411,13 @@ void _o2_signpost_interval_end_v(_o2_log_t* log, _o2_signpost_id_t id, char cons
   _o2_activity_t* activity = &log->activities[i];
   int leading = activity->indentation * 2;
   char prebuffer[4096];
-  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16sE>%*c", id.id, name, leading, ' ');
+  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16sE>%*c", id.value, name, leading, ' ');
   vsnprintf(prebuffer + s, 4096 - s, format, args);
   O2_LOG_MACRO("%s", prebuffer);
   // Clear the slot
   activity->indentation = -1;
   activity->name = nullptr;
-  log->ids[i].id = -1;
+  log->ids[i].value = -1;
   // Put back the slot
   log->current_indentation--;
   _o2_signpost_index_t signpost_index{i};
@@ -468,29 +440,50 @@ void _o2_log_set_stacktrace(_o2_log_t* log, int stacktrace)
 }
 #endif // O2_SIGNPOST_IMPLEMENTATION
 
-/// Dynamic logs need to be enabled via the O2_LOG_ENABLE_DYNAMIC macro. Notice this will only work
+#if defined(__APPLE__) || defined(O2_FORCE_SIGNPOSTS) || !defined(NDEBUG)
+/// Dynamic logs need to be enabled via the O2_LOG_ENABLE macro. Notice this will only work
 /// for the logger based logging, since the Apple version needs instruments to enable them.
-#define O2_DECLARE_DYNAMIC_LOG(name) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 0)
+#define O2_DECLARE_DYNAMIC_LOG(name) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 1)
 /// For the moment we do not support logs with a stacktrace.
-#define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(name) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 0)
+#define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(name) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 64)
 #define O2_DECLARE_LOG(name, category) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 1)
-#define O2_LOG_ENABLE_DYNAMIC(log) _o2_log_set_stacktrace(private_o2_log_##log, 1)
-// We print out only the first 64 frames.
-#define O2_LOG_ENABLE_STACKTRACE(log) _o2_log_set_stacktrace(private_o2_log_##log, 64)
+// When we enable the log, we set the stacktrace to the default value.
+#define O2_LOG_ENABLE(log) _o2_log_set_stacktrace(private_o2_log_##log, private_o2_log_##log->defaultStacktrace)
+#define O2_LOG_DISABLE(log) _o2_log_set_stacktrace(private_o2_log_##log, 0)
 // For the moment we simply use LOG DEBUG. We should have proper activities so that we can
 // turn on and off the printing.
 #define O2_LOG_DEBUG(log, ...) O2_LOG_MACRO(__VA_ARGS__)
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer) _o2_signpost_id_t name = _o2_signpost_id_make_with_pointer(private_o2_log_##log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log) _o2_signpost_id_t name = _o2_signpost_id_generate_local(private_o2_log_##log)
-#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
-#define O2_SIGNPOST_START(log, id, name, format, ...) _o2_signpost_interval_begin(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
-#define O2_SIGNPOST_END(log, id, name, format, ...) _o2_signpost_interval_end(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
+// In case Instruments is attached, we switch to the Apple signpost API otherwise, both one
+// mac and on linux we use our own implementation, using the logger. We can use the same ids because
+// they are compatible between the two implementations, we also use remove_engineering_type to remove
+// the engineering types from the format string, so that we can use the same format string for both.
+#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) __extension__({                                          \
+  if (O2_SIGNPOST_ENABLED_MAC(log)) {                                                                               \
+    O2_SIGNPOST_EVENT_EMIT_MAC(log, id, name, format, ##__VA_ARGS__);                                               \
+  } else if (private_o2_log_##log->stacktrace) {                                                                    \
+    _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
+  }                                                                                                                 \
+})
+#define O2_SIGNPOST_START(log, id, name, format, ...)                                                                   \
+  if (O2_SIGNPOST_ENABLED_MAC(log)) {                                                                                   \
+    O2_SIGNPOST_START_MAC(log, id, name, format, ##__VA_ARGS__);                                                        \
+  } else if (private_o2_log_##log->stacktrace) {                                                                        \
+    _o2_signpost_interval_begin(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
+  }
+#define O2_SIGNPOST_END(log, id, name, format, ...)                                                                   \
+  if (O2_SIGNPOST_ENABLED_MAC(log)) {                                                                                 \
+    O2_SIGNPOST_END_MAC(log, id, name, format, ##__VA_ARGS__);                                                        \
+  } else if (private_o2_log_##log->stacktrace) {                                                                      \
+    _o2_signpost_interval_end(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
+  }
 #else // This is the release implementation, it does nothing.
 #define O2_DECLARE_DYNAMIC_LOG(x)
 #define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(x)
 #define O2_DECLARE_LOG(x, category)
-#define O2_LOG_ENABLE_DYNAMIC(log)
-#define O2_LOG_ENABLE_STACKTRACE(log)
+#define O2_LOG_ENABLE(log)
+#define O2_LOG_DISABLE(log)
 #define O2_LOG_DEBUG(log, ...)
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log)
diff --git a/Framework/Foundation/test/test_Signpost.cxx b/Framework/Foundation/test/test_Signpost.cxx
index 24b6afaec5c3d..f9b8d4ec0e13a 100644
--- a/Framework/Foundation/test/test_Signpost.cxx
+++ b/Framework/Foundation/test/test_Signpost.cxx
@@ -43,7 +43,7 @@ int main(int argc, char** argv)
 
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
   O2_SIGNPOST_END(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
-  O2_LOG_ENABLE_DYNAMIC(test_SignpostDynamic);
+  O2_LOG_ENABLE(test_SignpostDynamic);
 #ifdef __APPLE__
   // On Apple there is no way to turn on signposts in the logger, so we do not display this message
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will see, because we turned them on");
diff --git a/Framework/Foundation/test/test_SignpostLogger.cxx b/Framework/Foundation/test/test_SignpostLogger.cxx
index 74da35abf7c70..ed917ea7cadf5 100644
--- a/Framework/Foundation/test/test_SignpostLogger.cxx
+++ b/Framework/Foundation/test/test_SignpostLogger.cxx
@@ -51,7 +51,7 @@ int main(int argc, char** argv)
 
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
   O2_SIGNPOST_END(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
-  O2_LOG_ENABLE_DYNAMIC(test_SignpostDynamic);
+  O2_LOG_ENABLE(test_SignpostDynamic);
 #ifdef __APPLE__
   // On Apple there is no way to turn on signposts in the logger, so we do not display this message
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will see, because we turned them on");

From d102e70351f89e1e2556340fb1c50374f5e1cfdf Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 11:22:27 +0100
Subject: [PATCH 227/726] DPL: add proper tracing for DataProcessingDevice
 socket callbacks

---
 Framework/Core/src/DataProcessingDevice.cxx | 40 +++++++++++++--------
 1 file changed, 26 insertions(+), 14 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index b03904c5b3776..366bfafd7d801 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -35,6 +35,9 @@
 #include "Framework/TMessageSerializer.h"
 #include "Framework/InputRecord.h"
 #include "Framework/InputSpan.h"
+#if defined(__APPLE__) || defined(NDEBUG)
+#define O2_SIGNPOST_IMPLEMENTATION
+#endif
 #include "Framework/Signpost.h"
 #include "Framework/TimingHelpers.h"
 #include "Framework/SourceInfoHeader.h"
@@ -80,6 +83,8 @@
 #include <sstream>
 #include <boost/property_tree/json_parser.hpp>
 
+O2_DECLARE_DYNAMIC_LOG(device);
+
 using namespace o2::framework;
 using ConfigurationInterface = o2::configuration::ConfigurationInterface;
 using DataHeader = o2::header::DataHeader;
@@ -274,21 +279,22 @@ struct PollerContext {
 void on_socket_polled(uv_poll_t* poller, int status, int events)
 {
   auto* context = (PollerContext*)poller->data;
+  assert(context);
+  O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
   context->state->loopReason |= DeviceState::DATA_SOCKET_POLLED;
   switch (events) {
     case UV_READABLE: {
-      ZoneScopedN("socket readable event");
-      LOG(debug) << "socket polled UV_READABLE: " << context->name;
+      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "Data pending on socket for channel %{public}s", context->name);
       context->state->loopReason |= DeviceState::DATA_INCOMING;
     } break;
     case UV_WRITABLE: {
-      ZoneScopedN("socket writeable");
+      O2_SIGNPOST_END(device, sid, "socket_state", "Socket connected for channel %{public}s", context->name);
       if (context->read) {
-        LOG(debug) << "socket polled UV_CONNECT" << context->name;
+        O2_SIGNPOST_START(device, sid, "socket_state", "Socket connected for read in context %{public}s", context->name);
         uv_poll_start(poller, UV_READABLE | UV_DISCONNECT | UV_PRIORITIZED, &on_socket_polled);
         context->state->loopReason |= DeviceState::DATA_CONNECTED;
       } else {
-        LOG(debug) << "socket polled UV_WRITABLE" << context->name;
+        O2_SIGNPOST_START(device, sid, "socket_state", "Socket connected for write for channel %{public}s", context->name);
         context->state->loopReason |= DeviceState::DATA_OUTGOING;
         // If the socket is writable, fairmq will handle the rest, so we can stop polling and
         // just wait for the disconnect.
@@ -297,12 +303,10 @@ void on_socket_polled(uv_poll_t* poller, int status, int events)
       context->pollerState = PollerContext::PollerState::Connected;
     } break;
     case UV_DISCONNECT: {
-      ZoneScopedN("socket disconnect");
-      LOG(debug) << "socket polled UV_DISCONNECT";
+      O2_SIGNPOST_END(device, sid, "socket_state", "Socket disconnected in context %{public}s", context->name);
     } break;
     case UV_PRIORITIZED: {
-      ZoneScopedN("socket prioritized");
-      LOG(debug) << "socket polled UV_PRIORITIZED";
+      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "Data pending on socket for context %{public}s", context->name);
     } break;
   }
   // We do nothing, all the logic for now stays in DataProcessingDevice::doRun()
@@ -873,7 +877,9 @@ void DataProcessingDevice::startPollers()
   auto& deviceContext = ref.get<DeviceContext>();
   auto& state = ref.get<DeviceState>();
 
-  for (auto& poller : state.activeInputPollers) {
+  for (auto* poller : state.activeInputPollers) {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
+    O2_SIGNPOST_START(device, sid, "socket_state", "Input socket waiting for connection.");
     uv_poll_start(poller, UV_WRITABLE, &on_socket_polled);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Disconnected;
   }
@@ -881,7 +887,9 @@ void DataProcessingDevice::startPollers()
     uv_poll_start(poller, UV_WRITABLE, &on_out_of_band_polled);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Disconnected;
   }
-  for (auto& poller : state.activeOutputPollers) {
+  for (auto* poller : state.activeOutputPollers) {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
+    O2_SIGNPOST_START(device, sid, "socket_state", "Output socket waiting for connection.");
     uv_poll_start(poller, UV_WRITABLE, &on_socket_polled);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Disconnected;
   }
@@ -897,17 +905,21 @@ void DataProcessingDevice::stopPollers()
   auto& deviceContext = ref.get<DeviceContext>();
   auto& state = ref.get<DeviceState>();
   LOGP(detail, "Stopping {} input pollers", state.activeInputPollers.size());
-  for (auto& poller : state.activeInputPollers) {
+  for (auto* poller : state.activeInputPollers) {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
+    O2_SIGNPOST_END(device, sid, "socket_state", "Output socket closed.");
     uv_poll_stop(poller);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Stopped;
   }
   LOGP(detail, "Stopping {} out of band pollers", state.activeOutOfBandPollers.size());
-  for (auto& poller : state.activeOutOfBandPollers) {
+  for (auto* poller : state.activeOutOfBandPollers) {
     uv_poll_stop(poller);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Stopped;
   }
   LOGP(detail, "Stopping {} output pollers", state.activeOutOfBandPollers.size());
-  for (auto& poller : state.activeOutputPollers) {
+  for (auto* poller : state.activeOutputPollers) {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
+    O2_SIGNPOST_END(device, sid, "socket_state", "Output socket closed.");
     uv_poll_stop(poller);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Stopped;
   }

From 7607e2111442968e483732fcf2405f9467efb343 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 09:24:03 +0100
Subject: [PATCH 228/726] DPL use Signposts to debug consumeWhenAll

---
 Framework/Core/src/CompletionPolicyHelpers.cxx | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx
index 8c4b38ba3355c..a6240b1165b81 100644
--- a/Framework/Core/src/CompletionPolicyHelpers.cxx
+++ b/Framework/Core/src/CompletionPolicyHelpers.cxx
@@ -18,10 +18,13 @@
 #include "Framework/TimesliceIndex.h"
 #include "Framework/TimingInfo.h"
 #include "DecongestionService.h"
+#include "Framework/Signpost.h"
 
 #include <cassert>
 #include <regex>
 
+O2_DECLARE_DYNAMIC_LOG(completion);
+
 namespace o2::framework
 {
 
@@ -108,6 +111,8 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
 {
   auto callback = [](InputSpan const& inputs, std::vector<InputSpec> const& specs, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     assert(inputs.size() == specs.size());
+    O2_SIGNPOST_ID_GENERATE(sid, completion);
+    O2_SIGNPOST_START(completion, sid, "consumeWhenAll", "Completion policy invoked");
 
     size_t si = 0;
     bool missingSporadic = false;
@@ -116,15 +121,18 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
       assert(si < specs.size());
       auto& spec = specs[si++];
       if (input.header == nullptr && spec.lifetime != Lifetime::Sporadic) {
+        O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s due to missing input %lu", "Wait", si);
         return CompletionPolicy::CompletionOp::Wait;
       }
       if (input.header == nullptr && spec.lifetime == Lifetime::Sporadic) {
+        O2_SIGNPOST_EVENT_EMIT(completion, sid, "consumeWhenAll", "Missing sporadic found for route index %lu", si);
         missingSporadic = true;
       }
       if (input.header != nullptr && currentTimeslice == -1) {
         auto const* dph = framework::DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(input);
         if (dph && !TimingInfo::timesliceIsTimer(dph->startTime)) {
           currentTimeslice = dph->startTime;
+          O2_SIGNPOST_EVENT_EMIT(completion, sid, "consumeWhenAll", "currentTimeslice %lu from route index %lu", currentTimeslice, si);
         }
       }
     }
@@ -134,8 +142,10 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
     auto oldestPossibleTimeslice = timesliceIndex.getOldestPossibleInput().timeslice.value;
 
     if (missingSporadic && currentTimeslice >= oldestPossibleTimeslice) {
+      O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s for timeslice %lu > oldestPossibleTimeslice %lu", "Retry", currentTimeslice, oldestPossibleTimeslice);
       return CompletionPolicy::CompletionOp::Retry;
     }
+    O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s for timeslice %lu <= oldestPossibleTimeslice %lu", "Consume", currentTimeslice, oldestPossibleTimeslice);
     return CompletionPolicy::CompletionOp::Consume;
   };
   return CompletionPolicy{name, matcher, callback};

From 12f7e25fe2f434c30605ae94b52ce7e89e7300de Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 09:41:37 +0100
Subject: [PATCH 229/726] Ability to turn on / off the logger from the GUI.

---
 .../Core/include/Framework/DeviceControl.h    |  2 +
 .../Core/include/Framework/DeviceState.h      |  9 +++++
 Framework/Core/src/WSDriverClient.cxx         | 40 +++++++++++++++++++
 .../src/FrameworkGUIDeviceInspector.cxx       | 11 +++++
 .../test/test_DetectMissingTimeframe.cxx      |  1 +
 5 files changed, 63 insertions(+)

diff --git a/Framework/Core/include/Framework/DeviceControl.h b/Framework/Core/include/Framework/DeviceControl.h
index 589de3937ee55..ce946e8e77fbf 100644
--- a/Framework/Core/include/Framework/DeviceControl.h
+++ b/Framework/Core/include/Framework/DeviceControl.h
@@ -48,6 +48,8 @@ struct DeviceControl {
   DeviceController* controller = nullptr;
   /// What kind of events should run with the TRACE level
   int tracingFlags = 0;
+  /// What kind of log streams should be enabled
+  int logStreams = 0;
   /// An incremental number to identify the device state
   int requestedState = 0;
 };
diff --git a/Framework/Core/include/Framework/DeviceState.h b/Framework/Core/include/Framework/DeviceState.h
index 3be0302fd4113..4fa72a84cad71 100644
--- a/Framework/Core/include/Framework/DeviceState.h
+++ b/Framework/Core/include/Framework/DeviceState.h
@@ -57,6 +57,13 @@ struct DeviceState {
     DATA_CONNECTED = 1 << 19,     // Data channel connected
   };
 
+  enum LogStreams : int {
+    NO_LOG = 0,
+    DEVICE_LOG = 1 << 0,             // Log for Data Processing Device activities.
+    COMPLETION_LOG = 1 << 1,         // Log for the completion policy of the device.
+    MONITORING_SERVICE_LOG = 1 << 2, // Log for the monitoring service flushing.
+  };
+
   std::vector<InputChannelInfo> inputChannelInfos;
   StreamingState streaming = StreamingState::Streaming;
   bool quitRequested = false;
@@ -93,6 +100,8 @@ struct DeviceState {
   int loopReason = 0;
   /// Bitmask of LoopReason to trace
   int tracingFlags = 0;
+  /// Bitmask of log streams which are available
+  int logStreams = 0;
   /// Stack of the severity, so that we can display only
   /// the bits we are interested in.
   std::vector<int> severityStack;
diff --git a/Framework/Core/src/WSDriverClient.cxx b/Framework/Core/src/WSDriverClient.cxx
index d4ed77b9a004e..ac2e3db41fcef 100644
--- a/Framework/Core/src/WSDriverClient.cxx
+++ b/Framework/Core/src/WSDriverClient.cxx
@@ -16,10 +16,15 @@
 #include "Framework/DeviceSpec.h"
 #include "DriverClientContext.h"
 #include "DPLWebSocket.h"
+#include "Framework/Signpost.h"
 #include <uv.h>
 #include <string_view>
 #include <charconv>
 
+O2_DECLARE_DYNAMIC_LOG(device);
+O2_DECLARE_DYNAMIC_LOG(completion);
+O2_DECLARE_DYNAMIC_LOG(monitoring_service);
+
 namespace o2::framework
 {
 
@@ -152,6 +157,41 @@ void on_connect(uv_connect_t* connection, int status)
     state.tracingFlags = tracingFlags;
   });
 
+  client->observe("/log-streams", [ref = context->ref](std::string_view cmd) {
+    auto& state = ref.get<DeviceState>();
+    static constexpr int prefixSize = std::string_view{"/log-streams "}.size();
+    if (prefixSize > cmd.size()) {
+      LOG(error) << "Malformed log-streams request";
+      return;
+    }
+    cmd.remove_prefix(prefixSize);
+    int logStreams = 0;
+
+    auto error = std::from_chars(cmd.data(), cmd.data() + cmd.size(), logStreams);
+    if (error.ec != std::errc()) {
+      LOG(error) << "Malformed log-streams mask";
+      return;
+    }
+    LOGP(info, "Logstreams flags set to {}", logStreams);
+    state.logStreams = logStreams;
+    if ((state.logStreams & DeviceState::LogStreams::DEVICE_LOG) != 0) {
+      O2_LOG_ENABLE(device);
+    } else {
+      O2_LOG_DISABLE(device);
+    }
+    if ((state.logStreams & DeviceState::LogStreams::COMPLETION_LOG) != 0) {
+      O2_LOG_ENABLE(completion);
+    } else {
+      O2_LOG_DISABLE(completion);
+    }
+
+    if ((state.logStreams & DeviceState::LogStreams::MONITORING_SERVICE_LOG) != 0) {
+      O2_LOG_ENABLE(monitoring_service);
+    } else {
+      O2_LOG_DISABLE(monitoring_service);
+    }
+  });
+
   // Client will be filled in the line after. I can probably have a single
   // client per device.
   auto dplClient = std::make_unique<WSDPLClient>();
diff --git a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
index a82753eb5af1f..fe93ca6d0f07f 100644
--- a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
@@ -400,6 +400,17 @@ void displayDeviceInspector(DeviceSpec const& spec,
     }
   }
 
+  bool logsChanged = false;
+  if (ImGui::CollapsingHeader("Signposts", ImGuiTreeNodeFlags_DefaultOpen)) {
+    logsChanged = ImGui::CheckboxFlags("Device", &control.logStreams, DeviceState::LogStreams::DEVICE_LOG);
+    logsChanged = ImGui::CheckboxFlags("Completion", &control.logStreams, DeviceState::LogStreams::COMPLETION_LOG);
+    logsChanged = ImGui::CheckboxFlags("Monitoring", &control.logStreams, DeviceState::LogStreams::MONITORING_SERVICE_LOG);
+    if (logsChanged && control.controller) {
+      std::string cmd = fmt::format("/log-streams {}", control.logStreams);
+      control.controller->write(cmd.c_str(), cmd.size());
+    }
+  }
+
   bool flagsChanged = false;
   if (ImGui::CollapsingHeader("Event loop tracing", ImGuiTreeNodeFlags_DefaultOpen)) {
     flagsChanged |= ImGui::CheckboxFlags("METRICS_MUST_FLUSH", &control.tracingFlags, DeviceState::LoopReason::METRICS_MUST_FLUSH);
diff --git a/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx b/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
index 3374cd9ad6cf2..7ef1370f040b4 100644
--- a/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
+++ b/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
@@ -43,6 +43,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& specs)
         if (i++ % 2 == 0) {
           outputs.make<int>(OutputRef{"a2"}, 1);
         }
+        sleep(1);
       })},
   };
   DataProcessorSpec d{

From b88a52e5631f03c19fb49503de5e91d8256bd587 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 10:17:25 +0100
Subject: [PATCH 230/726] DPL GUI: add GUI to enable / disable driver signposts

---
 Framework/GUISupport/src/FrameworkGUIDebugger.cxx | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
index 35e0bc563a22e..005f7082d77b4 100644
--- a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
@@ -23,6 +23,7 @@
 #include "FrameworkGUIDataRelayerUsage.h"
 #include "PaletteHelpers.h"
 #include "FrameworkGUIState.h"
+#include "Framework/Signpost.h"
 #include <DebugGUI/icons_font_awesome.h>
 
 #include <fmt/format.h>
@@ -34,6 +35,7 @@
 #include <cinttypes>
 #include <numeric>
 
+O2_DECLARE_DYNAMIC_LOG(driver);
 // Make sure we can use aggregated initialisers.
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wpedantic"
@@ -506,7 +508,7 @@ void displayDeviceMetrics(const char* label,
     case MetricsDisplayStyle::Lines: {
       auto xAxisFlags = ImPlotAxisFlags_None;
       auto yAxisFlags = ImPlotAxisFlags_LockMin;
-      //ImPlot::FitNextPlotAxes(true, true, true, true);
+      // ImPlot::FitNextPlotAxes(true, true, true, true);
       if (ImPlot::BeginPlot("##Some plot", {-1, -1}, axisFlags)) {
         ImPlot::SetupAxes("time", "value", xAxisFlags, yAxisFlags);
         ImPlot::SetupAxisFormat(ImAxis_Y1, formatSI, nullptr);
@@ -1093,6 +1095,17 @@ void displayDriverInfo(DriverInfo const& driverInfo, DriverControl& driverContro
   }
 #endif
 
+  static bool selectedSignpost = false;
+  if (ImGui::CollapsingHeader("Signposts", ImGuiTreeNodeFlags_DefaultOpen)) {
+    if (ImGui::Checkbox("Driver", &selectedSignpost)) {
+      if (selectedSignpost) {
+        O2_LOG_ENABLE(driver);
+      } else {
+        O2_LOG_DISABLE(driver);
+      }
+    }
+  }
+
   for (size_t i = 0; i < driverInfo.states.size(); ++i) {
     ImGui::Text("#%lu: %s", i, DriverInfoHelper::stateToString(driverInfo.states[i]));
   }

From 7978c5c54716ee48cb324f091363f435ce12ac3b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 11:20:50 +0100
Subject: [PATCH 231/726] DPL: do not validate lifetime when consumer uses
 consumeWhenAny

---
 Framework/Core/src/DeviceSpecHelpers.cxx | 16 +++++++++++++---
 Framework/Core/src/WorkflowHelpers.cxx   | 22 ++++++++++++++++++----
 Framework/Core/src/WorkflowHelpers.h     |  6 ++++++
 3 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 6c2063cb59311..359b1a343cb7e 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1074,8 +1074,6 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
 
   WorkflowHelpers::constructGraph(workflow, logicalEdges, outputs, availableForwardsInfo);
 
-  WorkflowHelpers::validateEdges(workflow, logicalEdges, outputs);
-
   // We need to instanciate one device per (me, timeIndex) in the
   // DeviceConnectionEdge. For each device we need one new binding
   // server per (me, other) -> port Moreover for each (me, other,
@@ -1122,10 +1120,13 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
                        inActions, workflow, availableForwardsInfo, channelPolicies, channelPrefix, defaultOffer, overrideServices);
   // We apply the completion policies here since this is where we have all the
   // devices resolved.
-  for (auto& device : devices) {
+  std::map<std::string, DataProcessorPoliciesInfo> policies;
+  for (DeviceSpec& device : devices) {
     bool hasPolicy = false;
+    policies[device.name].completionPolicyName = "unknown";
     for (auto& policy : completionPolicies) {
       if (policy.matcher(device) == true) {
+        policies[policy.name].completionPolicyName = policy.name;
         device.completionPolicy = policy;
         hasPolicy = true;
         break;
@@ -1158,6 +1159,15 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
       throw runtime_error_f("Unable to find a resource policy for %s", device.id.c_str());
     }
   }
+  // Iterate of the workflow and create a consistent vector of DataProcessorPoliciesInfo
+  std::vector<DataProcessorPoliciesInfo> policiesVector;
+  for (size_t wi = 0; wi < workflow.size(); ++wi) {
+    auto& processor = workflow[wi];
+    auto& info = policies[processor.name];
+    policiesVector.push_back(info);
+  }
+
+  WorkflowHelpers::validateEdges(workflow, policiesVector, logicalEdges, outputs);
 
   for (auto& device : devices) {
     device.resourceMonitoringInterval = resourcesMonitoringInterval;
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 360480f80d4ef..6b4ad35f826be 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -1214,8 +1214,14 @@ std::vector<InputSpec> WorkflowHelpers::computeDanglingOutputs(WorkflowSpec cons
   return results;
 }
 
-bool validateLifetime(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
+bool validateLifetime(std::ostream& errors,
+                      DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorPoliciesInfo const& producerPolicies,
+                      DataProcessorSpec const& consumer, InputSpec const& input, DataProcessorPoliciesInfo const& consumerPolicies)
 {
+  // In case the completion policy is consume-any, we do not need to check anything.
+  if (consumerPolicies.completionPolicyName == "consume-any") {
+    return true;
+  }
   if (input.lifetime == Lifetime::Timeframe && output.lifetime == Lifetime::Sporadic) {
     errors << fmt::format("Input {} of {} has lifetime Timeframe, but output {} of {} has lifetime Sporadic\n",
                           DataSpecUtils::describe(input).c_str(), consumer.name,
@@ -1225,7 +1231,9 @@ bool validateLifetime(std::ostream& errors, DataProcessorSpec const& producer, O
   return true;
 }
 
-bool validateExpendable(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
+bool validateExpendable(std::ostream& errors,
+                        DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorPoliciesInfo const& producerPolicies,
+                        DataProcessorSpec const& consumer, InputSpec const& input, DataProcessorPoliciesInfo const& consumerPolicies)
 {
   auto isExpendable = [](DataProcessorLabel const& label) {
     return label.value == "expendable";
@@ -1244,8 +1252,12 @@ bool validateExpendable(std::ostream& errors, DataProcessorSpec const& producer,
   return true;
 }
 
-using Validator = std::function<bool(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)>;
+using Validator = std::function<bool(std::ostream& errors,
+                                     DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorPoliciesInfo const& producerPolicies,
+                                     DataProcessorSpec const& consumer, InputSpec const& input, DataProcessorPoliciesInfo const& consumerPolicies)>;
+
 void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
+                                    std::vector<DataProcessorPoliciesInfo> const& policies,
                                     std::vector<DeviceConnectionEdge> const& edges,
                                     std::vector<OutputSpec> const& outputs)
 {
@@ -1262,10 +1274,12 @@ void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
   for (auto& edge : edges) {
     DataProcessorSpec const& producer = workflow[edge.producer];
     DataProcessorSpec const& consumer = workflow[edge.consumer];
+    DataProcessorPoliciesInfo const& producerPolicies = policies[edge.producer];
+    DataProcessorPoliciesInfo const& consumerPolicies = policies[edge.consumer];
     OutputSpec const& output = outputs[edge.outputGlobalIndex];
     InputSpec const& input = consumer.inputs[edge.consumerInputIndex];
     for (auto& validator : defaultValidators) {
-      hasErrors |= !validator(errors, producer, output, consumer, input);
+      hasErrors |= !validator(errors, producer, output, producerPolicies, consumer, input, consumerPolicies);
     }
   }
   if (hasErrors) {
diff --git a/Framework/Core/src/WorkflowHelpers.h b/Framework/Core/src/WorkflowHelpers.h
index 0a3cd408b3424..055cae1d6011b 100644
--- a/Framework/Core/src/WorkflowHelpers.h
+++ b/Framework/Core/src/WorkflowHelpers.h
@@ -141,6 +141,11 @@ struct TopoIndexInfo {
   friend std::ostream& operator<<(std::ostream& out, TopoIndexInfo const& info);
 };
 
+// Information about the policies which were derived for a given data processor.
+struct DataProcessorPoliciesInfo {
+  std::string completionPolicyName;
+};
+
 struct OutputObj {
   InputSpec spec;
   bool isdangling;
@@ -234,6 +239,7 @@ struct WorkflowHelpers {
   /// For example we should make sure that Lifetime::Timeframe inputs of
   /// one node is not connected to an Output of Lifetime::Sporadic of another node.
   static void validateEdges(WorkflowSpec const& workflow,
+                            std::vector<DataProcessorPoliciesInfo> const& policiesInfos,
                             std::vector<DeviceConnectionEdge> const& edges,
                             std::vector<OutputSpec> const& outputs);
 };

From 7e0544cb8f55a3436e32a20191ccec163e614cba Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 14:19:18 +0100
Subject: [PATCH 232/726] DPL: discard timeframe in case there is nothing which
 needs processing

---
 Framework/Core/src/CompletionPolicyHelpers.cxx | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx
index 8c4b38ba3355c..8877870be1a34 100644
--- a/Framework/Core/src/CompletionPolicyHelpers.cxx
+++ b/Framework/Core/src/CompletionPolicyHelpers.cxx
@@ -111,6 +111,7 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
 
     size_t si = 0;
     bool missingSporadic = false;
+    bool needsProcessing = false;
     size_t currentTimeslice = -1;
     for (auto& input : inputs) {
       assert(si < specs.size());
@@ -126,6 +127,9 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
         if (dph && !TimingInfo::timesliceIsTimer(dph->startTime)) {
           currentTimeslice = dph->startTime;
         }
+        // If we have at least some data, we can consume, otherwise we can Discard once 
+        // we are sure that no more data will come.
+        needsProcessing |= (spec.lifetime != Lifetime::Condition);
       }
     }
     // If some sporadic inputs are missing, we wait for them util we are sure they will not come,
@@ -136,7 +140,8 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
     if (missingSporadic && currentTimeslice >= oldestPossibleTimeslice) {
       return CompletionPolicy::CompletionOp::Retry;
     }
-    return CompletionPolicy::CompletionOp::Consume;
+    // We only consume if we have something which needs processing.
+    return needsProcessing ? CompletionPolicy::CompletionOp::Consume : CompletionPolicy::CompletionOp::Discard;
   };
   return CompletionPolicy{name, matcher, callback};
 }

From 84a7e8eea3bd7dcb204b7e07e641e6ef50d97fa6 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 14:20:51 +0100
Subject: [PATCH 233/726] DPL: discard timeframe in case there is nothing which
 needs processing

---
 Framework/Core/src/CompletionPolicyHelpers.cxx | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx
index 8c4b38ba3355c..b2b2fe7323742 100644
--- a/Framework/Core/src/CompletionPolicyHelpers.cxx
+++ b/Framework/Core/src/CompletionPolicyHelpers.cxx
@@ -111,6 +111,7 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
 
     size_t si = 0;
     bool missingSporadic = false;
+    bool needsProcessing = false;
     size_t currentTimeslice = -1;
     for (auto& input : inputs) {
       assert(si < specs.size());
@@ -126,6 +127,9 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
         if (dph && !TimingInfo::timesliceIsTimer(dph->startTime)) {
           currentTimeslice = dph->startTime;
         }
+        // Prepare for processing if we have at least some data, we can consume, otherwise we can Discard once
+        // we are sure that no more data will come.
+        needsProcessing |= (spec.lifetime != Lifetime::Condition);
       }
     }
     // If some sporadic inputs are missing, we wait for them util we are sure they will not come,
@@ -136,7 +140,8 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
     if (missingSporadic && currentTimeslice >= oldestPossibleTimeslice) {
       return CompletionPolicy::CompletionOp::Retry;
     }
-    return CompletionPolicy::CompletionOp::Consume;
+    // We only consume if we have something which needs processing.
+    return needsProcessing ? CompletionPolicy::CompletionOp::Consume : CompletionPolicy::CompletionOp::Discard;
   };
   return CompletionPolicy{name, matcher, callback};
 }

From d0d260285468b27ade61c37b4f688e2e2e74f0bd Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 14:53:42 +0100
Subject: [PATCH 234/726] DPL: discard timeframe in case there is nothing which
 needs processing

---
 Framework/Core/src/CompletionPolicyHelpers.cxx | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx
index 8c4b38ba3355c..64b183d43f2e4 100644
--- a/Framework/Core/src/CompletionPolicyHelpers.cxx
+++ b/Framework/Core/src/CompletionPolicyHelpers.cxx
@@ -111,6 +111,7 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
 
     size_t si = 0;
     bool missingSporadic = false;
+    bool needsProcessing = false;
     size_t currentTimeslice = -1;
     for (auto& input : inputs) {
       assert(si < specs.size());
@@ -127,6 +128,9 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
           currentTimeslice = dph->startTime;
         }
       }
+      if (input.header != nullptr && spec.lifetime != Lifetime::Condition) {
+        needsProcessing = true;
+      }
     }
     // If some sporadic inputs are missing, we wait for them util we are sure they will not come,
     // i.e. until the oldest possible timeslice is beyond the timeslice of the input.
@@ -136,7 +140,8 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
     if (missingSporadic && currentTimeslice >= oldestPossibleTimeslice) {
       return CompletionPolicy::CompletionOp::Retry;
     }
-    return CompletionPolicy::CompletionOp::Consume;
+    // We only consume if we have something which needs processing.
+    return needsProcessing ? CompletionPolicy::CompletionOp::Consume : CompletionPolicy::CompletionOp::Discard;
   };
   return CompletionPolicy{name, matcher, callback};
 }

From bf4962ab343ee141be67a08a50572110b8d91e53 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 16:20:14 +0100
Subject: [PATCH 235/726] DPL: drop unused o2_signpost_id_make_with_pointer
 method

Probably old remaining from a previous implementation.
---
 Framework/Foundation/include/Framework/Signpost.h | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index eca9e26f77469..64d92d67acbf7 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -218,7 +218,6 @@ bool _o2_lock_free_stack_push(_o2_lock_free_stack& stack, const int& value, bool
 bool _o2_lock_free_stack_pop(_o2_lock_free_stack& stack, int& value, bool spin = false);
 //_o2_signpost_id_t _o2_signpost_id_generate_local(_o2_log_t* log);
 //_o2_signpost_id_t _o2_signpost_id_make_with_pointer(_o2_log_t* log, void* pointer);
-_o2_signpost_index_t o2_signpost_id_make_with_pointer(_o2_log_t* log, void* pointer);
 void* _o2_log_create(char const* name, int stacktrace);
 void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...);
 void _o2_signpost_interval_begin(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...);
@@ -246,14 +245,6 @@ inline _o2_signpost_id_t _o2_signpost_id_make_with_pointer(_o2_log_t* log, void*
   return uniqueId;
 }
 
-inline _o2_signpost_index_t o2_signpost_id_make_with_pointer(_o2_log_t* log, void* pointer)
-{
-  _o2_signpost_index_t signpost_index;
-  _o2_lock_free_stack_pop(log->slots, signpost_index, true);
-  log->ids[signpost_index].id = (int64_t)pointer;
-  return signpost_index;
-}
-
 // Implementation start here. Include this file with O2_SIGNPOST_IMPLEMENTATION defined in one file of your
 // project.
 #ifdef O2_SIGNPOST_IMPLEMENTATION

From f17429804bd33b45919f44f7213bb5c0b802b697 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 16:20:14 +0100
Subject: [PATCH 236/726] DPL: rework signposts support

The logger based Signposts are now behaving the same on macOS and linux when
running normally.

The difference between the two implementations will now be that if Instruments
enables some OSLog namespace, we stop printing the messages and use the os_signpost
API instead.
---
 .../Foundation/include/Framework/Signpost.h   | 191 +++++++++---------
 Framework/Foundation/test/test_Signpost.cxx   |   2 +-
 .../Foundation/test/test_SignpostLogger.cxx   |   2 +-
 3 files changed, 94 insertions(+), 101 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index 64d92d67acbf7..d2bdec0ec8246 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -13,6 +13,9 @@
 
 #include <atomic>
 #include <array>
+#ifdef __APPLE__
+#include <os/log.h>
+#endif
 
 struct o2_log_handle_t {
   char const* name = nullptr;
@@ -22,6 +25,7 @@ struct o2_log_handle_t {
 
 // Helper function which replaces engineering types with a printf
 // compatible format string.
+// FIXME: make this consteval when available in C++20
 template <auto N>
 constexpr auto remove_engineering_type(char const (&src)[N])
 {
@@ -82,78 +86,37 @@ o2_log_handle_t* o2_walk_logs(bool (*callback)(char const* name, void* log, void
 }
 #endif
 
-#if !defined(O2_FORCE_LOGGER_SIGNPOST) && defined(__APPLE__) && (!defined(NDEBUG) || defined(O2_FORCE_SIGNPOSTS))
+#if defined(__APPLE__)
 #include <os/log.h>
 #include <os/signpost.h>
 #include <cstring>
-void* _o2_log_create(char const* name, char const* category);
-#define O2_DECLARE_DYNAMIC_LOG(x) static os_log_t private_o2_log_##x = (os_log_t)_o2_log_create("ch.cern.aliceo2." #x, OS_LOG_CATEGORY_DYNAMIC_TRACING)
-#define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(x) static os_log_t private_o2_log_##x = (os_log_t)_o2_log_create("ch.cern.aliceo2." #x, OS_LOG_CATEGORY_DYNAMIC_STACK_TRACING)
-// This is a no-op on macOS using the os_signpost API because only external instruments can enable/disable dynamic signposts
-#define O2_LOG_ENABLE_DYNAMIC(log)
-// This is a no-op on macOS using the os_signpost API because only external instruments can enable/disable dynamic signposts
-#define O2_LOG_ENABLE_STACKTRACE(log)
-#define O2_DECLARE_LOG(x, category) static os_log_t private_o2_log_##x = (os_log_t)_o2_log_create("ch.cern.aliceo2." #x, #category)
-#define O2_LOG_DEBUG(log, ...) os_log_debug(private_o2_log_##log, __VA_ARGS__)
-#define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer) os_signpost_id_t name = os_signpost_id_make_with_pointer(private_o2_log_##log, pointer)
-#define O2_SIGNPOST_ID_GENERATE(name, log) os_signpost_id_t name = os_signpost_id_generate(private_o2_log_##log)
+#define O2_LOG_DEBUG_MAC(log, ...) os_log_debug(private_o2_log_##log, __VA_ARGS__)
 // FIXME: use __VA_OPT__ when available in C++20
-#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) os_signpost_event_emit(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
-#define O2_SIGNPOST_START(log, id, name, format, ...) os_signpost_interval_begin(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
-#define O2_SIGNPOST_END(log, id, name, format, ...) os_signpost_interval_end(private_o2_log_##log, id, name, format, ##__VA_ARGS__)
-
-#ifdef O2_SIGNPOST_IMPLEMENTATION
-/// We use a wrapper so that we can keep track of the logs.
-void* _o2_log_create(char const* name, char const* category)
-{
-  // iterate over the list of logs and check if we already have
-  // one with the same name.
-  auto findLogByName = [](char const* name, void* log, void* context) -> bool {
-    char const* currentName = (char const*)context;
-    if (strcmp(name, currentName) == 0) {
-      return false;
-    }
-    return true;
-  };
-
-  o2_log_handle_t* handle = o2_walk_logs(findLogByName, (void*)name);
-
-  // If we found one, return it.
-  if (handle) {
-    return handle->log;
-  }
-  // Otherwise, create a new one and add it to the end of the list.
-  os_log_t log = os_log_create(name, category);
-  o2_log_handle_t* newHandle = new o2_log_handle_t();
-  newHandle->log = log;
-  newHandle->name = strdup(name);
-  newHandle->next = o2_get_logs_tail().load();
-  // Until I manage to replace the log I have in next, keep trying.
-  // Notice this does not protect against two threads trying to insert
-  // a log with the same name. I should probably do a sorted insert for that.
-  while (!o2_get_logs_tail().compare_exchange_weak(newHandle->next, newHandle,
-                                                   std::memory_order_release,
-                                                   std::memory_order_relaxed)) {
-    newHandle->next = o2_get_logs_tail();
-  }
-
-  return log;
-}
+#define O2_SIGNPOST_EVENT_EMIT_MAC(log, id, name, format, ...) os_signpost_event_emit(private_o2_log_##log->os_log, (uint64_t)id.value, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_START_MAC(log, id, name, format, ...) os_signpost_interval_begin(private_o2_log_##log->os_log, (uint64_t)id.value, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_END_MAC(log, id, name, format, ...) os_signpost_interval_end(private_o2_log_##log->os_log, (uint64_t)id.value, name, format, ##__VA_ARGS__)
+#define O2_SIGNPOST_ENABLED_MAC(log) os_signpost_enabled(private_o2_log_##log->os_log)
+#else
+// These are no-ops on linux.
+#define O2_DECLARE_LOG_MAC(x, category)
+#define O2_LOG_DEBUG_MAC(log, ...)
+#define O2_SIGNPOST_EVENT_EMIT_MAC(log, id, name, format, ...)
+#define O2_SIGNPOST_START_MAC(log, id, name, format, ...)
+#define O2_SIGNPOST_END_MAC(log, id, name, format, ...)
+#define O2_SIGNPOST_ENABLED_MAC(log) false
+#endif // __APPLE__
+
+// Unless we are on apple we enable checking for signposts only if in debug mode or if we force them.
+#if defined(__APPLE__) || defined(O2_FORCE_SIGNPOSTS) || !defined(NDEBUG)
+#define O2_LOG_ENABLED(log) private_o2_log_##log->stacktrace
+#else
+#define O2_LOG_ENABLED(log) false
 #endif
 
-#elif !defined(NDEBUG) || defined(O2_FORCE_LOGGER_SIGNPOST) || defined(O2_FORCE_SIGNPOSTS)
-
-#ifndef O2_LOG_MACRO
-#if __has_include("Framework/Logger.h")
+#if !defined(O2_LOG_MACRO) && __has_include("Framework/Logger.h")
 #include "Framework/Logger.h"
-// If NDEBUG is not defined, we use the logger to print out the signposts at the debug level.
-#if !defined(NDEBUG)
-#define O2_LOG_MACRO(...) LOGF(debug, __VA_ARGS__)
-#elif defined(O2_FORCE_LOGGER_SIGNPOST) || defined(O2_FORCE_SIGNPOSTS)
-// If we force the presence of the logger, we use it to print out the signposts at the detail level, which is not optimized out.
 #define O2_LOG_MACRO(...) LOGF(info, __VA_ARGS__)
-#endif
-#else
+#elif !defined(O2_LOG_MACRO)
 // If we do not have the fairlogger, we simply print out the signposts to the console.
 // This is useful for things like the tests, which this way do not need to depend on the FairLogger.
 #define O2_LOG_MACRO(...) \
@@ -161,8 +124,9 @@ void* _o2_log_create(char const* name, char const* category)
     printf(__VA_ARGS__);  \
     printf("\n");         \
   } while (0)
-#endif
-#endif
+#else
+#define O2_LOG_MACRO(...)
+#endif // O2_LOG_MACRO
 
 // This is the linux implementation, it is not as nice as the apple one and simply prints out
 // the signpost information to the log.
@@ -181,7 +145,7 @@ struct _o2_lock_free_stack {
 // A log is simply an inbox which keeps track of the available id, so that we can print out different signposts
 // with different indentation levels.
 // supports up to 1024 paralle signposts before it spinlocks.
-typedef int _o2_signpost_index_t;
+using _o2_signpost_index_t = int;
 
 struct _o2_activity_t {
   // How much the activity is indented in the output log.
@@ -191,10 +155,13 @@ struct _o2_activity_t {
 
 struct _o2_signpost_id_t {
   // The id of the activity.
-  int64_t id = -1;
+  int64_t value = -1;
 };
 
 struct _o2_log_t {
+#ifdef __APPLE__
+  os_log_t os_log = nullptr;
+#endif
   // A circular buffer of available slots. Each unique interval pulls an id from this buffer.
   _o2_lock_free_stack slots;
   // Up to 256 activities can be active at the same time.
@@ -211,7 +178,10 @@ struct _o2_log_t {
   // 0 means the log is disabled.
   // 1 means only the current signpost is printed.
   // >1 means the current signpost and n levels of the stacktrace are printed.
-  std::atomic<int> stacktrace = 1;
+  std::atomic<int> stacktrace = 0;
+
+  // Default stacktrace level for the log, when enabled.
+  int defaultStacktrace = 1;
 };
 
 bool _o2_lock_free_stack_push(_o2_lock_free_stack& stack, const int& value, bool spin = false);
@@ -310,7 +280,7 @@ bool _o2_lock_free_stack_pop(_o2_lock_free_stack& stack, int& value, bool spin)
   }
 }
 
-void* _o2_log_create(char const* name, int stacktrace)
+void* _o2_log_create(char const* name, int defaultStacktrace)
 {
   // iterate over the list of logs and check if we already have
   // one with the same name.
@@ -328,7 +298,7 @@ void* _o2_log_create(char const* name, int stacktrace)
     return handle->log;
   }
   // Otherwise, create a new one and add it to the end of the list.
-  _o2_log_t* log = new _o2_log_t();
+  auto* log = new _o2_log_t();
   // Write the initial 256 ids to the inbox, in reverse, so that the
   // linear search below is just for an handful of elements.
   int n = _o2_lock_free_stack::N;
@@ -336,9 +306,18 @@ void* _o2_log_create(char const* name, int stacktrace)
     _o2_signpost_index_t signpost_index{n - 1 - i};
     _o2_lock_free_stack_push(log->slots, signpost_index, true);
   }
-  log->stacktrace = stacktrace;
-  o2_log_handle_t* newHandle = new o2_log_handle_t();
+  log->defaultStacktrace = defaultStacktrace;
+  auto* newHandle = new o2_log_handle_t();
   newHandle->log = log;
+#ifdef __APPLE__
+  // On macOS, we use the os_signpost API so that when we are
+  // using instruments we can see the messages there.
+  if (defaultStacktrace > 1) {
+    log->os_log = os_log_create(name, OS_LOG_CATEGORY_DYNAMIC_STACK_TRACING);
+  } else {
+    log->os_log = os_log_create(name, OS_LOG_CATEGORY_DYNAMIC_TRACING);
+  }
+#endif
   newHandle->name = strdup(name);
   newHandle->next = o2_get_logs_tail().load();
   // Until I manage to replace the log I have in next, keep trying.
@@ -357,10 +336,6 @@ void* _o2_log_create(char const* name, int stacktrace)
 // If the slot is empty, it will return the id and increment the indentation level.
 void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...)
 {
-  // Nothing to be done
-  if (log->stacktrace == 0) {
-    return;
-  }
   va_list args;
   va_start(args, format);
 
@@ -368,10 +343,10 @@ void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* n
   int leading = 0;
 
   // This is the equivalent of exclusive
-  if (id.id != 0) {
+  if (id.value != 0) {
     int i = 0;
     for (i = 0; i < log->ids.size(); ++i) {
-      if (log->ids[i].id == id.id) {
+      if (log->ids[i].value == id.value) {
         break;
       }
     }
@@ -385,7 +360,7 @@ void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* n
   }
 
   char prebuffer[4096];
-  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16s*>%*c", id.id, name, leading, ' ');
+  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16s*>%*c", id.value, name, leading, ' ');
   vsnprintf(prebuffer + s, 4096 - s, format, args);
   va_end(args);
   O2_LOG_MACRO("%s", prebuffer);
@@ -395,22 +370,19 @@ void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* n
 // If the slot is empty, it will return the id and increment the indentation level.
 void _o2_signpost_interval_begin(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...)
 {
-  if (log->stacktrace == 0) {
-    return;
-  }
   va_list args;
   va_start(args, format);
   // This is a unique slot for this interval.
   _o2_signpost_index_t signpost_index;
   _o2_lock_free_stack_pop(log->slots, signpost_index, true);
   // Put the id in the slot, to close things or to attach signposts to a given interval
-  log->ids[signpost_index].id = id.id;
+  log->ids[signpost_index].value = id.value;
   auto* activity = &log->activities[signpost_index];
   activity->indentation = log->current_indentation++;
   activity->name = name;
   int leading = activity->indentation * 2;
   char prebuffer[4096];
-  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16sS>%*c", id.id, name, leading, ' ');
+  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16sS>%*c", id.value, name, leading, ' ');
   vsnprintf(prebuffer + s, 4096 - s, format, args);
   va_end(args);
   O2_LOG_MACRO("%s", prebuffer);
@@ -424,7 +396,7 @@ void _o2_signpost_interval_end_v(_o2_log_t* log, _o2_signpost_id_t id, char cons
   // Find the index of the activity
   int i = 0;
   for (i = 0; i < log->ids.size(); ++i) {
-    if (log->ids[i].id == id.id) {
+    if (log->ids[i].value == id.value) {
       break;
     }
   }
@@ -439,13 +411,13 @@ void _o2_signpost_interval_end_v(_o2_log_t* log, _o2_signpost_id_t id, char cons
   _o2_activity_t* activity = &log->activities[i];
   int leading = activity->indentation * 2;
   char prebuffer[4096];
-  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16sE>%*c", id.id, name, leading, ' ');
+  int s = snprintf(prebuffer, 4096, "id%.16" PRIx64 ":%-16sE>%*c", id.value, name, leading, ' ');
   vsnprintf(prebuffer + s, 4096 - s, format, args);
   O2_LOG_MACRO("%s", prebuffer);
   // Clear the slot
   activity->indentation = -1;
   activity->name = nullptr;
-  log->ids[i].id = -1;
+  log->ids[i].value = -1;
   // Put back the slot
   log->current_indentation--;
   _o2_signpost_index_t signpost_index{i};
@@ -468,29 +440,50 @@ void _o2_log_set_stacktrace(_o2_log_t* log, int stacktrace)
 }
 #endif // O2_SIGNPOST_IMPLEMENTATION
 
-/// Dynamic logs need to be enabled via the O2_LOG_ENABLE_DYNAMIC macro. Notice this will only work
+#if defined(__APPLE__) || defined(O2_FORCE_SIGNPOSTS) || !defined(NDEBUG)
+/// Dynamic logs need to be enabled via the O2_LOG_ENABLE macro. Notice this will only work
 /// for the logger based logging, since the Apple version needs instruments to enable them.
-#define O2_DECLARE_DYNAMIC_LOG(name) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 0)
+#define O2_DECLARE_DYNAMIC_LOG(name) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 1)
 /// For the moment we do not support logs with a stacktrace.
-#define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(name) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 0)
+#define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(name) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 64)
 #define O2_DECLARE_LOG(name, category) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 1)
-#define O2_LOG_ENABLE_DYNAMIC(log) _o2_log_set_stacktrace(private_o2_log_##log, 1)
-// We print out only the first 64 frames.
-#define O2_LOG_ENABLE_STACKTRACE(log) _o2_log_set_stacktrace(private_o2_log_##log, 64)
+// When we enable the log, we set the stacktrace to the default value.
+#define O2_LOG_ENABLE(log) _o2_log_set_stacktrace(private_o2_log_##log, private_o2_log_##log->defaultStacktrace)
+#define O2_LOG_DISABLE(log) _o2_log_set_stacktrace(private_o2_log_##log, 0)
 // For the moment we simply use LOG DEBUG. We should have proper activities so that we can
 // turn on and off the printing.
 #define O2_LOG_DEBUG(log, ...) O2_LOG_MACRO(__VA_ARGS__)
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer) _o2_signpost_id_t name = _o2_signpost_id_make_with_pointer(private_o2_log_##log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log) _o2_signpost_id_t name = _o2_signpost_id_generate_local(private_o2_log_##log)
-#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
-#define O2_SIGNPOST_START(log, id, name, format, ...) _o2_signpost_interval_begin(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
-#define O2_SIGNPOST_END(log, id, name, format, ...) _o2_signpost_interval_end(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__)
+// In case Instruments is attached, we switch to the Apple signpost API otherwise, both one
+// mac and on linux we use our own implementation, using the logger. We can use the same ids because
+// they are compatible between the two implementations, we also use remove_engineering_type to remove
+// the engineering types from the format string, so that we can use the same format string for both.
+#define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) __extension__({                                          \
+  if (O2_SIGNPOST_ENABLED_MAC(log)) {                                                                               \
+    O2_SIGNPOST_EVENT_EMIT_MAC(log, id, name, format, ##__VA_ARGS__);                                               \
+  } else if (private_o2_log_##log->stacktrace) {                                                                    \
+    _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
+  }                                                                                                                 \
+})
+#define O2_SIGNPOST_START(log, id, name, format, ...)                                                                   \
+  if (O2_SIGNPOST_ENABLED_MAC(log)) {                                                                                   \
+    O2_SIGNPOST_START_MAC(log, id, name, format, ##__VA_ARGS__);                                                        \
+  } else if (private_o2_log_##log->stacktrace) {                                                                        \
+    _o2_signpost_interval_begin(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
+  }
+#define O2_SIGNPOST_END(log, id, name, format, ...)                                                                   \
+  if (O2_SIGNPOST_ENABLED_MAC(log)) {                                                                                 \
+    O2_SIGNPOST_END_MAC(log, id, name, format, ##__VA_ARGS__);                                                        \
+  } else if (private_o2_log_##log->stacktrace) {                                                                      \
+    _o2_signpost_interval_end(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
+  }
 #else // This is the release implementation, it does nothing.
 #define O2_DECLARE_DYNAMIC_LOG(x)
 #define O2_DECLARE_DYNAMIC_STACKTRACE_LOG(x)
 #define O2_DECLARE_LOG(x, category)
-#define O2_LOG_ENABLE_DYNAMIC(log)
-#define O2_LOG_ENABLE_STACKTRACE(log)
+#define O2_LOG_ENABLE(log)
+#define O2_LOG_DISABLE(log)
 #define O2_LOG_DEBUG(log, ...)
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log)
diff --git a/Framework/Foundation/test/test_Signpost.cxx b/Framework/Foundation/test/test_Signpost.cxx
index 24b6afaec5c3d..f9b8d4ec0e13a 100644
--- a/Framework/Foundation/test/test_Signpost.cxx
+++ b/Framework/Foundation/test/test_Signpost.cxx
@@ -43,7 +43,7 @@ int main(int argc, char** argv)
 
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
   O2_SIGNPOST_END(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
-  O2_LOG_ENABLE_DYNAMIC(test_SignpostDynamic);
+  O2_LOG_ENABLE(test_SignpostDynamic);
 #ifdef __APPLE__
   // On Apple there is no way to turn on signposts in the logger, so we do not display this message
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will see, because we turned them on");
diff --git a/Framework/Foundation/test/test_SignpostLogger.cxx b/Framework/Foundation/test/test_SignpostLogger.cxx
index 74da35abf7c70..ed917ea7cadf5 100644
--- a/Framework/Foundation/test/test_SignpostLogger.cxx
+++ b/Framework/Foundation/test/test_SignpostLogger.cxx
@@ -51,7 +51,7 @@ int main(int argc, char** argv)
 
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
   O2_SIGNPOST_END(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will not see, because they are off by default");
-  O2_LOG_ENABLE_DYNAMIC(test_SignpostDynamic);
+  O2_LOG_ENABLE(test_SignpostDynamic);
 #ifdef __APPLE__
   // On Apple there is no way to turn on signposts in the logger, so we do not display this message
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will see, because we turned them on");

From 4e66267bd9a19c41cd65437285bab67fd2c2fba7 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 16:21:04 +0100
Subject: [PATCH 237/726] DPL: add proper tracing for DataProcessingDevice
 socket callbacks

---
 Framework/Core/src/DataProcessingDevice.cxx | 40 +++++++++++++--------
 1 file changed, 26 insertions(+), 14 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index b03904c5b3776..366bfafd7d801 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -35,6 +35,9 @@
 #include "Framework/TMessageSerializer.h"
 #include "Framework/InputRecord.h"
 #include "Framework/InputSpan.h"
+#if defined(__APPLE__) || defined(NDEBUG)
+#define O2_SIGNPOST_IMPLEMENTATION
+#endif
 #include "Framework/Signpost.h"
 #include "Framework/TimingHelpers.h"
 #include "Framework/SourceInfoHeader.h"
@@ -80,6 +83,8 @@
 #include <sstream>
 #include <boost/property_tree/json_parser.hpp>
 
+O2_DECLARE_DYNAMIC_LOG(device);
+
 using namespace o2::framework;
 using ConfigurationInterface = o2::configuration::ConfigurationInterface;
 using DataHeader = o2::header::DataHeader;
@@ -274,21 +279,22 @@ struct PollerContext {
 void on_socket_polled(uv_poll_t* poller, int status, int events)
 {
   auto* context = (PollerContext*)poller->data;
+  assert(context);
+  O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
   context->state->loopReason |= DeviceState::DATA_SOCKET_POLLED;
   switch (events) {
     case UV_READABLE: {
-      ZoneScopedN("socket readable event");
-      LOG(debug) << "socket polled UV_READABLE: " << context->name;
+      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "Data pending on socket for channel %{public}s", context->name);
       context->state->loopReason |= DeviceState::DATA_INCOMING;
     } break;
     case UV_WRITABLE: {
-      ZoneScopedN("socket writeable");
+      O2_SIGNPOST_END(device, sid, "socket_state", "Socket connected for channel %{public}s", context->name);
       if (context->read) {
-        LOG(debug) << "socket polled UV_CONNECT" << context->name;
+        O2_SIGNPOST_START(device, sid, "socket_state", "Socket connected for read in context %{public}s", context->name);
         uv_poll_start(poller, UV_READABLE | UV_DISCONNECT | UV_PRIORITIZED, &on_socket_polled);
         context->state->loopReason |= DeviceState::DATA_CONNECTED;
       } else {
-        LOG(debug) << "socket polled UV_WRITABLE" << context->name;
+        O2_SIGNPOST_START(device, sid, "socket_state", "Socket connected for write for channel %{public}s", context->name);
         context->state->loopReason |= DeviceState::DATA_OUTGOING;
         // If the socket is writable, fairmq will handle the rest, so we can stop polling and
         // just wait for the disconnect.
@@ -297,12 +303,10 @@ void on_socket_polled(uv_poll_t* poller, int status, int events)
       context->pollerState = PollerContext::PollerState::Connected;
     } break;
     case UV_DISCONNECT: {
-      ZoneScopedN("socket disconnect");
-      LOG(debug) << "socket polled UV_DISCONNECT";
+      O2_SIGNPOST_END(device, sid, "socket_state", "Socket disconnected in context %{public}s", context->name);
     } break;
     case UV_PRIORITIZED: {
-      ZoneScopedN("socket prioritized");
-      LOG(debug) << "socket polled UV_PRIORITIZED";
+      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "Data pending on socket for context %{public}s", context->name);
     } break;
   }
   // We do nothing, all the logic for now stays in DataProcessingDevice::doRun()
@@ -873,7 +877,9 @@ void DataProcessingDevice::startPollers()
   auto& deviceContext = ref.get<DeviceContext>();
   auto& state = ref.get<DeviceState>();
 
-  for (auto& poller : state.activeInputPollers) {
+  for (auto* poller : state.activeInputPollers) {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
+    O2_SIGNPOST_START(device, sid, "socket_state", "Input socket waiting for connection.");
     uv_poll_start(poller, UV_WRITABLE, &on_socket_polled);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Disconnected;
   }
@@ -881,7 +887,9 @@ void DataProcessingDevice::startPollers()
     uv_poll_start(poller, UV_WRITABLE, &on_out_of_band_polled);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Disconnected;
   }
-  for (auto& poller : state.activeOutputPollers) {
+  for (auto* poller : state.activeOutputPollers) {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
+    O2_SIGNPOST_START(device, sid, "socket_state", "Output socket waiting for connection.");
     uv_poll_start(poller, UV_WRITABLE, &on_socket_polled);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Disconnected;
   }
@@ -897,17 +905,21 @@ void DataProcessingDevice::stopPollers()
   auto& deviceContext = ref.get<DeviceContext>();
   auto& state = ref.get<DeviceState>();
   LOGP(detail, "Stopping {} input pollers", state.activeInputPollers.size());
-  for (auto& poller : state.activeInputPollers) {
+  for (auto* poller : state.activeInputPollers) {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
+    O2_SIGNPOST_END(device, sid, "socket_state", "Output socket closed.");
     uv_poll_stop(poller);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Stopped;
   }
   LOGP(detail, "Stopping {} out of band pollers", state.activeOutOfBandPollers.size());
-  for (auto& poller : state.activeOutOfBandPollers) {
+  for (auto* poller : state.activeOutOfBandPollers) {
     uv_poll_stop(poller);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Stopped;
   }
   LOGP(detail, "Stopping {} output pollers", state.activeOutOfBandPollers.size());
-  for (auto& poller : state.activeOutputPollers) {
+  for (auto* poller : state.activeOutputPollers) {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
+    O2_SIGNPOST_END(device, sid, "socket_state", "Output socket closed.");
     uv_poll_stop(poller);
     ((PollerContext*)poller->data)->pollerState = PollerContext::PollerState::Stopped;
   }

From b69354a7f54e476b5c87bee8a7dcbe3f8e68c11e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 16:20:14 +0100
Subject: [PATCH 238/726] DPL use Signposts to debug consumeWhenAll

---
 Framework/Core/src/CompletionPolicyHelpers.cxx | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx
index 64b183d43f2e4..585046bc89763 100644
--- a/Framework/Core/src/CompletionPolicyHelpers.cxx
+++ b/Framework/Core/src/CompletionPolicyHelpers.cxx
@@ -18,10 +18,13 @@
 #include "Framework/TimesliceIndex.h"
 #include "Framework/TimingInfo.h"
 #include "DecongestionService.h"
+#include "Framework/Signpost.h"
 
 #include <cassert>
 #include <regex>
 
+O2_DECLARE_DYNAMIC_LOG(completion);
+
 namespace o2::framework
 {
 
@@ -108,6 +111,8 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
 {
   auto callback = [](InputSpan const& inputs, std::vector<InputSpec> const& specs, ServiceRegistryRef& ref) -> CompletionPolicy::CompletionOp {
     assert(inputs.size() == specs.size());
+    O2_SIGNPOST_ID_GENERATE(sid, completion);
+    O2_SIGNPOST_START(completion, sid, "consumeWhenAll", "Completion policy invoked");
 
     size_t si = 0;
     bool missingSporadic = false;
@@ -117,15 +122,18 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
       assert(si < specs.size());
       auto& spec = specs[si++];
       if (input.header == nullptr && spec.lifetime != Lifetime::Sporadic) {
+        O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s due to missing input %lu", "Wait", si);
         return CompletionPolicy::CompletionOp::Wait;
       }
       if (input.header == nullptr && spec.lifetime == Lifetime::Sporadic) {
+        O2_SIGNPOST_EVENT_EMIT(completion, sid, "consumeWhenAll", "Missing sporadic found for route index %lu", si);
         missingSporadic = true;
       }
       if (input.header != nullptr && currentTimeslice == -1) {
         auto const* dph = framework::DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(input);
         if (dph && !TimingInfo::timesliceIsTimer(dph->startTime)) {
           currentTimeslice = dph->startTime;
+          O2_SIGNPOST_EVENT_EMIT(completion, sid, "consumeWhenAll", "currentTimeslice %lu from route index %lu", currentTimeslice, si);
         }
       }
       if (input.header != nullptr && spec.lifetime != Lifetime::Condition) {
@@ -138,9 +146,10 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
     auto oldestPossibleTimeslice = timesliceIndex.getOldestPossibleInput().timeslice.value;
 
     if (missingSporadic && currentTimeslice >= oldestPossibleTimeslice) {
+      O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s for timeslice %lu > oldestPossibleTimeslice %lu", "Retry", currentTimeslice, oldestPossibleTimeslice);
       return CompletionPolicy::CompletionOp::Retry;
     }
-    // We only consume if we have something which needs processing.
+    O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s for timeslice %lu <= oldestPossibleTimeslice %lu", needsProcessing ? "Consume" : "Discard", currentTimeslice, oldestPossibleTimeslice);
     return needsProcessing ? CompletionPolicy::CompletionOp::Consume : CompletionPolicy::CompletionOp::Discard;
   };
   return CompletionPolicy{name, matcher, callback};

From ab530c0085516f3640ae4b2ca7ec349b8f86d2b9 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 16:21:05 +0100
Subject: [PATCH 239/726] Ability to turn on / off the logger from the GUI.

---
 .../Core/include/Framework/DeviceControl.h    |  2 +
 .../Core/include/Framework/DeviceState.h      |  9 +++++
 Framework/Core/src/WSDriverClient.cxx         | 40 +++++++++++++++++++
 .../src/FrameworkGUIDeviceInspector.cxx       | 11 +++++
 .../test/test_DetectMissingTimeframe.cxx      |  1 +
 5 files changed, 63 insertions(+)

diff --git a/Framework/Core/include/Framework/DeviceControl.h b/Framework/Core/include/Framework/DeviceControl.h
index 589de3937ee55..ce946e8e77fbf 100644
--- a/Framework/Core/include/Framework/DeviceControl.h
+++ b/Framework/Core/include/Framework/DeviceControl.h
@@ -48,6 +48,8 @@ struct DeviceControl {
   DeviceController* controller = nullptr;
   /// What kind of events should run with the TRACE level
   int tracingFlags = 0;
+  /// What kind of log streams should be enabled
+  int logStreams = 0;
   /// An incremental number to identify the device state
   int requestedState = 0;
 };
diff --git a/Framework/Core/include/Framework/DeviceState.h b/Framework/Core/include/Framework/DeviceState.h
index 3be0302fd4113..4fa72a84cad71 100644
--- a/Framework/Core/include/Framework/DeviceState.h
+++ b/Framework/Core/include/Framework/DeviceState.h
@@ -57,6 +57,13 @@ struct DeviceState {
     DATA_CONNECTED = 1 << 19,     // Data channel connected
   };
 
+  enum LogStreams : int {
+    NO_LOG = 0,
+    DEVICE_LOG = 1 << 0,             // Log for Data Processing Device activities.
+    COMPLETION_LOG = 1 << 1,         // Log for the completion policy of the device.
+    MONITORING_SERVICE_LOG = 1 << 2, // Log for the monitoring service flushing.
+  };
+
   std::vector<InputChannelInfo> inputChannelInfos;
   StreamingState streaming = StreamingState::Streaming;
   bool quitRequested = false;
@@ -93,6 +100,8 @@ struct DeviceState {
   int loopReason = 0;
   /// Bitmask of LoopReason to trace
   int tracingFlags = 0;
+  /// Bitmask of log streams which are available
+  int logStreams = 0;
   /// Stack of the severity, so that we can display only
   /// the bits we are interested in.
   std::vector<int> severityStack;
diff --git a/Framework/Core/src/WSDriverClient.cxx b/Framework/Core/src/WSDriverClient.cxx
index d4ed77b9a004e..ac2e3db41fcef 100644
--- a/Framework/Core/src/WSDriverClient.cxx
+++ b/Framework/Core/src/WSDriverClient.cxx
@@ -16,10 +16,15 @@
 #include "Framework/DeviceSpec.h"
 #include "DriverClientContext.h"
 #include "DPLWebSocket.h"
+#include "Framework/Signpost.h"
 #include <uv.h>
 #include <string_view>
 #include <charconv>
 
+O2_DECLARE_DYNAMIC_LOG(device);
+O2_DECLARE_DYNAMIC_LOG(completion);
+O2_DECLARE_DYNAMIC_LOG(monitoring_service);
+
 namespace o2::framework
 {
 
@@ -152,6 +157,41 @@ void on_connect(uv_connect_t* connection, int status)
     state.tracingFlags = tracingFlags;
   });
 
+  client->observe("/log-streams", [ref = context->ref](std::string_view cmd) {
+    auto& state = ref.get<DeviceState>();
+    static constexpr int prefixSize = std::string_view{"/log-streams "}.size();
+    if (prefixSize > cmd.size()) {
+      LOG(error) << "Malformed log-streams request";
+      return;
+    }
+    cmd.remove_prefix(prefixSize);
+    int logStreams = 0;
+
+    auto error = std::from_chars(cmd.data(), cmd.data() + cmd.size(), logStreams);
+    if (error.ec != std::errc()) {
+      LOG(error) << "Malformed log-streams mask";
+      return;
+    }
+    LOGP(info, "Logstreams flags set to {}", logStreams);
+    state.logStreams = logStreams;
+    if ((state.logStreams & DeviceState::LogStreams::DEVICE_LOG) != 0) {
+      O2_LOG_ENABLE(device);
+    } else {
+      O2_LOG_DISABLE(device);
+    }
+    if ((state.logStreams & DeviceState::LogStreams::COMPLETION_LOG) != 0) {
+      O2_LOG_ENABLE(completion);
+    } else {
+      O2_LOG_DISABLE(completion);
+    }
+
+    if ((state.logStreams & DeviceState::LogStreams::MONITORING_SERVICE_LOG) != 0) {
+      O2_LOG_ENABLE(monitoring_service);
+    } else {
+      O2_LOG_DISABLE(monitoring_service);
+    }
+  });
+
   // Client will be filled in the line after. I can probably have a single
   // client per device.
   auto dplClient = std::make_unique<WSDPLClient>();
diff --git a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
index a82753eb5af1f..fe93ca6d0f07f 100644
--- a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
@@ -400,6 +400,17 @@ void displayDeviceInspector(DeviceSpec const& spec,
     }
   }
 
+  bool logsChanged = false;
+  if (ImGui::CollapsingHeader("Signposts", ImGuiTreeNodeFlags_DefaultOpen)) {
+    logsChanged = ImGui::CheckboxFlags("Device", &control.logStreams, DeviceState::LogStreams::DEVICE_LOG);
+    logsChanged = ImGui::CheckboxFlags("Completion", &control.logStreams, DeviceState::LogStreams::COMPLETION_LOG);
+    logsChanged = ImGui::CheckboxFlags("Monitoring", &control.logStreams, DeviceState::LogStreams::MONITORING_SERVICE_LOG);
+    if (logsChanged && control.controller) {
+      std::string cmd = fmt::format("/log-streams {}", control.logStreams);
+      control.controller->write(cmd.c_str(), cmd.size());
+    }
+  }
+
   bool flagsChanged = false;
   if (ImGui::CollapsingHeader("Event loop tracing", ImGuiTreeNodeFlags_DefaultOpen)) {
     flagsChanged |= ImGui::CheckboxFlags("METRICS_MUST_FLUSH", &control.tracingFlags, DeviceState::LoopReason::METRICS_MUST_FLUSH);
diff --git a/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx b/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
index 3374cd9ad6cf2..7ef1370f040b4 100644
--- a/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
+++ b/Framework/TestWorkflows/test/test_DetectMissingTimeframe.cxx
@@ -43,6 +43,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& specs)
         if (i++ % 2 == 0) {
           outputs.make<int>(OutputRef{"a2"}, 1);
         }
+        sleep(1);
       })},
   };
   DataProcessorSpec d{

From f38b25347476b5b3ba4581c0e98c6e8a4b8cf4a8 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 16:20:14 +0100
Subject: [PATCH 240/726] DPL GUI: add GUI to enable / disable driver signposts

---
 Framework/GUISupport/src/FrameworkGUIDebugger.cxx | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
index 35e0bc563a22e..005f7082d77b4 100644
--- a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
@@ -23,6 +23,7 @@
 #include "FrameworkGUIDataRelayerUsage.h"
 #include "PaletteHelpers.h"
 #include "FrameworkGUIState.h"
+#include "Framework/Signpost.h"
 #include <DebugGUI/icons_font_awesome.h>
 
 #include <fmt/format.h>
@@ -34,6 +35,7 @@
 #include <cinttypes>
 #include <numeric>
 
+O2_DECLARE_DYNAMIC_LOG(driver);
 // Make sure we can use aggregated initialisers.
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wpedantic"
@@ -506,7 +508,7 @@ void displayDeviceMetrics(const char* label,
     case MetricsDisplayStyle::Lines: {
       auto xAxisFlags = ImPlotAxisFlags_None;
       auto yAxisFlags = ImPlotAxisFlags_LockMin;
-      //ImPlot::FitNextPlotAxes(true, true, true, true);
+      // ImPlot::FitNextPlotAxes(true, true, true, true);
       if (ImPlot::BeginPlot("##Some plot", {-1, -1}, axisFlags)) {
         ImPlot::SetupAxes("time", "value", xAxisFlags, yAxisFlags);
         ImPlot::SetupAxisFormat(ImAxis_Y1, formatSI, nullptr);
@@ -1093,6 +1095,17 @@ void displayDriverInfo(DriverInfo const& driverInfo, DriverControl& driverContro
   }
 #endif
 
+  static bool selectedSignpost = false;
+  if (ImGui::CollapsingHeader("Signposts", ImGuiTreeNodeFlags_DefaultOpen)) {
+    if (ImGui::Checkbox("Driver", &selectedSignpost)) {
+      if (selectedSignpost) {
+        O2_LOG_ENABLE(driver);
+      } else {
+        O2_LOG_DISABLE(driver);
+      }
+    }
+  }
+
   for (size_t i = 0; i < driverInfo.states.size(); ++i) {
     ImGui::Text("#%lu: %s", i, DriverInfoHelper::stateToString(driverInfo.states[i]));
   }

From e1c1265ad943ddeab401fefbc1a0b85c9e06ae22 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 16:18:30 +0100
Subject: [PATCH 241/726] DPL: add ability to disable inputs programmatically

Disabled inputs will not result in an actual route for the data, however it
will be stored in the configuration, so that analysis workflows will not need
to have the configuration available at every step.
---
 Framework/Core/include/Framework/InputSpec.h  |  3 +++
 Framework/Core/include/Framework/OutputSpec.h |  4 ++++
 Framework/Core/src/DeviceSpecHelpers.cxx      | 15 ++++++++++--
 Framework/Core/src/WorkflowHelpers.cxx        | 12 +++++++---
 .../Core/src/WorkflowSerializationHelpers.cxx | 24 ++++++++++++++++++-
 5 files changed, 52 insertions(+), 6 deletions(-)

diff --git a/Framework/Core/include/Framework/InputSpec.h b/Framework/Core/include/Framework/InputSpec.h
index 9be95e6853da2..7858aacb80664 100644
--- a/Framework/Core/include/Framework/InputSpec.h
+++ b/Framework/Core/include/Framework/InputSpec.h
@@ -74,6 +74,9 @@ struct InputSpec {
 
   /// A set of configurables which can be used to customise the InputSpec.
   std::vector<ConfigParamSpec> metadata;
+  /// Wether or not the input is to be considered enabled.
+  /// Useful to programmatically disable inputs e.g. for the ProcessorOptions.
+  bool enabled = true;
 
   friend std::ostream& operator<<(std::ostream& stream, InputSpec const& arg);
   bool operator==(InputSpec const& that) const;
diff --git a/Framework/Core/include/Framework/OutputSpec.h b/Framework/Core/include/Framework/OutputSpec.h
index e92792faa6b14..51527bce4e339 100644
--- a/Framework/Core/include/Framework/OutputSpec.h
+++ b/Framework/Core/include/Framework/OutputSpec.h
@@ -86,6 +86,10 @@ struct OutputSpec {
   /// A set of configurables which can be used to customise the InputSpec.
   std::vector<ConfigParamSpec> metadata;
 
+  /// Wether or not this output is enabled. This is useful to decide programmatically
+  /// wether or not to produce a given output.
+  bool enabled = true;
+
   friend std::ostream& operator<<(std::ostream& stream, OutputSpec const& arg);
 };
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 6c2063cb59311..2b846c91b46aa 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -679,19 +679,27 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
     assert(policyPtr != nullptr);
 
     if (edge.isForward == false) {
+      auto &matcher = outputsMatchers[edge.outputGlobalIndex];
+      if (matcher.enabled == false) {
+        throw runtime_error_f("Output %s is disabled but it was still used in topology", DataSpecUtils::describe(matcher).data());
+      }
       OutputRoute route{
         edge.timeIndex,
         consumer.maxInputTimeslices,
-        outputsMatchers[edge.outputGlobalIndex],
+        matcher,
         channel.name,
         policyPtr,
       };
       device.outputs.emplace_back(route);
     } else {
+      auto &matcher = workflow[edge.consumer].inputs[edge.consumerInputIndex];
+      if (matcher.enabled == false) {
+        throw runtime_error_f("Output %s is disabled but it was still used in topology", DataSpecUtils::describe(matcher).data());
+      }
       ForwardRoute route{
         edge.timeIndex,
         consumer.maxInputTimeslices,
-        workflow[edge.consumer].inputs[edge.consumerInputIndex],
+        matcher,
         channel.name};
       device.forwards.emplace_back(route);
     }
@@ -915,6 +923,9 @@ void DeviceSpecHelpers::processInEdgeActions(std::vector<DeviceSpec>& devices,
 
     auto const& inputSpec = consumer.inputs[edge.consumerInputIndex];
     auto const& sourceChannel = consumerDevice.inputChannels[ci].name;
+    if (inputSpec.enabled == false) {
+      throw runtime_error_f("Input %s is disabled but it was still used in topology", DataSpecUtils::describe(inputSpec).data());
+    }
 
     InputRoute route{
       inputSpec,
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 360480f80d4ef..a4e1bd82f2ad8 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -841,11 +841,13 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
     for (size_t wi = 0; wi < workflow.size(); ++wi) {
       auto& producer = workflow[wi];
 
-      for (size_t oi = 0; oi < producer.outputs.size(); ++oi) {
-        auto& out = producer.outputs[oi];
+      for (auto& output : producer.outputs) {
+        if (output.enabled == false) {
+          continue;
+        }
         auto uniqueOutputId = outputs.size();
         availableOutputsInfo.emplace_back(LogicalOutputInfo{wi, uniqueOutputId, false});
-        outputs.push_back(out);
+        outputs.push_back(output);
       }
     }
   };
@@ -879,6 +881,10 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
   std::vector<bool> matches(constOutputs.size());
   for (size_t consumer = 0; consumer < workflow.size(); ++consumer) {
     for (size_t input = 0; input < workflow[consumer].inputs.size(); ++input) {
+      // Skip disabled inputs.
+      if (workflow[consumer].inputs[input].enabled == false) {
+        continue;
+      }
       forwards.clear();
       for (size_t i = 0; i < constOutputs.size(); i++) {
         matches[i] = DataSpecUtils::match(workflow[consumer].inputs[input], constOutputs[i]);
diff --git a/Framework/Core/src/WorkflowSerializationHelpers.cxx b/Framework/Core/src/WorkflowSerializationHelpers.cxx
index ac182a27a70c5..499d22348e1ba 100644
--- a/Framework/Core/src/WorkflowSerializationHelpers.cxx
+++ b/Framework/Core/src/WorkflowSerializationHelpers.cxx
@@ -58,6 +58,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
     IN_INPUT_ORIGIN,
     IN_INPUT_DESCRIPTION,
     IN_INPUT_SUBSPEC,
+    IN_INPUT_ENABLED,
     IN_INPUT_ORIGIN_REF,
     IN_INPUT_DESCRIPTION_REF,
     IN_INPUT_SUBSPEC_REF,
@@ -72,6 +73,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
     IN_OUTPUT_BINDING,
     IN_OUTPUT_ORIGIN,
     IN_OUTPUT_DESCRIPTION,
+    IN_OUTPUT_ENABLED,
     IN_OUTPUT_SUBSPEC,
     IN_OUTPUT_LIFETIME,
     IN_OUTPUT_OPTIONS,
@@ -164,6 +166,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       case State::IN_INPUT_SUBSPEC:
         s << "IN_INPUT_SUBSPEC";
         break;
+      case State::IN_INPUT_ENABLED:
+        s << "IN_INPUT_ENABLED";
+        break;
       case State::IN_INPUT_ORIGIN_REF:
         s << "IN_INPUT_ORIGIN_REF";
         break;
@@ -215,6 +220,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       case State::IN_OUTPUT_OPTIONS:
         s << "IN_OUTPUT_OPTIONS";
         break;
+      case WorkflowImporter::State::IN_OUTPUT_ENABLED:
+        s << "IN_OUTPUT_ENABLED";
+        break;
       case State::IN_OPTION:
         s << "IN_OPTION";
         break;
@@ -367,9 +375,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       } else {
         dataProcessors.back().inputs.push_back(InputSpec({binding}, std::move(*matcher), lifetime, inputOptions));
       }
+      dataProcessors.back().inputs.back().enabled = enabled;
       inputMatcherNodes.clear();
       inputOptions.clear();
-
     } else if (in(State::IN_INPUT_MATCHER) && inputMatcherNodes.size() > 1) {
       data_matcher::Node child = std::move(inputMatcherNodes.back());
       inputMatcherNodes.pop_back();
@@ -430,6 +438,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       } else {
         dataProcessors.back().outputs.push_back(OutputSpec({binding}, {origin, description}, lifetime));
       }
+      dataProcessors.back().outputs.back().enabled = enabled;
       outputHasSubSpec = false;
     } else if (in(State::IN_OPTION)) {
       std::unique_ptr<ConfigParamSpec> opt{nullptr};
@@ -599,6 +608,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       push(State::IN_INPUT_DESCRIPTION_REF);
     } else if (in(State::IN_INPUT) && strncmp(str, "subspecRef", length) == 0) {
       push(State::IN_INPUT_SUBSPEC_REF);
+    } else if (in(State::IN_INPUT) && strncmp(str, "enabled", length) == 0) {
+      push(State::IN_INPUT_ENABLED);
     } else if (in(State::IN_INPUT) && strncmp(str, "matcher", length) == 0) {
       // the outermost matcher is starting here
       // we create a placeholder which is being updated later
@@ -731,6 +742,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       metadata.back().executable = s;
     } else if (in(State::IN_INPUT_BINDING)) {
       binding = s;
+    } else if (in(State::IN_INPUT_ENABLED)) {
+      enabled = (s == "true");
     } else if (in(State::IN_INPUT_ORIGIN)) {
       origin.runtimeInit(s.c_str(), std::min(s.size(), 4UL));
       std::string v(s.c_str(), std::min(s.size(), 4UL));
@@ -838,6 +851,10 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       dataProcessors.back().inputTimeSliceId = i;
     } else if (in(State::IN_DATAPROCESSOR_MAX_TIMESLICES)) {
       dataProcessors.back().maxInputTimeslices = i;
+    } else if (in(State::IN_INPUT_ENABLED)) {
+      enabled = (i == 1);
+    } else if (in(State::IN_OUTPUT_ENABLED)) {
+      enabled = (i == 1);
     }
     pop();
     return true;
@@ -908,6 +925,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
   header::DataDescription description;
   size_t subspec;
   size_t ref;
+  bool enabled;
   Lifetime lifetime;
   std::string metadatumKey;
   std::string metadatumValue;
@@ -1096,6 +1114,8 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
       }
       w.Key("lifetime");
       w.Uint((int)input.lifetime);
+      w.Key("enabled");
+      w.Uint((int)input.enabled);
       if (input.metadata.empty() == false) {
         w.Key("metadata");
         w.StartArray();
@@ -1145,6 +1165,8 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
       }
       w.Key("lifetime");
       w.Uint((int)output.lifetime);
+      w.Key("enabled");
+      w.Uint((int)output.enabled);
       if (output.metadata.empty() == false) {
         w.Key("metadata");
         w.StartArray();

From ab2eb7097b6d88d7471cd989d396c25c53b44f95 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 16:24:10 +0100
Subject: [PATCH 242/726] DPL: add ability to disable inputs programmatically

Disabled inputs will not result in an actual route for the data, however it
will be stored in the configuration, so that analysis workflows will not need
to have the configuration available at every step.
---
 Framework/Core/include/Framework/InputSpec.h  |  3 +++
 Framework/Core/include/Framework/OutputSpec.h |  4 ++++
 Framework/Core/src/DeviceSpecHelpers.cxx      | 15 ++++++++++--
 Framework/Core/src/WorkflowHelpers.cxx        | 12 +++++++---
 .../Core/src/WorkflowSerializationHelpers.cxx | 24 ++++++++++++++++++-
 5 files changed, 52 insertions(+), 6 deletions(-)

diff --git a/Framework/Core/include/Framework/InputSpec.h b/Framework/Core/include/Framework/InputSpec.h
index 9be95e6853da2..7858aacb80664 100644
--- a/Framework/Core/include/Framework/InputSpec.h
+++ b/Framework/Core/include/Framework/InputSpec.h
@@ -74,6 +74,9 @@ struct InputSpec {
 
   /// A set of configurables which can be used to customise the InputSpec.
   std::vector<ConfigParamSpec> metadata;
+  /// Wether or not the input is to be considered enabled.
+  /// Useful to programmatically disable inputs e.g. for the ProcessorOptions.
+  bool enabled = true;
 
   friend std::ostream& operator<<(std::ostream& stream, InputSpec const& arg);
   bool operator==(InputSpec const& that) const;
diff --git a/Framework/Core/include/Framework/OutputSpec.h b/Framework/Core/include/Framework/OutputSpec.h
index e92792faa6b14..51527bce4e339 100644
--- a/Framework/Core/include/Framework/OutputSpec.h
+++ b/Framework/Core/include/Framework/OutputSpec.h
@@ -86,6 +86,10 @@ struct OutputSpec {
   /// A set of configurables which can be used to customise the InputSpec.
   std::vector<ConfigParamSpec> metadata;
 
+  /// Wether or not this output is enabled. This is useful to decide programmatically
+  /// wether or not to produce a given output.
+  bool enabled = true;
+
   friend std::ostream& operator<<(std::ostream& stream, OutputSpec const& arg);
 };
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 6c2063cb59311..bedcda35873f9 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -679,19 +679,27 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
     assert(policyPtr != nullptr);
 
     if (edge.isForward == false) {
+      auto& matcher = outputsMatchers[edge.outputGlobalIndex];
+      if (matcher.enabled == false) {
+        throw runtime_error_f("Output %s is disabled but it was still used in topology", DataSpecUtils::describe(matcher).data());
+      }
       OutputRoute route{
         edge.timeIndex,
         consumer.maxInputTimeslices,
-        outputsMatchers[edge.outputGlobalIndex],
+        matcher,
         channel.name,
         policyPtr,
       };
       device.outputs.emplace_back(route);
     } else {
+      auto& matcher = workflow[edge.consumer].inputs[edge.consumerInputIndex];
+      if (matcher.enabled == false) {
+        throw runtime_error_f("Output %s is disabled but it was still used in topology", DataSpecUtils::describe(matcher).data());
+      }
       ForwardRoute route{
         edge.timeIndex,
         consumer.maxInputTimeslices,
-        workflow[edge.consumer].inputs[edge.consumerInputIndex],
+        matcher,
         channel.name};
       device.forwards.emplace_back(route);
     }
@@ -915,6 +923,9 @@ void DeviceSpecHelpers::processInEdgeActions(std::vector<DeviceSpec>& devices,
 
     auto const& inputSpec = consumer.inputs[edge.consumerInputIndex];
     auto const& sourceChannel = consumerDevice.inputChannels[ci].name;
+    if (inputSpec.enabled == false) {
+      throw runtime_error_f("Input %s is disabled but it was still used in topology", DataSpecUtils::describe(inputSpec).data());
+    }
 
     InputRoute route{
       inputSpec,
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 360480f80d4ef..a4e1bd82f2ad8 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -841,11 +841,13 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
     for (size_t wi = 0; wi < workflow.size(); ++wi) {
       auto& producer = workflow[wi];
 
-      for (size_t oi = 0; oi < producer.outputs.size(); ++oi) {
-        auto& out = producer.outputs[oi];
+      for (auto& output : producer.outputs) {
+        if (output.enabled == false) {
+          continue;
+        }
         auto uniqueOutputId = outputs.size();
         availableOutputsInfo.emplace_back(LogicalOutputInfo{wi, uniqueOutputId, false});
-        outputs.push_back(out);
+        outputs.push_back(output);
       }
     }
   };
@@ -879,6 +881,10 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
   std::vector<bool> matches(constOutputs.size());
   for (size_t consumer = 0; consumer < workflow.size(); ++consumer) {
     for (size_t input = 0; input < workflow[consumer].inputs.size(); ++input) {
+      // Skip disabled inputs.
+      if (workflow[consumer].inputs[input].enabled == false) {
+        continue;
+      }
       forwards.clear();
       for (size_t i = 0; i < constOutputs.size(); i++) {
         matches[i] = DataSpecUtils::match(workflow[consumer].inputs[input], constOutputs[i]);
diff --git a/Framework/Core/src/WorkflowSerializationHelpers.cxx b/Framework/Core/src/WorkflowSerializationHelpers.cxx
index ac182a27a70c5..499d22348e1ba 100644
--- a/Framework/Core/src/WorkflowSerializationHelpers.cxx
+++ b/Framework/Core/src/WorkflowSerializationHelpers.cxx
@@ -58,6 +58,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
     IN_INPUT_ORIGIN,
     IN_INPUT_DESCRIPTION,
     IN_INPUT_SUBSPEC,
+    IN_INPUT_ENABLED,
     IN_INPUT_ORIGIN_REF,
     IN_INPUT_DESCRIPTION_REF,
     IN_INPUT_SUBSPEC_REF,
@@ -72,6 +73,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
     IN_OUTPUT_BINDING,
     IN_OUTPUT_ORIGIN,
     IN_OUTPUT_DESCRIPTION,
+    IN_OUTPUT_ENABLED,
     IN_OUTPUT_SUBSPEC,
     IN_OUTPUT_LIFETIME,
     IN_OUTPUT_OPTIONS,
@@ -164,6 +166,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       case State::IN_INPUT_SUBSPEC:
         s << "IN_INPUT_SUBSPEC";
         break;
+      case State::IN_INPUT_ENABLED:
+        s << "IN_INPUT_ENABLED";
+        break;
       case State::IN_INPUT_ORIGIN_REF:
         s << "IN_INPUT_ORIGIN_REF";
         break;
@@ -215,6 +220,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       case State::IN_OUTPUT_OPTIONS:
         s << "IN_OUTPUT_OPTIONS";
         break;
+      case WorkflowImporter::State::IN_OUTPUT_ENABLED:
+        s << "IN_OUTPUT_ENABLED";
+        break;
       case State::IN_OPTION:
         s << "IN_OPTION";
         break;
@@ -367,9 +375,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       } else {
         dataProcessors.back().inputs.push_back(InputSpec({binding}, std::move(*matcher), lifetime, inputOptions));
       }
+      dataProcessors.back().inputs.back().enabled = enabled;
       inputMatcherNodes.clear();
       inputOptions.clear();
-
     } else if (in(State::IN_INPUT_MATCHER) && inputMatcherNodes.size() > 1) {
       data_matcher::Node child = std::move(inputMatcherNodes.back());
       inputMatcherNodes.pop_back();
@@ -430,6 +438,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       } else {
         dataProcessors.back().outputs.push_back(OutputSpec({binding}, {origin, description}, lifetime));
       }
+      dataProcessors.back().outputs.back().enabled = enabled;
       outputHasSubSpec = false;
     } else if (in(State::IN_OPTION)) {
       std::unique_ptr<ConfigParamSpec> opt{nullptr};
@@ -599,6 +608,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       push(State::IN_INPUT_DESCRIPTION_REF);
     } else if (in(State::IN_INPUT) && strncmp(str, "subspecRef", length) == 0) {
       push(State::IN_INPUT_SUBSPEC_REF);
+    } else if (in(State::IN_INPUT) && strncmp(str, "enabled", length) == 0) {
+      push(State::IN_INPUT_ENABLED);
     } else if (in(State::IN_INPUT) && strncmp(str, "matcher", length) == 0) {
       // the outermost matcher is starting here
       // we create a placeholder which is being updated later
@@ -731,6 +742,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       metadata.back().executable = s;
     } else if (in(State::IN_INPUT_BINDING)) {
       binding = s;
+    } else if (in(State::IN_INPUT_ENABLED)) {
+      enabled = (s == "true");
     } else if (in(State::IN_INPUT_ORIGIN)) {
       origin.runtimeInit(s.c_str(), std::min(s.size(), 4UL));
       std::string v(s.c_str(), std::min(s.size(), 4UL));
@@ -838,6 +851,10 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       dataProcessors.back().inputTimeSliceId = i;
     } else if (in(State::IN_DATAPROCESSOR_MAX_TIMESLICES)) {
       dataProcessors.back().maxInputTimeslices = i;
+    } else if (in(State::IN_INPUT_ENABLED)) {
+      enabled = (i == 1);
+    } else if (in(State::IN_OUTPUT_ENABLED)) {
+      enabled = (i == 1);
     }
     pop();
     return true;
@@ -908,6 +925,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
   header::DataDescription description;
   size_t subspec;
   size_t ref;
+  bool enabled;
   Lifetime lifetime;
   std::string metadatumKey;
   std::string metadatumValue;
@@ -1096,6 +1114,8 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
       }
       w.Key("lifetime");
       w.Uint((int)input.lifetime);
+      w.Key("enabled");
+      w.Uint((int)input.enabled);
       if (input.metadata.empty() == false) {
         w.Key("metadata");
         w.StartArray();
@@ -1145,6 +1165,8 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
       }
       w.Key("lifetime");
       w.Uint((int)output.lifetime);
+      w.Key("enabled");
+      w.Uint((int)output.enabled);
       if (output.metadata.empty() == false) {
         w.Key("metadata");
         w.StartArray();

From 536881a3c9b82b4c3c1d7d46c8faa25bfb8a7fc7 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 16 Jan 2024 16:32:22 +0100
Subject: [PATCH 243/726] DPL: add ability to disable inputs programmatically

Disabled inputs will not result in an actual route for the data, however it
will be stored in the configuration, so that analysis workflows will not need
to have the configuration available at every step.
---
 Framework/Core/include/Framework/InputSpec.h  |  3 ++
 Framework/Core/include/Framework/OutputSpec.h |  4 +++
 Framework/Core/src/DeviceSpecHelpers.cxx      | 15 ++++++++--
 Framework/Core/src/WorkflowHelpers.cxx        | 12 ++++++--
 .../Core/src/WorkflowSerializationHelpers.cxx | 30 ++++++++++++++++++-
 5 files changed, 58 insertions(+), 6 deletions(-)

diff --git a/Framework/Core/include/Framework/InputSpec.h b/Framework/Core/include/Framework/InputSpec.h
index 9be95e6853da2..7858aacb80664 100644
--- a/Framework/Core/include/Framework/InputSpec.h
+++ b/Framework/Core/include/Framework/InputSpec.h
@@ -74,6 +74,9 @@ struct InputSpec {
 
   /// A set of configurables which can be used to customise the InputSpec.
   std::vector<ConfigParamSpec> metadata;
+  /// Wether or not the input is to be considered enabled.
+  /// Useful to programmatically disable inputs e.g. for the ProcessorOptions.
+  bool enabled = true;
 
   friend std::ostream& operator<<(std::ostream& stream, InputSpec const& arg);
   bool operator==(InputSpec const& that) const;
diff --git a/Framework/Core/include/Framework/OutputSpec.h b/Framework/Core/include/Framework/OutputSpec.h
index e92792faa6b14..51527bce4e339 100644
--- a/Framework/Core/include/Framework/OutputSpec.h
+++ b/Framework/Core/include/Framework/OutputSpec.h
@@ -86,6 +86,10 @@ struct OutputSpec {
   /// A set of configurables which can be used to customise the InputSpec.
   std::vector<ConfigParamSpec> metadata;
 
+  /// Wether or not this output is enabled. This is useful to decide programmatically
+  /// wether or not to produce a given output.
+  bool enabled = true;
+
   friend std::ostream& operator<<(std::ostream& stream, OutputSpec const& arg);
 };
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 6c2063cb59311..bedcda35873f9 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -679,19 +679,27 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
     assert(policyPtr != nullptr);
 
     if (edge.isForward == false) {
+      auto& matcher = outputsMatchers[edge.outputGlobalIndex];
+      if (matcher.enabled == false) {
+        throw runtime_error_f("Output %s is disabled but it was still used in topology", DataSpecUtils::describe(matcher).data());
+      }
       OutputRoute route{
         edge.timeIndex,
         consumer.maxInputTimeslices,
-        outputsMatchers[edge.outputGlobalIndex],
+        matcher,
         channel.name,
         policyPtr,
       };
       device.outputs.emplace_back(route);
     } else {
+      auto& matcher = workflow[edge.consumer].inputs[edge.consumerInputIndex];
+      if (matcher.enabled == false) {
+        throw runtime_error_f("Output %s is disabled but it was still used in topology", DataSpecUtils::describe(matcher).data());
+      }
       ForwardRoute route{
         edge.timeIndex,
         consumer.maxInputTimeslices,
-        workflow[edge.consumer].inputs[edge.consumerInputIndex],
+        matcher,
         channel.name};
       device.forwards.emplace_back(route);
     }
@@ -915,6 +923,9 @@ void DeviceSpecHelpers::processInEdgeActions(std::vector<DeviceSpec>& devices,
 
     auto const& inputSpec = consumer.inputs[edge.consumerInputIndex];
     auto const& sourceChannel = consumerDevice.inputChannels[ci].name;
+    if (inputSpec.enabled == false) {
+      throw runtime_error_f("Input %s is disabled but it was still used in topology", DataSpecUtils::describe(inputSpec).data());
+    }
 
     InputRoute route{
       inputSpec,
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 360480f80d4ef..a4e1bd82f2ad8 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -841,11 +841,13 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
     for (size_t wi = 0; wi < workflow.size(); ++wi) {
       auto& producer = workflow[wi];
 
-      for (size_t oi = 0; oi < producer.outputs.size(); ++oi) {
-        auto& out = producer.outputs[oi];
+      for (auto& output : producer.outputs) {
+        if (output.enabled == false) {
+          continue;
+        }
         auto uniqueOutputId = outputs.size();
         availableOutputsInfo.emplace_back(LogicalOutputInfo{wi, uniqueOutputId, false});
-        outputs.push_back(out);
+        outputs.push_back(output);
       }
     }
   };
@@ -879,6 +881,10 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
   std::vector<bool> matches(constOutputs.size());
   for (size_t consumer = 0; consumer < workflow.size(); ++consumer) {
     for (size_t input = 0; input < workflow[consumer].inputs.size(); ++input) {
+      // Skip disabled inputs.
+      if (workflow[consumer].inputs[input].enabled == false) {
+        continue;
+      }
       forwards.clear();
       for (size_t i = 0; i < constOutputs.size(); i++) {
         matches[i] = DataSpecUtils::match(workflow[consumer].inputs[input], constOutputs[i]);
diff --git a/Framework/Core/src/WorkflowSerializationHelpers.cxx b/Framework/Core/src/WorkflowSerializationHelpers.cxx
index ac182a27a70c5..dc0eb3243fd03 100644
--- a/Framework/Core/src/WorkflowSerializationHelpers.cxx
+++ b/Framework/Core/src/WorkflowSerializationHelpers.cxx
@@ -58,6 +58,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
     IN_INPUT_ORIGIN,
     IN_INPUT_DESCRIPTION,
     IN_INPUT_SUBSPEC,
+    IN_INPUT_ENABLED,
     IN_INPUT_ORIGIN_REF,
     IN_INPUT_DESCRIPTION_REF,
     IN_INPUT_SUBSPEC_REF,
@@ -72,6 +73,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
     IN_OUTPUT_BINDING,
     IN_OUTPUT_ORIGIN,
     IN_OUTPUT_DESCRIPTION,
+    IN_OUTPUT_ENABLED,
     IN_OUTPUT_SUBSPEC,
     IN_OUTPUT_LIFETIME,
     IN_OUTPUT_OPTIONS,
@@ -164,6 +166,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       case State::IN_INPUT_SUBSPEC:
         s << "IN_INPUT_SUBSPEC";
         break;
+      case State::IN_INPUT_ENABLED:
+        s << "IN_INPUT_ENABLED";
+        break;
       case State::IN_INPUT_ORIGIN_REF:
         s << "IN_INPUT_ORIGIN_REF";
         break;
@@ -215,6 +220,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       case State::IN_OUTPUT_OPTIONS:
         s << "IN_OUTPUT_OPTIONS";
         break;
+      case WorkflowImporter::State::IN_OUTPUT_ENABLED:
+        s << "IN_OUTPUT_ENABLED";
+        break;
       case State::IN_OPTION:
         s << "IN_OPTION";
         break;
@@ -299,6 +307,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       dataProcessors.push_back(DataProcessorSpec{});
     } else if (in(State::IN_INPUTS)) {
       push(State::IN_INPUT);
+      enabled = true;
       inputMatcherNodes.clear();
     } else if (in(State::IN_INPUT_MATCHER)) {
       // start a new embedded matcher
@@ -310,6 +319,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       // will be merged into the parent matcher
     } else if (in(State::IN_OUTPUTS)) {
       push(State::IN_OUTPUT);
+      enabled = true;
       outputHasSubSpec = false;
     } else if (in(State::IN_OPTIONS)) {
       push(State::IN_OPTION);
@@ -367,9 +377,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       } else {
         dataProcessors.back().inputs.push_back(InputSpec({binding}, std::move(*matcher), lifetime, inputOptions));
       }
+      dataProcessors.back().inputs.back().enabled = enabled;
       inputMatcherNodes.clear();
       inputOptions.clear();
-
     } else if (in(State::IN_INPUT_MATCHER) && inputMatcherNodes.size() > 1) {
       data_matcher::Node child = std::move(inputMatcherNodes.back());
       inputMatcherNodes.pop_back();
@@ -430,6 +440,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       } else {
         dataProcessors.back().outputs.push_back(OutputSpec({binding}, {origin, description}, lifetime));
       }
+      dataProcessors.back().outputs.back().enabled = enabled;
       outputHasSubSpec = false;
     } else if (in(State::IN_OPTION)) {
       std::unique_ptr<ConfigParamSpec> opt{nullptr};
@@ -542,12 +553,14 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       push(State::IN_DATAPROCESSORS);
     } else if (in(State::IN_INPUTS)) {
       push(State::IN_INPUT);
+      enabled = true;
     } else if (in(State::IN_INPUT_OPTIONS)) {
       push(State::IN_OPTION);
     } else if (in(State::IN_OUTPUT_OPTIONS)) {
       push(State::IN_OPTION);
     } else if (in(State::IN_OUTPUTS)) {
       push(State::IN_OUTPUT);
+      enabled = true;
       outputHasSubSpec = false;
     } else if (in(State::IN_OPTIONS)) {
       push(State::IN_OPTION);
@@ -599,6 +612,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       push(State::IN_INPUT_DESCRIPTION_REF);
     } else if (in(State::IN_INPUT) && strncmp(str, "subspecRef", length) == 0) {
       push(State::IN_INPUT_SUBSPEC_REF);
+    } else if (in(State::IN_INPUT) && strncmp(str, "enabled", length) == 0) {
+      push(State::IN_INPUT_ENABLED);
     } else if (in(State::IN_INPUT) && strncmp(str, "matcher", length) == 0) {
       // the outermost matcher is starting here
       // we create a placeholder which is being updated later
@@ -731,6 +746,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       metadata.back().executable = s;
     } else if (in(State::IN_INPUT_BINDING)) {
       binding = s;
+    } else if (in(State::IN_INPUT_ENABLED)) {
+      enabled = (s == "true");
     } else if (in(State::IN_INPUT_ORIGIN)) {
       origin.runtimeInit(s.c_str(), std::min(s.size(), 4UL));
       std::string v(s.c_str(), std::min(s.size(), 4UL));
@@ -838,6 +855,10 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       dataProcessors.back().inputTimeSliceId = i;
     } else if (in(State::IN_DATAPROCESSOR_MAX_TIMESLICES)) {
       dataProcessors.back().maxInputTimeslices = i;
+    } else if (in(State::IN_INPUT_ENABLED)) {
+      enabled = (i == 1);
+    } else if (in(State::IN_OUTPUT_ENABLED)) {
+      enabled = (i == 1);
     }
     pop();
     return true;
@@ -908,6 +929,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
   header::DataDescription description;
   size_t subspec;
   size_t ref;
+  // Keep track of the enabled state of the input/output
+  // Unless specified, inputs are enabled by default.
+  bool enabled = true;
   Lifetime lifetime;
   std::string metadatumKey;
   std::string metadatumValue;
@@ -1096,6 +1120,8 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
       }
       w.Key("lifetime");
       w.Uint((int)input.lifetime);
+      w.Key("enabled");
+      w.Uint((int)input.enabled);
       if (input.metadata.empty() == false) {
         w.Key("metadata");
         w.StartArray();
@@ -1145,6 +1171,8 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
       }
       w.Key("lifetime");
       w.Uint((int)output.lifetime);
+      w.Key("enabled");
+      w.Uint((int)output.enabled);
       if (output.metadata.empty() == false) {
         w.Key("metadata");
         w.StartArray();

From b86551b19d0becdc225586d98904b8c85cb42e75 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 17 Jan 2024 09:16:58 +0100
Subject: [PATCH 244/726] DPL: do not reset mDidDispatch too early

---
 Framework/Core/include/Framework/MessageContext.h |  9 ++-------
 Framework/Core/src/MessageContext.cxx             | 11 +++++++++--
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/Framework/Core/include/Framework/MessageContext.h b/Framework/Core/include/Framework/MessageContext.h
index 9e7268f614fda..407bac0ceb00a 100644
--- a/Framework/Core/include/Framework/MessageContext.h
+++ b/Framework/Core/include/Framework/MessageContext.h
@@ -464,12 +464,7 @@ class MessageContext
   /// Prepares the context to create messages for the given timeslice. This
   /// expects that the previous context was already sent and can be completely
   /// discarded.
-  void clear()
-  {
-    // Verify that everything has been sent on clear.
-    assert(std::all_of(mMessages.begin(), mMessages.end(), [](auto& m) { return m->empty(); }));
-    mMessages.clear();
-  }
+  void clear();
 
   FairMQDeviceProxy& proxy()
   {
@@ -494,7 +489,7 @@ class MessageContext
   /// return the headers of the 1st (from the end) matching message checking first in mMessages then in mScheduledMessages
   o2::header::DataHeader* findMessageHeader(const Output& spec);
   o2::header::Stack* findMessageHeaderStack(const Output& spec);
-  int countDeviceOutputs(bool excludeDPLOrigin = false);
+  [[nodiscard]] int countDeviceOutputs(bool excludeDPLOrigin = false) const;
   void fakeDispatch() { mDidDispatch = true; }
   bool didDispatch() { return mDidDispatch; }
   o2::framework::DataProcessingHeader* findMessageDataProcessingHeader(const Output& spec);
diff --git a/Framework/Core/src/MessageContext.cxx b/Framework/Core/src/MessageContext.cxx
index a7365594785c7..59dfc15837210 100644
--- a/Framework/Core/src/MessageContext.cxx
+++ b/Framework/Core/src/MessageContext.cxx
@@ -80,12 +80,11 @@ o2::header::Stack* MessageContext::findMessageHeaderStack(const Output& spec)
   return nullptr;
 }
 
-int MessageContext::countDeviceOutputs(bool excludeDPLOrigin)
+int MessageContext::countDeviceOutputs(bool excludeDPLOrigin) const
 {
   // If we dispatched some messages before the end of the callback
   // we need to account for them as well.
   int noutputs = mDidDispatch ? 1 : 0;
-  mDidDispatch = false;
   constexpr o2::header::DataOrigin DataOriginDPL{"DPL"};
   for (auto it = mMessages.rbegin(); it != mMessages.rend(); ++it) {
     if (!excludeDPLOrigin || (*it)->header()->dataOrigin != DataOriginDPL) {
@@ -100,6 +99,14 @@ int MessageContext::countDeviceOutputs(bool excludeDPLOrigin)
   return noutputs;
 }
 
+void MessageContext::clear()
+{
+  // Verify that everything has been sent on clear.
+  assert(std::all_of(mMessages.begin(), mMessages.end(), [](auto& m) { return m->empty(); }));
+  mDidDispatch = false;
+  mMessages.clear();
+}
+
 int64_t MessageContext::addToCache(std::unique_ptr<fair::mq::Message>& toCache)
 {
   auto&& cached = toCache->GetTransport()->CreateMessage();

From fd2f8bd3d4cb7edb42121f1f78761db26eb0e9c6 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 17 Jan 2024 11:35:28 +0100
Subject: [PATCH 245/726] DPL: optimize for the case debugging is disabled

---
 Framework/Foundation/include/Framework/Signpost.h | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index d2bdec0ec8246..76da9f395f6a9 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -11,6 +11,7 @@
 #ifndef O2_FRAMEWORK_SIGNPOST_H_
 #define O2_FRAMEWORK_SIGNPOST_H_
 
+#include "Framework/CompilerBuiltins.h"
 #include <atomic>
 #include <array>
 #ifdef __APPLE__
@@ -460,22 +461,22 @@ void _o2_log_set_stacktrace(_o2_log_t* log, int stacktrace)
 // they are compatible between the two implementations, we also use remove_engineering_type to remove
 // the engineering types from the format string, so that we can use the same format string for both.
 #define O2_SIGNPOST_EVENT_EMIT(log, id, name, format, ...) __extension__({                                          \
-  if (O2_SIGNPOST_ENABLED_MAC(log)) {                                                                               \
+  if (O2_BUILTIN_UNLIKELY(O2_SIGNPOST_ENABLED_MAC(log))) {                                                          \
     O2_SIGNPOST_EVENT_EMIT_MAC(log, id, name, format, ##__VA_ARGS__);                                               \
-  } else if (private_o2_log_##log->stacktrace) {                                                                    \
+  } else if (O2_BUILTIN_UNLIKELY(private_o2_log_##log->stacktrace)) {                                               \
     _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
   }                                                                                                                 \
 })
 #define O2_SIGNPOST_START(log, id, name, format, ...)                                                                   \
-  if (O2_SIGNPOST_ENABLED_MAC(log)) {                                                                                   \
+  if (O2_BUILTIN_UNLIKELY(O2_SIGNPOST_ENABLED_MAC(log))) {                                                              \
     O2_SIGNPOST_START_MAC(log, id, name, format, ##__VA_ARGS__);                                                        \
-  } else if (private_o2_log_##log->stacktrace) {                                                                        \
+  } else if (O2_BUILTIN_UNLIKELY(private_o2_log_##log->stacktrace)) {                                                   \
     _o2_signpost_interval_begin(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
   }
 #define O2_SIGNPOST_END(log, id, name, format, ...)                                                                   \
-  if (O2_SIGNPOST_ENABLED_MAC(log)) {                                                                                 \
+  if (O2_BUILTIN_UNLIKELY(O2_SIGNPOST_ENABLED_MAC(log))) {                                                            \
     O2_SIGNPOST_END_MAC(log, id, name, format, ##__VA_ARGS__);                                                        \
-  } else if (private_o2_log_##log->stacktrace) {                                                                      \
+  } else if (O2_BUILTIN_UNLIKELY(private_o2_log_##log->stacktrace)) {                                                 \
     _o2_signpost_interval_end(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
   }
 #else // This is the release implementation, it does nothing.

From 1c8fb9dfe6568cb3515fab1a11a2af9f5dc9682d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 17 Jan 2024 13:14:59 +0100
Subject: [PATCH 246/726] DPL: helper function to enable loggers from gdb /
 lldb

---
 Framework/Core/COOKBOOK.md                        | 14 ++++++++++++++
 Framework/Foundation/include/Framework/Signpost.h |  7 +++++++
 2 files changed, 21 insertions(+)

diff --git a/Framework/Core/COOKBOOK.md b/Framework/Core/COOKBOOK.md
index 7de2a6b7ca9c4..1a27079c3dc13 100644
--- a/Framework/Core/COOKBOOK.md
+++ b/Framework/Core/COOKBOOK.md
@@ -538,3 +538,17 @@ perf script -i perf.data > profile.linux-perf.txt
 ```
 
 and then you can either upload it to https://www.speedscope.app or use chrome://tracing.
+
+## Internal debug log streams
+
+Debug log entries for several DPL components are now provided via the Signpost API.
+
+Such streams can be enabled using the "Signpost" part of the Device Inspector GUI.
+
+Alternatively for the case were the GUI is not available, you can attach to a process via GDB / LLDB and invoke the `o2_debug_log_set_stacktrace` function, passing it the log you want to enable and the amount of stacktrace you want to see. E.g. to enable a log created with `O2_DECLARE_DYNAMIC_LOG(completion)` you can use:
+
+```
+p o2_debug_log_set_stacktrace(private_o2_log_completion, 1)
+```
+
+If you are on a mac, you can also use Instruments to visualise your Signpost, just like any other macOS application. In order to do so you need to enable the "Signpost" instrument, making sure you add `ch.cern.aliceo2.completion` to the list of loggers to watch.
diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index d2bdec0ec8246..72d956eaa727f 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -438,6 +438,13 @@ void _o2_log_set_stacktrace(_o2_log_t* log, int stacktrace)
 {
   log->stacktrace = stacktrace;
 }
+// A C function which can be used to enable the signposts
+extern "C" {
+void o2_debug_log_set_stacktrace(_o2_log_t* log, int stacktrace)
+{
+  log->stacktrace = stacktrace;
+}
+}
 #endif // O2_SIGNPOST_IMPLEMENTATION
 
 #if defined(__APPLE__) || defined(O2_FORCE_SIGNPOSTS) || !defined(NDEBUG)

From 306e2de8ab445df3c5cef412d70c4964e71245ae Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 17 Jan 2024 14:49:14 +0100
Subject: [PATCH 247/726] DPL: account for the case in which outputs are not
 user created

---
 .../Core/include/Framework/StreamContext.h      |  2 +-
 Framework/Core/src/CommonServices.cxx           | 17 +++++++++++------
 Framework/Core/src/DataAllocator.cxx            |  2 +-
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/Framework/Core/include/Framework/StreamContext.h b/Framework/Core/include/Framework/StreamContext.h
index 198802179a419..8300ce4ce7ac8 100644
--- a/Framework/Core/include/Framework/StreamContext.h
+++ b/Framework/Core/include/Framework/StreamContext.h
@@ -69,7 +69,7 @@ struct StreamContext {
   // for a given iteration.
   // This is in the stream context to allow tracking data creation on a per thread
   // basis.
-  std::vector<bool> routeCreated;
+  std::vector<bool> routeUserCreated;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 4656ddeb12e63..176a60a44bd36 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -169,9 +169,9 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
       // the same stream might be referring to different data processors.
       // We should probably have a context which is per stream of a specific
       // data processor.
-      stream->routeCreated.resize(routes.size());
-      // Reset the routeCreated at every processing step
-      std::fill(stream->routeCreated.begin(), stream->routeCreated.end(), false); },
+      stream->routeUserCreated.resize(routes.size());
+      // Reset the routeUserCreated at every processing step
+      std::fill(stream->routeUserCreated.begin(), stream->routeUserCreated.end(), false); },
     .postProcessing = [](ProcessingContext& processingContext, void* service) {
       auto* stream = (StreamContext*)service;
       auto& routes = processingContext.services().get<DeviceSpec const>().outputs;
@@ -182,7 +182,7 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
       // it means it was created out of band.
       bool didCreate = false;
       for (size_t ri = 0; ri < routes.size(); ++ri) {
-        if (stream->routeCreated[ri] == true) {
+        if (stream->routeUserCreated[ri] == true) {
           didCreate = true;
           break;
         }
@@ -192,7 +192,7 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
         return;
       }
       for (size_t ri = 0; ri < routes.size(); ++ri) {
-        if (stream->routeCreated[ri] == true) {
+        if (stream->routeUserCreated[ri] == true) {
           continue;
         }
         auto &route = routes[ri];
@@ -458,7 +458,9 @@ o2::framework::ServiceSpec CommonServices::ccdbSupportSpec()
       // For any output that is a FLP/DISTSUBTIMEFRAME with subspec != 0,
       // we create a new message.
       InputSpec matcher{"matcher", ConcreteDataTypeMatcher{"FLP", "DISTSUBTIMEFRAME"}};
-      for (auto& output : pc.services().get<DeviceSpec const>().outputs) {
+      auto &streamContext = pc.services().get<StreamContext>();
+      for (size_t oi = 0; oi < pc.services().get<DeviceSpec const>().outputs.size(); ++oi) {
+        OutputRoute const& output = pc.services().get<DeviceSpec const>().outputs[oi];
         if ((output.timeslice % output.maxTimeslices) != 0) {
           continue;
         }
@@ -471,6 +473,9 @@ o2::framework::ServiceSpec CommonServices::ccdbSupportSpec()
           stfDist.id = timingInfo.timeslice;
           stfDist.firstOrbit = timingInfo.firstTForbit;
           stfDist.runNumber = timingInfo.runNumber;
+          // We mark it as not created, because we do should not account for it when
+          // checking if we created all the data for a timeslice.
+          streamContext.routeUserCreated[oi] = false;
         }
       } },
     .kind = ServiceKind::Global};
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index 9ce72aed8c1cf..a6b13ceeda101 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -53,7 +53,7 @@ RouteIndex DataAllocator::matchDataHeader(const Output& spec, size_t timeslice)
   for (auto ri = 0; ri < allowedOutputRoutes.size(); ++ri) {
     auto& route = allowedOutputRoutes[ri];
     if (DataSpecUtils::match(route.matcher, spec.origin, spec.description, spec.subSpec) && ((timeslice % route.maxTimeslices) == route.timeslice)) {
-      stream.routeCreated[ri] = true;
+      stream.routeUserCreated[ri] = true;
       return RouteIndex{ri};
     }
   }

From 4ef0e664fbd2984eeb24b6b3f94e9ec42f99e493 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 17 Jan 2024 14:58:33 +0100
Subject: [PATCH 248/726] DPL: account for the case in which outputs are not
 user created

---
 .../Core/include/Framework/StreamContext.h      |  2 +-
 Framework/Core/src/CommonServices.cxx           | 17 +++++++++++------
 Framework/Core/src/DataAllocator.cxx            |  2 +-
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/Framework/Core/include/Framework/StreamContext.h b/Framework/Core/include/Framework/StreamContext.h
index 198802179a419..8300ce4ce7ac8 100644
--- a/Framework/Core/include/Framework/StreamContext.h
+++ b/Framework/Core/include/Framework/StreamContext.h
@@ -69,7 +69,7 @@ struct StreamContext {
   // for a given iteration.
   // This is in the stream context to allow tracking data creation on a per thread
   // basis.
-  std::vector<bool> routeCreated;
+  std::vector<bool> routeUserCreated;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 4656ddeb12e63..fd02a4c9ccb88 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -169,9 +169,9 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
       // the same stream might be referring to different data processors.
       // We should probably have a context which is per stream of a specific
       // data processor.
-      stream->routeCreated.resize(routes.size());
-      // Reset the routeCreated at every processing step
-      std::fill(stream->routeCreated.begin(), stream->routeCreated.end(), false); },
+      stream->routeUserCreated.resize(routes.size());
+      // Reset the routeUserCreated at every processing step
+      std::fill(stream->routeUserCreated.begin(), stream->routeUserCreated.end(), false); },
     .postProcessing = [](ProcessingContext& processingContext, void* service) {
       auto* stream = (StreamContext*)service;
       auto& routes = processingContext.services().get<DeviceSpec const>().outputs;
@@ -182,7 +182,7 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
       // it means it was created out of band.
       bool didCreate = false;
       for (size_t ri = 0; ri < routes.size(); ++ri) {
-        if (stream->routeCreated[ri] == true) {
+        if (stream->routeUserCreated[ri] == true) {
           didCreate = true;
           break;
         }
@@ -192,7 +192,7 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
         return;
       }
       for (size_t ri = 0; ri < routes.size(); ++ri) {
-        if (stream->routeCreated[ri] == true) {
+        if (stream->routeUserCreated[ri] == true) {
           continue;
         }
         auto &route = routes[ri];
@@ -458,7 +458,9 @@ o2::framework::ServiceSpec CommonServices::ccdbSupportSpec()
       // For any output that is a FLP/DISTSUBTIMEFRAME with subspec != 0,
       // we create a new message.
       InputSpec matcher{"matcher", ConcreteDataTypeMatcher{"FLP", "DISTSUBTIMEFRAME"}};
-      for (auto& output : pc.services().get<DeviceSpec const>().outputs) {
+      auto& streamContext = pc.services().get<StreamContext>();
+      for (size_t oi = 0; oi < pc.services().get<DeviceSpec const>().outputs.size(); ++oi) {
+        OutputRoute const& output = pc.services().get<DeviceSpec const>().outputs[oi];
         if ((output.timeslice % output.maxTimeslices) != 0) {
           continue;
         }
@@ -471,6 +473,9 @@ o2::framework::ServiceSpec CommonServices::ccdbSupportSpec()
           stfDist.id = timingInfo.timeslice;
           stfDist.firstOrbit = timingInfo.firstTForbit;
           stfDist.runNumber = timingInfo.runNumber;
+          // We mark it as not created, because we do should not account for it when
+          // checking if we created all the data for a timeslice.
+          streamContext.routeUserCreated[oi] = false;
         }
       } },
     .kind = ServiceKind::Global};
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index 9ce72aed8c1cf..a6b13ceeda101 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -53,7 +53,7 @@ RouteIndex DataAllocator::matchDataHeader(const Output& spec, size_t timeslice)
   for (auto ri = 0; ri < allowedOutputRoutes.size(); ++ri) {
     auto& route = allowedOutputRoutes[ri];
     if (DataSpecUtils::match(route.matcher, spec.origin, spec.description, spec.subSpec) && ((timeslice % route.maxTimeslices) == route.timeslice)) {
-      stream.routeCreated[ri] = true;
+      stream.routeUserCreated[ri] = true;
       return RouteIndex{ri};
     }
   }

From 0c273197dad569c2fa831ff1f4366708cd4ddaf4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 17 Jan 2024 16:24:00 +0100
Subject: [PATCH 249/726] DPL: invoke fakeDispatch when creating data out of
 band in test

---
 Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx b/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx
index c105a9d0f662c..249e7fc304949 100644
--- a/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx
+++ b/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx
@@ -174,7 +174,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
       callbacks.set<CallbackService::Id::Start>(producerChannelInit);
     }
     // the compute callback of the producer
-    auto producerCallback = [nRolls, channelName, proxyMode, counter = std::make_shared<size_t>()](DataAllocator& outputs, ControlService& control, RawDeviceService& rds) {
+    auto producerCallback = [nRolls, channelName, proxyMode, counter = std::make_shared<size_t>()](DataAllocator& outputs, ControlService& control, RawDeviceService& rds, MessageContext& messageContext) {
       int data = *counter;
       // outputs.make<int>(OutputRef{"data", 0}) = data;
 
@@ -233,6 +233,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
       }
       // using utility from ExternalFairMQDeviceProxy
       o2::framework::sendOnChannel(device, messages, *channelName, (size_t)-1);
+      messageContext.fakeDispatch();
 
       if (++(*counter) >= nRolls) {
         // send the end of stream signal, this is transferred by the proxies
@@ -256,6 +257,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
           // add empty payload message
           out.AddPart(std::move(device.NewMessageFor(*channelName, 0, 0)));
           o2::framework::sendOnChannel(device, out, *channelName, (size_t)-1);
+          messageContext.fakeDispatch();
         }
       }
     };
@@ -401,7 +403,9 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
         output.AddPart(std::move(inputs.At(msgidx)));
       }
     }
+    auto &messageContext = services.get<MessageContext>();
     o2::framework::sendOnChannel(*device, output, channelName, (size_t)-1);
+    messageContext.fakeDispatch();
     return output.Size() != 0;
   };
 

From 77dac53b0a43ec0a07292fa4abf1b3d0540975c1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 17 Jan 2024 20:22:18 +0100
Subject: [PATCH 250/726] DPL: add Signposts for some of the DataProcessing /
 Stream callbacks

---
 .../Core/include/Framework/DeviceState.h      |   8 +-
 Framework/Core/src/CommonServices.cxx         |   6 +
 Framework/Core/src/DataProcessingContext.cxx  | 122 +++++++++---------
 Framework/Core/src/StreamContext.cxx          |  11 +-
 Framework/Core/src/WSDriverClient.cxx         |  13 +-
 .../src/FrameworkGUIDeviceInspector.cxx       |   2 +
 6 files changed, 95 insertions(+), 67 deletions(-)

diff --git a/Framework/Core/include/Framework/DeviceState.h b/Framework/Core/include/Framework/DeviceState.h
index 4fa72a84cad71..89961b3e92dc7 100644
--- a/Framework/Core/include/Framework/DeviceState.h
+++ b/Framework/Core/include/Framework/DeviceState.h
@@ -59,9 +59,11 @@ struct DeviceState {
 
   enum LogStreams : int {
     NO_LOG = 0,
-    DEVICE_LOG = 1 << 0,             // Log for Data Processing Device activities.
-    COMPLETION_LOG = 1 << 1,         // Log for the completion policy of the device.
-    MONITORING_SERVICE_LOG = 1 << 2, // Log for the monitoring service flushing.
+    DEVICE_LOG = 1 << 0,                 // Log for Data Processing Device activities.
+    COMPLETION_LOG = 1 << 1,             // Log for the completion policy of the device.
+    MONITORING_SERVICE_LOG = 1 << 2,     // Log for the monitoring service flushing.
+    DATA_PROCESSOR_CONTEXT_LOG = 1 << 3, // Log for the DataProcessorContext callbacks
+    STREAM_CONTEXT_LOG = 1 << 4,         // Log for the StreamContext callbacks
   };
 
   std::vector<InputChannelInfo> inputChannelInfos;
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index fd02a4c9ccb88..d264124fe25af 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -57,6 +57,7 @@
 #include <Configuration/ConfigurationInterface.h>
 #include <Configuration/ConfigurationFactory.h>
 #include <Monitoring/MonitoringFactory.h>
+#include "Framework/Signpost.h"
 
 #include <fairmq/Device.h>
 #include <fairmq/shmem/Monitor.h>
@@ -79,6 +80,9 @@ using Value = o2::monitoring::tags::Value;
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wpedantic"
 
+O2_DECLARE_DYNAMIC_LOG(data_processor_context);
+O2_DECLARE_DYNAMIC_LOG(stream_context);
+
 namespace o2::framework
 {
 
@@ -188,6 +192,8 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
         }
       }
       if (didCreate == false && messageContext.didDispatch() == true) {
+        O2_SIGNPOST_ID_FROM_POINTER(cid, stream_context, service);
+        O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Data created out of band");
         LOGP(debug, "Data created out of band");
         return;
       }
diff --git a/Framework/Core/src/DataProcessingContext.cxx b/Framework/Core/src/DataProcessingContext.cxx
index 9c023e5021a9b..73223b3d6ff89 100644
--- a/Framework/Core/src/DataProcessingContext.cxx
+++ b/Framework/Core/src/DataProcessingContext.cxx
@@ -10,145 +10,143 @@
 // or submit itself to any jurisdiction.
 
 #include "Framework/DataProcessingContext.h"
+#include "Framework/DataProcessorSpec.h"
+#include "Framework/Signpost.h"
 
+O2_DECLARE_DYNAMIC_LOG(data_processor_context);
 namespace o2::framework
 {
+
+namespace {
+template <typename T, typename... ARGS>
+void invokeAll(T& handles, char const* callbackName, o2::framework::DataProcessorSpec *spec, ARGS&... args)
+{
+  assert(callbackName);
+  O2_SIGNPOST_ID_FROM_POINTER(dpid, data_processor_context, spec);
+  // FIXME: for now spec is nullptr because we don't have a list of possible DataProcessorSpecs
+  // per device.
+  char const* dataProcessorName = spec ? spec->name.c_str() : "DataProcessorContext";
+  O2_SIGNPOST_START(data_processor_context, dpid, "callbacks", "Starting %{public}s::%{public}s", dataProcessorName, callbackName);
+  for (auto& handle : handles) {
+    O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, handle.service);
+    O2_SIGNPOST_START(data_processor_context, cid, "callbacks", "Starting %{public}s::%{public}s::%{public}s", dataProcessorName, handle.spec.name.c_str(), callbackName);
+    handle.callback(args..., handle.service);
+    O2_SIGNPOST_END(data_processor_context, cid, "callbacks", "Ending %{public}s::%{public}s::%{public}s", dataProcessorName, handle.spec.name.c_str(), callbackName);
+  }
+  O2_SIGNPOST_END(data_processor_context, dpid, "callbacks", "Ending %{public}s::%{public}s", dataProcessorName, callbackName);
+}
+}
+
 /// Invoke callbacks to be executed before every dangling check
 void DataProcessorContext::preProcessingCallbacks(ProcessingContext& ctx)
 {
-  for (auto& handle : preProcessingHandlers) {
-    LOGP(debug, "Invoking preDanglingCallback for service {}", handle.spec.name);
-    handle.callback(ctx, handle.service);
-  }
+  invokeAll(preProcessingHandlers, "preProcessingCallbacks", spec, ctx);
 }
 
 void DataProcessorContext::finaliseOutputsCallbacks(ProcessingContext& ctx)
 {
-  for (auto& handle : finaliseOutputsHandles) {
-    LOGP(debug, "Invoking postProcessingCallback for service {}", handle.spec.name);
-    handle.callback(ctx, handle.service);
-  }
+  invokeAll(finaliseOutputsHandles, "finaliseOutputsCallbacks", spec, ctx);
 }
 
 /// Invoke callbacks to be executed before every dangling check
 void DataProcessorContext::postProcessingCallbacks(ProcessingContext& ctx)
 {
-  for (auto& handle : postProcessingHandlers) {
-    LOGP(debug, "Invoking postProcessingCallback for service {}", handle.spec.name);
-    handle.callback(ctx, handle.service);
-  }
+  invokeAll(postProcessingHandlers, "postProcessingCallbacks", spec, ctx);
 }
 
 /// Invoke callbacks to be executed before every dangling check
-void DataProcessorContext::preDanglingCallbacks(DanglingContext& danglingContext)
+void DataProcessorContext::preDanglingCallbacks(DanglingContext& ctx)
 {
-  for (auto& handle : preDanglingHandles) {
-    LOGP(debug, "Invoking preDanglingCallback for service {}", handle.spec.name);
-    handle.callback(danglingContext, handle.service);
-  }
+  invokeAll(preDanglingHandles, "preDanglingCallbacks", spec, ctx);
 }
 
 /// Invoke callbacks to be executed after every dangling check
-void DataProcessorContext::postDanglingCallbacks(DanglingContext& danglingContext)
+void DataProcessorContext::postDanglingCallbacks(DanglingContext& ctx)
 {
-  for (auto& handle : postDanglingHandles) {
-    LOGP(debug, "Invoking postDanglingCallback for service {}", handle.spec.name);
-    handle.callback(danglingContext, handle.service);
-  }
+  invokeAll(postDanglingHandles, "postDanglingCallbacks", spec, ctx);
 }
 
 /// Invoke callbacks to be executed before every EOS user callback invokation
-void DataProcessorContext::preEOSCallbacks(EndOfStreamContext& eosContext)
+void DataProcessorContext::preEOSCallbacks(EndOfStreamContext& ctx)
 {
-  for (auto& handle : preEOSHandles) {
-    LOGP(detail, "Invoking preEosCallback for service {}", handle.spec.name);
-    handle.callback(eosContext, handle.service);
-  }
+  invokeAll(preEOSHandles, "preEOSCallbacks", spec, ctx);
 }
 
 /// Invoke callbacks to be executed after every EOS user callback invokation
-void DataProcessorContext::postEOSCallbacks(EndOfStreamContext& eosContext)
+void DataProcessorContext::postEOSCallbacks(EndOfStreamContext& ctx)
 {
-  for (auto& handle : postEOSHandles) {
-    LOGP(detail, "Invoking postEoSCallback for service {}", handle.spec.name);
-    handle.callback(eosContext, handle.service);
-  }
+  invokeAll(postEOSHandles, "postEOSCallbacks", spec, ctx);
 }
 
 /// Invoke callbacks to be executed after every data Dispatching
-void DataProcessorContext::postDispatchingCallbacks(ProcessingContext& processContext)
+void DataProcessorContext::postDispatchingCallbacks(ProcessingContext& ctx)
 {
-  for (auto& handle : postDispatchingHandles) {
-    LOGP(debug, "Invoking postDispatchingCallback for service {}", handle.spec.name);
-    handle.callback(processContext, handle.service);
-  }
+  invokeAll(postDispatchingHandles, "postDispatchingCallbacks", spec, ctx);
 }
 
 /// Invoke callbacks to be executed after every data Dispatching
-void DataProcessorContext::postForwardingCallbacks(ProcessingContext& processContext)
+void DataProcessorContext::postForwardingCallbacks(ProcessingContext& ctx)
 {
-  for (auto& handle : postForwardingHandles) {
-    LOGP(debug, "Invoking postForwardingCallback for service {}", handle.spec.name);
-    handle.callback(processContext, handle.service);
-  }
+  invokeAll(postForwardingHandles, "postForwardingCallbacks", spec, ctx);
 }
 
 /// Callbacks to be called in fair::mq::Device::PreRun()
 void DataProcessorContext::preStartCallbacks(ServiceRegistryRef ref)
 {
-  for (auto& handle : preStartHandles) {
-    LOGP(detail, "Invoking preStartCallback for service {}", handle.spec.name);
-    handle.callback(ref, handle.service);
-  }
+  invokeAll(preStartHandles, "preStartCallbacks", spec, ref);
 }
 
 void DataProcessorContext::postStopCallbacks(ServiceRegistryRef ref)
 {
-  // FIXME: we need to call the callback only once for the global services
-  /// I guess...
-  for (auto& handle : postStopHandles) {
-    LOGP(detail, "Invoking postStopCallback for service {}", handle.spec.name);
-    handle.callback(ref, handle.service);
-  }
+  invokeAll(postStopHandles, "postStopCallbacks", spec, ref);
 }
 
 /// Invoke callback to be executed on exit, in reverse order.
 void DataProcessorContext::preExitCallbacks(std::vector<ServiceExitHandle> handles, ServiceRegistryRef ref)
 {
+  O2_SIGNPOST_ID_FROM_POINTER(dpid, data_processor_context, &ref);
+  O2_SIGNPOST_START(data_processor_context, dpid, "callbacks", "Starting DataProcessorContext preExitCallbacks");
   // FIXME: we need to call the callback only once for the global services
   /// I guess...
   for (auto handle = handles.rbegin(); handle != handles.rend(); ++handle) {
-    LOGP(detail, "Invoking preExitCallback for service {}", handle->spec.name);
+    O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, handle->service);
+    O2_SIGNPOST_START(data_processor_context, cid, "callbacks", "Starting DataProcessorContext::preExitCallbacks for service %{public}s", handle->spec.name.c_str());
     handle->callback(ref, handle->service);
+    O2_SIGNPOST_END(data_processor_context, cid, "callbacks", "Ending DataProcessorContext::preExitCallbacks for service %{public}s", handle->spec.name.c_str());
   }
+  O2_SIGNPOST_END(data_processor_context, dpid, "callbacks", "Ending DataProcessorContext preExitCallbacks");
 }
 
 /// Invoke callback to be executed on exit, in reverse order.
 void DataProcessorContext::preLoopCallbacks(ServiceRegistryRef ref)
 {
-  // FIXME: we need to call the callback only once for the global services
-  /// I guess...
-  LOGP(debug, "Invoking preLoopCallbacks");
-  for (auto& handle : preLoopHandles) {
-    LOGP(debug, "Invoking preLoopCallback for service {}", handle.spec.name);
-    handle.callback(ref, handle.service);
-  }
+  invokeAll(preLoopHandles, "preLoopCallbacks", spec, ref);
 }
 
 void DataProcessorContext::domainInfoUpdatedCallback(ServiceRegistryRef ref, size_t oldestPossibleTimeslice, ChannelIndex channelIndex)
 {
+  O2_SIGNPOST_ID_FROM_POINTER(dpid, data_processor_context, this);
+  O2_SIGNPOST_START(data_processor_context, dpid, "callbacks", "Starting DataProcessorContext domainInfoUpdatedCallback");
   for (auto& handle : domainInfoHandles) {
-    LOGP(debug, "Invoking domainInfoHandles for service {}", handle.spec.name);
+    O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, handle.service);
+    O2_SIGNPOST_START(data_processor_context, cid, "callbacks", "Starting DataProcessorContext::domainInfoUpdatedCallback for service %{public}s", handle.spec.name.c_str());
     handle.callback(ref, oldestPossibleTimeslice, channelIndex);
+    O2_SIGNPOST_END(data_processor_context, cid, "callbacks", "Ending DataProcessorContext::domainInfoUpdatedCallback for service %{public}s", handle.spec.name.c_str());
   }
+  O2_SIGNPOST_END(data_processor_context, dpid, "callbacks", "Ending DataProcessorContext domainInfoUpdatedCallback");
 }
 
 void DataProcessorContext::preSendingMessagesCallbacks(ServiceRegistryRef ref, fair::mq::Parts& parts, ChannelIndex channelIndex)
 {
+  O2_SIGNPOST_ID_FROM_POINTER(dpid, data_processor_context, this);
+  O2_SIGNPOST_START(data_processor_context, dpid, "callbacks", "Starting DataProcessorContext preSendingMessagesCallbacks");
   for (auto& handle : preSendingMessagesHandles) {
-    LOGP(debug, "Invoking preSending for service {}", handle.spec.name);
+    O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, handle.service);
+    O2_SIGNPOST_START(data_processor_context, cid, "callbacks", "Starting DataProcessorContext::preSendingMessagesCallbacks for service %{public}s", handle.spec.name.c_str());
     handle.callback(ref, parts, channelIndex);
+    O2_SIGNPOST_END(data_processor_context, cid, "callbacks", "Ending DataProcessorContext::preSendingMessagesCallbacks for service %{public}s", handle.spec.name.c_str());
   }
+  O2_SIGNPOST_END(data_processor_context, dpid, "callbacks", "Ending DataProcessorContext preSendingMessagesCallbacks");
 }
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/StreamContext.cxx b/Framework/Core/src/StreamContext.cxx
index c7f28a3dbde1a..2ce0c1c427cbd 100644
--- a/Framework/Core/src/StreamContext.cxx
+++ b/Framework/Core/src/StreamContext.cxx
@@ -11,6 +11,10 @@
 
 #include "Framework/StreamContext.h"
 
+#include "Framework/Signpost.h"
+
+O2_DECLARE_DYNAMIC_LOG(stream_context);
+
 namespace o2::framework
 {
 
@@ -49,12 +53,17 @@ void StreamContext::finaliseOutputsCallbacks(ProcessingContext& pcx)
 /// Invoke callbacks to be executed after every process method invokation
 void StreamContext::postProcessingCallbacks(ProcessingContext& pcx)
 {
+  O2_SIGNPOST_ID_FROM_POINTER(dpid, stream_context, &pcx);
+  O2_SIGNPOST_START(stream_context, dpid, "callbacks", "Starting StreamContext postProcessingCallbacks");
   for (auto& handle : postProcessingHandles) {
-    LOG(debug) << "Invoking postProcessingCallbacks for " << handle.service;
+    O2_SIGNPOST_ID_FROM_POINTER(cid, stream_context, handle.service);
+    O2_SIGNPOST_START(stream_context, cid, "callbacks", "Starting StreamContext::postProcessingCallbacks for service %{public}s", handle.spec.name.c_str());
     assert(handle.service);
     assert(handle.callback);
     handle.callback(pcx, handle.service);
+    O2_SIGNPOST_END(stream_context, cid, "callbacks", "Ending StreamContext::postProcessingCallbacks for service %{public}s", handle.spec.name.c_str());
   }
+  O2_SIGNPOST_END(stream_context, dpid, "callbacks", "Ending StreamContext postProcessingCallbacks");
 }
 
 /// Invoke callbacks to be executed before every EOS user callback invokation
diff --git a/Framework/Core/src/WSDriverClient.cxx b/Framework/Core/src/WSDriverClient.cxx
index ac2e3db41fcef..179b13bf91d76 100644
--- a/Framework/Core/src/WSDriverClient.cxx
+++ b/Framework/Core/src/WSDriverClient.cxx
@@ -24,6 +24,8 @@
 O2_DECLARE_DYNAMIC_LOG(device);
 O2_DECLARE_DYNAMIC_LOG(completion);
 O2_DECLARE_DYNAMIC_LOG(monitoring_service);
+O2_DECLARE_DYNAMIC_LOG(data_processor_context);
+O2_DECLARE_DYNAMIC_LOG(stream_context);
 
 namespace o2::framework
 {
@@ -184,12 +186,21 @@ void on_connect(uv_connect_t* connection, int status)
     } else {
       O2_LOG_DISABLE(completion);
     }
-
     if ((state.logStreams & DeviceState::LogStreams::MONITORING_SERVICE_LOG) != 0) {
       O2_LOG_ENABLE(monitoring_service);
     } else {
       O2_LOG_DISABLE(monitoring_service);
     }
+    if ((state.logStreams & DeviceState::LogStreams::DATA_PROCESSOR_CONTEXT_LOG) != 0) {
+      O2_LOG_ENABLE(data_processor_context);
+    } else {
+      O2_LOG_DISABLE(data_processor_context);
+    }
+    if ((state.logStreams & DeviceState::LogStreams::STREAM_CONTEXT_LOG) != 0) {
+      O2_LOG_ENABLE(stream_context);
+    } else {
+      O2_LOG_DISABLE(stream_context);
+    }
   });
 
   // Client will be filled in the line after. I can probably have a single
diff --git a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
index fe93ca6d0f07f..f54e74d6ac019 100644
--- a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
@@ -405,6 +405,8 @@ void displayDeviceInspector(DeviceSpec const& spec,
     logsChanged = ImGui::CheckboxFlags("Device", &control.logStreams, DeviceState::LogStreams::DEVICE_LOG);
     logsChanged = ImGui::CheckboxFlags("Completion", &control.logStreams, DeviceState::LogStreams::COMPLETION_LOG);
     logsChanged = ImGui::CheckboxFlags("Monitoring", &control.logStreams, DeviceState::LogStreams::MONITORING_SERVICE_LOG);
+    logsChanged = ImGui::CheckboxFlags("DataProcessorContext", &control.logStreams, DeviceState::LogStreams::DATA_PROCESSOR_CONTEXT_LOG);
+    logsChanged = ImGui::CheckboxFlags("StreamContext", &control.logStreams, DeviceState::LogStreams::STREAM_CONTEXT_LOG);
     if (logsChanged && control.controller) {
       std::string cmd = fmt::format("/log-streams {}", control.logStreams);
       control.controller->write(cmd.c_str(), cmd.size());

From 8a93fe99b4709d262c0f1395362459bf29a520d0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 18 Jan 2024 10:14:58 +0100
Subject: [PATCH 251/726] DPL: add Signposts for some of the DataProcessing /
 Stream callbacks

---
 .../Core/include/Framework/DeviceState.h      |   8 +-
 Framework/Core/src/CommonServices.cxx         |   6 +
 Framework/Core/src/DataProcessingContext.cxx  | 123 +++++++++---------
 Framework/Core/src/StreamContext.cxx          |  11 +-
 Framework/Core/src/WSDriverClient.cxx         |  13 +-
 .../src/FrameworkGUIDeviceInspector.cxx       |   2 +
 6 files changed, 96 insertions(+), 67 deletions(-)

diff --git a/Framework/Core/include/Framework/DeviceState.h b/Framework/Core/include/Framework/DeviceState.h
index 4fa72a84cad71..89961b3e92dc7 100644
--- a/Framework/Core/include/Framework/DeviceState.h
+++ b/Framework/Core/include/Framework/DeviceState.h
@@ -59,9 +59,11 @@ struct DeviceState {
 
   enum LogStreams : int {
     NO_LOG = 0,
-    DEVICE_LOG = 1 << 0,             // Log for Data Processing Device activities.
-    COMPLETION_LOG = 1 << 1,         // Log for the completion policy of the device.
-    MONITORING_SERVICE_LOG = 1 << 2, // Log for the monitoring service flushing.
+    DEVICE_LOG = 1 << 0,                 // Log for Data Processing Device activities.
+    COMPLETION_LOG = 1 << 1,             // Log for the completion policy of the device.
+    MONITORING_SERVICE_LOG = 1 << 2,     // Log for the monitoring service flushing.
+    DATA_PROCESSOR_CONTEXT_LOG = 1 << 3, // Log for the DataProcessorContext callbacks
+    STREAM_CONTEXT_LOG = 1 << 4,         // Log for the StreamContext callbacks
   };
 
   std::vector<InputChannelInfo> inputChannelInfos;
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index fd02a4c9ccb88..d264124fe25af 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -57,6 +57,7 @@
 #include <Configuration/ConfigurationInterface.h>
 #include <Configuration/ConfigurationFactory.h>
 #include <Monitoring/MonitoringFactory.h>
+#include "Framework/Signpost.h"
 
 #include <fairmq/Device.h>
 #include <fairmq/shmem/Monitor.h>
@@ -79,6 +80,9 @@ using Value = o2::monitoring::tags::Value;
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wpedantic"
 
+O2_DECLARE_DYNAMIC_LOG(data_processor_context);
+O2_DECLARE_DYNAMIC_LOG(stream_context);
+
 namespace o2::framework
 {
 
@@ -188,6 +192,8 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
         }
       }
       if (didCreate == false && messageContext.didDispatch() == true) {
+        O2_SIGNPOST_ID_FROM_POINTER(cid, stream_context, service);
+        O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Data created out of band");
         LOGP(debug, "Data created out of band");
         return;
       }
diff --git a/Framework/Core/src/DataProcessingContext.cxx b/Framework/Core/src/DataProcessingContext.cxx
index 9c023e5021a9b..86409bf5434bb 100644
--- a/Framework/Core/src/DataProcessingContext.cxx
+++ b/Framework/Core/src/DataProcessingContext.cxx
@@ -10,145 +10,144 @@
 // or submit itself to any jurisdiction.
 
 #include "Framework/DataProcessingContext.h"
+#include "Framework/DataProcessorSpec.h"
+#include "Framework/Signpost.h"
 
+O2_DECLARE_DYNAMIC_LOG(data_processor_context);
 namespace o2::framework
 {
+
+namespace
+{
+template <typename T, typename... ARGS>
+void invokeAll(T& handles, char const* callbackName, o2::framework::DataProcessorSpec* spec, ARGS&... args)
+{
+  assert(callbackName);
+  O2_SIGNPOST_ID_FROM_POINTER(dpid, data_processor_context, spec);
+  // FIXME: for now spec is nullptr because we don't have a list of possible DataProcessorSpecs
+  // per device.
+  char const* dataProcessorName = spec ? spec->name.c_str() : "DataProcessorContext";
+  O2_SIGNPOST_START(data_processor_context, dpid, "callbacks", "Starting %{public}s::%{public}s", dataProcessorName, callbackName);
+  for (auto& handle : handles) {
+    O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, handle.service);
+    O2_SIGNPOST_START(data_processor_context, cid, "callbacks", "Starting %{public}s::%{public}s::%{public}s", dataProcessorName, handle.spec.name.c_str(), callbackName);
+    handle.callback(args..., handle.service);
+    O2_SIGNPOST_END(data_processor_context, cid, "callbacks", "Ending %{public}s::%{public}s::%{public}s", dataProcessorName, handle.spec.name.c_str(), callbackName);
+  }
+  O2_SIGNPOST_END(data_processor_context, dpid, "callbacks", "Ending %{public}s::%{public}s", dataProcessorName, callbackName);
+}
+} // namespace
+
 /// Invoke callbacks to be executed before every dangling check
 void DataProcessorContext::preProcessingCallbacks(ProcessingContext& ctx)
 {
-  for (auto& handle : preProcessingHandlers) {
-    LOGP(debug, "Invoking preDanglingCallback for service {}", handle.spec.name);
-    handle.callback(ctx, handle.service);
-  }
+  invokeAll(preProcessingHandlers, "preProcessingCallbacks", spec, ctx);
 }
 
 void DataProcessorContext::finaliseOutputsCallbacks(ProcessingContext& ctx)
 {
-  for (auto& handle : finaliseOutputsHandles) {
-    LOGP(debug, "Invoking postProcessingCallback for service {}", handle.spec.name);
-    handle.callback(ctx, handle.service);
-  }
+  invokeAll(finaliseOutputsHandles, "finaliseOutputsCallbacks", spec, ctx);
 }
 
 /// Invoke callbacks to be executed before every dangling check
 void DataProcessorContext::postProcessingCallbacks(ProcessingContext& ctx)
 {
-  for (auto& handle : postProcessingHandlers) {
-    LOGP(debug, "Invoking postProcessingCallback for service {}", handle.spec.name);
-    handle.callback(ctx, handle.service);
-  }
+  invokeAll(postProcessingHandlers, "postProcessingCallbacks", spec, ctx);
 }
 
 /// Invoke callbacks to be executed before every dangling check
-void DataProcessorContext::preDanglingCallbacks(DanglingContext& danglingContext)
+void DataProcessorContext::preDanglingCallbacks(DanglingContext& ctx)
 {
-  for (auto& handle : preDanglingHandles) {
-    LOGP(debug, "Invoking preDanglingCallback for service {}", handle.spec.name);
-    handle.callback(danglingContext, handle.service);
-  }
+  invokeAll(preDanglingHandles, "preDanglingCallbacks", spec, ctx);
 }
 
 /// Invoke callbacks to be executed after every dangling check
-void DataProcessorContext::postDanglingCallbacks(DanglingContext& danglingContext)
+void DataProcessorContext::postDanglingCallbacks(DanglingContext& ctx)
 {
-  for (auto& handle : postDanglingHandles) {
-    LOGP(debug, "Invoking postDanglingCallback for service {}", handle.spec.name);
-    handle.callback(danglingContext, handle.service);
-  }
+  invokeAll(postDanglingHandles, "postDanglingCallbacks", spec, ctx);
 }
 
 /// Invoke callbacks to be executed before every EOS user callback invokation
-void DataProcessorContext::preEOSCallbacks(EndOfStreamContext& eosContext)
+void DataProcessorContext::preEOSCallbacks(EndOfStreamContext& ctx)
 {
-  for (auto& handle : preEOSHandles) {
-    LOGP(detail, "Invoking preEosCallback for service {}", handle.spec.name);
-    handle.callback(eosContext, handle.service);
-  }
+  invokeAll(preEOSHandles, "preEOSCallbacks", spec, ctx);
 }
 
 /// Invoke callbacks to be executed after every EOS user callback invokation
-void DataProcessorContext::postEOSCallbacks(EndOfStreamContext& eosContext)
+void DataProcessorContext::postEOSCallbacks(EndOfStreamContext& ctx)
 {
-  for (auto& handle : postEOSHandles) {
-    LOGP(detail, "Invoking postEoSCallback for service {}", handle.spec.name);
-    handle.callback(eosContext, handle.service);
-  }
+  invokeAll(postEOSHandles, "postEOSCallbacks", spec, ctx);
 }
 
 /// Invoke callbacks to be executed after every data Dispatching
-void DataProcessorContext::postDispatchingCallbacks(ProcessingContext& processContext)
+void DataProcessorContext::postDispatchingCallbacks(ProcessingContext& ctx)
 {
-  for (auto& handle : postDispatchingHandles) {
-    LOGP(debug, "Invoking postDispatchingCallback for service {}", handle.spec.name);
-    handle.callback(processContext, handle.service);
-  }
+  invokeAll(postDispatchingHandles, "postDispatchingCallbacks", spec, ctx);
 }
 
 /// Invoke callbacks to be executed after every data Dispatching
-void DataProcessorContext::postForwardingCallbacks(ProcessingContext& processContext)
+void DataProcessorContext::postForwardingCallbacks(ProcessingContext& ctx)
 {
-  for (auto& handle : postForwardingHandles) {
-    LOGP(debug, "Invoking postForwardingCallback for service {}", handle.spec.name);
-    handle.callback(processContext, handle.service);
-  }
+  invokeAll(postForwardingHandles, "postForwardingCallbacks", spec, ctx);
 }
 
 /// Callbacks to be called in fair::mq::Device::PreRun()
 void DataProcessorContext::preStartCallbacks(ServiceRegistryRef ref)
 {
-  for (auto& handle : preStartHandles) {
-    LOGP(detail, "Invoking preStartCallback for service {}", handle.spec.name);
-    handle.callback(ref, handle.service);
-  }
+  invokeAll(preStartHandles, "preStartCallbacks", spec, ref);
 }
 
 void DataProcessorContext::postStopCallbacks(ServiceRegistryRef ref)
 {
-  // FIXME: we need to call the callback only once for the global services
-  /// I guess...
-  for (auto& handle : postStopHandles) {
-    LOGP(detail, "Invoking postStopCallback for service {}", handle.spec.name);
-    handle.callback(ref, handle.service);
-  }
+  invokeAll(postStopHandles, "postStopCallbacks", spec, ref);
 }
 
 /// Invoke callback to be executed on exit, in reverse order.
 void DataProcessorContext::preExitCallbacks(std::vector<ServiceExitHandle> handles, ServiceRegistryRef ref)
 {
+  O2_SIGNPOST_ID_FROM_POINTER(dpid, data_processor_context, &ref);
+  O2_SIGNPOST_START(data_processor_context, dpid, "callbacks", "Starting DataProcessorContext preExitCallbacks");
   // FIXME: we need to call the callback only once for the global services
   /// I guess...
   for (auto handle = handles.rbegin(); handle != handles.rend(); ++handle) {
-    LOGP(detail, "Invoking preExitCallback for service {}", handle->spec.name);
+    O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, handle->service);
+    O2_SIGNPOST_START(data_processor_context, cid, "callbacks", "Starting DataProcessorContext::preExitCallbacks for service %{public}s", handle->spec.name.c_str());
     handle->callback(ref, handle->service);
+    O2_SIGNPOST_END(data_processor_context, cid, "callbacks", "Ending DataProcessorContext::preExitCallbacks for service %{public}s", handle->spec.name.c_str());
   }
+  O2_SIGNPOST_END(data_processor_context, dpid, "callbacks", "Ending DataProcessorContext preExitCallbacks");
 }
 
 /// Invoke callback to be executed on exit, in reverse order.
 void DataProcessorContext::preLoopCallbacks(ServiceRegistryRef ref)
 {
-  // FIXME: we need to call the callback only once for the global services
-  /// I guess...
-  LOGP(debug, "Invoking preLoopCallbacks");
-  for (auto& handle : preLoopHandles) {
-    LOGP(debug, "Invoking preLoopCallback for service {}", handle.spec.name);
-    handle.callback(ref, handle.service);
-  }
+  invokeAll(preLoopHandles, "preLoopCallbacks", spec, ref);
 }
 
 void DataProcessorContext::domainInfoUpdatedCallback(ServiceRegistryRef ref, size_t oldestPossibleTimeslice, ChannelIndex channelIndex)
 {
+  O2_SIGNPOST_ID_FROM_POINTER(dpid, data_processor_context, this);
+  O2_SIGNPOST_START(data_processor_context, dpid, "callbacks", "Starting DataProcessorContext domainInfoUpdatedCallback");
   for (auto& handle : domainInfoHandles) {
-    LOGP(debug, "Invoking domainInfoHandles for service {}", handle.spec.name);
+    O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, handle.service);
+    O2_SIGNPOST_START(data_processor_context, cid, "callbacks", "Starting DataProcessorContext::domainInfoUpdatedCallback for service %{public}s", handle.spec.name.c_str());
     handle.callback(ref, oldestPossibleTimeslice, channelIndex);
+    O2_SIGNPOST_END(data_processor_context, cid, "callbacks", "Ending DataProcessorContext::domainInfoUpdatedCallback for service %{public}s", handle.spec.name.c_str());
   }
+  O2_SIGNPOST_END(data_processor_context, dpid, "callbacks", "Ending DataProcessorContext domainInfoUpdatedCallback");
 }
 
 void DataProcessorContext::preSendingMessagesCallbacks(ServiceRegistryRef ref, fair::mq::Parts& parts, ChannelIndex channelIndex)
 {
+  O2_SIGNPOST_ID_FROM_POINTER(dpid, data_processor_context, this);
+  O2_SIGNPOST_START(data_processor_context, dpid, "callbacks", "Starting DataProcessorContext preSendingMessagesCallbacks");
   for (auto& handle : preSendingMessagesHandles) {
-    LOGP(debug, "Invoking preSending for service {}", handle.spec.name);
+    O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, handle.service);
+    O2_SIGNPOST_START(data_processor_context, cid, "callbacks", "Starting DataProcessorContext::preSendingMessagesCallbacks for service %{public}s", handle.spec.name.c_str());
     handle.callback(ref, parts, channelIndex);
+    O2_SIGNPOST_END(data_processor_context, cid, "callbacks", "Ending DataProcessorContext::preSendingMessagesCallbacks for service %{public}s", handle.spec.name.c_str());
   }
+  O2_SIGNPOST_END(data_processor_context, dpid, "callbacks", "Ending DataProcessorContext preSendingMessagesCallbacks");
 }
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/StreamContext.cxx b/Framework/Core/src/StreamContext.cxx
index c7f28a3dbde1a..2ce0c1c427cbd 100644
--- a/Framework/Core/src/StreamContext.cxx
+++ b/Framework/Core/src/StreamContext.cxx
@@ -11,6 +11,10 @@
 
 #include "Framework/StreamContext.h"
 
+#include "Framework/Signpost.h"
+
+O2_DECLARE_DYNAMIC_LOG(stream_context);
+
 namespace o2::framework
 {
 
@@ -49,12 +53,17 @@ void StreamContext::finaliseOutputsCallbacks(ProcessingContext& pcx)
 /// Invoke callbacks to be executed after every process method invokation
 void StreamContext::postProcessingCallbacks(ProcessingContext& pcx)
 {
+  O2_SIGNPOST_ID_FROM_POINTER(dpid, stream_context, &pcx);
+  O2_SIGNPOST_START(stream_context, dpid, "callbacks", "Starting StreamContext postProcessingCallbacks");
   for (auto& handle : postProcessingHandles) {
-    LOG(debug) << "Invoking postProcessingCallbacks for " << handle.service;
+    O2_SIGNPOST_ID_FROM_POINTER(cid, stream_context, handle.service);
+    O2_SIGNPOST_START(stream_context, cid, "callbacks", "Starting StreamContext::postProcessingCallbacks for service %{public}s", handle.spec.name.c_str());
     assert(handle.service);
     assert(handle.callback);
     handle.callback(pcx, handle.service);
+    O2_SIGNPOST_END(stream_context, cid, "callbacks", "Ending StreamContext::postProcessingCallbacks for service %{public}s", handle.spec.name.c_str());
   }
+  O2_SIGNPOST_END(stream_context, dpid, "callbacks", "Ending StreamContext postProcessingCallbacks");
 }
 
 /// Invoke callbacks to be executed before every EOS user callback invokation
diff --git a/Framework/Core/src/WSDriverClient.cxx b/Framework/Core/src/WSDriverClient.cxx
index ac2e3db41fcef..179b13bf91d76 100644
--- a/Framework/Core/src/WSDriverClient.cxx
+++ b/Framework/Core/src/WSDriverClient.cxx
@@ -24,6 +24,8 @@
 O2_DECLARE_DYNAMIC_LOG(device);
 O2_DECLARE_DYNAMIC_LOG(completion);
 O2_DECLARE_DYNAMIC_LOG(monitoring_service);
+O2_DECLARE_DYNAMIC_LOG(data_processor_context);
+O2_DECLARE_DYNAMIC_LOG(stream_context);
 
 namespace o2::framework
 {
@@ -184,12 +186,21 @@ void on_connect(uv_connect_t* connection, int status)
     } else {
       O2_LOG_DISABLE(completion);
     }
-
     if ((state.logStreams & DeviceState::LogStreams::MONITORING_SERVICE_LOG) != 0) {
       O2_LOG_ENABLE(monitoring_service);
     } else {
       O2_LOG_DISABLE(monitoring_service);
     }
+    if ((state.logStreams & DeviceState::LogStreams::DATA_PROCESSOR_CONTEXT_LOG) != 0) {
+      O2_LOG_ENABLE(data_processor_context);
+    } else {
+      O2_LOG_DISABLE(data_processor_context);
+    }
+    if ((state.logStreams & DeviceState::LogStreams::STREAM_CONTEXT_LOG) != 0) {
+      O2_LOG_ENABLE(stream_context);
+    } else {
+      O2_LOG_DISABLE(stream_context);
+    }
   });
 
   // Client will be filled in the line after. I can probably have a single
diff --git a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
index fe93ca6d0f07f..f54e74d6ac019 100644
--- a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
@@ -405,6 +405,8 @@ void displayDeviceInspector(DeviceSpec const& spec,
     logsChanged = ImGui::CheckboxFlags("Device", &control.logStreams, DeviceState::LogStreams::DEVICE_LOG);
     logsChanged = ImGui::CheckboxFlags("Completion", &control.logStreams, DeviceState::LogStreams::COMPLETION_LOG);
     logsChanged = ImGui::CheckboxFlags("Monitoring", &control.logStreams, DeviceState::LogStreams::MONITORING_SERVICE_LOG);
+    logsChanged = ImGui::CheckboxFlags("DataProcessorContext", &control.logStreams, DeviceState::LogStreams::DATA_PROCESSOR_CONTEXT_LOG);
+    logsChanged = ImGui::CheckboxFlags("StreamContext", &control.logStreams, DeviceState::LogStreams::STREAM_CONTEXT_LOG);
     if (logsChanged && control.controller) {
       std::string cmd = fmt::format("/log-streams {}", control.logStreams);
       control.controller->write(cmd.c_str(), cmd.size());

From d712036dc6e37184fecfa0d8694e01f55acc4e78 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 18 Jan 2024 12:36:12 +0100
Subject: [PATCH 252/726] DPL: make log messages optional as well

---
 Framework/Foundation/include/Framework/Signpost.h | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index 8bd54975d1622..8d0107e68ecd6 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -460,7 +460,13 @@ void o2_debug_log_set_stacktrace(_o2_log_t* log, int stacktrace)
 #define O2_LOG_DISABLE(log) _o2_log_set_stacktrace(private_o2_log_##log, 0)
 // For the moment we simply use LOG DEBUG. We should have proper activities so that we can
 // turn on and off the printing.
-#define O2_LOG_DEBUG(log, ...) O2_LOG_MACRO(__VA_ARGS__)
+#define O2_LOG_DEBUG(log, ...) __extension__({                        \
+  if (O2_BUILTIN_UNLIKELY(O2_LOG_ENABLED(log))) {                     \
+    O2_LOG_MACRO(__VA_ARGS__);                                        \
+  } else if (O2_BUILTIN_UNLIKELY(private_o2_log_##log->stacktrace)) { \
+    O2_LOG_MACRO(__VA_ARGS__);                                        \
+  }                                                                   \
+})
 #define O2_SIGNPOST_ID_FROM_POINTER(name, log, pointer) _o2_signpost_id_t name = _o2_signpost_id_make_with_pointer(private_o2_log_##log, pointer)
 #define O2_SIGNPOST_ID_GENERATE(name, log) _o2_signpost_id_t name = _o2_signpost_id_generate_local(private_o2_log_##log)
 // In case Instruments is attached, we switch to the Apple signpost API otherwise, both one

From 7610e035bc1c413930df1585fe1456147236fa4b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 18 Jan 2024 12:36:12 +0100
Subject: [PATCH 253/726] DPL: improve debugging of WorkflowImporter

Using Signposts allow us to have a nested view of the state processing.
---
 .../Core/src/WorkflowSerializationHelpers.cxx | 23 ++++++++++++-------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/Framework/Core/src/WorkflowSerializationHelpers.cxx b/Framework/Core/src/WorkflowSerializationHelpers.cxx
index ac182a27a70c5..e20e23f98c90b 100644
--- a/Framework/Core/src/WorkflowSerializationHelpers.cxx
+++ b/Framework/Core/src/WorkflowSerializationHelpers.cxx
@@ -18,6 +18,7 @@
 #include "Framework/DataDescriptorMatcher.h"
 #include "Framework/DataMatcherWalker.h"
 #include "Framework/Logger.h"
+#include "Framework/Signpost.h"
 
 #include <rapidjson/reader.h>
 #include <rapidjson/prettywriter.h>
@@ -27,6 +28,8 @@
 #include <algorithm>
 #include <memory>
 
+O2_DECLARE_DYNAMIC_LOG(workflow_importer);
+
 namespace o2::framework
 {
 
@@ -811,7 +814,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
 
   bool Uint(unsigned i)
   {
-    debug << "Uint(" << i << ")" << std::endl;
+    O2_SIGNPOST_EVENT_EMIT(workflow_importer, _o2_signpost_id_t{(int64_t)states.size()}, "import", "Uint(%d)", i);
     if (in(State::IN_INPUT_SUBSPEC)) {
       subspec = i;
       inputMatcherNodes.push_back(SubSpecificationTypeValueMatcher{i});
@@ -845,28 +848,31 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
 
   bool Int(int i)
   {
-    debug << "Int(" << i << ")" << std::endl;
+    O2_SIGNPOST_EVENT_EMIT(workflow_importer, _o2_signpost_id_t{(int64_t)states.size()}, "import", "Int(%d)", i);
     return true;
   }
   bool Uint64(uint64_t u)
   {
-    debug << "Uint64(" << u << ")" << std::endl;
+    O2_SIGNPOST_EVENT_EMIT(workflow_importer, _o2_signpost_id_t{(int64_t)states.size()}, "import", "Uint64(%" PRIu64 ")", u);
     return true;
   }
   bool Double(double d)
   {
-    debug << "Double(" << d << ")" << std::endl;
+    O2_SIGNPOST_EVENT_EMIT(workflow_importer, _o2_signpost_id_t{(int64_t)states.size()}, "import", "Double(%f)", d);
     return true;
   }
 
   void enter(char const* what)
   {
-    debug << "ENTER: " << what << std::endl;
+    O2_SIGNPOST_EVENT_EMIT(workflow_importer, _o2_signpost_id_t{(int64_t)states.size()}, "import", "ENTER: %s", what);
   }
+
   void push(State state)
   {
-    debug << "PUSH: " << state << std::endl;
+    debug.str("");
+    debug << state;
     states.push_back(state);
+    O2_SIGNPOST_START(workflow_importer, _o2_signpost_id_t{(int64_t)states.size()}, "import", "PUSH: %s", debug.str().c_str());
   }
 
   State pop()
@@ -877,11 +883,12 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
     }
     auto result = states.back();
     states.pop_back();
-    debug << "POP: " << result;
+    debug.str("");
+    debug << result;
     if (!states.empty()) {
       debug << " now in " << states.back();
     }
-    debug << std::endl;
+    O2_SIGNPOST_END(workflow_importer, _o2_signpost_id_t{(int64_t)states.size()+1}, "import", "POP: %s", debug.str().c_str());
     return result;
   }
   bool in(State o)

From cf1002b505386b1366c5b1d3bf3e2d1ac521ed58 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 18 Jan 2024 13:31:59 +0100
Subject: [PATCH 254/726] DPL: add ability to disable inputs programmatically

Disabled inputs will not result in an actual route for the data, however it
will be stored in the configuration, so that analysis workflows will not need
to have the configuration available at every step.
---
 Framework/Core/include/Framework/InputSpec.h  |  3 ++
 Framework/Core/include/Framework/OutputSpec.h |  4 ++
 Framework/Core/src/DeviceSpecHelpers.cxx      | 15 +++++++-
 Framework/Core/src/WorkflowHelpers.cxx        | 12 ++++--
 .../Core/src/WorkflowSerializationHelpers.cxx | 32 +++++++++++++++-
 .../Core/test/test_WorkflowSerialization.cxx  | 38 +++++++++++++++++++
 6 files changed, 98 insertions(+), 6 deletions(-)

diff --git a/Framework/Core/include/Framework/InputSpec.h b/Framework/Core/include/Framework/InputSpec.h
index 9be95e6853da2..7858aacb80664 100644
--- a/Framework/Core/include/Framework/InputSpec.h
+++ b/Framework/Core/include/Framework/InputSpec.h
@@ -74,6 +74,9 @@ struct InputSpec {
 
   /// A set of configurables which can be used to customise the InputSpec.
   std::vector<ConfigParamSpec> metadata;
+  /// Wether or not the input is to be considered enabled.
+  /// Useful to programmatically disable inputs e.g. for the ProcessorOptions.
+  bool enabled = true;
 
   friend std::ostream& operator<<(std::ostream& stream, InputSpec const& arg);
   bool operator==(InputSpec const& that) const;
diff --git a/Framework/Core/include/Framework/OutputSpec.h b/Framework/Core/include/Framework/OutputSpec.h
index e92792faa6b14..51527bce4e339 100644
--- a/Framework/Core/include/Framework/OutputSpec.h
+++ b/Framework/Core/include/Framework/OutputSpec.h
@@ -86,6 +86,10 @@ struct OutputSpec {
   /// A set of configurables which can be used to customise the InputSpec.
   std::vector<ConfigParamSpec> metadata;
 
+  /// Wether or not this output is enabled. This is useful to decide programmatically
+  /// wether or not to produce a given output.
+  bool enabled = true;
+
   friend std::ostream& operator<<(std::ostream& stream, OutputSpec const& arg);
 };
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 6c2063cb59311..bedcda35873f9 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -679,19 +679,27 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
     assert(policyPtr != nullptr);
 
     if (edge.isForward == false) {
+      auto& matcher = outputsMatchers[edge.outputGlobalIndex];
+      if (matcher.enabled == false) {
+        throw runtime_error_f("Output %s is disabled but it was still used in topology", DataSpecUtils::describe(matcher).data());
+      }
       OutputRoute route{
         edge.timeIndex,
         consumer.maxInputTimeslices,
-        outputsMatchers[edge.outputGlobalIndex],
+        matcher,
         channel.name,
         policyPtr,
       };
       device.outputs.emplace_back(route);
     } else {
+      auto& matcher = workflow[edge.consumer].inputs[edge.consumerInputIndex];
+      if (matcher.enabled == false) {
+        throw runtime_error_f("Output %s is disabled but it was still used in topology", DataSpecUtils::describe(matcher).data());
+      }
       ForwardRoute route{
         edge.timeIndex,
         consumer.maxInputTimeslices,
-        workflow[edge.consumer].inputs[edge.consumerInputIndex],
+        matcher,
         channel.name};
       device.forwards.emplace_back(route);
     }
@@ -915,6 +923,9 @@ void DeviceSpecHelpers::processInEdgeActions(std::vector<DeviceSpec>& devices,
 
     auto const& inputSpec = consumer.inputs[edge.consumerInputIndex];
     auto const& sourceChannel = consumerDevice.inputChannels[ci].name;
+    if (inputSpec.enabled == false) {
+      throw runtime_error_f("Input %s is disabled but it was still used in topology", DataSpecUtils::describe(inputSpec).data());
+    }
 
     InputRoute route{
       inputSpec,
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 360480f80d4ef..a4e1bd82f2ad8 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -841,11 +841,13 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
     for (size_t wi = 0; wi < workflow.size(); ++wi) {
       auto& producer = workflow[wi];
 
-      for (size_t oi = 0; oi < producer.outputs.size(); ++oi) {
-        auto& out = producer.outputs[oi];
+      for (auto& output : producer.outputs) {
+        if (output.enabled == false) {
+          continue;
+        }
         auto uniqueOutputId = outputs.size();
         availableOutputsInfo.emplace_back(LogicalOutputInfo{wi, uniqueOutputId, false});
-        outputs.push_back(out);
+        outputs.push_back(output);
       }
     }
   };
@@ -879,6 +881,10 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
   std::vector<bool> matches(constOutputs.size());
   for (size_t consumer = 0; consumer < workflow.size(); ++consumer) {
     for (size_t input = 0; input < workflow[consumer].inputs.size(); ++input) {
+      // Skip disabled inputs.
+      if (workflow[consumer].inputs[input].enabled == false) {
+        continue;
+      }
       forwards.clear();
       for (size_t i = 0; i < constOutputs.size(); i++) {
         matches[i] = DataSpecUtils::match(workflow[consumer].inputs[input], constOutputs[i]);
diff --git a/Framework/Core/src/WorkflowSerializationHelpers.cxx b/Framework/Core/src/WorkflowSerializationHelpers.cxx
index e20e23f98c90b..56fd43df4ca54 100644
--- a/Framework/Core/src/WorkflowSerializationHelpers.cxx
+++ b/Framework/Core/src/WorkflowSerializationHelpers.cxx
@@ -61,6 +61,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
     IN_INPUT_ORIGIN,
     IN_INPUT_DESCRIPTION,
     IN_INPUT_SUBSPEC,
+    IN_INPUT_ENABLED,
     IN_INPUT_ORIGIN_REF,
     IN_INPUT_DESCRIPTION_REF,
     IN_INPUT_SUBSPEC_REF,
@@ -75,6 +76,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
     IN_OUTPUT_BINDING,
     IN_OUTPUT_ORIGIN,
     IN_OUTPUT_DESCRIPTION,
+    IN_OUTPUT_ENABLED,
     IN_OUTPUT_SUBSPEC,
     IN_OUTPUT_LIFETIME,
     IN_OUTPUT_OPTIONS,
@@ -167,6 +169,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       case State::IN_INPUT_SUBSPEC:
         s << "IN_INPUT_SUBSPEC";
         break;
+      case State::IN_INPUT_ENABLED:
+        s << "IN_INPUT_ENABLED";
+        break;
       case State::IN_INPUT_ORIGIN_REF:
         s << "IN_INPUT_ORIGIN_REF";
         break;
@@ -218,6 +223,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       case State::IN_OUTPUT_OPTIONS:
         s << "IN_OUTPUT_OPTIONS";
         break;
+      case WorkflowImporter::State::IN_OUTPUT_ENABLED:
+        s << "IN_OUTPUT_ENABLED";
+        break;
       case State::IN_OPTION:
         s << "IN_OPTION";
         break;
@@ -302,6 +310,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       dataProcessors.push_back(DataProcessorSpec{});
     } else if (in(State::IN_INPUTS)) {
       push(State::IN_INPUT);
+      enabled = true;
       inputMatcherNodes.clear();
     } else if (in(State::IN_INPUT_MATCHER)) {
       // start a new embedded matcher
@@ -313,6 +322,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       // will be merged into the parent matcher
     } else if (in(State::IN_OUTPUTS)) {
       push(State::IN_OUTPUT);
+      enabled = true;
       outputHasSubSpec = false;
     } else if (in(State::IN_OPTIONS)) {
       push(State::IN_OPTION);
@@ -370,9 +380,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       } else {
         dataProcessors.back().inputs.push_back(InputSpec({binding}, std::move(*matcher), lifetime, inputOptions));
       }
+      dataProcessors.back().inputs.back().enabled = enabled;
       inputMatcherNodes.clear();
       inputOptions.clear();
-
     } else if (in(State::IN_INPUT_MATCHER) && inputMatcherNodes.size() > 1) {
       data_matcher::Node child = std::move(inputMatcherNodes.back());
       inputMatcherNodes.pop_back();
@@ -433,6 +443,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       } else {
         dataProcessors.back().outputs.push_back(OutputSpec({binding}, {origin, description}, lifetime));
       }
+      dataProcessors.back().outputs.back().enabled = enabled;
       outputHasSubSpec = false;
     } else if (in(State::IN_OPTION)) {
       std::unique_ptr<ConfigParamSpec> opt{nullptr};
@@ -545,12 +556,14 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       push(State::IN_DATAPROCESSORS);
     } else if (in(State::IN_INPUTS)) {
       push(State::IN_INPUT);
+      enabled = true;
     } else if (in(State::IN_INPUT_OPTIONS)) {
       push(State::IN_OPTION);
     } else if (in(State::IN_OUTPUT_OPTIONS)) {
       push(State::IN_OPTION);
     } else if (in(State::IN_OUTPUTS)) {
       push(State::IN_OUTPUT);
+      enabled = true;
       outputHasSubSpec = false;
     } else if (in(State::IN_OPTIONS)) {
       push(State::IN_OPTION);
@@ -602,6 +615,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       push(State::IN_INPUT_DESCRIPTION_REF);
     } else if (in(State::IN_INPUT) && strncmp(str, "subspecRef", length) == 0) {
       push(State::IN_INPUT_SUBSPEC_REF);
+    } else if (in(State::IN_INPUT) && strncmp(str, "enabled", length) == 0) {
+      push(State::IN_INPUT_ENABLED);
     } else if (in(State::IN_INPUT) && strncmp(str, "matcher", length) == 0) {
       // the outermost matcher is starting here
       // we create a placeholder which is being updated later
@@ -664,6 +679,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       push(State::IN_OUTPUT_LIFETIME);
     } else if (in(State::IN_OUTPUT) && strncmp(str, "metadata", length) == 0) {
       push(State::IN_OUTPUT_OPTIONS);
+    } else if (in(State::IN_OUTPUT) && strncmp(str, "enabled", length) == 0) {
+      push(State::IN_OUTPUT_ENABLED);
     } else if (in(State::IN_DATAPROCESSOR) && strncmp(str, "name", length) == 0) {
       push(State::IN_DATAPROCESSOR_NAME);
     } else if (in(State::IN_DATAPROCESSOR) && strncmp(str, "ranks", length) == 0) {
@@ -734,6 +751,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       metadata.back().executable = s;
     } else if (in(State::IN_INPUT_BINDING)) {
       binding = s;
+    } else if (in(State::IN_INPUT_ENABLED)) {
+      enabled = (s == "true");
     } else if (in(State::IN_INPUT_ORIGIN)) {
       origin.runtimeInit(s.c_str(), std::min(s.size(), 4UL));
       std::string v(s.c_str(), std::min(s.size(), 4UL));
@@ -841,6 +860,10 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       dataProcessors.back().inputTimeSliceId = i;
     } else if (in(State::IN_DATAPROCESSOR_MAX_TIMESLICES)) {
       dataProcessors.back().maxInputTimeslices = i;
+    } else if (in(State::IN_INPUT_ENABLED)) {
+      enabled = (i == 1);
+    } else if (in(State::IN_OUTPUT_ENABLED)) {
+      enabled = (i == 1);
     }
     pop();
     return true;
@@ -915,6 +938,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
   header::DataDescription description;
   size_t subspec;
   size_t ref;
+  // Keep track of the enabled state of the input/output
+  // Unless specified, inputs are enabled by default.
+  bool enabled = true;
   Lifetime lifetime;
   std::string metadatumKey;
   std::string metadatumValue;
@@ -1103,6 +1129,8 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
       }
       w.Key("lifetime");
       w.Uint((int)input.lifetime);
+      w.Key("enabled");
+      w.Uint((int)input.enabled);
       if (input.metadata.empty() == false) {
         w.Key("metadata");
         w.StartArray();
@@ -1152,6 +1180,8 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
       }
       w.Key("lifetime");
       w.Uint((int)output.lifetime);
+      w.Key("enabled");
+      w.Uint((int)output.enabled);
       if (output.metadata.empty() == false) {
         w.Key("metadata");
         w.StartArray();
diff --git a/Framework/Core/test/test_WorkflowSerialization.cxx b/Framework/Core/test/test_WorkflowSerialization.cxx
index 6e541f7d22f07..e7f39e7e38cbd 100644
--- a/Framework/Core/test/test_WorkflowSerialization.cxx
+++ b/Framework/Core/test/test_WorkflowSerialization.cxx
@@ -120,3 +120,41 @@ TEST_CASE("TestVerifyWildcard")
   // also check if the conversion to ConcreteDataMatcher is working at import
   // REQUIRE(std::get_if<ConcreteDataTypeMatcher>(&w1[0].inputs[0].matcher) != nullptr);;
 }
+
+/// Verify that disabled inputs / outputs are serialised correctly
+TEST_CASE("TestDisabled")
+{
+  using namespace o2::framework;
+  WorkflowSpec w0{
+    DataProcessorSpec{
+      .name = "A",
+      .inputs = {{"clbPayload", "CLP"}, {"clbWrapper", "CLW"}},
+      .outputs = {{"CP2", "1"}, {"CL2", "2"}},
+    }};
+
+  w0[0].inputs[0].enabled = false;
+  w0[0].outputs[1].enabled = false;
+
+  std::vector<DataProcessorInfo> dataProcessorInfoOut{
+    {"A", "test_Framework_test_SerializationWorkflow", {}},
+  };
+
+  CommandInfo commandInfoOut{"o2-dpl-workflow -b --option 1 --option 2"};
+
+  std::vector<DataProcessorInfo> dataProcessorInfoIn{};
+  CommandInfo commandInfoIn;
+
+  std::ostringstream firstDump;
+  WorkflowSerializationHelpers::dump(firstDump, w0, dataProcessorInfoOut, commandInfoOut);
+  std::istringstream is;
+  is.str(firstDump.str());
+  WorkflowSpec w1;
+  WorkflowSerializationHelpers::import(is, w1, dataProcessorInfoIn, commandInfoIn);
+  REQUIRE(w1.size() == 1);
+  REQUIRE(w1[0].inputs.size() == 2);
+  REQUIRE(w1[0].inputs[0].enabled == false);
+  REQUIRE(w1[0].inputs[1].enabled == true);
+  REQUIRE(w1[0].outputs.size() == 2);
+  REQUIRE(w1[0].outputs[0].enabled == true);
+  REQUIRE(w1[0].outputs[1].enabled == false);
+}

From 3cb857b0c7adcb96cd83aadc8d12df5488db1021 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 18 Jan 2024 13:31:59 +0100
Subject: [PATCH 255/726] DPL: add ability to disable inputs programmatically

Disabled inputs will not result in an actual route for the data, however it
will be stored in the configuration, so that analysis workflows will not need
to have the configuration available at every step.
---
 Framework/Core/include/Framework/InputSpec.h  |  3 ++
 Framework/Core/include/Framework/OutputSpec.h |  4 ++
 Framework/Core/src/DeviceSpecHelpers.cxx      | 15 +++++-
 Framework/Core/src/WorkflowHelpers.cxx        | 12 +++--
 .../Core/src/WorkflowSerializationHelpers.cxx | 32 ++++++++++++-
 .../Core/test/test_FrameworkDataFlowToDDS.cxx | 48 ++++++++++++-------
 .../Core/test/test_WorkflowSerialization.cxx  | 38 +++++++++++++++
 7 files changed, 130 insertions(+), 22 deletions(-)

diff --git a/Framework/Core/include/Framework/InputSpec.h b/Framework/Core/include/Framework/InputSpec.h
index 9be95e6853da2..7858aacb80664 100644
--- a/Framework/Core/include/Framework/InputSpec.h
+++ b/Framework/Core/include/Framework/InputSpec.h
@@ -74,6 +74,9 @@ struct InputSpec {
 
   /// A set of configurables which can be used to customise the InputSpec.
   std::vector<ConfigParamSpec> metadata;
+  /// Wether or not the input is to be considered enabled.
+  /// Useful to programmatically disable inputs e.g. for the ProcessorOptions.
+  bool enabled = true;
 
   friend std::ostream& operator<<(std::ostream& stream, InputSpec const& arg);
   bool operator==(InputSpec const& that) const;
diff --git a/Framework/Core/include/Framework/OutputSpec.h b/Framework/Core/include/Framework/OutputSpec.h
index e92792faa6b14..51527bce4e339 100644
--- a/Framework/Core/include/Framework/OutputSpec.h
+++ b/Framework/Core/include/Framework/OutputSpec.h
@@ -86,6 +86,10 @@ struct OutputSpec {
   /// A set of configurables which can be used to customise the InputSpec.
   std::vector<ConfigParamSpec> metadata;
 
+  /// Wether or not this output is enabled. This is useful to decide programmatically
+  /// wether or not to produce a given output.
+  bool enabled = true;
+
   friend std::ostream& operator<<(std::ostream& stream, OutputSpec const& arg);
 };
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 6c2063cb59311..bedcda35873f9 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -679,19 +679,27 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
     assert(policyPtr != nullptr);
 
     if (edge.isForward == false) {
+      auto& matcher = outputsMatchers[edge.outputGlobalIndex];
+      if (matcher.enabled == false) {
+        throw runtime_error_f("Output %s is disabled but it was still used in topology", DataSpecUtils::describe(matcher).data());
+      }
       OutputRoute route{
         edge.timeIndex,
         consumer.maxInputTimeslices,
-        outputsMatchers[edge.outputGlobalIndex],
+        matcher,
         channel.name,
         policyPtr,
       };
       device.outputs.emplace_back(route);
     } else {
+      auto& matcher = workflow[edge.consumer].inputs[edge.consumerInputIndex];
+      if (matcher.enabled == false) {
+        throw runtime_error_f("Output %s is disabled but it was still used in topology", DataSpecUtils::describe(matcher).data());
+      }
       ForwardRoute route{
         edge.timeIndex,
         consumer.maxInputTimeslices,
-        workflow[edge.consumer].inputs[edge.consumerInputIndex],
+        matcher,
         channel.name};
       device.forwards.emplace_back(route);
     }
@@ -915,6 +923,9 @@ void DeviceSpecHelpers::processInEdgeActions(std::vector<DeviceSpec>& devices,
 
     auto const& inputSpec = consumer.inputs[edge.consumerInputIndex];
     auto const& sourceChannel = consumerDevice.inputChannels[ci].name;
+    if (inputSpec.enabled == false) {
+      throw runtime_error_f("Input %s is disabled but it was still used in topology", DataSpecUtils::describe(inputSpec).data());
+    }
 
     InputRoute route{
       inputSpec,
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 360480f80d4ef..a4e1bd82f2ad8 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -841,11 +841,13 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
     for (size_t wi = 0; wi < workflow.size(); ++wi) {
       auto& producer = workflow[wi];
 
-      for (size_t oi = 0; oi < producer.outputs.size(); ++oi) {
-        auto& out = producer.outputs[oi];
+      for (auto& output : producer.outputs) {
+        if (output.enabled == false) {
+          continue;
+        }
         auto uniqueOutputId = outputs.size();
         availableOutputsInfo.emplace_back(LogicalOutputInfo{wi, uniqueOutputId, false});
-        outputs.push_back(out);
+        outputs.push_back(output);
       }
     }
   };
@@ -879,6 +881,10 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
   std::vector<bool> matches(constOutputs.size());
   for (size_t consumer = 0; consumer < workflow.size(); ++consumer) {
     for (size_t input = 0; input < workflow[consumer].inputs.size(); ++input) {
+      // Skip disabled inputs.
+      if (workflow[consumer].inputs[input].enabled == false) {
+        continue;
+      }
       forwards.clear();
       for (size_t i = 0; i < constOutputs.size(); i++) {
         matches[i] = DataSpecUtils::match(workflow[consumer].inputs[input], constOutputs[i]);
diff --git a/Framework/Core/src/WorkflowSerializationHelpers.cxx b/Framework/Core/src/WorkflowSerializationHelpers.cxx
index e20e23f98c90b..56fd43df4ca54 100644
--- a/Framework/Core/src/WorkflowSerializationHelpers.cxx
+++ b/Framework/Core/src/WorkflowSerializationHelpers.cxx
@@ -61,6 +61,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
     IN_INPUT_ORIGIN,
     IN_INPUT_DESCRIPTION,
     IN_INPUT_SUBSPEC,
+    IN_INPUT_ENABLED,
     IN_INPUT_ORIGIN_REF,
     IN_INPUT_DESCRIPTION_REF,
     IN_INPUT_SUBSPEC_REF,
@@ -75,6 +76,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
     IN_OUTPUT_BINDING,
     IN_OUTPUT_ORIGIN,
     IN_OUTPUT_DESCRIPTION,
+    IN_OUTPUT_ENABLED,
     IN_OUTPUT_SUBSPEC,
     IN_OUTPUT_LIFETIME,
     IN_OUTPUT_OPTIONS,
@@ -167,6 +169,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       case State::IN_INPUT_SUBSPEC:
         s << "IN_INPUT_SUBSPEC";
         break;
+      case State::IN_INPUT_ENABLED:
+        s << "IN_INPUT_ENABLED";
+        break;
       case State::IN_INPUT_ORIGIN_REF:
         s << "IN_INPUT_ORIGIN_REF";
         break;
@@ -218,6 +223,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       case State::IN_OUTPUT_OPTIONS:
         s << "IN_OUTPUT_OPTIONS";
         break;
+      case WorkflowImporter::State::IN_OUTPUT_ENABLED:
+        s << "IN_OUTPUT_ENABLED";
+        break;
       case State::IN_OPTION:
         s << "IN_OPTION";
         break;
@@ -302,6 +310,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       dataProcessors.push_back(DataProcessorSpec{});
     } else if (in(State::IN_INPUTS)) {
       push(State::IN_INPUT);
+      enabled = true;
       inputMatcherNodes.clear();
     } else if (in(State::IN_INPUT_MATCHER)) {
       // start a new embedded matcher
@@ -313,6 +322,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       // will be merged into the parent matcher
     } else if (in(State::IN_OUTPUTS)) {
       push(State::IN_OUTPUT);
+      enabled = true;
       outputHasSubSpec = false;
     } else if (in(State::IN_OPTIONS)) {
       push(State::IN_OPTION);
@@ -370,9 +380,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       } else {
         dataProcessors.back().inputs.push_back(InputSpec({binding}, std::move(*matcher), lifetime, inputOptions));
       }
+      dataProcessors.back().inputs.back().enabled = enabled;
       inputMatcherNodes.clear();
       inputOptions.clear();
-
     } else if (in(State::IN_INPUT_MATCHER) && inputMatcherNodes.size() > 1) {
       data_matcher::Node child = std::move(inputMatcherNodes.back());
       inputMatcherNodes.pop_back();
@@ -433,6 +443,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       } else {
         dataProcessors.back().outputs.push_back(OutputSpec({binding}, {origin, description}, lifetime));
       }
+      dataProcessors.back().outputs.back().enabled = enabled;
       outputHasSubSpec = false;
     } else if (in(State::IN_OPTION)) {
       std::unique_ptr<ConfigParamSpec> opt{nullptr};
@@ -545,12 +556,14 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       push(State::IN_DATAPROCESSORS);
     } else if (in(State::IN_INPUTS)) {
       push(State::IN_INPUT);
+      enabled = true;
     } else if (in(State::IN_INPUT_OPTIONS)) {
       push(State::IN_OPTION);
     } else if (in(State::IN_OUTPUT_OPTIONS)) {
       push(State::IN_OPTION);
     } else if (in(State::IN_OUTPUTS)) {
       push(State::IN_OUTPUT);
+      enabled = true;
       outputHasSubSpec = false;
     } else if (in(State::IN_OPTIONS)) {
       push(State::IN_OPTION);
@@ -602,6 +615,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       push(State::IN_INPUT_DESCRIPTION_REF);
     } else if (in(State::IN_INPUT) && strncmp(str, "subspecRef", length) == 0) {
       push(State::IN_INPUT_SUBSPEC_REF);
+    } else if (in(State::IN_INPUT) && strncmp(str, "enabled", length) == 0) {
+      push(State::IN_INPUT_ENABLED);
     } else if (in(State::IN_INPUT) && strncmp(str, "matcher", length) == 0) {
       // the outermost matcher is starting here
       // we create a placeholder which is being updated later
@@ -664,6 +679,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       push(State::IN_OUTPUT_LIFETIME);
     } else if (in(State::IN_OUTPUT) && strncmp(str, "metadata", length) == 0) {
       push(State::IN_OUTPUT_OPTIONS);
+    } else if (in(State::IN_OUTPUT) && strncmp(str, "enabled", length) == 0) {
+      push(State::IN_OUTPUT_ENABLED);
     } else if (in(State::IN_DATAPROCESSOR) && strncmp(str, "name", length) == 0) {
       push(State::IN_DATAPROCESSOR_NAME);
     } else if (in(State::IN_DATAPROCESSOR) && strncmp(str, "ranks", length) == 0) {
@@ -734,6 +751,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       metadata.back().executable = s;
     } else if (in(State::IN_INPUT_BINDING)) {
       binding = s;
+    } else if (in(State::IN_INPUT_ENABLED)) {
+      enabled = (s == "true");
     } else if (in(State::IN_INPUT_ORIGIN)) {
       origin.runtimeInit(s.c_str(), std::min(s.size(), 4UL));
       std::string v(s.c_str(), std::min(s.size(), 4UL));
@@ -841,6 +860,10 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       dataProcessors.back().inputTimeSliceId = i;
     } else if (in(State::IN_DATAPROCESSOR_MAX_TIMESLICES)) {
       dataProcessors.back().maxInputTimeslices = i;
+    } else if (in(State::IN_INPUT_ENABLED)) {
+      enabled = (i == 1);
+    } else if (in(State::IN_OUTPUT_ENABLED)) {
+      enabled = (i == 1);
     }
     pop();
     return true;
@@ -915,6 +938,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
   header::DataDescription description;
   size_t subspec;
   size_t ref;
+  // Keep track of the enabled state of the input/output
+  // Unless specified, inputs are enabled by default.
+  bool enabled = true;
   Lifetime lifetime;
   std::string metadatumKey;
   std::string metadatumValue;
@@ -1103,6 +1129,8 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
       }
       w.Key("lifetime");
       w.Uint((int)input.lifetime);
+      w.Key("enabled");
+      w.Uint((int)input.enabled);
       if (input.metadata.empty() == false) {
         w.Key("metadata");
         w.StartArray();
@@ -1152,6 +1180,8 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
       }
       w.Key("lifetime");
       w.Uint((int)output.lifetime);
+      w.Key("enabled");
+      w.Uint((int)output.enabled);
       if (output.metadata.empty() == false) {
         w.Key("metadata");
         w.StartArray();
diff --git a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
index 49420e4bd99ad..1a2fed675536a 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
@@ -169,14 +169,16 @@ TEST_CASE("TestDDS")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;A1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 },
                 {
                     &quot;binding&quot;: &quot;TST/A2/0&quot;,
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;A2&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;options&quot;: [],
@@ -195,7 +197,8 @@ TEST_CASE("TestDDS")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;A1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;outputs&quot;: [
@@ -204,7 +207,8 @@ TEST_CASE("TestDDS")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;B1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;options&quot;: [],
@@ -223,7 +227,8 @@ TEST_CASE("TestDDS")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;A2&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;outputs&quot;: [
@@ -232,7 +237,8 @@ TEST_CASE("TestDDS")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;C1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;options&quot;: [],
@@ -251,14 +257,16 @@ TEST_CASE("TestDDS")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;B1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 },
                 {
                     &quot;binding&quot;: &quot;y&quot;,
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;C1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;outputs&quot;: [],
@@ -433,14 +441,16 @@ TEST_CASE("TestDDSExpendable")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;A1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 },
                 {
                     &quot;binding&quot;: &quot;TST/A2/0&quot;,
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;A2&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;options&quot;: [],
@@ -459,7 +469,8 @@ TEST_CASE("TestDDSExpendable")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;A1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;outputs&quot;: [
@@ -468,7 +479,8 @@ TEST_CASE("TestDDSExpendable")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;B1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;options&quot;: [],
@@ -487,7 +499,8 @@ TEST_CASE("TestDDSExpendable")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;A2&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;outputs&quot;: [
@@ -496,7 +509,8 @@ TEST_CASE("TestDDSExpendable")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;C1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;options&quot;: [],
@@ -515,14 +529,16 @@ TEST_CASE("TestDDSExpendable")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;B1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 },
                 {
                     &quot;binding&quot;: &quot;y&quot;,
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;C1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;outputs&quot;: [],
diff --git a/Framework/Core/test/test_WorkflowSerialization.cxx b/Framework/Core/test/test_WorkflowSerialization.cxx
index 6e541f7d22f07..e7f39e7e38cbd 100644
--- a/Framework/Core/test/test_WorkflowSerialization.cxx
+++ b/Framework/Core/test/test_WorkflowSerialization.cxx
@@ -120,3 +120,41 @@ TEST_CASE("TestVerifyWildcard")
   // also check if the conversion to ConcreteDataMatcher is working at import
   // REQUIRE(std::get_if<ConcreteDataTypeMatcher>(&w1[0].inputs[0].matcher) != nullptr);;
 }
+
+/// Verify that disabled inputs / outputs are serialised correctly
+TEST_CASE("TestDisabled")
+{
+  using namespace o2::framework;
+  WorkflowSpec w0{
+    DataProcessorSpec{
+      .name = "A",
+      .inputs = {{"clbPayload", "CLP"}, {"clbWrapper", "CLW"}},
+      .outputs = {{"CP2", "1"}, {"CL2", "2"}},
+    }};
+
+  w0[0].inputs[0].enabled = false;
+  w0[0].outputs[1].enabled = false;
+
+  std::vector<DataProcessorInfo> dataProcessorInfoOut{
+    {"A", "test_Framework_test_SerializationWorkflow", {}},
+  };
+
+  CommandInfo commandInfoOut{"o2-dpl-workflow -b --option 1 --option 2"};
+
+  std::vector<DataProcessorInfo> dataProcessorInfoIn{};
+  CommandInfo commandInfoIn;
+
+  std::ostringstream firstDump;
+  WorkflowSerializationHelpers::dump(firstDump, w0, dataProcessorInfoOut, commandInfoOut);
+  std::istringstream is;
+  is.str(firstDump.str());
+  WorkflowSpec w1;
+  WorkflowSerializationHelpers::import(is, w1, dataProcessorInfoIn, commandInfoIn);
+  REQUIRE(w1.size() == 1);
+  REQUIRE(w1[0].inputs.size() == 2);
+  REQUIRE(w1[0].inputs[0].enabled == false);
+  REQUIRE(w1[0].inputs[1].enabled == true);
+  REQUIRE(w1[0].outputs.size() == 2);
+  REQUIRE(w1[0].outputs[0].enabled == true);
+  REQUIRE(w1[0].outputs[1].enabled == false);
+}

From 4a65d7c6ce0580e508f094f3820b50768196bb69 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 18 Jan 2024 14:20:06 +0100
Subject: [PATCH 256/726] DPL: get rid of warning

---
 Framework/Core/src/FairMQDeviceProxy.cxx | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Framework/Core/src/FairMQDeviceProxy.cxx b/Framework/Core/src/FairMQDeviceProxy.cxx
index a83057e9578be..c25a58e47ef2d 100644
--- a/Framework/Core/src/FairMQDeviceProxy.cxx
+++ b/Framework/Core/src/FairMQDeviceProxy.cxx
@@ -320,10 +320,7 @@ void FairMQDeviceProxy::bind(std::vector<OutputRoute> const& outputs, std::vecto
       mInputRoutes.emplace_back(RouteState{channelIndex, false});
       ri++;
     }
-    for (auto& route : mInputRoutes) {
-      assert(route.channel.value != -1);
-      assert(route.channel.value < mInputChannels.size());
-    }
+    assert(std::all_of(mInputRoutes.begin(), mInputRoutes.end(), [s = mInputChannels.size()](RouteState const& route) { return route.channel.value != -1 && route.channel.value < s; }));
     LOGP(detail, "Total input channels found {}, total routes {}", mInputChannels.size(), mInputRoutes.size());
     assert(mInputRoutes.size() == inputs.size());
   }

From c829bb9b39c20d10ca96a4b114af3b8903e3f322 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 18 Jan 2024 14:55:02 +0100
Subject: [PATCH 257/726] DPL: use signposts to debug data creation

---
 Framework/Core/src/CommonServices.cxx | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index d264124fe25af..d1cf0d2f714ab 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -185,6 +185,7 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
       // if we did not, but we still have didDispatched set to true
       // it means it was created out of band.
       bool didCreate = false;
+      O2_SIGNPOST_ID_FROM_POINTER(cid, stream_context, service);
       for (size_t ri = 0; ri < routes.size(); ++ri) {
         if (stream->routeUserCreated[ri] == true) {
           didCreate = true;
@@ -192,24 +193,29 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
         }
       }
       if (didCreate == false && messageContext.didDispatch() == true) {
-        O2_SIGNPOST_ID_FROM_POINTER(cid, stream_context, service);
-        O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Data created out of band");
-        LOGP(debug, "Data created out of band");
+        O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Data created out of band didCreate == %b && messageContext.didDispatch == %b",
+                               didCreate,
+                               messageContext.didDispatch());
         return;
       }
       for (size_t ri = 0; ri < routes.size(); ++ri) {
+        auto &route = routes[ri];
+        auto &matcher = route.matcher;
         if (stream->routeUserCreated[ri] == true) {
+          O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Data created by user. ri = %" PRIu64 ", %{public}s",
+                                 (uint64_t)ri, DataSpecUtils::describe(matcher).c_str());
           continue;
         }
-        auto &route = routes[ri];
-        auto &matcher = route.matcher;
         if ((timeslice % route.maxTimeslices) != route.timeslice) {
+          O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Route ri = %" PRIu64 ", skipped",
+                                 (uint64_t)ri);
           continue;
         }
         if (matcher.lifetime == Lifetime::Timeframe) {
           LOGP(error, "Expected Lifetime::Timeframe data {} was not created for timeslice {} and might result in dropped timeframes", DataSpecUtils::describe(matcher), timeslice);
         }
-      } },
+      } 
+      },
     .kind = ServiceKind::Stream};
 }
 
@@ -510,10 +516,11 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
     },
     .postForwarding = [](ProcessingContext& ctx, void* service) {
       auto* decongestion = reinterpret_cast<DecongestionService*>(service);
-      if (decongestion->isFirstInTopology == false) {
-        LOGP(debug, "We are not the first in the topology, do not update the oldest possible timeslice");
+      if (O2_BUILTIN_LIKELY(decongestion->isFirstInTopology == false)) {
         return;
       }
+      O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, service);
+      O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "postForwardingCallbacks", "We are the first one in the topology, we need to update the oldest possible timeslice");
       auto& timesliceIndex = ctx.services().get<TimesliceIndex>();
       auto& relayer = ctx.services().get<DataRelayer>();
       timesliceIndex.updateOldestPossibleOutput();

From ec29274420b075f3330dcdc32902911b456d0775 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 18 Jan 2024 14:57:00 +0100
Subject: [PATCH 258/726] Debug data creation

---
 Framework/Core/src/CommonServices.cxx | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index d1cf0d2f714ab..59a6488f6235a 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -192,6 +192,7 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
           break;
         }
       }
+      O2_LOG_ENABLE(stream_context);
       if (didCreate == false && messageContext.didDispatch() == true) {
         O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Data created out of band didCreate == %b && messageContext.didDispatch == %b",
                                didCreate,
@@ -215,6 +216,7 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
           LOGP(error, "Expected Lifetime::Timeframe data {} was not created for timeslice {} and might result in dropped timeframes", DataSpecUtils::describe(matcher), timeslice);
         }
       } 
+      O2_LOG_DISABLE(stream_context);
       },
     .kind = ServiceKind::Stream};
 }

From 96a619b5e175233dd3b612e9db63e282d1132eb1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 18 Jan 2024 16:13:58 +0100
Subject: [PATCH 259/726] DPL: do not run the check for the case data was not
 created nor dispatched

---
 Framework/Core/src/CommonServices.cxx | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index d264124fe25af..f8783a6c3a21b 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -197,6 +197,11 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
         LOGP(debug, "Data created out of band");
         return;
       }
+      if (didCreate == false && messageContext.didDispatch() == false) {
+        O2_SIGNPOST_ID_FROM_POINTER(cid, stream_context, service);
+        O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "No data created");
+        return;
+      }
       for (size_t ri = 0; ri < routes.size(); ++ri) {
         if (stream->routeUserCreated[ri] == true) {
           continue;

From 94486b9efa0d56f5b05b710117e302ba5c666dca Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 18 Jan 2024 16:27:55 +0100
Subject: [PATCH 260/726] DPL: drop commented code

---
 .../Core/test/benchmark_ExternalFairMQDeviceProxies.cxx     | 6 ------
 Framework/Core/test/test_DataAllocator.cxx                  | 1 -
 Framework/Core/test/test_DataDescriptorMatcherWorkflow.cxx  | 1 -
 Framework/Core/test/test_DataOutputDirector.cxx             | 3 ---
 Framework/Core/test/test_Parallel.cxx                       | 1 -
 Framework/Core/test/test_ParallelPipeline.cxx               | 3 ---
 Framework/Core/test/test_Root2ArrowTable.cxx                | 5 -----
 Framework/Core/test/test_StaticFor.cxx                      | 2 --
 .../Core/test/test_VariablePayloadSequenceWorkflow.cxx      | 1 -
 9 files changed, 23 deletions(-)

diff --git a/Framework/Core/test/benchmark_ExternalFairMQDeviceProxies.cxx b/Framework/Core/test/benchmark_ExternalFairMQDeviceProxies.cxx
index bdb4deb443021..c116ed81537ce 100644
--- a/Framework/Core/test/benchmark_ExternalFairMQDeviceProxies.cxx
+++ b/Framework/Core/test/benchmark_ExternalFairMQDeviceProxies.cxx
@@ -450,14 +450,8 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
   Inputs checkerInputs;
   if (bypassProxies != ProxyBypass::None) {
     checkerInputs.emplace_back(InputSpec{"datain", ConcreteDataTypeMatcher{"TST", "DATA"}, Lifetime::Timeframe});
-    //for (unsigned int i = 0; i < pState->nChannels; i++) {
-    //  checkerInputs.emplace_back(InputSpec{{"datain"}, "TST", "DATA", i, Lifetime::Timeframe});
-    //}
   } else {
     checkerInputs.emplace_back(InputSpec{"datain", ConcreteDataTypeMatcher{"PRX", "DATA"}, Lifetime::Timeframe});
-    //for (unsigned int i = 0; i < pState->nChannels; i++) {
-    //  checkerInputs.emplace_back(InputSpec{{"datain"}, "PRX", "DATA", i, Lifetime::Timeframe});
-    //}
   }
   workflow.emplace_back(DataProcessorSpec{"checker",
                                           std::move(checkerInputs),
diff --git a/Framework/Core/test/test_DataAllocator.cxx b/Framework/Core/test/test_DataAllocator.cxx
index 894fb5f38368a..5c29040a37450 100644
--- a/Framework/Core/test/test_DataAllocator.cxx
+++ b/Framework/Core/test/test_DataAllocator.cxx
@@ -338,7 +338,6 @@ DataProcessorSpec getSinkSpec()
 
     LOG(info) << "extracting POD vector";
     // TODO: use the ReturnType helper once implemented
-    //InputRecord::ReturnType<std::vector<int>> podvector;
     decltype(std::declval<InputRecord>().get<std::vector<int>>(DataRef{nullptr, nullptr, nullptr})) podvector;
     podvector = pc.inputs().get<std::vector<int>>("inputPODvector");
     ASSERT_ERROR(podvector.size() == 3);
diff --git a/Framework/Core/test/test_DataDescriptorMatcherWorkflow.cxx b/Framework/Core/test/test_DataDescriptorMatcherWorkflow.cxx
index ef792afa4cc79..e8bac109985c4 100644
--- a/Framework/Core/test/test_DataDescriptorMatcherWorkflow.cxx
+++ b/Framework/Core/test/test_DataDescriptorMatcherWorkflow.cxx
@@ -21,7 +21,6 @@
 #include "Framework/DataAllocator.h"
 
 // we need to add workflow options before including Framework/runDataProcessing
-//void customize(std::vector<ConfigParamSpec>& workflowOptions)
 //{
 //}
 
diff --git a/Framework/Core/test/test_DataOutputDirector.cxx b/Framework/Core/test/test_DataOutputDirector.cxx
index c158c52932db4..1834dd0781897 100644
--- a/Framework/Core/test/test_DataOutputDirector.cxx
+++ b/Framework/Core/test/test_DataOutputDirector.cxx
@@ -29,7 +29,6 @@ TEST_CASE("TestDataOutputDirector")
   std::string keepString("AOD/UNO/0:tr1:c1/c2/c3:fn1,AOD/UNO/0::c4");
   dod.readString(keepString);
   dod.setFilenameBase(mydfn);
-  //dod.printOut(); printf("\n\n");
 
   auto ds = dod.getDataOutputDescriptors(dh);
 
@@ -59,7 +58,6 @@ TEST_CASE("TestDataOutputDirector")
 
   dod.reset();
   std::tie(rdn, dfn, fmode, mfs, ntf) = dod.readJsonString(jsonString);
-  //dod.printOut(); printf("\n\n");
   ds = dod.getDataOutputDescriptors(dh);
 
   REQUIRE(ds.size() == 1);
@@ -106,7 +104,6 @@ TEST_CASE("TestDataOutputDirector")
   dod.reset();
   std::tie(rdn, dfn, fmode, mfs, ntf) = dod.readJson(jsonFile);
   dod.setFilenameBase("AnalysisResults");
-  //dod.printOut(); printf("\n\n");
   ds = dod.getDataOutputDescriptors(dh);
 
   REQUIRE(ds.size() == 2);
diff --git a/Framework/Core/test/test_Parallel.cxx b/Framework/Core/test/test_Parallel.cxx
index 6e8ebb2a2d1f0..4fe0c920426e2 100644
--- a/Framework/Core/test/test_Parallel.cxx
+++ b/Framework/Core/test/test_Parallel.cxx
@@ -19,7 +19,6 @@
 
 #include <boost/algorithm/string.hpp>
 
-
 using namespace o2::framework;
 
 struct FakeCluster {
diff --git a/Framework/Core/test/test_ParallelPipeline.cxx b/Framework/Core/test/test_ParallelPipeline.cxx
index 47106855a8f74..b79ee3a73cd24 100644
--- a/Framework/Core/test/test_ParallelPipeline.cxx
+++ b/Framework/Core/test/test_ParallelPipeline.cxx
@@ -63,7 +63,6 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
          LOG(debug) << "instance " << parallelContext.index1D() << " of " << parallelContext.index1DSize() << ": "
                     << *input.spec << ": " << *((int*)input.payload);
          auto const* dataheader = DataRefUtils::getHeader<o2::header::DataHeader*>(input);
-         //auto& data = ctx.outputs().make<int>(OutputRef{"output", dataheader->subSpecification});
          auto& data = ctx.outputs().make<int>(Output{"TST", "PREPROC", dataheader->subSpecification});
          ASSERT_ERROR(ctx.inputs().get<int>(input.spec->binding.c_str()) == parallelContext.index1D());
          data = parallelContext.index1D();
@@ -84,10 +83,8 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
          auto const* dataheader = DataRefUtils::getHeader<o2::header::DataHeader*>(input);
          // TODO: there is a bug in the API for using OutputRef, returns an rvalue which can not be bound to
          // lvalue reference
-         //auto& data = ctx.outputs().make<int>(OutputRef{"output", dataheader->subSpecification});
          auto& data = ctx.outputs().make<int>(Output{"TST", "DATA", dataheader->subSpecification});
          data = ctx.inputs().get<int>(input.spec->binding.c_str());
-         //auto& meta = ctx.outputs().make<int>(OutputRef{"metadt", dataheader->subSpecification});
          auto& meta = ctx.outputs().make<int>(Output{"TST", "META", dataheader->subSpecification});
          meta = dataheader->subSpecification;
        }
diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx
index b22432d8565a5..a3f48957d7044 100644
--- a/Framework/Core/test/test_Root2ArrowTable.cxx
+++ b/Framework/Core/test/test_Root2ArrowTable.cxx
@@ -47,7 +47,6 @@ TEST_CASE("RootTree2Table")
   t1.Branch("ij", ij, "ij[2]/I");
   //fill the tree
   for (Int_t i = 0; i < 1000; i++) {
-    //gRandom->Rannor(xyz[0], xyz[1]);
     xyz[0] = 1;
     xyz[1] = 2;
     xyz[2] = 3;
@@ -92,16 +91,12 @@ TEST_CASE("RootTree2Table")
     REQUIRE(array->length() == 3000);
     const float* c = reinterpret_cast<float const*>(array->values()->data());
 
-    //auto array = std::static_pointer_cast<arrow::FixedSizeBinaryArray>(table->column(0)->chunk(0));
-    //CHECK_EQUAL(array->byte_width(), sizeof(float[3]));
-    //const float* c = reinterpret_cast<float const*>(array->Value(0));
 
     CHECK(c[0] == 1);
     CHECK(c[1] == 2);
     CHECK(c[2] == 1);
   }
   {
-    //auto values = std::static_pointer_cast<arrow::FixedSizeBinaryArray>(table->column(1)->chunk(0));
     auto chunkToUse = table->column(1)->chunk(0);
     chunkToUse = std::dynamic_pointer_cast<arrow::FixedSizeListArray>(chunkToUse)->values();
     auto array = std::static_pointer_cast<arrow::Int32Array>(chunkToUse);
diff --git a/Framework/Core/test/test_StaticFor.cxx b/Framework/Core/test/test_StaticFor.cxx
index bacdcaeefe86e..ee66c5b4aaf5b 100644
--- a/Framework/Core/test/test_StaticFor.cxx
+++ b/Framework/Core/test/test_StaticFor.cxx
@@ -57,8 +57,6 @@ TEST_CASE("TestStaticFor")
     constexpr int index = i.value;
 
     // compiler will complain if constexpr is not enforced for index access:
-    //CONST_STR(staticNames[index]);    // works
-    //CONST_STR(staticNames[i.value]);  // fails
 
     constexpr auto sayHello = CONST_STR("Hello ") + CONST_STR(staticNames[index]);
 
diff --git a/Framework/Core/test/test_VariablePayloadSequenceWorkflow.cxx b/Framework/Core/test/test_VariablePayloadSequenceWorkflow.cxx
index 1024df1218485..9945eab037780 100644
--- a/Framework/Core/test/test_VariablePayloadSequenceWorkflow.cxx
+++ b/Framework/Core/test/test_VariablePayloadSequenceWorkflow.cxx
@@ -217,7 +217,6 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
         ASSERT_ERROR(*reinterpret_cast<size_t const*>(ref.payload) == sd->initialValue + nSequencePayloads);
         ++nSequencePayloads;
       }
-      //LOG(info) << "input " << ref.spec->binding << " has data {" << dh->dataOrigin.as<std::string>() << "/" << dh->dataDescription.as<std::string>() << "/" << dh->subSpecification << "}: " << *reinterpret_cast<size_t const*>(ref.payload);
     }
     for (auto const& [channel, count] : active) {
       ++counters[channel];

From deaf51be8461e455167b9c17ca64e9a17a24d4ea Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 18 Jan 2024 16:31:08 +0100
Subject: [PATCH 261/726] Fix formatting in tests

---
 Framework/Core/test/test_Expressions.cxx                    | 2 +-
 Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx   | 2 +-
 Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx   | 2 +-
 Framework/Core/test/test_O2DatabasePDG.cxx                  | 2 +-
 Framework/Core/test/test_Parallel.cxx                       | 6 +++---
 Framework/Core/test/test_Root2ArrowTable.cxx                | 4 ++--
 Framework/Core/test/test_Services.cxx                       | 2 +-
 Framework/Core/test/test_StaggeringWorkflow.cxx             | 2 +-
 Framework/Core/test/test_TableBuilder.cxx                   | 4 ++--
 Framework/Core/test/test_TreeToTable.cxx                    | 2 +-
 .../Core/test/test_VariablePayloadSequenceWorkflow.cxx      | 2 +-
 11 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/Framework/Core/test/test_Expressions.cxx b/Framework/Core/test/test_Expressions.cxx
index 0bd4fdc564946..176db6a259a7c 100644
--- a/Framework/Core/test/test_Expressions.cxx
+++ b/Framework/Core/test/test_Expressions.cxx
@@ -32,7 +32,7 @@ static BindingNode testInt{"testInt", 6, atype::INT32};
 
 namespace o2::aod::track
 {
-DECLARE_SOA_EXPRESSION_COLUMN(Pze, pz, float, o2::aod::track::tgl*(1.f / o2::aod::track::signed1Pt));
+DECLARE_SOA_EXPRESSION_COLUMN(Pze, pz, float, o2::aod::track::tgl * (1.f / o2::aod::track::signed1Pt));
 } // namespace o2::aod::track
 
 TEST_CASE("TestTreeParsing")
diff --git a/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx b/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx
index 249e7fc304949..d4d204f206e0a 100644
--- a/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx
+++ b/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx
@@ -403,7 +403,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
         output.AddPart(std::move(inputs.At(msgidx)));
       }
     }
-    auto &messageContext = services.get<MessageContext>();
+    auto& messageContext = services.get<MessageContext>();
     o2::framework::sendOnChannel(*device, output, channelName, (size_t)-1);
     messageContext.fakeDispatch();
     return output.Size() != 0;
diff --git a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
index 7b0ffa462a9db..d15eebd26bdf7 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
@@ -31,7 +31,7 @@ namespace
 {
 WorkflowSpec defineDataProcessing()
 {
-  return {{.name = "A",                                                                  //
+  return {{.name = "A",                                                          //
            .outputs = Outputs{OutputSpec{"TST", "A1"}, OutputSpec{"TST", "A2"}}, // A1 will be consumed twice, A2 is dangling
            .algorithm = AlgorithmSpec{},                                         //
            .options = {ConfigParamSpec{"channel-config", VariantType::String,    // raw input channel
diff --git a/Framework/Core/test/test_O2DatabasePDG.cxx b/Framework/Core/test/test_O2DatabasePDG.cxx
index c6f6f3c9c57e7..11b07d1473390 100644
--- a/Framework/Core/test/test_O2DatabasePDG.cxx
+++ b/Framework/Core/test/test_O2DatabasePDG.cxx
@@ -40,7 +40,7 @@ struct PdgTest {
     TParticlePDG* p = pdgOld->GetParticle(300553);
     ASSERT_ERROR(p != nullptr);
     ASSERT_ERROR(p->Mass() == 10.580);
-    ASSERT_ERROR(p->Stable() ==  kFALSE);
+    ASSERT_ERROR(p->Stable() == kFALSE);
     ASSERT_ERROR(p->Charge() == 0);
     ASSERT_ERROR(p->Width() == 0.000);
 
diff --git a/Framework/Core/test/test_Parallel.cxx b/Framework/Core/test/test_Parallel.cxx
index 4fe0c920426e2..7967f55a7b93d 100644
--- a/Framework/Core/test/test_Parallel.cxx
+++ b/Framework/Core/test/test_Parallel.cxx
@@ -57,7 +57,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
       Outputs{
         {"TPC", "CLUSTERS_P", 0, Lifetime::Timeframe}},
       AlgorithmSpec{
-        //CLion says it ambiguous without (AlgorithmSpec::ProcessCallback), but cmake compiles fine anyway.
+        // CLion says it ambiguous without (AlgorithmSpec::ProcessCallback), but cmake compiles fine anyway.
         (AlgorithmSpec::ProcessCallback)someProcessingStageAlgorithm}},
     parallelSize,
     [](DataProcessorSpec& spec, size_t index) {
@@ -95,7 +95,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
     auto matcher = DataSpecUtils::asConcreteDataMatcher(*inputSpec);
     o2::header::DataDescription outputDescription = matcher.description;
 
-    //todo: better sampled data flagging
+    // todo: better sampled data flagging
     size_t len = strlen(outputDescription.str);
     if (len < outputDescription.size - 2) {
       outputDescription.str[len] = '_';
@@ -113,7 +113,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
     const auto* inputHeader = DataRefUtils::getHeader<o2::header::DataHeader*>(input);
     auto& output = ctx.outputs().make<char>(description, inputHeader->size());
 
-    //todo: use some std function or adopt(), when it is available for POD data
+    // todo: use some std function or adopt(), when it is available for POD data
     const char* input_ptr = input.payload;
     for (char& it : output) {
       it = *input_ptr++;
diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx
index a3f48957d7044..bcd0730734eb8 100644
--- a/Framework/Core/test/test_Root2ArrowTable.cxx
+++ b/Framework/Core/test/test_Root2ArrowTable.cxx
@@ -45,7 +45,7 @@ TEST_CASE("RootTree2Table")
   t1.Branch("ev", &ev, "ev/I");
   t1.Branch("xyz", xyz, "xyz[3]/F");
   t1.Branch("ij", ij, "ij[2]/I");
-  //fill the tree
+  // fill the tree
   for (Int_t i = 0; i < 1000; i++) {
     xyz[0] = 1;
     xyz[1] = 2;
@@ -146,7 +146,7 @@ TEST_CASE("RootTree2TableViaASoA")
   t2.Branch("ev", &ev, "ev/I");
   t2.Branch("xyz", xyz, "xyz[3]/F");
   t2.Branch("ij", ij, "ij[2]/I");
-  //fill the tree
+  // fill the tree
   for (Int_t i = 0; i < 1000; i++) {
     gRandom->Rannor(xyz[0], xyz[1]);
     gRandom->Rannor(px, py);
diff --git a/Framework/Core/test/test_Services.cxx b/Framework/Core/test/test_Services.cxx
index d1a8a3592cd10..23092127fb37b 100644
--- a/Framework/Core/test/test_Services.cxx
+++ b/Framework/Core/test/test_Services.cxx
@@ -94,7 +94,7 @@ static ServiceRegistry::Salt salt_1 = ServiceRegistry::Salt{1, 0};
 static ServiceRegistry::Salt salt_2 = ServiceRegistry::Salt{2, 0};
 static ServiceRegistry::Salt salt_3 = ServiceRegistry::Salt{3, 0};
 static ServiceRegistry::Salt salt_1_1 = ServiceRegistry::Salt{1, 1};
-}
+} // namespace o2::framework
 
 TEST_CASE("TestSerialServices")
 {
diff --git a/Framework/Core/test/test_StaggeringWorkflow.cxx b/Framework/Core/test/test_StaggeringWorkflow.cxx
index 590ce83cef467..cef3ec90c2042 100644
--- a/Framework/Core/test/test_StaggeringWorkflow.cxx
+++ b/Framework/Core/test/test_StaggeringWorkflow.cxx
@@ -53,7 +53,7 @@ void customize(std::vector<o2::framework::CompletionPolicy>& policies)
                         // search for spec names starting with "processor"
                         return spec.name.find("processor") == 0;
                       },
-                      [](auto const&, auto const&, auto &) { return o2::framework::CompletionPolicy::CompletionOp::Consume; }});
+                      [](auto const&, auto const&, auto&) { return o2::framework::CompletionPolicy::CompletionOp::Consume; }});
 }
 
 #include "Framework/runDataProcessing.h"
diff --git a/Framework/Core/test/test_TableBuilder.cxx b/Framework/Core/test/test_TableBuilder.cxx
index 26a0f5a930b73..1035a0e0008b2 100644
--- a/Framework/Core/test/test_TableBuilder.cxx
+++ b/Framework/Core/test/test_TableBuilder.cxx
@@ -32,7 +32,7 @@ namespace test2
 DECLARE_SOA_COLUMN_FULL(X, x, uint64_t, "x");
 DECLARE_SOA_COLUMN_FULL(Y, y, uint64_t, "y");
 DECLARE_SOA_COLUMN_FULL(Pos, pos, int[4], "pos");
-} // namespace test
+} // namespace test2
 
 using TestTable = o2::soa::Table<test2::X, test2::Y>;
 using ArrayTable = o2::soa::Table<test2::Pos>;
@@ -295,7 +295,7 @@ TEST_CASE("TestColumnCount")
 
 TEST_CASE("TestMakeFields")
 {
-  auto fields = TableBuilderHelpers::makeFields<int, float>({ "i", "f" });
+  auto fields = TableBuilderHelpers::makeFields<int, float>({"i", "f"});
   REQUIRE(fields.size() == 2);
   REQUIRE(fields[0]->name() == "i");
   REQUIRE(fields[1]->name() == "f");
diff --git a/Framework/Core/test/test_TreeToTable.cxx b/Framework/Core/test/test_TreeToTable.cxx
index 65569b72b9f35..4746eead57c1e 100644
--- a/Framework/Core/test/test_TreeToTable.cxx
+++ b/Framework/Core/test/test_TreeToTable.cxx
@@ -53,7 +53,7 @@ TEST_CASE("TreeToTableConversion")
   t1.Branch("xyzw", xyzw, "xyzw[96]/F");
   t1.Branch("small", &b, "small/b");
 
-  //fill the tree
+  // fill the tree
   int ntruein[2] = {0};
   for (int i = 0; i < ndp; i++) {
     ok = (i % 2) == 0;
diff --git a/Framework/Core/test/test_VariablePayloadSequenceWorkflow.cxx b/Framework/Core/test/test_VariablePayloadSequenceWorkflow.cxx
index 9945eab037780..0df6d626f1b49 100644
--- a/Framework/Core/test/test_VariablePayloadSequenceWorkflow.cxx
+++ b/Framework/Core/test/test_VariablePayloadSequenceWorkflow.cxx
@@ -56,7 +56,7 @@ namespace test
 // a header with the information expected in the payload
 // will be sent on the header stack
 struct SequenceDesc : public o2::header::BaseHeader {
-  //static data for this header type/version
+  // static data for this header type/version
   static constexpr uint32_t sVersion{1};
   static constexpr o2::header::HeaderType sHeaderType{o2::header::String2<uint64_t>("SequDesc")};
   static constexpr o2::header::SerializationMethod sSerializationMethod{o2::header::gSerializationMethodNone};

From 1b5ad1dfcd13d72ba7e7543cfeb7e4cb9800cb47 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 19 Jan 2024 08:41:04 +0100
Subject: [PATCH 262/726] DPL: drop commented code

---
 .../Core/test/benchmark_ExternalFairMQDeviceProxies.cxx     | 6 ------
 Framework/Core/test/test_DataAllocator.cxx                  | 1 -
 Framework/Core/test/test_DataDescriptorMatcherWorkflow.cxx  | 1 -
 Framework/Core/test/test_DataOutputDirector.cxx             | 3 ---
 Framework/Core/test/test_Parallel.cxx                       | 1 -
 Framework/Core/test/test_ParallelPipeline.cxx               | 3 ---
 Framework/Core/test/test_Root2ArrowTable.cxx                | 5 -----
 Framework/Core/test/test_StaticFor.cxx                      | 2 --
 .../Core/test/test_VariablePayloadSequenceWorkflow.cxx      | 1 -
 9 files changed, 23 deletions(-)

diff --git a/Framework/Core/test/benchmark_ExternalFairMQDeviceProxies.cxx b/Framework/Core/test/benchmark_ExternalFairMQDeviceProxies.cxx
index bdb4deb443021..c116ed81537ce 100644
--- a/Framework/Core/test/benchmark_ExternalFairMQDeviceProxies.cxx
+++ b/Framework/Core/test/benchmark_ExternalFairMQDeviceProxies.cxx
@@ -450,14 +450,8 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
   Inputs checkerInputs;
   if (bypassProxies != ProxyBypass::None) {
     checkerInputs.emplace_back(InputSpec{"datain", ConcreteDataTypeMatcher{"TST", "DATA"}, Lifetime::Timeframe});
-    //for (unsigned int i = 0; i < pState->nChannels; i++) {
-    //  checkerInputs.emplace_back(InputSpec{{"datain"}, "TST", "DATA", i, Lifetime::Timeframe});
-    //}
   } else {
     checkerInputs.emplace_back(InputSpec{"datain", ConcreteDataTypeMatcher{"PRX", "DATA"}, Lifetime::Timeframe});
-    //for (unsigned int i = 0; i < pState->nChannels; i++) {
-    //  checkerInputs.emplace_back(InputSpec{{"datain"}, "PRX", "DATA", i, Lifetime::Timeframe});
-    //}
   }
   workflow.emplace_back(DataProcessorSpec{"checker",
                                           std::move(checkerInputs),
diff --git a/Framework/Core/test/test_DataAllocator.cxx b/Framework/Core/test/test_DataAllocator.cxx
index 894fb5f38368a..5c29040a37450 100644
--- a/Framework/Core/test/test_DataAllocator.cxx
+++ b/Framework/Core/test/test_DataAllocator.cxx
@@ -338,7 +338,6 @@ DataProcessorSpec getSinkSpec()
 
     LOG(info) << "extracting POD vector";
     // TODO: use the ReturnType helper once implemented
-    //InputRecord::ReturnType<std::vector<int>> podvector;
     decltype(std::declval<InputRecord>().get<std::vector<int>>(DataRef{nullptr, nullptr, nullptr})) podvector;
     podvector = pc.inputs().get<std::vector<int>>("inputPODvector");
     ASSERT_ERROR(podvector.size() == 3);
diff --git a/Framework/Core/test/test_DataDescriptorMatcherWorkflow.cxx b/Framework/Core/test/test_DataDescriptorMatcherWorkflow.cxx
index ef792afa4cc79..e8bac109985c4 100644
--- a/Framework/Core/test/test_DataDescriptorMatcherWorkflow.cxx
+++ b/Framework/Core/test/test_DataDescriptorMatcherWorkflow.cxx
@@ -21,7 +21,6 @@
 #include "Framework/DataAllocator.h"
 
 // we need to add workflow options before including Framework/runDataProcessing
-//void customize(std::vector<ConfigParamSpec>& workflowOptions)
 //{
 //}
 
diff --git a/Framework/Core/test/test_DataOutputDirector.cxx b/Framework/Core/test/test_DataOutputDirector.cxx
index c158c52932db4..1834dd0781897 100644
--- a/Framework/Core/test/test_DataOutputDirector.cxx
+++ b/Framework/Core/test/test_DataOutputDirector.cxx
@@ -29,7 +29,6 @@ TEST_CASE("TestDataOutputDirector")
   std::string keepString("AOD/UNO/0:tr1:c1/c2/c3:fn1,AOD/UNO/0::c4");
   dod.readString(keepString);
   dod.setFilenameBase(mydfn);
-  //dod.printOut(); printf("\n\n");
 
   auto ds = dod.getDataOutputDescriptors(dh);
 
@@ -59,7 +58,6 @@ TEST_CASE("TestDataOutputDirector")
 
   dod.reset();
   std::tie(rdn, dfn, fmode, mfs, ntf) = dod.readJsonString(jsonString);
-  //dod.printOut(); printf("\n\n");
   ds = dod.getDataOutputDescriptors(dh);
 
   REQUIRE(ds.size() == 1);
@@ -106,7 +104,6 @@ TEST_CASE("TestDataOutputDirector")
   dod.reset();
   std::tie(rdn, dfn, fmode, mfs, ntf) = dod.readJson(jsonFile);
   dod.setFilenameBase("AnalysisResults");
-  //dod.printOut(); printf("\n\n");
   ds = dod.getDataOutputDescriptors(dh);
 
   REQUIRE(ds.size() == 2);
diff --git a/Framework/Core/test/test_Parallel.cxx b/Framework/Core/test/test_Parallel.cxx
index 6e8ebb2a2d1f0..4fe0c920426e2 100644
--- a/Framework/Core/test/test_Parallel.cxx
+++ b/Framework/Core/test/test_Parallel.cxx
@@ -19,7 +19,6 @@
 
 #include <boost/algorithm/string.hpp>
 
-
 using namespace o2::framework;
 
 struct FakeCluster {
diff --git a/Framework/Core/test/test_ParallelPipeline.cxx b/Framework/Core/test/test_ParallelPipeline.cxx
index 47106855a8f74..b79ee3a73cd24 100644
--- a/Framework/Core/test/test_ParallelPipeline.cxx
+++ b/Framework/Core/test/test_ParallelPipeline.cxx
@@ -63,7 +63,6 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
          LOG(debug) << "instance " << parallelContext.index1D() << " of " << parallelContext.index1DSize() << ": "
                     << *input.spec << ": " << *((int*)input.payload);
          auto const* dataheader = DataRefUtils::getHeader<o2::header::DataHeader*>(input);
-         //auto& data = ctx.outputs().make<int>(OutputRef{"output", dataheader->subSpecification});
          auto& data = ctx.outputs().make<int>(Output{"TST", "PREPROC", dataheader->subSpecification});
          ASSERT_ERROR(ctx.inputs().get<int>(input.spec->binding.c_str()) == parallelContext.index1D());
          data = parallelContext.index1D();
@@ -84,10 +83,8 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
          auto const* dataheader = DataRefUtils::getHeader<o2::header::DataHeader*>(input);
          // TODO: there is a bug in the API for using OutputRef, returns an rvalue which can not be bound to
          // lvalue reference
-         //auto& data = ctx.outputs().make<int>(OutputRef{"output", dataheader->subSpecification});
          auto& data = ctx.outputs().make<int>(Output{"TST", "DATA", dataheader->subSpecification});
          data = ctx.inputs().get<int>(input.spec->binding.c_str());
-         //auto& meta = ctx.outputs().make<int>(OutputRef{"metadt", dataheader->subSpecification});
          auto& meta = ctx.outputs().make<int>(Output{"TST", "META", dataheader->subSpecification});
          meta = dataheader->subSpecification;
        }
diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx
index b22432d8565a5..a3f48957d7044 100644
--- a/Framework/Core/test/test_Root2ArrowTable.cxx
+++ b/Framework/Core/test/test_Root2ArrowTable.cxx
@@ -47,7 +47,6 @@ TEST_CASE("RootTree2Table")
   t1.Branch("ij", ij, "ij[2]/I");
   //fill the tree
   for (Int_t i = 0; i < 1000; i++) {
-    //gRandom->Rannor(xyz[0], xyz[1]);
     xyz[0] = 1;
     xyz[1] = 2;
     xyz[2] = 3;
@@ -92,16 +91,12 @@ TEST_CASE("RootTree2Table")
     REQUIRE(array->length() == 3000);
     const float* c = reinterpret_cast<float const*>(array->values()->data());
 
-    //auto array = std::static_pointer_cast<arrow::FixedSizeBinaryArray>(table->column(0)->chunk(0));
-    //CHECK_EQUAL(array->byte_width(), sizeof(float[3]));
-    //const float* c = reinterpret_cast<float const*>(array->Value(0));
 
     CHECK(c[0] == 1);
     CHECK(c[1] == 2);
     CHECK(c[2] == 1);
   }
   {
-    //auto values = std::static_pointer_cast<arrow::FixedSizeBinaryArray>(table->column(1)->chunk(0));
     auto chunkToUse = table->column(1)->chunk(0);
     chunkToUse = std::dynamic_pointer_cast<arrow::FixedSizeListArray>(chunkToUse)->values();
     auto array = std::static_pointer_cast<arrow::Int32Array>(chunkToUse);
diff --git a/Framework/Core/test/test_StaticFor.cxx b/Framework/Core/test/test_StaticFor.cxx
index bacdcaeefe86e..ee66c5b4aaf5b 100644
--- a/Framework/Core/test/test_StaticFor.cxx
+++ b/Framework/Core/test/test_StaticFor.cxx
@@ -57,8 +57,6 @@ TEST_CASE("TestStaticFor")
     constexpr int index = i.value;
 
     // compiler will complain if constexpr is not enforced for index access:
-    //CONST_STR(staticNames[index]);    // works
-    //CONST_STR(staticNames[i.value]);  // fails
 
     constexpr auto sayHello = CONST_STR("Hello ") + CONST_STR(staticNames[index]);
 
diff --git a/Framework/Core/test/test_VariablePayloadSequenceWorkflow.cxx b/Framework/Core/test/test_VariablePayloadSequenceWorkflow.cxx
index 1024df1218485..9945eab037780 100644
--- a/Framework/Core/test/test_VariablePayloadSequenceWorkflow.cxx
+++ b/Framework/Core/test/test_VariablePayloadSequenceWorkflow.cxx
@@ -217,7 +217,6 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
         ASSERT_ERROR(*reinterpret_cast<size_t const*>(ref.payload) == sd->initialValue + nSequencePayloads);
         ++nSequencePayloads;
       }
-      //LOG(info) << "input " << ref.spec->binding << " has data {" << dh->dataOrigin.as<std::string>() << "/" << dh->dataDescription.as<std::string>() << "/" << dh->subSpecification << "}: " << *reinterpret_cast<size_t const*>(ref.payload);
     }
     for (auto const& [channel, count] : active) {
       ++counters[channel];

From 9a893773bcefdbfba6f5f21fbab85d2d70d76663 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 19 Jan 2024 08:41:04 +0100
Subject: [PATCH 263/726] Fix formatting in tests

---
 Framework/Core/test/test_Expressions.cxx                    | 2 +-
 Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx   | 2 +-
 Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx   | 2 +-
 Framework/Core/test/test_O2DatabasePDG.cxx                  | 2 +-
 Framework/Core/test/test_Parallel.cxx                       | 6 +++---
 Framework/Core/test/test_Root2ArrowTable.cxx                | 5 ++---
 Framework/Core/test/test_Services.cxx                       | 2 +-
 Framework/Core/test/test_StaggeringWorkflow.cxx             | 2 +-
 Framework/Core/test/test_TableBuilder.cxx                   | 4 ++--
 Framework/Core/test/test_TreeToTable.cxx                    | 2 +-
 .../Core/test/test_VariablePayloadSequenceWorkflow.cxx      | 2 +-
 11 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/Framework/Core/test/test_Expressions.cxx b/Framework/Core/test/test_Expressions.cxx
index 0bd4fdc564946..176db6a259a7c 100644
--- a/Framework/Core/test/test_Expressions.cxx
+++ b/Framework/Core/test/test_Expressions.cxx
@@ -32,7 +32,7 @@ static BindingNode testInt{"testInt", 6, atype::INT32};
 
 namespace o2::aod::track
 {
-DECLARE_SOA_EXPRESSION_COLUMN(Pze, pz, float, o2::aod::track::tgl*(1.f / o2::aod::track::signed1Pt));
+DECLARE_SOA_EXPRESSION_COLUMN(Pze, pz, float, o2::aod::track::tgl * (1.f / o2::aod::track::signed1Pt));
 } // namespace o2::aod::track
 
 TEST_CASE("TestTreeParsing")
diff --git a/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx b/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx
index 249e7fc304949..d4d204f206e0a 100644
--- a/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx
+++ b/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx
@@ -403,7 +403,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
         output.AddPart(std::move(inputs.At(msgidx)));
       }
     }
-    auto &messageContext = services.get<MessageContext>();
+    auto& messageContext = services.get<MessageContext>();
     o2::framework::sendOnChannel(*device, output, channelName, (size_t)-1);
     messageContext.fakeDispatch();
     return output.Size() != 0;
diff --git a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
index 7b0ffa462a9db..d15eebd26bdf7 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
@@ -31,7 +31,7 @@ namespace
 {
 WorkflowSpec defineDataProcessing()
 {
-  return {{.name = "A",                                                                  //
+  return {{.name = "A",                                                          //
            .outputs = Outputs{OutputSpec{"TST", "A1"}, OutputSpec{"TST", "A2"}}, // A1 will be consumed twice, A2 is dangling
            .algorithm = AlgorithmSpec{},                                         //
            .options = {ConfigParamSpec{"channel-config", VariantType::String,    // raw input channel
diff --git a/Framework/Core/test/test_O2DatabasePDG.cxx b/Framework/Core/test/test_O2DatabasePDG.cxx
index c6f6f3c9c57e7..11b07d1473390 100644
--- a/Framework/Core/test/test_O2DatabasePDG.cxx
+++ b/Framework/Core/test/test_O2DatabasePDG.cxx
@@ -40,7 +40,7 @@ struct PdgTest {
     TParticlePDG* p = pdgOld->GetParticle(300553);
     ASSERT_ERROR(p != nullptr);
     ASSERT_ERROR(p->Mass() == 10.580);
-    ASSERT_ERROR(p->Stable() ==  kFALSE);
+    ASSERT_ERROR(p->Stable() == kFALSE);
     ASSERT_ERROR(p->Charge() == 0);
     ASSERT_ERROR(p->Width() == 0.000);
 
diff --git a/Framework/Core/test/test_Parallel.cxx b/Framework/Core/test/test_Parallel.cxx
index 4fe0c920426e2..7967f55a7b93d 100644
--- a/Framework/Core/test/test_Parallel.cxx
+++ b/Framework/Core/test/test_Parallel.cxx
@@ -57,7 +57,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
       Outputs{
         {"TPC", "CLUSTERS_P", 0, Lifetime::Timeframe}},
       AlgorithmSpec{
-        //CLion says it ambiguous without (AlgorithmSpec::ProcessCallback), but cmake compiles fine anyway.
+        // CLion says it ambiguous without (AlgorithmSpec::ProcessCallback), but cmake compiles fine anyway.
         (AlgorithmSpec::ProcessCallback)someProcessingStageAlgorithm}},
     parallelSize,
     [](DataProcessorSpec& spec, size_t index) {
@@ -95,7 +95,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
     auto matcher = DataSpecUtils::asConcreteDataMatcher(*inputSpec);
     o2::header::DataDescription outputDescription = matcher.description;
 
-    //todo: better sampled data flagging
+    // todo: better sampled data flagging
     size_t len = strlen(outputDescription.str);
     if (len < outputDescription.size - 2) {
       outputDescription.str[len] = '_';
@@ -113,7 +113,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const&)
     const auto* inputHeader = DataRefUtils::getHeader<o2::header::DataHeader*>(input);
     auto& output = ctx.outputs().make<char>(description, inputHeader->size());
 
-    //todo: use some std function or adopt(), when it is available for POD data
+    // todo: use some std function or adopt(), when it is available for POD data
     const char* input_ptr = input.payload;
     for (char& it : output) {
       it = *input_ptr++;
diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx
index a3f48957d7044..109d9718f5f09 100644
--- a/Framework/Core/test/test_Root2ArrowTable.cxx
+++ b/Framework/Core/test/test_Root2ArrowTable.cxx
@@ -45,7 +45,7 @@ TEST_CASE("RootTree2Table")
   t1.Branch("ev", &ev, "ev/I");
   t1.Branch("xyz", xyz, "xyz[3]/F");
   t1.Branch("ij", ij, "ij[2]/I");
-  //fill the tree
+  // fill the tree
   for (Int_t i = 0; i < 1000; i++) {
     xyz[0] = 1;
     xyz[1] = 2;
@@ -91,7 +91,6 @@ TEST_CASE("RootTree2Table")
     REQUIRE(array->length() == 3000);
     const float* c = reinterpret_cast<float const*>(array->values()->data());
 
-
     CHECK(c[0] == 1);
     CHECK(c[1] == 2);
     CHECK(c[2] == 1);
@@ -146,7 +145,7 @@ TEST_CASE("RootTree2TableViaASoA")
   t2.Branch("ev", &ev, "ev/I");
   t2.Branch("xyz", xyz, "xyz[3]/F");
   t2.Branch("ij", ij, "ij[2]/I");
-  //fill the tree
+  // fill the tree
   for (Int_t i = 0; i < 1000; i++) {
     gRandom->Rannor(xyz[0], xyz[1]);
     gRandom->Rannor(px, py);
diff --git a/Framework/Core/test/test_Services.cxx b/Framework/Core/test/test_Services.cxx
index d1a8a3592cd10..23092127fb37b 100644
--- a/Framework/Core/test/test_Services.cxx
+++ b/Framework/Core/test/test_Services.cxx
@@ -94,7 +94,7 @@ static ServiceRegistry::Salt salt_1 = ServiceRegistry::Salt{1, 0};
 static ServiceRegistry::Salt salt_2 = ServiceRegistry::Salt{2, 0};
 static ServiceRegistry::Salt salt_3 = ServiceRegistry::Salt{3, 0};
 static ServiceRegistry::Salt salt_1_1 = ServiceRegistry::Salt{1, 1};
-}
+} // namespace o2::framework
 
 TEST_CASE("TestSerialServices")
 {
diff --git a/Framework/Core/test/test_StaggeringWorkflow.cxx b/Framework/Core/test/test_StaggeringWorkflow.cxx
index 590ce83cef467..cef3ec90c2042 100644
--- a/Framework/Core/test/test_StaggeringWorkflow.cxx
+++ b/Framework/Core/test/test_StaggeringWorkflow.cxx
@@ -53,7 +53,7 @@ void customize(std::vector<o2::framework::CompletionPolicy>& policies)
                         // search for spec names starting with "processor"
                         return spec.name.find("processor") == 0;
                       },
-                      [](auto const&, auto const&, auto &) { return o2::framework::CompletionPolicy::CompletionOp::Consume; }});
+                      [](auto const&, auto const&, auto&) { return o2::framework::CompletionPolicy::CompletionOp::Consume; }});
 }
 
 #include "Framework/runDataProcessing.h"
diff --git a/Framework/Core/test/test_TableBuilder.cxx b/Framework/Core/test/test_TableBuilder.cxx
index 26a0f5a930b73..1035a0e0008b2 100644
--- a/Framework/Core/test/test_TableBuilder.cxx
+++ b/Framework/Core/test/test_TableBuilder.cxx
@@ -32,7 +32,7 @@ namespace test2
 DECLARE_SOA_COLUMN_FULL(X, x, uint64_t, "x");
 DECLARE_SOA_COLUMN_FULL(Y, y, uint64_t, "y");
 DECLARE_SOA_COLUMN_FULL(Pos, pos, int[4], "pos");
-} // namespace test
+} // namespace test2
 
 using TestTable = o2::soa::Table<test2::X, test2::Y>;
 using ArrayTable = o2::soa::Table<test2::Pos>;
@@ -295,7 +295,7 @@ TEST_CASE("TestColumnCount")
 
 TEST_CASE("TestMakeFields")
 {
-  auto fields = TableBuilderHelpers::makeFields<int, float>({ "i", "f" });
+  auto fields = TableBuilderHelpers::makeFields<int, float>({"i", "f"});
   REQUIRE(fields.size() == 2);
   REQUIRE(fields[0]->name() == "i");
   REQUIRE(fields[1]->name() == "f");
diff --git a/Framework/Core/test/test_TreeToTable.cxx b/Framework/Core/test/test_TreeToTable.cxx
index 65569b72b9f35..4746eead57c1e 100644
--- a/Framework/Core/test/test_TreeToTable.cxx
+++ b/Framework/Core/test/test_TreeToTable.cxx
@@ -53,7 +53,7 @@ TEST_CASE("TreeToTableConversion")
   t1.Branch("xyzw", xyzw, "xyzw[96]/F");
   t1.Branch("small", &b, "small/b");
 
-  //fill the tree
+  // fill the tree
   int ntruein[2] = {0};
   for (int i = 0; i < ndp; i++) {
     ok = (i % 2) == 0;
diff --git a/Framework/Core/test/test_VariablePayloadSequenceWorkflow.cxx b/Framework/Core/test/test_VariablePayloadSequenceWorkflow.cxx
index 9945eab037780..0df6d626f1b49 100644
--- a/Framework/Core/test/test_VariablePayloadSequenceWorkflow.cxx
+++ b/Framework/Core/test/test_VariablePayloadSequenceWorkflow.cxx
@@ -56,7 +56,7 @@ namespace test
 // a header with the information expected in the payload
 // will be sent on the header stack
 struct SequenceDesc : public o2::header::BaseHeader {
-  //static data for this header type/version
+  // static data for this header type/version
   static constexpr uint32_t sVersion{1};
   static constexpr o2::header::HeaderType sHeaderType{o2::header::String2<uint64_t>("SequDesc")};
   static constexpr o2::header::SerializationMethod sSerializationMethod{o2::header::gSerializationMethodNone};

From 4ec5e8adb24f41c2d499c22346041a9fbc2d6ed0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 19 Jan 2024 23:30:41 +0100
Subject: [PATCH 264/726] DPL: enable Signposts by default also on linux

---
 Framework/Foundation/include/Framework/Signpost.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index 8d0107e68ecd6..d7b8766de83c4 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -108,7 +108,7 @@ o2_log_handle_t* o2_walk_logs(bool (*callback)(char const* name, void* log, void
 #endif // __APPLE__
 
 // Unless we are on apple we enable checking for signposts only if in debug mode or if we force them.
-#if defined(__APPLE__) || defined(O2_FORCE_SIGNPOSTS) || !defined(NDEBUG)
+#if defined(__APPLE__) || defined(O2_FORCE_SIGNPOSTS) || !defined(O2_NSIGNPOSTS)
 #define O2_LOG_ENABLED(log) private_o2_log_##log->stacktrace
 #else
 #define O2_LOG_ENABLED(log) false
@@ -448,7 +448,7 @@ void o2_debug_log_set_stacktrace(_o2_log_t* log, int stacktrace)
 }
 #endif // O2_SIGNPOST_IMPLEMENTATION
 
-#if defined(__APPLE__) || defined(O2_FORCE_SIGNPOSTS) || !defined(NDEBUG)
+#if defined(__APPLE__) || defined(O2_FORCE_SIGNPOSTS) || !defined(O2_NSIGNPOSTS)
 /// Dynamic logs need to be enabled via the O2_LOG_ENABLE macro. Notice this will only work
 /// for the logger based logging, since the Apple version needs instruments to enable them.
 #define O2_DECLARE_DYNAMIC_LOG(name) static _o2_log_t* private_o2_log_##name = (_o2_log_t*)_o2_log_create("ch.cern.aliceo2." #name, 1)

From aa947aad463843fb3c7f6003cd7bf522baf4dc6d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 22 Jan 2024 12:45:45 +0100
Subject: [PATCH 265/726] DPL: fix typo in help message

---
 Framework/Core/src/WorkflowCustomizationHelpers.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/src/WorkflowCustomizationHelpers.cxx b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
index da21168a8fe44..ffad8a8720019 100644
--- a/Framework/Core/src/WorkflowCustomizationHelpers.cxx
+++ b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
@@ -49,7 +49,7 @@ std::vector<ConfigParamSpec> WorkflowCustomizationHelpers::requiredWorkflowOptio
            {"workflow-suffix", VariantType::String, "", {"suffix to add to all dataprocessors"}},
 
            // options for TF rate limiting
-           {"timeframes-rate-limit-ipcid", VariantType::String, "-1", {"Suffix for IPC channel for metrix-feedback, -1 = disable"}},
+           {"timeframes-rate-limit-ipcid", VariantType::String, "-1", {"Suffix for IPC channel for metric-feedback, -1 = disable"}},
 
            // options for AOD rate limiting
            {"aod-memory-rate-limit", VariantType::Int64, 0LL, {"Rate limit AOD processing based on memory"}},

From 538614f13978993f3c13c8ab4b701d630dc6c146 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 22 Jan 2024 13:56:03 +0100
Subject: [PATCH 266/726] DPL: fix formatting and format string

---
 Framework/Core/src/CommonServices.cxx | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index f9d327d14107d..8ac8315e949da 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -193,7 +193,7 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
         }
       }
       if (didCreate == false && messageContext.didDispatch() == true) {
-        O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Data created out of band didCreate == %b && messageContext.didDispatch == %b",
+        O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Data created out of band didCreate == %d && messageContext.didDispatch == %d",
                                didCreate,
                                messageContext.didDispatch());
         return;
@@ -219,8 +219,7 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
         if (matcher.lifetime == Lifetime::Timeframe) {
           LOGP(error, "Expected Lifetime::Timeframe data {} was not created for timeslice {} and might result in dropped timeframes", DataSpecUtils::describe(matcher), timeslice);
         }
-      } 
-      },
+      } },
     .kind = ServiceKind::Stream};
 }
 

From a50b44593d025044911a6520f7a310c150b7dbd9 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 22 Jan 2024 23:56:00 +0100
Subject: [PATCH 267/726] DPL: handle corner case for consumeWhenAll

This should avoid computation if only sporadic inputs are expected
and they are all missing.
---
 .../Core/src/CompletionPolicyHelpers.cxx      | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx
index 585046bc89763..018dbb150fc92 100644
--- a/Framework/Core/src/CompletionPolicyHelpers.cxx
+++ b/Framework/Core/src/CompletionPolicyHelpers.cxx
@@ -115,20 +115,28 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
     O2_SIGNPOST_START(completion, sid, "consumeWhenAll", "Completion policy invoked");
 
     size_t si = 0;
-    bool missingSporadic = false;
+    int sporadicCount = 0;
+    int timeframeCount = 0;
+    int missingSporadicCount = 0;
     bool needsProcessing = false;
     size_t currentTimeslice = -1;
     for (auto& input : inputs) {
       assert(si < specs.size());
       auto& spec = specs[si++];
+      sporadicCount += spec.lifetime == Lifetime::Sporadic ? 1 : 0;
+      timeframeCount += spec.lifetime == Lifetime::Timeframe ? 1 : 0;
+      // If we are missing something which is not sporadic, we wait.
       if (input.header == nullptr && spec.lifetime != Lifetime::Sporadic) {
         O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s due to missing input %lu", "Wait", si);
         return CompletionPolicy::CompletionOp::Wait;
       }
+      // If we are missing something which is sporadic, we wait until we are sure it will not come.
       if (input.header == nullptr && spec.lifetime == Lifetime::Sporadic) {
         O2_SIGNPOST_EVENT_EMIT(completion, sid, "consumeWhenAll", "Missing sporadic found for route index %lu", si);
-        missingSporadic = true;
+        missingSporadicCount += 1;
       }
+      // If we have a header, we use it to determine the current timesliceIsTimer
+      // (unless this is a timer which does not enter the oldest possible timeslice).
       if (input.header != nullptr && currentTimeslice == -1) {
         auto const* dph = framework::DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(input);
         if (dph && !TimingInfo::timesliceIsTimer(dph->startTime)) {
@@ -136,6 +144,7 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
           O2_SIGNPOST_EVENT_EMIT(completion, sid, "consumeWhenAll", "currentTimeslice %lu from route index %lu", currentTimeslice, si);
         }
       }
+      // If we have a header, we need to process it if it is not a condition object.
       if (input.header != nullptr && spec.lifetime != Lifetime::Condition) {
         needsProcessing = true;
       }
@@ -145,11 +154,15 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
     auto& timesliceIndex = ref.get<TimesliceIndex>();
     auto oldestPossibleTimeslice = timesliceIndex.getOldestPossibleInput().timeslice.value;
 
-    if (missingSporadic && currentTimeslice >= oldestPossibleTimeslice) {
+    if (missingSporadicCount && currentTimeslice >= oldestPossibleTimeslice) {
       O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s for timeslice %lu > oldestPossibleTimeslice %lu", "Retry", currentTimeslice, oldestPossibleTimeslice);
       return CompletionPolicy::CompletionOp::Retry;
     }
     O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s for timeslice %lu <= oldestPossibleTimeslice %lu", needsProcessing ? "Consume" : "Discard", currentTimeslice, oldestPossibleTimeslice);
+    // No need to process if we have only sporadic inputs and they are all missing.
+    if (needsProcessing && (missingSporadicCount == sporadicCount) && (timeframeCount == 0)) {
+      return CompletionPolicy::CompletionOp::Discard;
+    }
     return needsProcessing ? CompletionPolicy::CompletionOp::Consume : CompletionPolicy::CompletionOp::Discard;
   };
   return CompletionPolicy{name, matcher, callback};

From dc32ec4ab20cd2a47506cc3425df9f67af0c1789 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 22 Jan 2024 14:14:39 +0100
Subject: [PATCH 268/726] DPL: add signposts to debug signals

---
 Framework/Core/src/DataProcessingDevice.cxx | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 366bfafd7d801..6ae4b62241930 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -495,11 +495,12 @@ void DataProcessingDevice::Init()
 
 void on_signal_callback(uv_signal_t* handle, int signum)
 {
-  ZoneScopedN("Signal callaback");
-  LOG(debug) << "Signal " << signum << " received.";
+  O2_SIGNPOST_ID_FROM_POINTER(sid, device, handle);
+  O2_SIGNPOST_START(device, sid, "signal_state", "Signal %d received.", signum);
+
   auto* registry = (ServiceRegistry*)handle->data;
   if (!registry) {
-    LOG(debug) << "No registry active. Ignoring signal";
+    O2_SIGNPOST_END(device, sid, "signal_state", "No registry active. Ignoring signal.");
     return;
   }
   ServiceRegistryRef ref{*registry};
@@ -516,6 +517,7 @@ void on_signal_callback(uv_signal_t* handle, int signum)
     //        available and being offered, however we
     //        want to get out of the woods for now.
     if (offer.valid && offer.sharedMemory != 0) {
+      O2_SIGNPOST_END(device, sid, "signal_state", "Memory already offered.");
       return;
     }
     ri++;
@@ -532,6 +534,7 @@ void on_signal_callback(uv_signal_t* handle, int signum)
     }
   }
   stats.updateStats({(int)ProcessingStatsId::TOTAL_SIGUSR1, DataProcessingStats::Op::Add, 1});
+  O2_SIGNPOST_END(device, sid, "signal_state", "Done processing signals.");
 }
 
 static auto toBeForwardedHeader = [](void* header) -> bool {

From f674031804a442af92e311aca5883b842048b4ba Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 23 Jan 2024 00:57:44 +0100
Subject: [PATCH 269/726] DPL: handle corner case for consumeWhenAll

This should avoid computation if only sporadic inputs are expected
and they are all missing.
---
 .../Core/src/CompletionPolicyHelpers.cxx      | 26 +++++++++++++++----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx
index 585046bc89763..9dd895a6fed6d 100644
--- a/Framework/Core/src/CompletionPolicyHelpers.cxx
+++ b/Framework/Core/src/CompletionPolicyHelpers.cxx
@@ -115,20 +115,28 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
     O2_SIGNPOST_START(completion, sid, "consumeWhenAll", "Completion policy invoked");
 
     size_t si = 0;
-    bool missingSporadic = false;
+    int sporadicCount = 0;
+    int timeframeCount = 0;
+    int missingSporadicCount = 0;
     bool needsProcessing = false;
     size_t currentTimeslice = -1;
     for (auto& input : inputs) {
       assert(si < specs.size());
       auto& spec = specs[si++];
+      sporadicCount += spec.lifetime == Lifetime::Sporadic ? 1 : 0;
+      timeframeCount += spec.lifetime == Lifetime::Timeframe ? 1 : 0;
+      // If we are missing something which is not sporadic, we wait.
       if (input.header == nullptr && spec.lifetime != Lifetime::Sporadic) {
         O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s due to missing input %lu", "Wait", si);
         return CompletionPolicy::CompletionOp::Wait;
       }
+      // If we are missing something which is sporadic, we wait until we are sure it will not come.
       if (input.header == nullptr && spec.lifetime == Lifetime::Sporadic) {
         O2_SIGNPOST_EVENT_EMIT(completion, sid, "consumeWhenAll", "Missing sporadic found for route index %lu", si);
-        missingSporadic = true;
+        missingSporadicCount += 1;
       }
+      // If we have a header, we use it to determine the current timesliceIsTimer
+      // (unless this is a timer which does not enter the oldest possible timeslice).
       if (input.header != nullptr && currentTimeslice == -1) {
         auto const* dph = framework::DataRefUtils::getHeader<o2::framework::DataProcessingHeader*>(input);
         if (dph && !TimingInfo::timesliceIsTimer(dph->startTime)) {
@@ -136,6 +144,7 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
           O2_SIGNPOST_EVENT_EMIT(completion, sid, "consumeWhenAll", "currentTimeslice %lu from route index %lu", currentTimeslice, si);
         }
       }
+      // If we have a header, we need to process it if it is not a condition object.
       if (input.header != nullptr && spec.lifetime != Lifetime::Condition) {
         needsProcessing = true;
       }
@@ -145,12 +154,19 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAll(const char* name, Compl
     auto& timesliceIndex = ref.get<TimesliceIndex>();
     auto oldestPossibleTimeslice = timesliceIndex.getOldestPossibleInput().timeslice.value;
 
-    if (missingSporadic && currentTimeslice >= oldestPossibleTimeslice) {
+    if (missingSporadicCount && currentTimeslice >= oldestPossibleTimeslice) {
       O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s for timeslice %lu > oldestPossibleTimeslice %lu", "Retry", currentTimeslice, oldestPossibleTimeslice);
       return CompletionPolicy::CompletionOp::Retry;
     }
-    O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s for timeslice %lu <= oldestPossibleTimeslice %lu", needsProcessing ? "Consume" : "Discard", currentTimeslice, oldestPossibleTimeslice);
-    return needsProcessing ? CompletionPolicy::CompletionOp::Consume : CompletionPolicy::CompletionOp::Discard;
+
+    // No need to process if we have only sporadic inputs and they are all missing.
+    if (needsProcessing && (sporadicCount > 0) && (missingSporadicCount == sporadicCount) && (timeframeCount == 0)) {
+      O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s for timeslice %lu", "Discard", currentTimeslice);
+      return CompletionPolicy::CompletionOp::Discard;
+    }
+    auto consumes = (needsProcessing || sporadicCount == 0);
+    O2_SIGNPOST_END(completion, sid, "consumeWhenAll", "Completion policy returned %{public}s for timeslice %lu", consumes ? "Consume" : "Discard", currentTimeslice);
+    return consumes ? CompletionPolicy::CompletionOp::Consume : CompletionPolicy::CompletionOp::Discard;
   };
   return CompletionPolicy{name, matcher, callback};
 }

From f6a0983f92a16b67f72e760ee7ed890e17e67355 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 23 Jan 2024 14:21:01 +0100
Subject: [PATCH 270/726] DPL: make sure we do not invoke user callback when
 Discarding

Not sure this is actually the correct thing to do.
---
 Framework/Core/src/DataProcessingDevice.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 6ae4b62241930..9f69d796f086f 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -2297,8 +2297,8 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       if (spec.forwards.empty() == false) {
         auto& timesliceIndex = ref.get<TimesliceIndex>();
         forwardInputs(ref, action.slot, currentSetOfInputs, timesliceIndex.getOldestPossibleOutput(), false);
-        continue;
       }
+      continue;
     }
     // If there is no optional inputs we canForwardEarly
     // the messages to that parallel processing can happen.

From dc30b66aa6936195616becf87338737f22eedced Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 23 Jan 2024 22:20:58 +0100
Subject: [PATCH 271/726] DPL: improve logging for processing loop

---
 Framework/Core/src/DataProcessingDevice.cxx | 67 ++++++++++++++-------
 1 file changed, 44 insertions(+), 23 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 9f69d796f086f..f5c9b7947afde 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -73,6 +73,7 @@
 #include <TMessage.h>
 #include <TClonesArray.h>
 
+#include <fmt/ostream.h>
 #include <algorithm>
 #include <vector>
 #include <numeric>
@@ -83,6 +84,14 @@
 #include <sstream>
 #include <boost/property_tree/json_parser.hpp>
 
+// Formatter to avoid having to rewrite the ostream operator for the enum
+namespace fmt
+{
+template <>
+struct formatter<o2::framework::CompletionPolicy::CompletionOp> : ostream_formatter {
+};
+} // namespace fmt
+
 O2_DECLARE_DYNAMIC_LOG(device);
 
 using namespace o2::framework;
@@ -2253,28 +2262,21 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   };
 
   // This is the main dispatching loop
-  LOGP(debug, "Processing actions:");
   auto& state = ref.get<DeviceState>();
   auto& spec = ref.get<DeviceSpec const>();
 
   auto& dpContext = ref.get<DataProcessorContext>();
   auto& streamContext = ref.get<StreamContext>();
+  O2_SIGNPOST_ID_GENERATE(sid, device);
+  O2_SIGNPOST_START(device, sid, "device", "Start processing ready actions");
   for (auto action : getReadyActions()) {
-    LOGP(debug, "  Begin action");
+    O2_SIGNPOST_ID_GENERATE(aid, device);
+    O2_SIGNPOST_START(device, aid, "device", "Processing action on slot %lu for action %{public}s", action.slot.index, fmt::format("{}", action.op).c_str());
     if (action.op == CompletionPolicy::CompletionOp::Wait) {
-      LOGP(debug, "  - Action is to Wait");
+      O2_SIGNPOST_END(device, aid, "device", "Waiting for more data.");
       continue;
     }
 
-    switch (action.op) {
-      case CompletionPolicy::CompletionOp::Consume:
-        LOG(debug) << "  - Action is to " << action.op << " " << action.slot.index;
-        break;
-      default:
-        LOG(debug) << "  - Action is to " << action.op << " " << action.slot.index;
-        break;
-    }
-
     prepareAllocatorForCurrentTimeSlice(TimesliceSlot{action.slot});
     bool shouldConsume = action.op == CompletionPolicy::CompletionOp::Consume ||
                          action.op == CompletionPolicy::CompletionOp::Discard;
@@ -2285,20 +2287,19 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
                        *context.registry};
     ProcessingContext processContext{record, ref, ref.get<DataAllocator>()};
     {
-      ZoneScopedN("service pre processing");
       // Notice this should be thread safe and reentrant
       // as it is called from many threads.
       streamContext.preProcessingCallbacks(processContext);
       dpContext.preProcessingCallbacks(processContext);
     }
     if (action.op == CompletionPolicy::CompletionOp::Discard) {
-      LOGP(debug, "  - Action is to Discard");
       context.postDispatchingCallbacks(processContext);
       if (spec.forwards.empty() == false) {
         auto& timesliceIndex = ref.get<TimesliceIndex>();
         forwardInputs(ref, action.slot, currentSetOfInputs, timesliceIndex.getOldestPossibleOutput(), false);
+        O2_SIGNPOST_END(device, aid, "device", "Forwarding inputs consume: %d.", false);
+        continue;
       }
-      continue;
     }
     // If there is no optional inputs we canForwardEarly
     // the messages to that parallel processing can happen.
@@ -2308,7 +2309,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
     bool consumeSomething = action.op == CompletionPolicy::CompletionOp::Consume || action.op == CompletionPolicy::CompletionOp::ConsumeExisting;
 
     if (context.canForwardEarly && hasForwards && consumeSomething) {
-      LOGP(debug, "  - Early forwarding");
+      O2_SIGNPOST_EVENT_EMIT(device, aid, "device", "Early forwainding: %{public}s.", fmt::format("{}", action.op).c_str());
       auto& timesliceIndex = ref.get<TimesliceIndex>();
       forwardInputs(ref, action.slot, currentSetOfInputs, timesliceIndex.getOldestPossibleOutput(), true, action.op == CompletionPolicy::CompletionOp::Consume);
     }
@@ -2325,9 +2326,20 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       auto& spec = ref.get<DeviceSpec const>();
       auto& streamContext = ref.get<StreamContext>();
       auto& dpContext = ref.get<DataProcessorContext>();
+      auto shouldProcess = [](DataRelayer::RecordAction& action) -> bool {
+        switch (action.op) {
+          case CompletionPolicy::CompletionOp::Consume:
+          case CompletionPolicy::CompletionOp::ConsumeExisting:
+          case CompletionPolicy::CompletionOp::ConsumeAndRescan:
+          case CompletionPolicy::CompletionOp::Process:
+            return true;
+            break;
+          default:
+            return false;
+        }
+      };
       if (state.quitRequested == false) {
         {
-          ZoneScopedN("service post processing");
           // Callbacks from services
           dpContext.preProcessingCallbacks(processContext);
           streamContext.preProcessingCallbacks(processContext);
@@ -2335,18 +2347,27 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
           // Callbacks from users
           ref.get<CallbackService>().call<CallbackService::Id::PreProcessing>(o2::framework::ServiceRegistryRef{ref}, (int)action.op);
         }
-        if (context.statefulProcess) {
-          ZoneScopedN("statefull process");
+        O2_SIGNPOST_ID_FROM_POINTER(pcid, device, &processContext);
+        if (context.statefulProcess && shouldProcess(action)) {
+          // This way, usercode can use the the same processing context to identify
+          // its signposts and we can map user code to device iterations.
+          O2_SIGNPOST_START(device, pcid, "device", "Stateful process");
           (context.statefulProcess)(processContext);
-        } else if (context.statelessProcess) {
-          ZoneScopedN("stateless process");
+          O2_SIGNPOST_END(device, pcid, "device", "Stateful process");
+        } else if (context.statelessProcess && shouldProcess(action)) {
+          O2_SIGNPOST_START(device, pcid, "device", "Stateful process");
           (context.statelessProcess)(processContext);
+          O2_SIGNPOST_END(device, pcid, "device", "Stateful process");
+        } else if (context.statelessProcess || context.statefulProcess) {
+          O2_SIGNPOST_EVENT_EMIT(device, pcid, "device", "Skipping processing because we are discarding.");
         } else {
+          O2_SIGNPOST_EVENT_EMIT(device, pcid, "device", "No processing callback provided. Switching to %{public}s.", "Idle");
           state.streaming = StreamingState::Idle;
         }
 
         // Notify the sink we just consumed some timeframe data
         if (context.isSink && action.op == CompletionPolicy::CompletionOp::Consume) {
+          O2_SIGNPOST_EVENT_EMIT(device, pcid, "device", "Sending dpl-summary");
           auto& allocator = ref.get<DataAllocator>();
           allocator.make<int>(OutputRef{"dpl-summary", compile_time_hash(spec.name.c_str())}, 1);
         }
@@ -2361,7 +2382,6 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
         }
 
         {
-          ZoneScopedN("service post processing");
           ref.get<CallbackService>().call<CallbackService::Id::PostProcessing>(o2::framework::ServiceRegistryRef{ref}, (int)action.op);
           dpContext.postProcessingCallbacks(processContext);
           streamContext.postProcessingCallbacks(processContext);
@@ -2408,7 +2428,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       ref.get<CallbackService>().call<CallbackService::Id::DataConsumed>(o2::framework::ServiceRegistryRef{ref});
     }
     if ((context.canForwardEarly == false) && hasForwards && consumeSomething) {
-      LOGP(debug, "Late forwarding");
+      O2_SIGNPOST_EVENT_EMIT(device, aid, "device", "Late forwarding");
       auto& timesliceIndex = ref.get<TimesliceIndex>();
       forwardInputs(ref, action.slot, currentSetOfInputs, timesliceIndex.getOldestPossibleOutput(), false, action.op == CompletionPolicy::CompletionOp::Consume);
     }
@@ -2421,6 +2441,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       cleanTimers(action.slot, record);
     }
   }
+  O2_SIGNPOST_END(device, sid, "device", "Start processing ready actions");
 
   // We now broadcast the end of stream if it was requested
   if (state.streaming == StreamingState::EndOfStreaming) {

From 28c9b891dec6a32184e45ebd66259930670fe82d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 25 Jan 2024 09:43:21 +0100
Subject: [PATCH 272/726] DPL: command line option to enable signposts

---
 Framework/Core/src/DeviceSpecHelpers.cxx |  2 ++
 Framework/Core/src/runDataProcessing.cxx | 25 ++++++++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 6c2063cb59311..17abf3fa7faad 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1664,6 +1664,8 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
     ("infologger-mode", bpo::value<std::string>(), "O2_INFOLOGGER_MODE override")                                                                                    //
     ("infologger-severity", bpo::value<std::string>(), "minimun FairLogger severity which goes to info logger")                                                      //
     ("dpl-tracing-flags", bpo::value<std::string>(), "pipe separated list of events to trace")                                                                       //
+    ("signposts", bpo::value<std::string>(),                                                                                                                         //
+     "comma separated list of signposts to enable (any of `completion`, `data_processor_context`, `stream_context`, `device`, `monitoring_service`)")                //
     ("child-driver", bpo::value<std::string>(), "external driver to start childs with (e.g. valgrind)");                                                             //
 
   return forwardedDeviceOptions;
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 325417235cd83..d98d4a5680e8d 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -977,10 +977,12 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
     }
     boost::program_options::options_description optsDesc;
     ConfigParamsHelper::populateBoostProgramOptions(optsDesc, spec.options, gHiddenDeviceOptions);
+    char const* defaultSignposts = getenv("DPL_SIGNPOSTS");
     optsDesc.add_options()("monitoring-backend", bpo::value<std::string>()->default_value("default"), "monitoring backend info")                                                           //
       ("driver-client-backend", bpo::value<std::string>()->default_value(defaultDriverClient), "backend for device -> driver communicataon: stdout://: use stdout, ws://: use websockets") //
       ("infologger-severity", bpo::value<std::string>()->default_value(""), "minimum FairLogger severity to send to InfoLogger")                                                           //
       ("dpl-tracing-flags", bpo::value<std::string>()->default_value(""), "pipe `|` separate list of events to be traced")                                                                 //
+      ("signposts", bpo::value<std::string>()->default_value(defaultSignposts ? defaultSignposts : ""), "comma separated list of signposts to enable")                                     //
       ("expected-region-callbacks", bpo::value<std::string>()->default_value("0"), "how many region callbacks we are expecting")                                                           //
       ("exit-transition-timeout", bpo::value<std::string>()->default_value(defaultExitTransitionTimeout), "how many second to wait before switching from RUN to READY")                    //
       ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframe can be in fly at the same moment (0 disables)")                                         //
@@ -2949,6 +2951,29 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
     }
   }
 
+  if (varmap.count("signposts")) {
+    auto signpostsToEnable = varmap["signposts"].as<std::string>();
+    auto matchingLogEnabler = [](char const* name, void* l, void* context) {
+      auto* log = (_o2_log_t*)l;
+      auto* selectedName = (char const*)context;
+      std::string prefix = "ch.cern.aliceo2.";
+      if (strcmp(name, (prefix + selectedName).data()) == 0) {
+        LOGP(info, "Enabling signposts for {}", *selectedName);
+        _o2_log_set_stacktrace(log, 1);
+        return false;
+      }
+      return true;
+    };
+    // Split signpostsToEnable by comma using strtok_r
+    char* saveptr;
+    char* src = const_cast<char*>(signpostsToEnable.data());
+    auto* token = strtok_r(src, ",", &saveptr);
+    while (token) {
+      o2_walk_logs(matchingLogEnabler, token);
+      token = strtok_r(nullptr, ",", &saveptr);
+    }
+  }
+
   auto evaluateBatchOption = [&varmap]() -> bool {
     if (varmap.count("no-batch") > 0) {
       return false;

From 1c8722551611250dc7e16c653c3019fb86c36f28 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 25 Jan 2024 10:30:00 +0100
Subject: [PATCH 273/726] DPL: use Signposts to debug oldest possible
 timeframe.

---
 Framework/Core/src/CommonServices.cxx | 39 ++++++++++++++++++---------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 8ac8315e949da..b3ee75a6a3886 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -536,7 +536,9 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
       }
 
       if (decongestion->lastTimeslice && oldestPossibleOutput.timeslice.value == decongestion->lastTimeslice) {
-        LOGP(debug, "Not sending already sent value");
+        O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice",
+                               "Not sending already sent value for oldest possible timeslice: %" PRIu64,
+                               (uint64_t)oldestPossibleOutput.timeslice.value);
         return;
       }
       if (oldestPossibleOutput.timeslice.value < decongestion->lastTimeslice) {
@@ -545,9 +547,10 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
         return;
       }
 
-      LOGP(debug, "Broadcasting oldest possible output {} due to {} ({})", oldestPossibleOutput.timeslice.value,
-           oldestPossibleOutput.slot.index == -1 ? "channel" : "slot",
-           oldestPossibleOutput.slot.index == -1 ? oldestPossibleOutput.channel.value : oldestPossibleOutput.slot.index);
+      O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Broadcasting oldest posssible output %" PRIu64 " due to %{public}s (%" PRIu64 ")",
+                             (uint64_t)oldestPossibleOutput.timeslice.value,
+                             oldestPossibleOutput.slot.index == -1 ? "channel" : "slot",
+                             (uint64_t)(oldestPossibleOutput.slot.index == -1 ? oldestPossibleOutput.channel.value : oldestPossibleOutput.slot.index));
       if (decongestion->orderedCompletionPolicyActive) {
         auto oldNextTimeslice = decongestion->nextTimeslice;
         decongestion->nextTimeslice = std::max(decongestion->nextTimeslice, (int64_t)oldestPossibleOutput.timeslice.value);
@@ -563,11 +566,13 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
         auto& state = proxy.getForwardChannelState(ChannelIndex{fi});
         // TODO: this we could cache in the proxy at the bind moment.
         if (info.channelType != ChannelAccountingType::DPL) {
-          LOG(debug) << "Skipping channel";
+          O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
           continue;
         }
         if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestPossibleOutput.timeslice.value)) {
-          LOGP(debug, "Forwarding to channel {} oldest possible timeslice {}, prio 20", info.name, oldestPossibleOutput.timeslice.value);
+          O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice",
+                                 "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
+                                 info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
         }
       }
       decongestion->lastTimeslice = oldestPossibleOutput.timeslice.value; },
@@ -588,13 +593,15 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
       auto& relayer = services.get<DataRelayer>();
       auto& timesliceIndex = services.get<TimesliceIndex>();
       auto& proxy = services.get<FairMQDeviceProxy>();
-      LOGP(debug, "Received oldest possible timeframe {} from channel {}", oldestPossibleTimeslice, channel.value);
+      O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, &decongestion);
+      O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Received oldest possible timeframe %" PRIu64 " from channel %d",
+                             (uint64_t)oldestPossibleTimeslice, channel.value);
       relayer.setOldestPossibleInput({oldestPossibleTimeslice}, channel);
       timesliceIndex.updateOldestPossibleOutput();
       auto oldestPossibleOutput = relayer.getOldestPossibleOutput();
 
       if (oldestPossibleOutput.timeslice.value == decongestion.lastTimeslice) {
-        LOGP(debug, "Not sending already sent value");
+        O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Not sending already sent value: %" PRIu64, (uint64_t)oldestPossibleOutput.timeslice.value);
         return;
       }
       if (oldestPossibleOutput.timeslice.value < decongestion.lastTimeslice) {
@@ -607,14 +614,18 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
       auto *device = services.get<RawDeviceService>().device();
       /// We use the oldest possible timeslice to debounce, so that only the latest one
       /// at the end of one iteration is sent.
-      LOGP(debug, "Queueing oldest possible timeslice {} propagation for execution.", oldestPossibleOutput.timeslice.value);
+      O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Queueing oldest possible timeslice %" PRIu64 " propagation for execution.",
+                             (uint64_t)oldestPossibleOutput.timeslice.value);
       AsyncQueueHelpers::post(
         queue, decongestion.oldestPossibleTimesliceTask, [oldestPossibleOutput, &decongestion, &proxy, &spec, device, &timesliceIndex]() {
+          O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, &decongestion);
           if (decongestion.lastTimeslice >= oldestPossibleOutput.timeslice.value) {
-            LOGP(debug, "Not sending already sent value {} >= {}", decongestion.lastTimeslice, oldestPossibleOutput.timeslice.value);
+            O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Not sending already sent value: %" PRIu64 "> %" PRIu64,
+                decongestion.lastTimeslice, (uint64_t)oldestPossibleOutput.timeslice.value);
             return;
           }
-          LOGP(debug, "Running oldest possible timeslice {} propagation.", oldestPossibleOutput.timeslice.value);
+          O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Running oldest possible timeslice %" PRIu64 " propagation.",
+                                 (uint64_t)oldestPossibleOutput.timeslice.value);
           DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, oldestPossibleOutput.timeslice.value);
 
           for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
@@ -622,11 +633,13 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
             auto& state = proxy.getForwardChannelState(ChannelIndex{fi});
             // TODO: this we could cache in the proxy at the bind moment.
             if (info.channelType != ChannelAccountingType::DPL) {
-              LOG(debug) << "Skipping channel";
+              O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
               continue;
             }
             if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestPossibleOutput.timeslice.value)) {
-              LOGP(debug, "Forwarding to channel {} oldest possible timeslice {}, prio 20", info.name, oldestPossibleOutput.timeslice.value);
+              O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice",
+                                     "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
+                                     info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
             }
           }
           decongestion.lastTimeslice = oldestPossibleOutput.timeslice.value;

From 781679104bf2d9bbd846d6b5afca1cb239dfcb4b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 25 Jan 2024 14:17:28 +0100
Subject: [PATCH 274/726] DPL: make stacktrace level not atomic

Not needed and simplifies overriding it from the outside via ptrace /
vm_write.
---
 Framework/Foundation/include/Framework/Signpost.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index d7b8766de83c4..6dad1ed9f6341 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -179,7 +179,7 @@ struct _o2_log_t {
   // 0 means the log is disabled.
   // 1 means only the current signpost is printed.
   // >1 means the current signpost and n levels of the stacktrace are printed.
-  std::atomic<int> stacktrace = 0;
+  int stacktrace = 0;
 
   // Default stacktrace level for the log, when enabled.
   int defaultStacktrace = 1;

From f2785c7b49ce967508e3e7b48a5d5b5bb6bee5ea Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 25 Jan 2024 14:17:28 +0100
Subject: [PATCH 275/726] DPL: allow signposts to be enabled via DPL_SIGNPOSTS

---
 Framework/Core/src/DeviceSpecHelpers.cxx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 17abf3fa7faad..756d4d57ef342 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1627,6 +1627,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
   // - rate is an option of FairMQ device for ConditionalRun
   // - child-driver is not a FairMQ device option but used per device to start to process
   bpo::options_description forwardedDeviceOptions;
+  char const* defaultSignposts = getenv("DPL_SIGNPOSTS") ? getenv("DPL_SIGNPOSTS") : "";
   forwardedDeviceOptions.add_options()                                                                                                                               //
     ("severity", bpo::value<std::string>()->default_value("info"), "severity level of the log")                                                                      //
     ("plugin,P", bpo::value<std::string>(), "FairMQ plugin list")                                                                                                    //

From f53182b7a4dc49e5984752a658986e2b7bd187ff Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 25 Jan 2024 14:24:37 +0100
Subject: [PATCH 276/726] DPL: add o2-log utility (for Mac, Linux coming)

Allow enabling signpost logs while running.
---
 Framework/Foundation/CMakeLists.txt   |  5 ++
 Framework/Foundation/src/o2LogMac.cxx | 91 +++++++++++++++++++++++++++
 2 files changed, 96 insertions(+)
 create mode 100644 Framework/Foundation/src/o2LogMac.cxx

diff --git a/Framework/Foundation/CMakeLists.txt b/Framework/Foundation/CMakeLists.txt
index aa7a073fca581..1405dbffc031a 100644
--- a/Framework/Foundation/CMakeLists.txt
+++ b/Framework/Foundation/CMakeLists.txt
@@ -41,6 +41,11 @@ add_executable(o2-test-framework-Signpost
 add_executable(o2-test-framework-ThreadSanitizer
                test/test_ThreadSanitizer.cxx)
 
+if (APPLE)
+add_executable(o2-log
+               src/o2LogMac.cxx)
+endif()
+
 target_link_libraries(o2-test-framework-Signpost PRIVATE O2::FrameworkFoundation)
 target_link_libraries(o2-test-framework-ThreadSanitizer
                       PRIVATE O2::FrameworkFoundation Threads::Threads)
diff --git a/Framework/Foundation/src/o2LogMac.cxx b/Framework/Foundation/src/o2LogMac.cxx
new file mode 100644
index 0000000000000..6ac9a17e9cda7
--- /dev/null
+++ b/Framework/Foundation/src/o2LogMac.cxx
@@ -0,0 +1,91 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#import <dlfcn.h>
+#include <mach-o/dyld.h>
+#include <mach/host_info.h>
+#include <mach/mach.h>
+#include <mach/mach_host.h>
+#include <sys/mman.h>
+#include <sys/sysctl.h>
+
+#include <cassert>
+#include <cerrno>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <getopt.h>
+
+int xnu_write(int pid, void* addr, unsigned char* data, size_t dsize)
+{
+  assert(dsize != 0);
+  assert(addr != nullptr);
+  assert(data != nullptr);
+
+  auto* ptxt = (unsigned char*)malloc(dsize);
+
+  assert(ptxt != nullptr);
+  memcpy(ptxt, data, dsize);
+
+  mach_port_t task;
+  mach_msg_type_number_t dataCunt = dsize;
+
+  kern_return_t kret = task_for_pid(mach_task_self(), pid, &task);
+  if (kret != KERN_SUCCESS) {
+    printf("task_for_pid failed: %s. Are you root?", mach_error_string(kret));
+    return 0;
+  }
+
+  vm_protect(task, (vm_address_t)addr, (vm_size_t)dsize, 0,
+             VM_PROT_READ | VM_PROT_WRITE | VM_PROT_ALL);
+
+  kret = vm_write(task, (vm_address_t)addr, (pointer_t)ptxt, dataCunt);
+
+  return kret;
+}
+
+// Writes a 4-byte value to the specified address in the target process
+// If the address in question points to private_o2_log_<signpost>::stacktrace
+// This will have the side effect of enabling the signpost.
+// 
+int main(int argc, char** argv)
+{
+  // Use getopt_long to parse command line arguments
+  // -p pid
+  // -a address
+  // -s stacktrace level (default 1, 0 to disable)
+  static struct option long_options[] = {
+    {"pid", required_argument, nullptr, 'p'},
+    {"address", required_argument, nullptr, 'a'},
+    {"stacktrace", required_argument, nullptr, 's'},
+    {nullptr, 0, nullptr, 0}};
+  int opt;
+  pid_t pid;
+  int stacktrace = 1;
+  void* addr;
+  while ((opt = getopt_long(argc, argv, "p:a:", long_options, nullptr)) != -1) {
+    switch (opt) {
+      case 'p':
+        pid = atoi(optarg);
+        break;
+      case 'a':
+        addr = (void*)strtoul(optarg, nullptr, 16);
+        break;
+      case 's':
+        stacktrace = strtol(optarg, nullptr, 10);
+        break;
+      default:
+        printf("Usage: %s -p pid -a address [-s level]\n", argv[0]);
+        exit(EXIT_FAILURE);
+    }
+  }
+  xnu_write(pid, addr, (unsigned char*)&stacktrace, 4);
+  return 0;
+}

From bd97b87a9ed17f8558b004acb5b60fdbfb232891 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 25 Jan 2024 14:17:28 +0100
Subject: [PATCH 277/726] DPL: support for Signposts in CCDB

---
 Framework/CCDBSupport/src/CCDBHelpers.cxx | 39 +++++++++++++++--------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/Framework/CCDBSupport/src/CCDBHelpers.cxx b/Framework/CCDBSupport/src/CCDBHelpers.cxx
index f2d329db39b6d..837b20cf4262b 100644
--- a/Framework/CCDBSupport/src/CCDBHelpers.cxx
+++ b/Framework/CCDBSupport/src/CCDBHelpers.cxx
@@ -19,11 +19,14 @@
 #include "Framework/DataSpecUtils.h"
 #include "CCDB/CcdbApi.h"
 #include "CommonConstants/LHCConstants.h"
+#include "Framework/Signpost.h"
 #include <typeinfo>
 #include <TError.h>
 #include <TMemFile.h>
 #include <functional>
 
+O2_DECLARE_DYNAMIC_LOG(ccdb);
+
 namespace o2::framework
 {
 
@@ -181,8 +184,12 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
 {
   std::string ccdbMetadataPrefix = "ccdb-metadata-";
   int objCnt = -1;
+  // We use the timeslice, so that we hook into the same interval as the rest of the
+  // callback.
+  auto sid = _o2_signpost_id_t{(int64_t)timingInfo.timeslice};
+  O2_SIGNPOST_START(ccdb, sid, "populateCacheWith", "Starting to populate cache with CCDB objects");
   for (auto& route : helper->routes) {
-    LOGP(debug, "Fetching object for route {}", DataSpecUtils::describe(route.matcher));
+    O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Fetching object for route %{public}s", DataSpecUtils::describe(route.matcher).data());
     objCnt++;
     auto concrete = DataSpecUtils::asConcreteDataMatcher(route.matcher);
     Output output{concrete.origin, concrete.description, concrete.subSpec};
@@ -201,7 +208,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       } else if (isPrefix(ccdbMetadataPrefix, meta.name)) {
         std::string key = meta.name.substr(ccdbMetadataPrefix.size());
         auto value = meta.defaultValue.get<std::string>();
-        LOGP(debug, "Adding metadata {}: {} to the request", key, value);
+        O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Adding metadata %{public}s: %{public}s to the request", key.data(), value.data());
         metadata[key] = value;
       } else if (meta.name == "ccdb-query-rate") {
         chRate = meta.defaultValue.get<int>() * helper->queryPeriodFactor;
@@ -215,7 +222,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       checkValidity = true; // never skip check if the cache is empty
     }
 
-    LOGP(debug, "checkValidity is {} for tfID {} of {}", checkValidity, timingInfo.tfCounter, path);
+    O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "checkValidity is %{public}s for tfID %d of %{public}s", checkValidity ? "true" : "false", timingInfo.tfCounter, path.data());
 
     const auto& api = helper->getAPI(path);
     if (checkValidity && (!api.isSnapshotMode() || etag.empty())) { // in the snapshot mode the object needs to be fetched only once
@@ -238,7 +245,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
         auto cacheId = allocator.adoptContainer(output, std::move(v), DataAllocator::CacheStrategy::Always, header::gSerializationMethodCCDB);
         helper->mapURL2DPLCache[path] = cacheId;
-        LOGP(debug, "Caching {} for {} (DPL id {})", path, headers["ETag"], cacheId.value);
+        O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Caching %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
         continue;
       }
       if (v.size()) { // but should be overridden by fresh object
@@ -249,7 +256,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
         auto cacheId = allocator.adoptContainer(output, std::move(v), DataAllocator::CacheStrategy::Always, header::gSerializationMethodCCDB);
         helper->mapURL2DPLCache[path] = cacheId;
-        LOGP(debug, "Caching {} for {} (DPL id {})", path, headers["ETag"], cacheId.value);
+        O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Caching %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
         // one could modify the    adoptContainer to take optional old cacheID to clean:
         // mapURL2DPLCache[URL] = ctx.outputs().adoptContainer(output, std::move(outputBuffer), DataAllocator::CacheStrategy::Always, mapURL2DPLCache[URL]);
         continue;
@@ -257,11 +264,12 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
     }
     // cached object is fine
     auto cacheId = helper->mapURL2DPLCache[path];
-    LOGP(debug, "Reusing {} for {}", cacheId.value, path);
+    O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Reusing %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
     helper->mapURL2UUID[path].cacheHit++;
     allocator.adoptFromCache(output, cacheId, header::gSerializationMethodCCDB);
     // the outputBuffer was not used, can we destroy it?
   }
+  O2_SIGNPOST_END(ccdb, sid, "populateCacheWith", "Finished populating cache with CCDB objects");
 };
 
 AlgorithmSpec CCDBHelpers::fetchFromCCDB()
@@ -276,6 +284,7 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
       helper->queryPeriodGlo = checkRate > 0 ? checkRate : std::numeric_limits<int>::max();
       helper->queryPeriodFactor = checkMult > 0 ? checkMult : 1;
       LOGP(info, "CCDB Backend at: {}, validity check for every {} TF{}", defHost, helper->queryPeriodGlo, helper->queryPeriodFactor == 1 ? std::string{} : fmt::format(", (query for high-rate objects downscaled by {})", helper->queryPeriodFactor));
+      LOGP(info, "Hook to enable signposts for CCDB messages at {}", (void*)&private_o2_log_ccdb->stacktrace);
       auto remapString = options.get<std::string>("condition-remap");
       ParserResult result = CCDBHelpers::parseRemappings(remapString.c_str());
       if (!result.error.empty()) {
@@ -322,6 +331,8 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
       });
 
       return adaptStateless([helper](DataTakingContext& dtc, DataAllocator& allocator, TimingInfo& timingInfo) {
+        auto sid = _o2_signpost_id_t{(int64_t)timingInfo.timeslice};
+        O2_SIGNPOST_START(ccdb, sid, "fetchFromCCDB", "Fetching CCDB objects for timeslice %" PRIu64, (uint64_t)timingInfo.timeslice);
         static Long64_t orbitResetTime = -1;
         static size_t lastTimeUsed = -1;
         if (timingInfo.creation & DataProcessingHeader::DUMMY_CREATION_TIME_OFFSET) {
@@ -342,7 +353,8 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
           } else {
             checkValidity = true; // never skip check if the cache is empty
           }
-          LOG(debug) << "checkValidity = " << checkValidity << " for TF " << timingInfo.timeslice;
+          O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "checkValidity is %{public}s for tfID %d of %{public}s",
+                                 checkValidity ? "true" : "false", timingInfo.tfCounter, path.data());
           Output output{"CTP", "OrbitReset", 0};
           Long64_t newOrbitResetTime = orbitResetTime;
           auto&& v = allocator.makeVector<char>(output);
@@ -363,7 +375,7 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
               newOrbitResetTime = getOrbitResetTime(v);
               auto cacheId = allocator.adoptContainer(output, std::move(v), DataAllocator::CacheStrategy::Always, header::gSerializationMethodNone);
               helper->mapURL2DPLCache[path] = cacheId;
-              LOGP(debug, "Caching {} for {} (DPL id {})", path, headers["ETag"], cacheId.value);
+              O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "Caching %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
             } else if (v.size()) { // but should be overridden by fresh object
               // somewhere here pruneFromCache should be called
               helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
@@ -373,19 +385,19 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
               newOrbitResetTime = getOrbitResetTime(v);
               auto cacheId = allocator.adoptContainer(output, std::move(v), DataAllocator::CacheStrategy::Always, header::gSerializationMethodNone);
               helper->mapURL2DPLCache[path] = cacheId;
-              LOGP(debug, "Caching {} for {} (DPL id {})", path, headers["ETag"], cacheId.value);
+              O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "Caching %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
               // one could modify the adoptContainer to take optional old cacheID to clean:
               // mapURL2DPLCache[URL] = ctx.outputs().adoptContainer(output, std::move(outputBuffer), DataAllocator::CacheStrategy::Always, mapURL2DPLCache[URL]);
             }
             // cached object is fine
           }
           auto cacheId = helper->mapURL2DPLCache[path];
-          LOGP(debug, "Reusing {} for {}", cacheId.value, path);
+          O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "Reusing %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
           helper->mapURL2UUID[path].cacheHit++;
           allocator.adoptFromCache(output, cacheId, header::gSerializationMethodNone);
 
           if (newOrbitResetTime != orbitResetTime) {
-            LOGP(debug, "Orbit reset time now at {} (was {})", newOrbitResetTime, orbitResetTime);
+            O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "Orbit reset time changed from %lld to %lld", orbitResetTime, newOrbitResetTime);
             orbitResetTime = newOrbitResetTime;
             dtc.orbitResetTimeMUS = orbitResetTime;
           }
@@ -403,10 +415,11 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
           timestamp = timingInfo.creation;
         }
         // Fetch the rest of the objects.
-        LOGP(debug, "Fetching objects. Run: {}. OrbitResetTime: {}, Creation: {}, Timestamp: {}, firstTForbit: {}",
-             dtc.runNumber, orbitResetTime, timingInfo.creation, timestamp, timingInfo.firstTForbit);
+        O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "Fetching objects. Run %{public}s. OrbitResetTime %lld. Creation %lld. Timestamp %lld. firstTForbit %" PRIu32,
+            dtc.runNumber.data(), orbitResetTime, timingInfo.creation, timestamp, timingInfo.firstTForbit);
 
         populateCacheWith(helper, timestamp, timingInfo, dtc, allocator);
+        O2_SIGNPOST_END(ccdb, _o2_signpost_id_t{(int64_t)timingInfo.timeslice}, "fetchFromCCDB", "Fetching CCDB objects");
       }); });
 }
 

From 856a01d469ce163f39a201179ae42dd8fde7669d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 25 Jan 2024 16:07:45 +0100
Subject: [PATCH 278/726] DPL: add o2-log utility (for Mac, Linux coming)

Allow enabling signpost logs while running.
---
 Framework/Foundation/CMakeLists.txt |   3 +
 Framework/Foundation/src/o2Log.cxx  | 122 ++++++++++++++++++++++++++++
 2 files changed, 125 insertions(+)
 create mode 100644 Framework/Foundation/src/o2Log.cxx

diff --git a/Framework/Foundation/CMakeLists.txt b/Framework/Foundation/CMakeLists.txt
index aa7a073fca581..0682d37b61306 100644
--- a/Framework/Foundation/CMakeLists.txt
+++ b/Framework/Foundation/CMakeLists.txt
@@ -41,6 +41,9 @@ add_executable(o2-test-framework-Signpost
 add_executable(o2-test-framework-ThreadSanitizer
                test/test_ThreadSanitizer.cxx)
 
+add_executable(o2-log
+               src/o2Log.cxx)
+
 target_link_libraries(o2-test-framework-Signpost PRIVATE O2::FrameworkFoundation)
 target_link_libraries(o2-test-framework-ThreadSanitizer
                       PRIVATE O2::FrameworkFoundation Threads::Threads)
diff --git a/Framework/Foundation/src/o2Log.cxx b/Framework/Foundation/src/o2Log.cxx
new file mode 100644
index 0000000000000..7806229b9be7c
--- /dev/null
+++ b/Framework/Foundation/src/o2Log.cxx
@@ -0,0 +1,122 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifdef __APPLE__
+#import <dlfcn.h>
+#include <mach-o/dyld.h>
+#include <mach/host_info.h>
+#include <mach/mach.h>
+#include <mach/mach_host.h>
+#include <sys/mman.h>
+#include <sys/sysctl.h>
+#else
+#include <sys/ptrace.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#endif
+
+#include <cassert>
+#include <cerrno>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <getopt.h>
+
+#ifdef __APPLE__
+int xnu_write(int pid, void* addr, unsigned char* data, size_t dsize)
+{
+  assert(dsize != 0);
+  assert(addr != nullptr);
+  assert(data != nullptr);
+
+  auto* ptxt = (unsigned char*)malloc(dsize);
+
+  assert(ptxt != nullptr);
+  memcpy(ptxt, data, dsize);
+
+  mach_port_t task;
+  mach_msg_type_number_t dataCunt = dsize;
+
+  kern_return_t kret = task_for_pid(mach_task_self(), pid, &task);
+  if (kret != KERN_SUCCESS) {
+    printf("task_for_pid failed: %s. Are you root?", mach_error_string(kret));
+    return 0;
+  }
+
+  vm_protect(task, (vm_address_t)addr, (vm_size_t)dsize, 0,
+             VM_PROT_READ | VM_PROT_WRITE | VM_PROT_ALL);
+
+  kret = vm_write(task, (vm_address_t)addr, (pointer_t)ptxt, dataCunt);
+
+  return kret;
+}
+#elif __linux__
+int xnu_write(int pid, void *addr, unsigned char *data, size_t dsize) {
+  if ((ptrace(PTRACE_ATTACH, pid, NULL, NULL)) < 0) {
+    perror("ptrace(ATTACH)");
+    exit(1);
+  }
+  int waitStat = 0;
+  int waitRes = waitpid(pid, &waitStat, WUNTRACED);
+  if (waitRes != pid || !WIFSTOPPED(waitStat)) {
+    perror("....:");
+    printf("Something went wrong...\n");
+    exit(1);
+  }
+
+  if ((ptrace(PTRACE_POKEDATA, pid, addr, data))) {
+    perror("pokedata");
+  }
+
+  if ((ptrace(PTRACE_DETACH, pid, NULL, NULL) < 0)) {
+    perror("ptrace(DETACH)");
+    exit(1);
+  }
+  return 0;
+}
+#endif
+
+// Writes a 4-byte value to the specified address in the target process
+// If the address in question points to private_o2_log_<signpost>::stacktrace
+// This will have the side effect of enabling the signpost.
+int main(int argc, char** argv)
+{
+  // Use getopt_long to parse command line arguments
+  // -p pid
+  // -a address
+  // -s stacktrace level (default 1, 0 to disable)
+  static struct option long_options[] = {
+    {"pid", required_argument, nullptr, 'p'},
+    {"address", required_argument, nullptr, 'a'},
+    {"stacktrace", required_argument, nullptr, 's'},
+    {nullptr, 0, nullptr, 0}};
+  int opt;
+  pid_t pid;
+  int stacktrace = 1;
+  void* addr;
+  while ((opt = getopt_long(argc, argv, "p:a:", long_options, nullptr)) != -1) {
+    switch (opt) {
+      case 'p':
+        pid = atoi(optarg);
+        break;
+      case 'a':
+        addr = (void*)strtoul(optarg, nullptr, 16);
+        break;
+      case 's':
+        stacktrace = strtol(optarg, nullptr, 10);
+        break;
+      default:
+        printf("Usage: %s -p pid -a address [-s level]\n", argv[0]);
+        exit(EXIT_FAILURE);
+    }
+  }
+  xnu_write(pid, addr, (unsigned char*)&stacktrace, 4);
+  return 0;
+}

From 48866d24c60a53a961e428511868e85d893753b6 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 25 Jan 2024 16:07:45 +0100
Subject: [PATCH 279/726] DPL: support for Signposts in CCDB

---
 Framework/CCDBSupport/src/CCDBHelpers.cxx | 39 +++++++++++++++--------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/Framework/CCDBSupport/src/CCDBHelpers.cxx b/Framework/CCDBSupport/src/CCDBHelpers.cxx
index f2d329db39b6d..837b20cf4262b 100644
--- a/Framework/CCDBSupport/src/CCDBHelpers.cxx
+++ b/Framework/CCDBSupport/src/CCDBHelpers.cxx
@@ -19,11 +19,14 @@
 #include "Framework/DataSpecUtils.h"
 #include "CCDB/CcdbApi.h"
 #include "CommonConstants/LHCConstants.h"
+#include "Framework/Signpost.h"
 #include <typeinfo>
 #include <TError.h>
 #include <TMemFile.h>
 #include <functional>
 
+O2_DECLARE_DYNAMIC_LOG(ccdb);
+
 namespace o2::framework
 {
 
@@ -181,8 +184,12 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
 {
   std::string ccdbMetadataPrefix = "ccdb-metadata-";
   int objCnt = -1;
+  // We use the timeslice, so that we hook into the same interval as the rest of the
+  // callback.
+  auto sid = _o2_signpost_id_t{(int64_t)timingInfo.timeslice};
+  O2_SIGNPOST_START(ccdb, sid, "populateCacheWith", "Starting to populate cache with CCDB objects");
   for (auto& route : helper->routes) {
-    LOGP(debug, "Fetching object for route {}", DataSpecUtils::describe(route.matcher));
+    O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Fetching object for route %{public}s", DataSpecUtils::describe(route.matcher).data());
     objCnt++;
     auto concrete = DataSpecUtils::asConcreteDataMatcher(route.matcher);
     Output output{concrete.origin, concrete.description, concrete.subSpec};
@@ -201,7 +208,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       } else if (isPrefix(ccdbMetadataPrefix, meta.name)) {
         std::string key = meta.name.substr(ccdbMetadataPrefix.size());
         auto value = meta.defaultValue.get<std::string>();
-        LOGP(debug, "Adding metadata {}: {} to the request", key, value);
+        O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Adding metadata %{public}s: %{public}s to the request", key.data(), value.data());
         metadata[key] = value;
       } else if (meta.name == "ccdb-query-rate") {
         chRate = meta.defaultValue.get<int>() * helper->queryPeriodFactor;
@@ -215,7 +222,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       checkValidity = true; // never skip check if the cache is empty
     }
 
-    LOGP(debug, "checkValidity is {} for tfID {} of {}", checkValidity, timingInfo.tfCounter, path);
+    O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "checkValidity is %{public}s for tfID %d of %{public}s", checkValidity ? "true" : "false", timingInfo.tfCounter, path.data());
 
     const auto& api = helper->getAPI(path);
     if (checkValidity && (!api.isSnapshotMode() || etag.empty())) { // in the snapshot mode the object needs to be fetched only once
@@ -238,7 +245,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
         auto cacheId = allocator.adoptContainer(output, std::move(v), DataAllocator::CacheStrategy::Always, header::gSerializationMethodCCDB);
         helper->mapURL2DPLCache[path] = cacheId;
-        LOGP(debug, "Caching {} for {} (DPL id {})", path, headers["ETag"], cacheId.value);
+        O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Caching %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
         continue;
       }
       if (v.size()) { // but should be overridden by fresh object
@@ -249,7 +256,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
         auto cacheId = allocator.adoptContainer(output, std::move(v), DataAllocator::CacheStrategy::Always, header::gSerializationMethodCCDB);
         helper->mapURL2DPLCache[path] = cacheId;
-        LOGP(debug, "Caching {} for {} (DPL id {})", path, headers["ETag"], cacheId.value);
+        O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Caching %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
         // one could modify the    adoptContainer to take optional old cacheID to clean:
         // mapURL2DPLCache[URL] = ctx.outputs().adoptContainer(output, std::move(outputBuffer), DataAllocator::CacheStrategy::Always, mapURL2DPLCache[URL]);
         continue;
@@ -257,11 +264,12 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
     }
     // cached object is fine
     auto cacheId = helper->mapURL2DPLCache[path];
-    LOGP(debug, "Reusing {} for {}", cacheId.value, path);
+    O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Reusing %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
     helper->mapURL2UUID[path].cacheHit++;
     allocator.adoptFromCache(output, cacheId, header::gSerializationMethodCCDB);
     // the outputBuffer was not used, can we destroy it?
   }
+  O2_SIGNPOST_END(ccdb, sid, "populateCacheWith", "Finished populating cache with CCDB objects");
 };
 
 AlgorithmSpec CCDBHelpers::fetchFromCCDB()
@@ -276,6 +284,7 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
       helper->queryPeriodGlo = checkRate > 0 ? checkRate : std::numeric_limits<int>::max();
       helper->queryPeriodFactor = checkMult > 0 ? checkMult : 1;
       LOGP(info, "CCDB Backend at: {}, validity check for every {} TF{}", defHost, helper->queryPeriodGlo, helper->queryPeriodFactor == 1 ? std::string{} : fmt::format(", (query for high-rate objects downscaled by {})", helper->queryPeriodFactor));
+      LOGP(info, "Hook to enable signposts for CCDB messages at {}", (void*)&private_o2_log_ccdb->stacktrace);
       auto remapString = options.get<std::string>("condition-remap");
       ParserResult result = CCDBHelpers::parseRemappings(remapString.c_str());
       if (!result.error.empty()) {
@@ -322,6 +331,8 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
       });
 
       return adaptStateless([helper](DataTakingContext& dtc, DataAllocator& allocator, TimingInfo& timingInfo) {
+        auto sid = _o2_signpost_id_t{(int64_t)timingInfo.timeslice};
+        O2_SIGNPOST_START(ccdb, sid, "fetchFromCCDB", "Fetching CCDB objects for timeslice %" PRIu64, (uint64_t)timingInfo.timeslice);
         static Long64_t orbitResetTime = -1;
         static size_t lastTimeUsed = -1;
         if (timingInfo.creation & DataProcessingHeader::DUMMY_CREATION_TIME_OFFSET) {
@@ -342,7 +353,8 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
           } else {
             checkValidity = true; // never skip check if the cache is empty
           }
-          LOG(debug) << "checkValidity = " << checkValidity << " for TF " << timingInfo.timeslice;
+          O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "checkValidity is %{public}s for tfID %d of %{public}s",
+                                 checkValidity ? "true" : "false", timingInfo.tfCounter, path.data());
           Output output{"CTP", "OrbitReset", 0};
           Long64_t newOrbitResetTime = orbitResetTime;
           auto&& v = allocator.makeVector<char>(output);
@@ -363,7 +375,7 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
               newOrbitResetTime = getOrbitResetTime(v);
               auto cacheId = allocator.adoptContainer(output, std::move(v), DataAllocator::CacheStrategy::Always, header::gSerializationMethodNone);
               helper->mapURL2DPLCache[path] = cacheId;
-              LOGP(debug, "Caching {} for {} (DPL id {})", path, headers["ETag"], cacheId.value);
+              O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "Caching %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
             } else if (v.size()) { // but should be overridden by fresh object
               // somewhere here pruneFromCache should be called
               helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
@@ -373,19 +385,19 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
               newOrbitResetTime = getOrbitResetTime(v);
               auto cacheId = allocator.adoptContainer(output, std::move(v), DataAllocator::CacheStrategy::Always, header::gSerializationMethodNone);
               helper->mapURL2DPLCache[path] = cacheId;
-              LOGP(debug, "Caching {} for {} (DPL id {})", path, headers["ETag"], cacheId.value);
+              O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "Caching %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
               // one could modify the adoptContainer to take optional old cacheID to clean:
               // mapURL2DPLCache[URL] = ctx.outputs().adoptContainer(output, std::move(outputBuffer), DataAllocator::CacheStrategy::Always, mapURL2DPLCache[URL]);
             }
             // cached object is fine
           }
           auto cacheId = helper->mapURL2DPLCache[path];
-          LOGP(debug, "Reusing {} for {}", cacheId.value, path);
+          O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "Reusing %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
           helper->mapURL2UUID[path].cacheHit++;
           allocator.adoptFromCache(output, cacheId, header::gSerializationMethodNone);
 
           if (newOrbitResetTime != orbitResetTime) {
-            LOGP(debug, "Orbit reset time now at {} (was {})", newOrbitResetTime, orbitResetTime);
+            O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "Orbit reset time changed from %lld to %lld", orbitResetTime, newOrbitResetTime);
             orbitResetTime = newOrbitResetTime;
             dtc.orbitResetTimeMUS = orbitResetTime;
           }
@@ -403,10 +415,11 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
           timestamp = timingInfo.creation;
         }
         // Fetch the rest of the objects.
-        LOGP(debug, "Fetching objects. Run: {}. OrbitResetTime: {}, Creation: {}, Timestamp: {}, firstTForbit: {}",
-             dtc.runNumber, orbitResetTime, timingInfo.creation, timestamp, timingInfo.firstTForbit);
+        O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "Fetching objects. Run %{public}s. OrbitResetTime %lld. Creation %lld. Timestamp %lld. firstTForbit %" PRIu32,
+            dtc.runNumber.data(), orbitResetTime, timingInfo.creation, timestamp, timingInfo.firstTForbit);
 
         populateCacheWith(helper, timestamp, timingInfo, dtc, allocator);
+        O2_SIGNPOST_END(ccdb, _o2_signpost_id_t{(int64_t)timingInfo.timeslice}, "fetchFromCCDB", "Fetching CCDB objects");
       }); });
 }
 

From 9d82a88a007ac5d9c12ded6473b59a01cf501acf Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 25 Jan 2024 16:11:39 +0100
Subject: [PATCH 280/726] DPL: add o2-log utility (for Mac, Linux coming)

Allow enabling signpost logs while running.
---
 Framework/Foundation/CMakeLists.txt |   3 +
 Framework/Foundation/src/o2Log.cxx  | 123 ++++++++++++++++++++++++++++
 2 files changed, 126 insertions(+)
 create mode 100644 Framework/Foundation/src/o2Log.cxx

diff --git a/Framework/Foundation/CMakeLists.txt b/Framework/Foundation/CMakeLists.txt
index aa7a073fca581..0682d37b61306 100644
--- a/Framework/Foundation/CMakeLists.txt
+++ b/Framework/Foundation/CMakeLists.txt
@@ -41,6 +41,9 @@ add_executable(o2-test-framework-Signpost
 add_executable(o2-test-framework-ThreadSanitizer
                test/test_ThreadSanitizer.cxx)
 
+add_executable(o2-log
+               src/o2Log.cxx)
+
 target_link_libraries(o2-test-framework-Signpost PRIVATE O2::FrameworkFoundation)
 target_link_libraries(o2-test-framework-ThreadSanitizer
                       PRIVATE O2::FrameworkFoundation Threads::Threads)
diff --git a/Framework/Foundation/src/o2Log.cxx b/Framework/Foundation/src/o2Log.cxx
new file mode 100644
index 0000000000000..c9cef04c7b7b6
--- /dev/null
+++ b/Framework/Foundation/src/o2Log.cxx
@@ -0,0 +1,123 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifdef __APPLE__
+#import <dlfcn.h>
+#include <mach-o/dyld.h>
+#include <mach/host_info.h>
+#include <mach/mach.h>
+#include <mach/mach_host.h>
+#include <sys/mman.h>
+#include <sys/sysctl.h>
+#else
+#include <sys/ptrace.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#endif
+
+#include <cassert>
+#include <cerrno>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <getopt.h>
+
+#ifdef __APPLE__
+int xnu_write(int pid, void* addr, unsigned char* data, size_t dsize)
+{
+  assert(dsize != 0);
+  assert(addr != nullptr);
+  assert(data != nullptr);
+
+  auto* ptxt = (unsigned char*)malloc(dsize);
+
+  assert(ptxt != nullptr);
+  memcpy(ptxt, data, dsize);
+
+  mach_port_t task;
+  mach_msg_type_number_t dataCunt = dsize;
+
+  kern_return_t kret = task_for_pid(mach_task_self(), pid, &task);
+  if (kret != KERN_SUCCESS) {
+    printf("task_for_pid failed: %s. Are you root?", mach_error_string(kret));
+    return 0;
+  }
+
+  vm_protect(task, (vm_address_t)addr, (vm_size_t)dsize, 0,
+             VM_PROT_READ | VM_PROT_WRITE | VM_PROT_ALL);
+
+  kret = vm_write(task, (vm_address_t)addr, (pointer_t)ptxt, dataCunt);
+
+  return kret;
+}
+#elif __linux__
+int xnu_write(int pid, void* addr, unsigned char* data, size_t dsize)
+{
+  if ((ptrace(PTRACE_ATTACH, pid, NULL, NULL)) < 0) {
+    perror("ptrace(ATTACH)");
+    exit(1);
+  }
+  int waitStat = 0;
+  int waitRes = waitpid(pid, &waitStat, WUNTRACED);
+  if (waitRes != pid || !WIFSTOPPED(waitStat)) {
+    perror("....:");
+    printf("Something went wrong...\n");
+    exit(1);
+  }
+
+  if ((ptrace(PTRACE_POKEDATA, pid, addr, data))) {
+    perror("pokedata");
+  }
+
+  if ((ptrace(PTRACE_DETACH, pid, NULL, NULL) < 0)) {
+    perror("ptrace(DETACH)");
+    exit(1);
+  }
+  return 0;
+}
+#endif
+
+// Writes a 4-byte value to the specified address in the target process
+// If the address in question points to private_o2_log_<signpost>::stacktrace
+// This will have the side effect of enabling the signpost.
+int main(int argc, char** argv)
+{
+  // Use getopt_long to parse command line arguments
+  // -p pid
+  // -a address
+  // -s stacktrace level (default 1, 0 to disable)
+  static struct option long_options[] = {
+    {"pid", required_argument, nullptr, 'p'},
+    {"address", required_argument, nullptr, 'a'},
+    {"stacktrace", required_argument, nullptr, 's'},
+    {nullptr, 0, nullptr, 0}};
+  int opt;
+  pid_t pid;
+  int stacktrace = 1;
+  void* addr;
+  while ((opt = getopt_long(argc, argv, "p:a:", long_options, nullptr)) != -1) {
+    switch (opt) {
+      case 'p':
+        pid = atoi(optarg);
+        break;
+      case 'a':
+        addr = (void*)strtoul(optarg, nullptr, 16);
+        break;
+      case 's':
+        stacktrace = strtol(optarg, nullptr, 10);
+        break;
+      default:
+        printf("Usage: %s -p pid -a address [-s level]\n", argv[0]);
+        exit(EXIT_FAILURE);
+    }
+  }
+  xnu_write(pid, addr, (unsigned char*)&stacktrace, 4);
+  return 0;
+}

From 6a86ab66f1c47581808e3cac551455501eff0ced Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 25 Jan 2024 16:11:39 +0100
Subject: [PATCH 281/726] DPL: support for Signposts in CCDB

---
 Framework/CCDBSupport/src/CCDBHelpers.cxx | 39 +++++++++++++++--------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/Framework/CCDBSupport/src/CCDBHelpers.cxx b/Framework/CCDBSupport/src/CCDBHelpers.cxx
index f2d329db39b6d..837b20cf4262b 100644
--- a/Framework/CCDBSupport/src/CCDBHelpers.cxx
+++ b/Framework/CCDBSupport/src/CCDBHelpers.cxx
@@ -19,11 +19,14 @@
 #include "Framework/DataSpecUtils.h"
 #include "CCDB/CcdbApi.h"
 #include "CommonConstants/LHCConstants.h"
+#include "Framework/Signpost.h"
 #include <typeinfo>
 #include <TError.h>
 #include <TMemFile.h>
 #include <functional>
 
+O2_DECLARE_DYNAMIC_LOG(ccdb);
+
 namespace o2::framework
 {
 
@@ -181,8 +184,12 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
 {
   std::string ccdbMetadataPrefix = "ccdb-metadata-";
   int objCnt = -1;
+  // We use the timeslice, so that we hook into the same interval as the rest of the
+  // callback.
+  auto sid = _o2_signpost_id_t{(int64_t)timingInfo.timeslice};
+  O2_SIGNPOST_START(ccdb, sid, "populateCacheWith", "Starting to populate cache with CCDB objects");
   for (auto& route : helper->routes) {
-    LOGP(debug, "Fetching object for route {}", DataSpecUtils::describe(route.matcher));
+    O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Fetching object for route %{public}s", DataSpecUtils::describe(route.matcher).data());
     objCnt++;
     auto concrete = DataSpecUtils::asConcreteDataMatcher(route.matcher);
     Output output{concrete.origin, concrete.description, concrete.subSpec};
@@ -201,7 +208,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       } else if (isPrefix(ccdbMetadataPrefix, meta.name)) {
         std::string key = meta.name.substr(ccdbMetadataPrefix.size());
         auto value = meta.defaultValue.get<std::string>();
-        LOGP(debug, "Adding metadata {}: {} to the request", key, value);
+        O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Adding metadata %{public}s: %{public}s to the request", key.data(), value.data());
         metadata[key] = value;
       } else if (meta.name == "ccdb-query-rate") {
         chRate = meta.defaultValue.get<int>() * helper->queryPeriodFactor;
@@ -215,7 +222,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       checkValidity = true; // never skip check if the cache is empty
     }
 
-    LOGP(debug, "checkValidity is {} for tfID {} of {}", checkValidity, timingInfo.tfCounter, path);
+    O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "checkValidity is %{public}s for tfID %d of %{public}s", checkValidity ? "true" : "false", timingInfo.tfCounter, path.data());
 
     const auto& api = helper->getAPI(path);
     if (checkValidity && (!api.isSnapshotMode() || etag.empty())) { // in the snapshot mode the object needs to be fetched only once
@@ -238,7 +245,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
         auto cacheId = allocator.adoptContainer(output, std::move(v), DataAllocator::CacheStrategy::Always, header::gSerializationMethodCCDB);
         helper->mapURL2DPLCache[path] = cacheId;
-        LOGP(debug, "Caching {} for {} (DPL id {})", path, headers["ETag"], cacheId.value);
+        O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Caching %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
         continue;
       }
       if (v.size()) { // but should be overridden by fresh object
@@ -249,7 +256,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
         auto cacheId = allocator.adoptContainer(output, std::move(v), DataAllocator::CacheStrategy::Always, header::gSerializationMethodCCDB);
         helper->mapURL2DPLCache[path] = cacheId;
-        LOGP(debug, "Caching {} for {} (DPL id {})", path, headers["ETag"], cacheId.value);
+        O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Caching %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
         // one could modify the    adoptContainer to take optional old cacheID to clean:
         // mapURL2DPLCache[URL] = ctx.outputs().adoptContainer(output, std::move(outputBuffer), DataAllocator::CacheStrategy::Always, mapURL2DPLCache[URL]);
         continue;
@@ -257,11 +264,12 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
     }
     // cached object is fine
     auto cacheId = helper->mapURL2DPLCache[path];
-    LOGP(debug, "Reusing {} for {}", cacheId.value, path);
+    O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Reusing %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
     helper->mapURL2UUID[path].cacheHit++;
     allocator.adoptFromCache(output, cacheId, header::gSerializationMethodCCDB);
     // the outputBuffer was not used, can we destroy it?
   }
+  O2_SIGNPOST_END(ccdb, sid, "populateCacheWith", "Finished populating cache with CCDB objects");
 };
 
 AlgorithmSpec CCDBHelpers::fetchFromCCDB()
@@ -276,6 +284,7 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
       helper->queryPeriodGlo = checkRate > 0 ? checkRate : std::numeric_limits<int>::max();
       helper->queryPeriodFactor = checkMult > 0 ? checkMult : 1;
       LOGP(info, "CCDB Backend at: {}, validity check for every {} TF{}", defHost, helper->queryPeriodGlo, helper->queryPeriodFactor == 1 ? std::string{} : fmt::format(", (query for high-rate objects downscaled by {})", helper->queryPeriodFactor));
+      LOGP(info, "Hook to enable signposts for CCDB messages at {}", (void*)&private_o2_log_ccdb->stacktrace);
       auto remapString = options.get<std::string>("condition-remap");
       ParserResult result = CCDBHelpers::parseRemappings(remapString.c_str());
       if (!result.error.empty()) {
@@ -322,6 +331,8 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
       });
 
       return adaptStateless([helper](DataTakingContext& dtc, DataAllocator& allocator, TimingInfo& timingInfo) {
+        auto sid = _o2_signpost_id_t{(int64_t)timingInfo.timeslice};
+        O2_SIGNPOST_START(ccdb, sid, "fetchFromCCDB", "Fetching CCDB objects for timeslice %" PRIu64, (uint64_t)timingInfo.timeslice);
         static Long64_t orbitResetTime = -1;
         static size_t lastTimeUsed = -1;
         if (timingInfo.creation & DataProcessingHeader::DUMMY_CREATION_TIME_OFFSET) {
@@ -342,7 +353,8 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
           } else {
             checkValidity = true; // never skip check if the cache is empty
           }
-          LOG(debug) << "checkValidity = " << checkValidity << " for TF " << timingInfo.timeslice;
+          O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "checkValidity is %{public}s for tfID %d of %{public}s",
+                                 checkValidity ? "true" : "false", timingInfo.tfCounter, path.data());
           Output output{"CTP", "OrbitReset", 0};
           Long64_t newOrbitResetTime = orbitResetTime;
           auto&& v = allocator.makeVector<char>(output);
@@ -363,7 +375,7 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
               newOrbitResetTime = getOrbitResetTime(v);
               auto cacheId = allocator.adoptContainer(output, std::move(v), DataAllocator::CacheStrategy::Always, header::gSerializationMethodNone);
               helper->mapURL2DPLCache[path] = cacheId;
-              LOGP(debug, "Caching {} for {} (DPL id {})", path, headers["ETag"], cacheId.value);
+              O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "Caching %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
             } else if (v.size()) { // but should be overridden by fresh object
               // somewhere here pruneFromCache should be called
               helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
@@ -373,19 +385,19 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
               newOrbitResetTime = getOrbitResetTime(v);
               auto cacheId = allocator.adoptContainer(output, std::move(v), DataAllocator::CacheStrategy::Always, header::gSerializationMethodNone);
               helper->mapURL2DPLCache[path] = cacheId;
-              LOGP(debug, "Caching {} for {} (DPL id {})", path, headers["ETag"], cacheId.value);
+              O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "Caching %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
               // one could modify the adoptContainer to take optional old cacheID to clean:
               // mapURL2DPLCache[URL] = ctx.outputs().adoptContainer(output, std::move(outputBuffer), DataAllocator::CacheStrategy::Always, mapURL2DPLCache[URL]);
             }
             // cached object is fine
           }
           auto cacheId = helper->mapURL2DPLCache[path];
-          LOGP(debug, "Reusing {} for {}", cacheId.value, path);
+          O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "Reusing %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
           helper->mapURL2UUID[path].cacheHit++;
           allocator.adoptFromCache(output, cacheId, header::gSerializationMethodNone);
 
           if (newOrbitResetTime != orbitResetTime) {
-            LOGP(debug, "Orbit reset time now at {} (was {})", newOrbitResetTime, orbitResetTime);
+            O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "Orbit reset time changed from %lld to %lld", orbitResetTime, newOrbitResetTime);
             orbitResetTime = newOrbitResetTime;
             dtc.orbitResetTimeMUS = orbitResetTime;
           }
@@ -403,10 +415,11 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
           timestamp = timingInfo.creation;
         }
         // Fetch the rest of the objects.
-        LOGP(debug, "Fetching objects. Run: {}. OrbitResetTime: {}, Creation: {}, Timestamp: {}, firstTForbit: {}",
-             dtc.runNumber, orbitResetTime, timingInfo.creation, timestamp, timingInfo.firstTForbit);
+        O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "Fetching objects. Run %{public}s. OrbitResetTime %lld. Creation %lld. Timestamp %lld. firstTForbit %" PRIu32,
+            dtc.runNumber.data(), orbitResetTime, timingInfo.creation, timestamp, timingInfo.firstTForbit);
 
         populateCacheWith(helper, timestamp, timingInfo, dtc, allocator);
+        O2_SIGNPOST_END(ccdb, _o2_signpost_id_t{(int64_t)timingInfo.timeslice}, "fetchFromCCDB", "Fetching CCDB objects");
       }); });
 }
 

From f43100f68b7c513ae8fd0a16376474cb6487e183 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 26 Jan 2024 11:36:20 +0100
Subject: [PATCH 282/726] DPL: install o2-log in the proper place

---
 Framework/Foundation/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Framework/Foundation/CMakeLists.txt b/Framework/Foundation/CMakeLists.txt
index 0682d37b61306..8babb4bc9a44e 100644
--- a/Framework/Foundation/CMakeLists.txt
+++ b/Framework/Foundation/CMakeLists.txt
@@ -59,6 +59,8 @@ set_property(TARGET o2-test-framework-foundation PROPERTY RUNTIME_OUTPUT_DIRECTO
 set_property(TARGET o2-test-framework-Signpost PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
 set_property(TARGET o2-test-framework-SignpostLogger PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
 set_property(TARGET o2-test-framework-ThreadSanitizer PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
+get_filename_component(bindir ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/../bin ABSOLUTE)
+set_property(TARGET o2-log PROPERTY RUNTIME_OUTPUT_DIRECTORY ${bindir})
 
 add_test(NAME framework:foundation COMMAND o2-test-framework-foundation)
 

From 56e810135e263ce7e8395e9c07d136c634c4b4d3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 26 Jan 2024 11:36:22 +0100
Subject: [PATCH 283/726] DPL: do not update the new run flag unless we do
 process data

Data might be dropped by the oldest possible timeframe mechanism for the
first timeslice which arrives, at which point we lose the information about
the new run.

Moving it down ensures that the check is done for the first timeframe which arrives.
---
 Framework/Core/src/DataProcessingDevice.cxx | 32 ++++++++++++---------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index f5c9b7947afde..15abefe42add0 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -2134,7 +2134,6 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   // create messages) because the messages need to have the timeslice id into
   // it.
   auto prepareAllocatorForCurrentTimeSlice = [ref](TimesliceSlot i) -> void {
-    auto& dataProcessorContext = ref.get<DataProcessorContext>();
     auto& relayer = ref.get<DataRelayer>();
     auto& timingInfo = ref.get<TimingInfo>();
     ZoneScopedN("DataProcessingDevice::prepareForCurrentTimeslice");
@@ -2145,17 +2144,6 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
     timingInfo.firstTForbit = relayer.getFirstTFOrbitForSlot(i);
     timingInfo.runNumber = relayer.getRunNumberForSlot(i);
     timingInfo.creation = relayer.getCreationTimeForSlot(i);
-    timingInfo.globalRunNumberChanged = !TimingInfo::timesliceIsTimer(timeslice.value) && dataProcessorContext.lastRunNumberProcessed != timingInfo.runNumber;
-    // A switch to runNumber=0 should not appear and thus does not set globalRunNumberChanged, unless it is seen in the first processed timeslice
-    timingInfo.globalRunNumberChanged &= (dataProcessorContext.lastRunNumberProcessed == -1 || timingInfo.runNumber != 0);
-    // We report wether or not this timing info refers to a new Run.
-    if (timingInfo.globalRunNumberChanged) {
-      dataProcessorContext.lastRunNumberProcessed = timingInfo.runNumber;
-    }
-    // FIXME: for now there is only one stream, however we
-    //        should calculate this correctly once we finally get the
-    //        the StreamContext in.
-    timingInfo.streamRunNumberChanged = timingInfo.globalRunNumberChanged;
   };
 
   // When processing them, timers will have to be cleaned up
@@ -2277,9 +2265,9 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       continue;
     }
 
-    prepareAllocatorForCurrentTimeSlice(TimesliceSlot{action.slot});
     bool shouldConsume = action.op == CompletionPolicy::CompletionOp::Consume ||
                          action.op == CompletionPolicy::CompletionOp::Discard;
+    prepareAllocatorForCurrentTimeSlice(TimesliceSlot{action.slot});
     InputSpan span = getInputSpan(action.slot, shouldConsume);
     auto& spec = ref.get<DeviceSpec const>();
     InputRecord record{spec.inputs,
@@ -2338,6 +2326,23 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
             return false;
         }
       };
+      auto updateRunInformation = [ref](TimesliceSlot i) -> void {
+        auto& dataProcessorContext = ref.get<DataProcessorContext>();
+        auto& relayer = ref.get<DataRelayer>();
+        auto& timingInfo = ref.get<TimingInfo>();
+        auto timeslice = relayer.getTimesliceForSlot(i);
+        timingInfo.globalRunNumberChanged = !TimingInfo::timesliceIsTimer(timeslice.value) && dataProcessorContext.lastRunNumberProcessed != timingInfo.runNumber;
+        // A switch to runNumber=0 should not appear and thus does not set globalRunNumberChanged, unless it is seen in the first processed timeslice
+        timingInfo.globalRunNumberChanged &= (dataProcessorContext.lastRunNumberProcessed == -1 || timingInfo.runNumber != 0);
+        // We report wether or not this timing info refers to a new Run.
+        if (timingInfo.globalRunNumberChanged) {
+          dataProcessorContext.lastRunNumberProcessed = timingInfo.runNumber;
+        }
+        // FIXME: for now there is only one stream, however we
+        //        should calculate this correctly once we finally get the
+        //        the StreamContext in.
+        timingInfo.streamRunNumberChanged = timingInfo.globalRunNumberChanged;
+      };
       if (state.quitRequested == false) {
         {
           // Callbacks from services
@@ -2349,6 +2354,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
         }
         O2_SIGNPOST_ID_FROM_POINTER(pcid, device, &processContext);
         if (context.statefulProcess && shouldProcess(action)) {
+          updateRunInformation(action.slot);
           // This way, usercode can use the the same processing context to identify
           // its signposts and we can map user code to device iterations.
           O2_SIGNPOST_START(device, pcid, "device", "Stateful process");

From 7e70b694465a29ecb05f0ab87c5c81bee0543706 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 26 Jan 2024 14:41:59 +0100
Subject: [PATCH 284/726] DPL: do not update the new run flag unless we do
 process data

Data might be dropped by the oldest possible timeframe mechanism for the
first timeslice which arrives, at which point we lose the information about
the new run.

Moving it down ensures that the check is done for the first timeframe which arrives.
---
 Framework/Core/src/DataProcessingDevice.cxx | 34 +++++++++++++--------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index f5c9b7947afde..6b1843f925127 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -2134,7 +2134,6 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   // create messages) because the messages need to have the timeslice id into
   // it.
   auto prepareAllocatorForCurrentTimeSlice = [ref](TimesliceSlot i) -> void {
-    auto& dataProcessorContext = ref.get<DataProcessorContext>();
     auto& relayer = ref.get<DataRelayer>();
     auto& timingInfo = ref.get<TimingInfo>();
     ZoneScopedN("DataProcessingDevice::prepareForCurrentTimeslice");
@@ -2145,17 +2144,6 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
     timingInfo.firstTForbit = relayer.getFirstTFOrbitForSlot(i);
     timingInfo.runNumber = relayer.getRunNumberForSlot(i);
     timingInfo.creation = relayer.getCreationTimeForSlot(i);
-    timingInfo.globalRunNumberChanged = !TimingInfo::timesliceIsTimer(timeslice.value) && dataProcessorContext.lastRunNumberProcessed != timingInfo.runNumber;
-    // A switch to runNumber=0 should not appear and thus does not set globalRunNumberChanged, unless it is seen in the first processed timeslice
-    timingInfo.globalRunNumberChanged &= (dataProcessorContext.lastRunNumberProcessed == -1 || timingInfo.runNumber != 0);
-    // We report wether or not this timing info refers to a new Run.
-    if (timingInfo.globalRunNumberChanged) {
-      dataProcessorContext.lastRunNumberProcessed = timingInfo.runNumber;
-    }
-    // FIXME: for now there is only one stream, however we
-    //        should calculate this correctly once we finally get the
-    //        the StreamContext in.
-    timingInfo.streamRunNumberChanged = timingInfo.globalRunNumberChanged;
   };
 
   // When processing them, timers will have to be cleaned up
@@ -2277,9 +2265,9 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       continue;
     }
 
-    prepareAllocatorForCurrentTimeSlice(TimesliceSlot{action.slot});
     bool shouldConsume = action.op == CompletionPolicy::CompletionOp::Consume ||
                          action.op == CompletionPolicy::CompletionOp::Discard;
+    prepareAllocatorForCurrentTimeSlice(TimesliceSlot{action.slot});
     InputSpan span = getInputSpan(action.slot, shouldConsume);
     auto& spec = ref.get<DeviceSpec const>();
     InputRecord record{spec.inputs,
@@ -2338,6 +2326,23 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
             return false;
         }
       };
+      auto updateRunInformation = [ref](TimesliceSlot i) -> void {
+        auto& dataProcessorContext = ref.get<DataProcessorContext>();
+        auto& relayer = ref.get<DataRelayer>();
+        auto& timingInfo = ref.get<TimingInfo>();
+        auto timeslice = relayer.getTimesliceForSlot(i);
+        timingInfo.globalRunNumberChanged = !TimingInfo::timesliceIsTimer(timeslice.value) && dataProcessorContext.lastRunNumberProcessed != timingInfo.runNumber;
+        // A switch to runNumber=0 should not appear and thus does not set globalRunNumberChanged, unless it is seen in the first processed timeslice
+        timingInfo.globalRunNumberChanged &= (dataProcessorContext.lastRunNumberProcessed == -1 || timingInfo.runNumber != 0);
+        // We report wether or not this timing info refers to a new Run.
+        if (timingInfo.globalRunNumberChanged) {
+          dataProcessorContext.lastRunNumberProcessed = timingInfo.runNumber;
+        }
+        // FIXME: for now there is only one stream, however we
+        //        should calculate this correctly once we finally get the
+        //        the StreamContext in.
+        timingInfo.streamRunNumberChanged = timingInfo.globalRunNumberChanged;
+      };
       if (state.quitRequested == false) {
         {
           // Callbacks from services
@@ -2349,18 +2354,21 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
         }
         O2_SIGNPOST_ID_FROM_POINTER(pcid, device, &processContext);
         if (context.statefulProcess && shouldProcess(action)) {
+          updateRunInformation(action.slot);
           // This way, usercode can use the the same processing context to identify
           // its signposts and we can map user code to device iterations.
           O2_SIGNPOST_START(device, pcid, "device", "Stateful process");
           (context.statefulProcess)(processContext);
           O2_SIGNPOST_END(device, pcid, "device", "Stateful process");
         } else if (context.statelessProcess && shouldProcess(action)) {
+          updateRunInformation(action.slot);
           O2_SIGNPOST_START(device, pcid, "device", "Stateful process");
           (context.statelessProcess)(processContext);
           O2_SIGNPOST_END(device, pcid, "device", "Stateful process");
         } else if (context.statelessProcess || context.statefulProcess) {
           O2_SIGNPOST_EVENT_EMIT(device, pcid, "device", "Skipping processing because we are discarding.");
         } else {
+          updateRunInformation(action.slot);
           O2_SIGNPOST_EVENT_EMIT(device, pcid, "device", "No processing callback provided. Switching to %{public}s.", "Idle");
           state.streaming = StreamingState::Idle;
         }

From 0cd7e702d12dfc453d646f6689192bf50a823356 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 26 Jan 2024 21:55:07 +0100
Subject: [PATCH 285/726] DPL: do not update the new run flag unless we do
 process data

Data might be dropped by the oldest possible timeframe mechanism for the
first timeslice which arrives, at which point we lose the information about
the new run.

Moving it down ensures that the check is done for the first timeframe which arrives.
---
 Framework/Core/src/DataProcessingDevice.cxx | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index f5c9b7947afde..cb4598daa661f 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -2134,7 +2134,6 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   // create messages) because the messages need to have the timeslice id into
   // it.
   auto prepareAllocatorForCurrentTimeSlice = [ref](TimesliceSlot i) -> void {
-    auto& dataProcessorContext = ref.get<DataProcessorContext>();
     auto& relayer = ref.get<DataRelayer>();
     auto& timingInfo = ref.get<TimingInfo>();
     ZoneScopedN("DataProcessingDevice::prepareForCurrentTimeslice");
@@ -2145,6 +2144,12 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
     timingInfo.firstTForbit = relayer.getFirstTFOrbitForSlot(i);
     timingInfo.runNumber = relayer.getRunNumberForSlot(i);
     timingInfo.creation = relayer.getCreationTimeForSlot(i);
+  };
+  auto updateRunInformation = [ref](TimesliceSlot i) -> void {
+    auto& dataProcessorContext = ref.get<DataProcessorContext>();
+    auto& relayer = ref.get<DataRelayer>();
+    auto& timingInfo = ref.get<TimingInfo>();
+    auto timeslice = relayer.getTimesliceForSlot(i);
     timingInfo.globalRunNumberChanged = !TimingInfo::timesliceIsTimer(timeslice.value) && dataProcessorContext.lastRunNumberProcessed != timingInfo.runNumber;
     // A switch to runNumber=0 should not appear and thus does not set globalRunNumberChanged, unless it is seen in the first processed timeslice
     timingInfo.globalRunNumberChanged &= (dataProcessorContext.lastRunNumberProcessed == -1 || timingInfo.runNumber != 0);
@@ -2277,9 +2282,14 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       continue;
     }
 
-    prepareAllocatorForCurrentTimeSlice(TimesliceSlot{action.slot});
     bool shouldConsume = action.op == CompletionPolicy::CompletionOp::Consume ||
                          action.op == CompletionPolicy::CompletionOp::Discard;
+    prepareAllocatorForCurrentTimeSlice(TimesliceSlot{action.slot});
+    if (action.op != CompletionPolicy::CompletionOp::Discard &&
+        action.op != CompletionPolicy::CompletionOp::Wait &&
+        action.op != CompletionPolicy::CompletionOp::Retry) {
+      updateRunInformation(TimesliceSlot{action.slot});
+    }
     InputSpan span = getInputSpan(action.slot, shouldConsume);
     auto& spec = ref.get<DeviceSpec const>();
     InputRecord record{spec.inputs,

From 92093d70dceb6974f19b430db77a224cb76bfbd1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 26 Jan 2024 21:55:17 +0100
Subject: [PATCH 286/726] DPL: also consume for the ConsumeExitisting case

---
 Framework/Core/src/DataProcessingDevice.cxx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index cb4598daa661f..15c625c3615e5 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -2283,6 +2283,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
     }
 
     bool shouldConsume = action.op == CompletionPolicy::CompletionOp::Consume ||
+                         action.op == CompletionPolicy::CompletionOp::ConsumeExisting ||
                          action.op == CompletionPolicy::CompletionOp::Discard;
     prepareAllocatorForCurrentTimeSlice(TimesliceSlot{action.slot});
     if (action.op != CompletionPolicy::CompletionOp::Discard &&

From c2b3d2d3330c106c55a244c171b760cd806a0730 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 26 Jan 2024 22:00:30 +0100
Subject: [PATCH 287/726] Move another macro to be compiled only

---
 Detectors/TOF/prototyping/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Detectors/TOF/prototyping/CMakeLists.txt b/Detectors/TOF/prototyping/CMakeLists.txt
index 767e44d70d6d4..1ce2268f1358a 100644
--- a/Detectors/TOF/prototyping/CMakeLists.txt
+++ b/Detectors/TOF/prototyping/CMakeLists.txt
@@ -39,7 +39,7 @@ o2_add_test_root_macro(findTOFclusterFromLabel.C
                                              O2::ReconstructionDataFormats
                                              O2::CommonDataFormat
                                              O2::TOFBase
-                       LABELS tof)
+                       LABELS tof COMPILE_ONLY)
 
 o2_add_test_root_macro(checkTS.C
                        PUBLIC_LINK_LIBRARIES O2::DataFormatsTOF

From b82e22c6dbbbb6b2ed391459085850baaef0060e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 26 Jan 2024 22:31:56 +0100
Subject: [PATCH 288/726] DPL: do not validate lifetime when consumer uses
 consumeWhenAny

---
 Framework/Core/src/DeviceSpecHelpers.cxx | 16 +++++++++++++---
 Framework/Core/src/WorkflowHelpers.cxx   | 22 ++++++++++++++++++----
 Framework/Core/src/WorkflowHelpers.h     |  6 ++++++
 3 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 756d4d57ef342..b1681a4779ceb 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1074,8 +1074,6 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
 
   WorkflowHelpers::constructGraph(workflow, logicalEdges, outputs, availableForwardsInfo);
 
-  WorkflowHelpers::validateEdges(workflow, logicalEdges, outputs);
-
   // We need to instanciate one device per (me, timeIndex) in the
   // DeviceConnectionEdge. For each device we need one new binding
   // server per (me, other) -> port Moreover for each (me, other,
@@ -1122,10 +1120,13 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
                        inActions, workflow, availableForwardsInfo, channelPolicies, channelPrefix, defaultOffer, overrideServices);
   // We apply the completion policies here since this is where we have all the
   // devices resolved.
-  for (auto& device : devices) {
+  std::map<std::string, DataProcessorPoliciesInfo> policies;
+  for (DeviceSpec& device : devices) {
     bool hasPolicy = false;
+    policies[device.name].completionPolicyName = "unknown";
     for (auto& policy : completionPolicies) {
       if (policy.matcher(device) == true) {
+        policies[policy.name].completionPolicyName = policy.name;
         device.completionPolicy = policy;
         hasPolicy = true;
         break;
@@ -1158,6 +1159,15 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
       throw runtime_error_f("Unable to find a resource policy for %s", device.id.c_str());
     }
   }
+  // Iterate of the workflow and create a consistent vector of DataProcessorPoliciesInfo
+  std::vector<DataProcessorPoliciesInfo> policiesVector;
+  for (size_t wi = 0; wi < workflow.size(); ++wi) {
+    auto& processor = workflow[wi];
+    auto& info = policies[processor.name];
+    policiesVector.push_back(info);
+  }
+
+  WorkflowHelpers::validateEdges(workflow, policiesVector, logicalEdges, outputs);
 
   for (auto& device : devices) {
     device.resourceMonitoringInterval = resourcesMonitoringInterval;
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 360480f80d4ef..6b4ad35f826be 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -1214,8 +1214,14 @@ std::vector<InputSpec> WorkflowHelpers::computeDanglingOutputs(WorkflowSpec cons
   return results;
 }
 
-bool validateLifetime(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
+bool validateLifetime(std::ostream& errors,
+                      DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorPoliciesInfo const& producerPolicies,
+                      DataProcessorSpec const& consumer, InputSpec const& input, DataProcessorPoliciesInfo const& consumerPolicies)
 {
+  // In case the completion policy is consume-any, we do not need to check anything.
+  if (consumerPolicies.completionPolicyName == "consume-any") {
+    return true;
+  }
   if (input.lifetime == Lifetime::Timeframe && output.lifetime == Lifetime::Sporadic) {
     errors << fmt::format("Input {} of {} has lifetime Timeframe, but output {} of {} has lifetime Sporadic\n",
                           DataSpecUtils::describe(input).c_str(), consumer.name,
@@ -1225,7 +1231,9 @@ bool validateLifetime(std::ostream& errors, DataProcessorSpec const& producer, O
   return true;
 }
 
-bool validateExpendable(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)
+bool validateExpendable(std::ostream& errors,
+                        DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorPoliciesInfo const& producerPolicies,
+                        DataProcessorSpec const& consumer, InputSpec const& input, DataProcessorPoliciesInfo const& consumerPolicies)
 {
   auto isExpendable = [](DataProcessorLabel const& label) {
     return label.value == "expendable";
@@ -1244,8 +1252,12 @@ bool validateExpendable(std::ostream& errors, DataProcessorSpec const& producer,
   return true;
 }
 
-using Validator = std::function<bool(std::ostream& errors, DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorSpec const& consumer, InputSpec const& input)>;
+using Validator = std::function<bool(std::ostream& errors,
+                                     DataProcessorSpec const& producer, OutputSpec const& output, DataProcessorPoliciesInfo const& producerPolicies,
+                                     DataProcessorSpec const& consumer, InputSpec const& input, DataProcessorPoliciesInfo const& consumerPolicies)>;
+
 void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
+                                    std::vector<DataProcessorPoliciesInfo> const& policies,
                                     std::vector<DeviceConnectionEdge> const& edges,
                                     std::vector<OutputSpec> const& outputs)
 {
@@ -1262,10 +1274,12 @@ void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow,
   for (auto& edge : edges) {
     DataProcessorSpec const& producer = workflow[edge.producer];
     DataProcessorSpec const& consumer = workflow[edge.consumer];
+    DataProcessorPoliciesInfo const& producerPolicies = policies[edge.producer];
+    DataProcessorPoliciesInfo const& consumerPolicies = policies[edge.consumer];
     OutputSpec const& output = outputs[edge.outputGlobalIndex];
     InputSpec const& input = consumer.inputs[edge.consumerInputIndex];
     for (auto& validator : defaultValidators) {
-      hasErrors |= !validator(errors, producer, output, consumer, input);
+      hasErrors |= !validator(errors, producer, output, producerPolicies, consumer, input, consumerPolicies);
     }
   }
   if (hasErrors) {
diff --git a/Framework/Core/src/WorkflowHelpers.h b/Framework/Core/src/WorkflowHelpers.h
index 0a3cd408b3424..055cae1d6011b 100644
--- a/Framework/Core/src/WorkflowHelpers.h
+++ b/Framework/Core/src/WorkflowHelpers.h
@@ -141,6 +141,11 @@ struct TopoIndexInfo {
   friend std::ostream& operator<<(std::ostream& out, TopoIndexInfo const& info);
 };
 
+// Information about the policies which were derived for a given data processor.
+struct DataProcessorPoliciesInfo {
+  std::string completionPolicyName;
+};
+
 struct OutputObj {
   InputSpec spec;
   bool isdangling;
@@ -234,6 +239,7 @@ struct WorkflowHelpers {
   /// For example we should make sure that Lifetime::Timeframe inputs of
   /// one node is not connected to an Output of Lifetime::Sporadic of another node.
   static void validateEdges(WorkflowSpec const& workflow,
+                            std::vector<DataProcessorPoliciesInfo> const& policiesInfos,
                             std::vector<DeviceConnectionEdge> const& edges,
                             std::vector<OutputSpec> const& outputs);
 };

From b59c981e85c82da2cb4dda7337b7051240572116 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 30 Jan 2024 11:23:30 +0100
Subject: [PATCH 289/726] Make sure the user routes are traced also on EoS

---
 Framework/Core/src/CommonServices.cxx | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index b3ee75a6a3886..e6d7ae7fcd159 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -220,6 +220,20 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
           LOGP(error, "Expected Lifetime::Timeframe data {} was not created for timeslice {} and might result in dropped timeframes", DataSpecUtils::describe(matcher), timeslice);
         }
       } },
+    .preEOS = [](EndOfStreamContext& context, void* service) {
+      // We need to reset the routeUserCreated because the end of stream
+      // uses a different context which does not know about the routes.
+      // FIXME: This should be fixed in a different way, but for now it will
+      // allow TPC IDC to work.
+      auto* stream = (StreamContext*)service;
+      auto& routes = context.services().get<DeviceSpec const>().outputs;
+      // Notice I need to do this here, because different invocation for
+      // the same stream might be referring to different data processors.
+      // We should probably have a context which is per stream of a specific
+      // data processor.
+      stream->routeUserCreated.resize(routes.size());
+      // Reset the routeUserCreated at every processing step
+      std::fill(stream->routeUserCreated.begin(), stream->routeUserCreated.end(), false); },
     .kind = ServiceKind::Stream};
 }
 

From 0a038a4f943087d9749195065eb9bcd235abd808 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 30 Jan 2024 15:11:18 +0100
Subject: [PATCH 290/726] DPL: avoid invoking matchDataHeader twice when
 sending data

Percolate precomputed value instead, given that payload and
header all go to the same route.
---
 Framework/Core/include/Framework/DataAllocator.h | 4 ++--
 Framework/Core/src/DataAllocator.cxx             | 8 +++-----
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/Framework/Core/include/Framework/DataAllocator.h b/Framework/Core/include/Framework/DataAllocator.h
index f80d6d0836e28..23b9c62458b5e 100644
--- a/Framework/Core/include/Framework/DataAllocator.h
+++ b/Framework/Core/include/Framework/DataAllocator.h
@@ -399,7 +399,7 @@ class DataAllocator
                     "\n - std::vector of messageable structures or pointers to those"
                     "\n - types with ROOT dictionary and implementing ROOT ClassDef interface");
     }
-    addPartToContext(std::move(payloadMessage), spec, serializationType);
+    addPartToContext(routeIndex, std::move(payloadMessage), spec, serializationType);
   }
 
   /// Take a snapshot of a raw data array which can be either POD or may contain a serialized
@@ -527,7 +527,7 @@ class DataAllocator
                                                size_t payloadSize);                                 //
 
   Output getOutputByBind(OutputRef&& ref);
-  void addPartToContext(fair::mq::MessagePtr&& payload,
+  void addPartToContext(RouteIndex routeIndex, fair::mq::MessagePtr&& payload,
                         const Output& spec,
                         o2::header::SerializationMethod serializationMethod);
 };
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index dbeeadad68b2b..474be319b4857 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -119,11 +119,9 @@ fair::mq::MessagePtr DataAllocator::headerMessageFromOutput(Output const& spec,
   return o2::pmr::getMessage(o2::header::Stack{channelAlloc, dh, dph, spec.metaHeader});
 }
 
-void DataAllocator::addPartToContext(fair::mq::MessagePtr&& payloadMessage, const Output& spec,
+void DataAllocator::addPartToContext(RouteIndex routeIndex, fair::mq::MessagePtr&& payloadMessage, const Output& spec,
                                      o2::header::SerializationMethod serializationMethod)
 {
-  auto& timingInfo = mRegistry.get<TimingInfo>();
-  RouteIndex routeIndex = matchDataHeader(spec, timingInfo.timeslice);
   auto headerMessage = headerMessageFromOutput(spec, routeIndex, serializationMethod, 0);
 
   // FIXME: this is kind of ugly, we know that we can change the content of the
@@ -284,7 +282,7 @@ void DataAllocator::snapshot(const Output& spec, const char* payload, size_t pay
   fair::mq::MessagePtr payloadMessage(proxy.createOutputMessage(routeIndex, payloadSize));
   memcpy(payloadMessage->GetData(), payload, payloadSize);
 
-  addPartToContext(std::move(payloadMessage), spec, serializationMethod);
+  addPartToContext(routeIndex, std::move(payloadMessage), spec, serializationMethod);
 }
 
 Output DataAllocator::getOutputByBind(OutputRef&& ref)
@@ -348,7 +346,7 @@ void DataAllocator::cookDeadBeef(const Output& spec)
   auto deadBeefOutput = Output{spec.origin, spec.description, 0xdeadbeef};
   auto headerMessage = headerMessageFromOutput(deadBeefOutput, routeIndex, header::gSerializationMethodNone, 0);
 
-  addPartToContext(proxy.createOutputMessage(routeIndex, 0), deadBeefOutput, header::gSerializationMethodNone);
+  addPartToContext(routeIndex, proxy.createOutputMessage(routeIndex, 0), deadBeefOutput, header::gSerializationMethodNone);
 }
 
 } // namespace o2::framework

From 54b809640566f568991d3571557f08b9990555a4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 31 Jan 2024 11:47:51 +0100
Subject: [PATCH 291/726] DPL: mark output proxy as resiliant, if actually the
 case

If the only inputs are sporadic, they can be missing by design.

If the completion policy is consumeWhenAny, it can for sure survive a
crashing upstream device.
---
 Framework/Utils/src/dpl-output-proxy.cxx | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Framework/Utils/src/dpl-output-proxy.cxx b/Framework/Utils/src/dpl-output-proxy.cxx
index fb70f7dfdf4ed..85bd869f3351c 100644
--- a/Framework/Utils/src/dpl-output-proxy.cxx
+++ b/Framework/Utils/src/dpl-output-proxy.cxx
@@ -92,11 +92,13 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
   if (inputs.size() == 0) {
     throw std::runtime_error("invalid dataspec '" + inputConfig + "'");
   }
+  bool isResilient = false;
   // we need to set the lifetime of the inputs to sporadic if requested
   if (sporadicInputs) {
     for (auto& input : inputs) {
       input.lifetime = Lifetime::Sporadic;
     }
+    isResilient = true;
   }
 
   // we build the default channel configuration from the binding of the first input
@@ -133,5 +135,11 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
 
   std::vector<DataProcessorSpec> workflow;
   workflow.emplace_back(std::move(specifyFairMQDeviceOutputProxy(processorName.c_str(), inputs, defaultChannelConfig.c_str())));
+  if (getenv("DPL_OUTPUT_PROXY_WHENANY") && atoi(getenv("DPL_OUTPUT_PROXY_WHENANY"))) {
+    isResilient = true;
+  }
+  if (isResilient) {
+    workflow.back().labels.push_back({"resilient"});
+  }
   return workflow;
 }

From 4c1d3786e3f2d414d9254070ecf60babbfd9038a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 31 Jan 2024 13:58:22 +0100
Subject: [PATCH 292/726] DPL: avoid invoking matchDataHeader twice

---
 Framework/Core/include/Framework/DataAllocator.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/include/Framework/DataAllocator.h b/Framework/Core/include/Framework/DataAllocator.h
index 23b9c62458b5e..8151d2f83c6c6 100644
--- a/Framework/Core/include/Framework/DataAllocator.h
+++ b/Framework/Core/include/Framework/DataAllocator.h
@@ -174,10 +174,10 @@ class DataAllocator
   decltype(auto) make(const Output& spec, Args... args)
   {
     auto& timingInfo = mRegistry.get<TimingInfo>();
-    auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
     auto& context = mRegistry.get<MessageContext>();
 
     if constexpr (is_specialization_v<T, UninitializedVector>) {
+      auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
       // plain buffer as polymorphic spectator std::vector, which does not run constructors / destructors
       using ValueType = typename T::value_type;
 
@@ -187,6 +187,7 @@ class DataAllocator
                       std::move(headerMessage), routeIndex, 0, std::forward<Args>(args)...)
         .get();
     } else if constexpr (is_specialization_v<T, std::vector> && has_messageable_value_type<T>::value) {
+      auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
       // this catches all std::vector objects with messageable value type before checking if is also
       // has a root dictionary, so non-serialized transmission is preferred
       using ValueType = typename T::value_type;
@@ -195,6 +196,7 @@ class DataAllocator
       fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, 0);
       return context.add<MessageContext::VectorObject<ValueType>>(std::move(headerMessage), routeIndex, 0, std::forward<Args>(args)...).get();
     } else if constexpr (has_root_dictionary<T>::value == true && is_messageable<T>::value == false) {
+      auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
       // Extended support for types implementing the Root ClassDef interface, both TObject
       // derived types and others
       if constexpr (enable_root_serialization<T>::value) {
@@ -233,6 +235,7 @@ class DataAllocator
         if constexpr (is_messageable<T>::value == true) {
           auto [nElements] = std::make_tuple(args...);
           auto size = nElements * sizeof(T);
+          auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
 
           fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, size);
           return context.add<MessageContext::SpanObject<T>>(std::move(headerMessage), routeIndex, 0, nElements).get();

From 00e80dec12fa3a43224780642874dda56fea7102 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 31 Jan 2024 20:14:46 +0100
Subject: [PATCH 293/726] DPL: make sure we correctly handle DPL created
 messages

This should fix the issue about the missing DISTSUBTIMEFRAME messages for good.
---
 .../Core/include/Framework/StreamContext.h    |  3 +-
 Framework/Core/src/CommonServices.cxx         | 51 ++++++++++++-------
 Framework/Core/src/DataAllocator.cxx          |  8 ++-
 3 files changed, 41 insertions(+), 21 deletions(-)

diff --git a/Framework/Core/include/Framework/StreamContext.h b/Framework/Core/include/Framework/StreamContext.h
index 8300ce4ce7ac8..79c8ad798836a 100644
--- a/Framework/Core/include/Framework/StreamContext.h
+++ b/Framework/Core/include/Framework/StreamContext.h
@@ -69,7 +69,8 @@ struct StreamContext {
   // for a given iteration.
   // This is in the stream context to allow tracking data creation on a per thread
   // basis.
-  std::vector<bool> routeUserCreated;
+  std::vector<bool> routeDPLCreated;
+  std::vector<bool> routeCreated;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index e6d7ae7fcd159..fc5127c9b05a3 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -173,9 +173,11 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
       // the same stream might be referring to different data processors.
       // We should probably have a context which is per stream of a specific
       // data processor.
-      stream->routeUserCreated.resize(routes.size());
-      // Reset the routeUserCreated at every processing step
-      std::fill(stream->routeUserCreated.begin(), stream->routeUserCreated.end(), false); },
+      stream->routeDPLCreated.resize(routes.size());
+      stream->routeCreated.resize(routes.size());
+      // Reset the routeDPLCreated at every processing step
+      std::fill(stream->routeDPLCreated.begin(), stream->routeDPLCreated.end(), false);
+      std::fill(stream->routeCreated.begin(), stream->routeCreated.end(), false); },
     .postProcessing = [](ProcessingContext& processingContext, void* service) {
       auto* stream = (StreamContext*)service;
       auto& routes = processingContext.services().get<DeviceSpec const>().outputs;
@@ -184,44 +186,50 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
       // Check if we never created any data for this timeslice
       // if we did not, but we still have didDispatched set to true
       // it means it was created out of band.
-      bool didCreate = false;
+      bool userDidCreate = false;
       O2_SIGNPOST_ID_FROM_POINTER(cid, stream_context, service);
       for (size_t ri = 0; ri < routes.size(); ++ri) {
-        if (stream->routeUserCreated[ri] == true) {
-          didCreate = true;
+        if (stream->routeCreated[ri] == true && stream->routeDPLCreated[ri] == false) {
+          userDidCreate = true;
           break;
         }
       }
-      if (didCreate == false && messageContext.didDispatch() == true) {
-        O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Data created out of band didCreate == %d && messageContext.didDispatch == %d",
-                               didCreate,
+      O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "userDidCreate == %d && didDispatch == %d",
+                             userDidCreate,
+                             messageContext.didDispatch());
+      if (userDidCreate == false && messageContext.didDispatch() == true) {
+        O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Data created out of band userDidCreate == %d && messageContext.didDispatch == %d",
+                               userDidCreate,
                                messageContext.didDispatch());
         return;
       }
-      if (didCreate == false && messageContext.didDispatch() == false) {
+      if (userDidCreate == false && messageContext.didDispatch() == false) {
         O2_SIGNPOST_ID_FROM_POINTER(cid, stream_context, service);
-        O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "No data created");
+        O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "No data created.");
         return;
       }
       for (size_t ri = 0; ri < routes.size(); ++ri) {
         auto &route = routes[ri];
         auto &matcher = route.matcher;
-        if (stream->routeUserCreated[ri] == true) {
-          O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Data created by user. ri = %" PRIu64 ", %{public}s",
+        if (stream->routeDPLCreated[ri] == true) {
+          O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Data created by DPL. ri = %" PRIu64 ", %{public}s",
                                  (uint64_t)ri, DataSpecUtils::describe(matcher).c_str());
           continue;
         }
         if ((timeslice % route.maxTimeslices) != route.timeslice) {
-          O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Route ri = %" PRIu64 ", skipped",
+          O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Route ri = %" PRIu64 ", skipped because of pipelining.",
                                  (uint64_t)ri);
           continue;
         }
         if (matcher.lifetime == Lifetime::Timeframe) {
+          O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks",
+                                 "Expected Lifetime::Timeframe data %{public}s was not created for timeslice %" PRIu64 " and might result in dropped timeframes",
+                                 DataSpecUtils::describe(matcher).c_str(), (uint64_t)timeslice);
           LOGP(error, "Expected Lifetime::Timeframe data {} was not created for timeslice {} and might result in dropped timeframes", DataSpecUtils::describe(matcher), timeslice);
         }
       } },
     .preEOS = [](EndOfStreamContext& context, void* service) {
-      // We need to reset the routeUserCreated because the end of stream
+      // We need to reset the routeDPLCreated / routeCreated because the end of stream
       // uses a different context which does not know about the routes.
       // FIXME: This should be fixed in a different way, but for now it will
       // allow TPC IDC to work.
@@ -231,9 +239,11 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
       // the same stream might be referring to different data processors.
       // We should probably have a context which is per stream of a specific
       // data processor.
-      stream->routeUserCreated.resize(routes.size());
-      // Reset the routeUserCreated at every processing step
-      std::fill(stream->routeUserCreated.begin(), stream->routeUserCreated.end(), false); },
+      stream->routeDPLCreated.resize(routes.size());
+      stream->routeCreated.resize(routes.size());
+      // Reset the routeCreated / routeDPLCreated at every processing step
+      std::fill(stream->routeCreated.begin(), stream->routeCreated.end(), false);
+      std::fill(stream->routeDPLCreated.begin(), stream->routeDPLCreated.end(), false); },
     .kind = ServiceKind::Stream};
 }
 
@@ -505,7 +515,10 @@ o2::framework::ServiceSpec CommonServices::ccdbSupportSpec()
           stfDist.runNumber = timingInfo.runNumber;
           // We mark it as not created, because we do should not account for it when
           // checking if we created all the data for a timeslice.
-          streamContext.routeUserCreated[oi] = false;
+          O2_SIGNPOST_ID_FROM_POINTER(sid, stream_context, &streamContext);
+          O2_SIGNPOST_EVENT_EMIT(stream_context, sid, "finaliseOutputs", "Route %" PRIu64 " (%{public}s) was created by DPL.", (uint64_t)oi,
+                                 DataSpecUtils::describe(output.matcher).c_str());
+          streamContext.routeDPLCreated[oi] = true;
         }
       } },
     .kind = ServiceKind::Global};
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index 474be319b4857..1171caa0b0e23 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -20,6 +20,7 @@
 #include "Framework/DataProcessingContext.h"
 #include "Framework/DeviceSpec.h"
 #include "Framework/StreamContext.h"
+#include "Framework/Signpost.h"
 #include "Headers/Stack.h"
 
 #include <fairmq/Device.h>
@@ -33,6 +34,8 @@
 
 #include <utility>
 
+O2_DECLARE_DYNAMIC_LOG(stream_context);
+
 namespace o2::framework
 {
 
@@ -53,7 +56,10 @@ RouteIndex DataAllocator::matchDataHeader(const Output& spec, size_t timeslice)
   for (auto ri = 0; ri < allowedOutputRoutes.size(); ++ri) {
     auto& route = allowedOutputRoutes[ri];
     if (DataSpecUtils::match(route.matcher, spec.origin, spec.description, spec.subSpec) && ((timeslice % route.maxTimeslices) == route.timeslice)) {
-      stream.routeUserCreated.at(ri) = true;
+      stream.routeCreated.at(ri) = true;
+      auto sid = _o2_signpost_id_t{(int64_t)&stream};
+      O2_SIGNPOST_EVENT_EMIT(stream_context, sid, "data_allocator", "Route %" PRIu64 " (%{public}s) created for timeslice %" PRIu64, 
+                             (uint64_t)ri, DataSpecUtils::describe(route.matcher).c_str(), (uint64_t)timeslice);
       return RouteIndex{ri};
     }
   }

From 9cbe4b647a0fe1db279c3051e3a82dd85b795e3e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 1 Feb 2024 13:14:48 +0100
Subject: [PATCH 294/726] DPL: print out how to enable disabled signposts

---
 Framework/Core/src/runDataProcessing.cxx | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index d98d4a5680e8d..9ca46e060b1a2 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -2951,6 +2951,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
     }
   }
 
+  static pid_t pid = getpid();
   if (varmap.count("signposts")) {
     auto signpostsToEnable = varmap["signposts"].as<std::string>();
     auto matchingLogEnabler = [](char const* name, void* l, void* context) {
@@ -2961,6 +2962,8 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
         LOGP(info, "Enabling signposts for {}", *selectedName);
         _o2_log_set_stacktrace(log, 1);
         return false;
+      } else {
+        LOGP(info, "Signpost stream \"{}\" disabled. Enable it with o2-log -p {} -a {}", name, pid, (void*)&log->stacktrace);
       }
       return true;
     };
@@ -2972,6 +2975,13 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
       o2_walk_logs(matchingLogEnabler, token);
       token = strtok_r(nullptr, ",", &saveptr);
     }
+  } else {
+    auto printAllSignposts = [](char const* name, void* l, void* context) {
+      auto* log = (_o2_log_t*)l;
+      LOGP(detail, "Signpost stream {} disabled. Enable it with o2-log -p {} -a {}", name, pid, (void*)&log->stacktrace);
+      return true;
+    };
+    o2_walk_logs(printAllSignposts, nullptr);
   }
 
   auto evaluateBatchOption = [&varmap]() -> bool {

From bf896bcfee9a95c15f415422bcad055c9904b472 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 1 Feb 2024 13:26:45 +0100
Subject: [PATCH 295/726] DPL: properly install o2-log

---
 Framework/Foundation/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Framework/Foundation/CMakeLists.txt b/Framework/Foundation/CMakeLists.txt
index 8babb4bc9a44e..bcde0f7a69f34 100644
--- a/Framework/Foundation/CMakeLists.txt
+++ b/Framework/Foundation/CMakeLists.txt
@@ -61,6 +61,7 @@ set_property(TARGET o2-test-framework-SignpostLogger PROPERTY RUNTIME_OUTPUT_DIR
 set_property(TARGET o2-test-framework-ThreadSanitizer PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
 get_filename_component(bindir ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/../bin ABSOLUTE)
 set_property(TARGET o2-log PROPERTY RUNTIME_OUTPUT_DIRECTORY ${bindir})
+install(TARGETS o2-log RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
 
 add_test(NAME framework:foundation COMMAND o2-test-framework-foundation)
 

From b14aad9b9f8e4617be75b208d2e83d6aceade0f7 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 1 Feb 2024 15:50:29 +0100
Subject: [PATCH 296/726] DPL: make sure we correctly handle DPL created
 messages

This should fix the issue about the missing DISTSUBTIMEFRAME messages for good.
---
 .../Core/include/Framework/StreamContext.h    |  3 +-
 Framework/Core/src/CommonServices.cxx         | 55 ++++++++++++-------
 Framework/Core/src/DataAllocator.cxx          |  8 ++-
 3 files changed, 44 insertions(+), 22 deletions(-)

diff --git a/Framework/Core/include/Framework/StreamContext.h b/Framework/Core/include/Framework/StreamContext.h
index 8300ce4ce7ac8..79c8ad798836a 100644
--- a/Framework/Core/include/Framework/StreamContext.h
+++ b/Framework/Core/include/Framework/StreamContext.h
@@ -69,7 +69,8 @@ struct StreamContext {
   // for a given iteration.
   // This is in the stream context to allow tracking data creation on a per thread
   // basis.
-  std::vector<bool> routeUserCreated;
+  std::vector<bool> routeDPLCreated;
+  std::vector<bool> routeCreated;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index e6d7ae7fcd159..b7fc3410a1931 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -173,9 +173,11 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
       // the same stream might be referring to different data processors.
       // We should probably have a context which is per stream of a specific
       // data processor.
-      stream->routeUserCreated.resize(routes.size());
-      // Reset the routeUserCreated at every processing step
-      std::fill(stream->routeUserCreated.begin(), stream->routeUserCreated.end(), false); },
+      stream->routeDPLCreated.resize(routes.size());
+      stream->routeCreated.resize(routes.size());
+      // Reset the routeDPLCreated at every processing step
+      std::fill(stream->routeDPLCreated.begin(), stream->routeDPLCreated.end(), false);
+      std::fill(stream->routeCreated.begin(), stream->routeCreated.end(), false); },
     .postProcessing = [](ProcessingContext& processingContext, void* service) {
       auto* stream = (StreamContext*)service;
       auto& routes = processingContext.services().get<DeviceSpec const>().outputs;
@@ -184,44 +186,52 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
       // Check if we never created any data for this timeslice
       // if we did not, but we still have didDispatched set to true
       // it means it was created out of band.
-      bool didCreate = false;
+      bool userDidCreate = false;
       O2_SIGNPOST_ID_FROM_POINTER(cid, stream_context, service);
       for (size_t ri = 0; ri < routes.size(); ++ri) {
-        if (stream->routeUserCreated[ri] == true) {
-          didCreate = true;
+        if (stream->routeCreated[ri] == true && stream->routeDPLCreated[ri] == false) {
+          userDidCreate = true;
           break;
         }
       }
-      if (didCreate == false && messageContext.didDispatch() == true) {
-        O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Data created out of band didCreate == %d && messageContext.didDispatch == %d",
-                               didCreate,
+      O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "userDidCreate == %d && didDispatch == %d",
+                             userDidCreate,
+                             messageContext.didDispatch());
+      if (userDidCreate == false && messageContext.didDispatch() == true) {
+        O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Data created out of band userDidCreate == %d && messageContext.didDispatch == %d",
+                               userDidCreate,
                                messageContext.didDispatch());
         return;
       }
-      if (didCreate == false && messageContext.didDispatch() == false) {
+      if (userDidCreate == false && messageContext.didDispatch() == false) {
         O2_SIGNPOST_ID_FROM_POINTER(cid, stream_context, service);
-        O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "No data created");
+        O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "No data created.");
         return;
       }
       for (size_t ri = 0; ri < routes.size(); ++ri) {
         auto &route = routes[ri];
         auto &matcher = route.matcher;
-        if (stream->routeUserCreated[ri] == true) {
-          O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Data created by user. ri = %" PRIu64 ", %{public}s",
+        if (stream->routeDPLCreated[ri] == true) {
+          O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Data created by DPL. ri = %" PRIu64 ", %{public}s",
                                  (uint64_t)ri, DataSpecUtils::describe(matcher).c_str());
           continue;
         }
-        if ((timeslice % route.maxTimeslices) != route.timeslice) {
-          O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Route ri = %" PRIu64 ", skipped",
+        if (stream->routeCreated[ri] == true) {
+          continue;
+        } if ((timeslice % route.maxTimeslices) != route.timeslice) {
+          O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks", "Route ri = %" PRIu64 ", skipped because of pipelining.",
                                  (uint64_t)ri);
           continue;
         }
         if (matcher.lifetime == Lifetime::Timeframe) {
+          O2_SIGNPOST_EVENT_EMIT(stream_context, cid, "postProcessingCallbacks",
+                                 "Expected Lifetime::Timeframe data %{public}s was not created for timeslice %" PRIu64 " and might result in dropped timeframes",
+                                 DataSpecUtils::describe(matcher).c_str(), (uint64_t)timeslice);
           LOGP(error, "Expected Lifetime::Timeframe data {} was not created for timeslice {} and might result in dropped timeframes", DataSpecUtils::describe(matcher), timeslice);
         }
       } },
     .preEOS = [](EndOfStreamContext& context, void* service) {
-      // We need to reset the routeUserCreated because the end of stream
+      // We need to reset the routeDPLCreated / routeCreated because the end of stream
       // uses a different context which does not know about the routes.
       // FIXME: This should be fixed in a different way, but for now it will
       // allow TPC IDC to work.
@@ -231,9 +241,11 @@ o2::framework::ServiceSpec CommonServices::streamContextSpec()
       // the same stream might be referring to different data processors.
       // We should probably have a context which is per stream of a specific
       // data processor.
-      stream->routeUserCreated.resize(routes.size());
-      // Reset the routeUserCreated at every processing step
-      std::fill(stream->routeUserCreated.begin(), stream->routeUserCreated.end(), false); },
+      stream->routeDPLCreated.resize(routes.size());
+      stream->routeCreated.resize(routes.size());
+      // Reset the routeCreated / routeDPLCreated at every processing step
+      std::fill(stream->routeCreated.begin(), stream->routeCreated.end(), false);
+      std::fill(stream->routeDPLCreated.begin(), stream->routeDPLCreated.end(), false); },
     .kind = ServiceKind::Stream};
 }
 
@@ -505,7 +517,10 @@ o2::framework::ServiceSpec CommonServices::ccdbSupportSpec()
           stfDist.runNumber = timingInfo.runNumber;
           // We mark it as not created, because we do should not account for it when
           // checking if we created all the data for a timeslice.
-          streamContext.routeUserCreated[oi] = false;
+          O2_SIGNPOST_ID_FROM_POINTER(sid, stream_context, &streamContext);
+          O2_SIGNPOST_EVENT_EMIT(stream_context, sid, "finaliseOutputs", "Route %" PRIu64 " (%{public}s) was created by DPL.", (uint64_t)oi,
+                                 DataSpecUtils::describe(output.matcher).c_str());
+          streamContext.routeDPLCreated[oi] = true;
         }
       } },
     .kind = ServiceKind::Global};
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index 474be319b4857..de500cd6d6c4e 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -20,6 +20,7 @@
 #include "Framework/DataProcessingContext.h"
 #include "Framework/DeviceSpec.h"
 #include "Framework/StreamContext.h"
+#include "Framework/Signpost.h"
 #include "Headers/Stack.h"
 
 #include <fairmq/Device.h>
@@ -33,6 +34,8 @@
 
 #include <utility>
 
+O2_DECLARE_DYNAMIC_LOG(stream_context);
+
 namespace o2::framework
 {
 
@@ -53,7 +56,10 @@ RouteIndex DataAllocator::matchDataHeader(const Output& spec, size_t timeslice)
   for (auto ri = 0; ri < allowedOutputRoutes.size(); ++ri) {
     auto& route = allowedOutputRoutes[ri];
     if (DataSpecUtils::match(route.matcher, spec.origin, spec.description, spec.subSpec) && ((timeslice % route.maxTimeslices) == route.timeslice)) {
-      stream.routeUserCreated.at(ri) = true;
+      stream.routeCreated.at(ri) = true;
+      auto sid = _o2_signpost_id_t{(int64_t)&stream};
+      O2_SIGNPOST_EVENT_EMIT(stream_context, sid, "data_allocator", "Route %" PRIu64 " (%{public}s) created for timeslice %" PRIu64,
+                             (uint64_t)ri, DataSpecUtils::describe(route.matcher).c_str(), (uint64_t)timeslice);
       return RouteIndex{ri};
     }
   }

From bb5c392d97bd97b7d6e117da03140aaa6eb6e558 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 2 Feb 2024 09:03:03 +0100
Subject: [PATCH 297/726] Use C++20 spaceship operator

See also:

https://en.cppreference.com/w/cpp/language/operator_comparison#Three-way_comparison
https://en.cppreference.com/w/cpp/language/default_comparisons
---
 Detectors/MUON/MID/CTF/include/MIDCTF/CTFHelper.h | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/Detectors/MUON/MID/CTF/include/MIDCTF/CTFHelper.h b/Detectors/MUON/MID/CTF/include/MIDCTF/CTFHelper.h
index 01e519701b501..8ca21658e0942 100644
--- a/Detectors/MUON/MID/CTF/include/MIDCTF/CTFHelper.h
+++ b/Detectors/MUON/MID/CTF/include/MIDCTF/CTFHelper.h
@@ -131,12 +131,8 @@ class CTFHelper
 
     inline friend I operator+(difference_type i, const I& iter) { return iter + i; };
 
-    bool operator!=(const I& other) const noexcept { return mIndex != other.mIndex; }
     bool operator==(const I& other) const noexcept { return mIndex == other.mIndex; }
-    bool operator>(const I& other) const noexcept { return mIndex > other.mIndex; }
-    bool operator<(const I& other) const noexcept { return mIndex < other.mIndex; }
-    bool operator>=(const I& other) const noexcept { return mIndex >= other.mIndex; }
-    bool operator<=(const I& other) const noexcept { return mIndex <= other.mIndex; }
+    bool operator<=>(const I& other) const noexcept { return mIndex <=> other.mIndex; }
 
    protected:
     gsl::span<const OrderRef> mOrder{};

From cad2296e920e1dd83ef59ed6f888225180a4c6c7 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 2 Feb 2024 11:26:18 +0100
Subject: [PATCH 298/726] DPL GUI: display labels for the device

---
 Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
index f54e74d6ac019..1887a41192bc4 100644
--- a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
@@ -354,6 +354,11 @@ void displayDeviceInspector(DeviceSpec const& spec,
   deviceInfoTable("Outputs:", ProcessingStateId::OUTPUT_MATCHERS, states, metrics);
   configurationTable(info.currentConfig, info.currentProvenance);
   optionsTable("Workflow Options", metadata.workflowOptions, control);
+  if (ImGui::CollapsingHeader("Labels", ImGuiTreeNodeFlags_DefaultOpen)) {
+    for (auto &label : spec.labels) {
+      ImGui::Text("%s", label.value.c_str());
+    }
+  }
   servicesTable("Services", spec.services);
   if (ImGui::CollapsingHeader("Command line arguments", ImGuiTreeNodeFlags_DefaultOpen)) {
     static ImGuiTextFilter filter;

From 345d0c5e7851c454f009a13bdb13d3bbe4b684da Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 2 Feb 2024 11:23:30 +0100
Subject: [PATCH 299/726] DPL: fix circular dependency between expendable /
 resilient tasks and output proxy

The output proxy was always sorted last, however this clearly conflicts in the case
there is expendable tasks or resilient one, where the behavior than depends on the
resiliency of the output proxy itself and the data dependencies.

This address the issue and hopefully fixes the problem for good.
---
 Framework/Core/src/TopologyPolicy.cxx | 58 +++++++++++++++++++++++----
 1 file changed, 51 insertions(+), 7 deletions(-)

diff --git a/Framework/Core/src/TopologyPolicy.cxx b/Framework/Core/src/TopologyPolicy.cxx
index fb96eff5af2fc..ebeb70c7950a3 100644
--- a/Framework/Core/src/TopologyPolicy.cxx
+++ b/Framework/Core/src/TopologyPolicy.cxx
@@ -10,9 +10,12 @@
 // or submit itself to any jurisdiction.
 #include "Framework/DataProcessorSpec.h"
 #include "Framework/TopologyPolicy.h"
+#include "Framework/Signpost.h"
 #include <string>
 #include <regex>
 
+O2_DECLARE_DYNAMIC_LOG(topology);
+
 namespace o2::framework
 {
 
@@ -68,15 +71,20 @@ bool dataDeps(DataProcessorSpec const& a, DataProcessorSpec const& b)
 
 bool expendableDataDeps(DataProcessorSpec const& a, DataProcessorSpec const& b)
 {
+  O2_SIGNPOST_ID_GENERATE(sid, topology);
+  O2_SIGNPOST_START(topology, sid, "expendableDataDeps", "Checking if %s depends on %s", a.name.c_str(), b.name.c_str());
   // We never put anything behind the dummy sink.
   if (b.name.find("internal-dpl-injected-dummy-sink") != std::string::npos) {
+    O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "false. %s is dummy sink and it nothing can depend on it.", b.name.c_str());
     return false;
   }
   if (a.name.find("internal-dpl-injected-dummy-sink") != std::string::npos) {
+    O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "true. %s is dummy sink and it nothing can depend on it.", a.name.c_str());
     return true;
   }
   /// If there is an actual dependency between a and b, we return true.
   if (dataDeps(a, b)) {
+    O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "true. %s has a data dependency on %s", a.name.c_str(), b.name.c_str());
     return true;
   }
   // If we are here we do not have any data dependency,
@@ -101,27 +109,35 @@ bool expendableDataDeps(DataProcessorSpec const& a, DataProcessorSpec const& b)
 
   // If none is expendable. We simply return false and sort as usual.
   if (!isAExpendable && !isBExpendable) {
-    LOGP(debug, "Neither {} nor {} are expendable. No dependency beyond data deps.", a.name, b.name);
+    O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "false. Neither %s nor %s are expendable. No dependency beyond data deps.",
+                    a.name.c_str(), b.name.c_str());
     return false;
   }
   // If both are expendable. We return false and sort as usual.
   if (isAExpendable && isBExpendable) {
-    LOGP(debug, "Both {} and {} are expendable. No dependency.", a.name, b.name);
+    O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "false. Both %s and %s are expendable. No dependency.",
+                    a.name.c_str(), b.name.c_str());
     return false;
   }
 
   // If b is expendable but b is resilient, we can keep the same order.
   if (isAExpendable && bResilient) {
-    LOGP(debug, "{} is expendable but b is resilient, no need to add an unneeded dependency", a.name, a.name, b.name);
+    O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "false. %s is expendable but %s is resilient, no need to add an unneeded dependency",
+                    a.name.c_str(), b.name.c_str());
     return false;
   }
   // If a is expendable we consider it as if there was a dependency from a to b,
   // however we still need to check if there is not one already from b to a.
   if (isAExpendable) {
-    LOGP(debug, "{} is expendable. Checking if there is a dependency from {} to {}.", a.name, b.name, a.name);
-    return !dataDeps(b, a);
+    bool hasDependency = dataDeps(b, a);
+    O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "%s is expendable. %s from %s to %s => %s.",
+                    a.name.c_str(), hasDependency ? "There is however an inverse dependency" : "No inverse dependency", b.name.c_str(), a.name.c_str(),
+                    !hasDependency ? "true" : "false");
+    return !hasDependency;
   }
   // b is expendable and a is not. We are fine with no dependency.
+  O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "false. %s is expendable but %s is not. No need to add an unneeded dependency.",
+                  b.name.c_str(), a.name.c_str());
   return false;
 };
 
@@ -135,18 +151,46 @@ TopologyPolicy::DependencyChecker TopologyPolicyHelpers::dataDependency()
 TopologyPolicy::DependencyChecker TopologyPolicyHelpers::alwaysDependent()
 {
   return [](DataProcessorSpec const& dependent, DataProcessorSpec const& ancestor) {
+    O2_SIGNPOST_ID_GENERATE(sid, topology);
+    O2_SIGNPOST_START(topology, sid, "alwaysDependent", "Checking if %s depends on %s", dependent.name.c_str(), ancestor.name.c_str());
     if (dependent.name == ancestor.name) {
+      O2_SIGNPOST_END(topology, sid, "alwaysDependent", "false. %s and %s are the same.", dependent.name.c_str(), ancestor.name.c_str());
       return false;
     }
     if (ancestor.name == "internal-dpl-injected-dummy-sink") {
+      O2_SIGNPOST_END(topology, sid, "alwaysDependent", "false. %s is a dummy sink.", ancestor.name.c_str());
       return false;
     }
     const std::regex matcher(".*output-proxy.*");
     // Check if regex applies
     std::cmatch m;
-    if (std::regex_match(ancestor.name.data(), m, matcher) && std::regex_match(ancestor.name.data(), m, matcher)) {
-      return dataDeps(dependent, ancestor);
+    bool isAncestorOutputProxy = std::regex_match(ancestor.name.data(), m, matcher);
+    // For now dependent is always an output proxy.
+    assert(std::regex_match(dependent.name.data(), m, matcher));
+    bool isAncestorExpendable = std::find_if(ancestor.labels.begin(), ancestor.labels.end(), [](DataProcessorLabel const& label) {
+                                  return label.value == "expendable";
+                                }) != ancestor.labels.end();
+
+    bool isDependentResilient = std::find_if(dependent.labels.begin(), dependent.labels.end(), [](DataProcessorLabel const& label) {
+                                  return label.value == "resilient";
+                                }) != dependent.labels.end();
+    bool isAncestorResilient = std::find_if(ancestor.labels.begin(), ancestor.labels.end(), [](DataProcessorLabel const& label) {
+                                 return label.value == "resilient";
+                               }) != ancestor.labels.end();
+
+    if (!isDependentResilient && isAncestorExpendable) {
+      O2_SIGNPOST_END(topology, sid, "alwaysDependent", "false. Ancestor %s is expendable while %s is non-resilient output proxy (dependent).",
+                      ancestor.name.c_str(), dependent.name.c_str());
+      return false;
+    }
+
+    if (isAncestorOutputProxy || (!isDependentResilient && isAncestorResilient)) {
+      bool hasDependency = dataDeps(dependent, ancestor);
+      O2_SIGNPOST_END(topology, sid, "alwaysDependent", "%s. Dependent %s %s a dependency on ancestor %s.",
+                      hasDependency ? "true" : "false", dependent.name.c_str(), hasDependency ? "has" : "has not", ancestor.name.c_str());
+      return hasDependency;
     }
+    O2_SIGNPOST_END(topology, sid, "alwaysDependent", "true by default. Ancestor %s is not an output proxy.", ancestor.name.c_str());
     return true;
   };
 }

From ef3f58af0917bc72ad1d92b3fb9653f0aad7e58c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 2 Feb 2024 11:42:03 +0100
Subject: [PATCH 300/726] DPL GUI: display labels for the device

---
 Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
index f54e74d6ac019..eb62fcc09d338 100644
--- a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
@@ -354,6 +354,11 @@ void displayDeviceInspector(DeviceSpec const& spec,
   deviceInfoTable("Outputs:", ProcessingStateId::OUTPUT_MATCHERS, states, metrics);
   configurationTable(info.currentConfig, info.currentProvenance);
   optionsTable("Workflow Options", metadata.workflowOptions, control);
+  if (ImGui::CollapsingHeader("Labels", ImGuiTreeNodeFlags_DefaultOpen)) {
+    for (auto& label : spec.labels) {
+      ImGui::Text("%s", label.value.c_str());
+    }
+  }
   servicesTable("Services", spec.services);
   if (ImGui::CollapsingHeader("Command line arguments", ImGuiTreeNodeFlags_DefaultOpen)) {
     static ImGuiTextFilter filter;

From 9a6cd8d52688914650dc36942b7748cd7bed96bd Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 2 Feb 2024 11:42:03 +0100
Subject: [PATCH 301/726] DPL: fix circular dependency between expendable /
 resilient tasks and output proxy

The output proxy was always sorted last, however this clearly conflicts in the case
there is expendable tasks or resilient one, where the behavior than depends on the
resiliency of the output proxy itself and the data dependencies.

This address the issue and hopefully fixes the problem for good.
---
 Framework/Core/src/TopologyPolicy.cxx | 58 +++++++++++++++++++++++----
 1 file changed, 51 insertions(+), 7 deletions(-)

diff --git a/Framework/Core/src/TopologyPolicy.cxx b/Framework/Core/src/TopologyPolicy.cxx
index fb96eff5af2fc..ebeb70c7950a3 100644
--- a/Framework/Core/src/TopologyPolicy.cxx
+++ b/Framework/Core/src/TopologyPolicy.cxx
@@ -10,9 +10,12 @@
 // or submit itself to any jurisdiction.
 #include "Framework/DataProcessorSpec.h"
 #include "Framework/TopologyPolicy.h"
+#include "Framework/Signpost.h"
 #include <string>
 #include <regex>
 
+O2_DECLARE_DYNAMIC_LOG(topology);
+
 namespace o2::framework
 {
 
@@ -68,15 +71,20 @@ bool dataDeps(DataProcessorSpec const& a, DataProcessorSpec const& b)
 
 bool expendableDataDeps(DataProcessorSpec const& a, DataProcessorSpec const& b)
 {
+  O2_SIGNPOST_ID_GENERATE(sid, topology);
+  O2_SIGNPOST_START(topology, sid, "expendableDataDeps", "Checking if %s depends on %s", a.name.c_str(), b.name.c_str());
   // We never put anything behind the dummy sink.
   if (b.name.find("internal-dpl-injected-dummy-sink") != std::string::npos) {
+    O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "false. %s is dummy sink and it nothing can depend on it.", b.name.c_str());
     return false;
   }
   if (a.name.find("internal-dpl-injected-dummy-sink") != std::string::npos) {
+    O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "true. %s is dummy sink and it nothing can depend on it.", a.name.c_str());
     return true;
   }
   /// If there is an actual dependency between a and b, we return true.
   if (dataDeps(a, b)) {
+    O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "true. %s has a data dependency on %s", a.name.c_str(), b.name.c_str());
     return true;
   }
   // If we are here we do not have any data dependency,
@@ -101,27 +109,35 @@ bool expendableDataDeps(DataProcessorSpec const& a, DataProcessorSpec const& b)
 
   // If none is expendable. We simply return false and sort as usual.
   if (!isAExpendable && !isBExpendable) {
-    LOGP(debug, "Neither {} nor {} are expendable. No dependency beyond data deps.", a.name, b.name);
+    O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "false. Neither %s nor %s are expendable. No dependency beyond data deps.",
+                    a.name.c_str(), b.name.c_str());
     return false;
   }
   // If both are expendable. We return false and sort as usual.
   if (isAExpendable && isBExpendable) {
-    LOGP(debug, "Both {} and {} are expendable. No dependency.", a.name, b.name);
+    O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "false. Both %s and %s are expendable. No dependency.",
+                    a.name.c_str(), b.name.c_str());
     return false;
   }
 
   // If b is expendable but b is resilient, we can keep the same order.
   if (isAExpendable && bResilient) {
-    LOGP(debug, "{} is expendable but b is resilient, no need to add an unneeded dependency", a.name, a.name, b.name);
+    O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "false. %s is expendable but %s is resilient, no need to add an unneeded dependency",
+                    a.name.c_str(), b.name.c_str());
     return false;
   }
   // If a is expendable we consider it as if there was a dependency from a to b,
   // however we still need to check if there is not one already from b to a.
   if (isAExpendable) {
-    LOGP(debug, "{} is expendable. Checking if there is a dependency from {} to {}.", a.name, b.name, a.name);
-    return !dataDeps(b, a);
+    bool hasDependency = dataDeps(b, a);
+    O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "%s is expendable. %s from %s to %s => %s.",
+                    a.name.c_str(), hasDependency ? "There is however an inverse dependency" : "No inverse dependency", b.name.c_str(), a.name.c_str(),
+                    !hasDependency ? "true" : "false");
+    return !hasDependency;
   }
   // b is expendable and a is not. We are fine with no dependency.
+  O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "false. %s is expendable but %s is not. No need to add an unneeded dependency.",
+                  b.name.c_str(), a.name.c_str());
   return false;
 };
 
@@ -135,18 +151,46 @@ TopologyPolicy::DependencyChecker TopologyPolicyHelpers::dataDependency()
 TopologyPolicy::DependencyChecker TopologyPolicyHelpers::alwaysDependent()
 {
   return [](DataProcessorSpec const& dependent, DataProcessorSpec const& ancestor) {
+    O2_SIGNPOST_ID_GENERATE(sid, topology);
+    O2_SIGNPOST_START(topology, sid, "alwaysDependent", "Checking if %s depends on %s", dependent.name.c_str(), ancestor.name.c_str());
     if (dependent.name == ancestor.name) {
+      O2_SIGNPOST_END(topology, sid, "alwaysDependent", "false. %s and %s are the same.", dependent.name.c_str(), ancestor.name.c_str());
       return false;
     }
     if (ancestor.name == "internal-dpl-injected-dummy-sink") {
+      O2_SIGNPOST_END(topology, sid, "alwaysDependent", "false. %s is a dummy sink.", ancestor.name.c_str());
       return false;
     }
     const std::regex matcher(".*output-proxy.*");
     // Check if regex applies
     std::cmatch m;
-    if (std::regex_match(ancestor.name.data(), m, matcher) && std::regex_match(ancestor.name.data(), m, matcher)) {
-      return dataDeps(dependent, ancestor);
+    bool isAncestorOutputProxy = std::regex_match(ancestor.name.data(), m, matcher);
+    // For now dependent is always an output proxy.
+    assert(std::regex_match(dependent.name.data(), m, matcher));
+    bool isAncestorExpendable = std::find_if(ancestor.labels.begin(), ancestor.labels.end(), [](DataProcessorLabel const& label) {
+                                  return label.value == "expendable";
+                                }) != ancestor.labels.end();
+
+    bool isDependentResilient = std::find_if(dependent.labels.begin(), dependent.labels.end(), [](DataProcessorLabel const& label) {
+                                  return label.value == "resilient";
+                                }) != dependent.labels.end();
+    bool isAncestorResilient = std::find_if(ancestor.labels.begin(), ancestor.labels.end(), [](DataProcessorLabel const& label) {
+                                 return label.value == "resilient";
+                               }) != ancestor.labels.end();
+
+    if (!isDependentResilient && isAncestorExpendable) {
+      O2_SIGNPOST_END(topology, sid, "alwaysDependent", "false. Ancestor %s is expendable while %s is non-resilient output proxy (dependent).",
+                      ancestor.name.c_str(), dependent.name.c_str());
+      return false;
+    }
+
+    if (isAncestorOutputProxy || (!isDependentResilient && isAncestorResilient)) {
+      bool hasDependency = dataDeps(dependent, ancestor);
+      O2_SIGNPOST_END(topology, sid, "alwaysDependent", "%s. Dependent %s %s a dependency on ancestor %s.",
+                      hasDependency ? "true" : "false", dependent.name.c_str(), hasDependency ? "has" : "has not", ancestor.name.c_str());
+      return hasDependency;
     }
+    O2_SIGNPOST_END(topology, sid, "alwaysDependent", "true by default. Ancestor %s is not an output proxy.", ancestor.name.c_str());
     return true;
   };
 }

From 004bc21cd626ed2bb287146068d5d9be771a8d6d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 2 Feb 2024 12:54:02 +0100
Subject: [PATCH 302/726] DPL Analysis: introduce PersistenColumn concept

This way the compiler does not need to instanciate the conjuction type
and it will point to exactly which column is not persistent, when needed.
---
 Framework/Core/include/Framework/ASoA.h | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h
index 4d07fd124a8fe..4be6455ad3b5e 100644
--- a/Framework/Core/include/Framework/ASoA.h
+++ b/Framework/Core/include/Framework/ASoA.h
@@ -1775,17 +1775,18 @@ void notBoundTable(const char* tableName);
 
 namespace row_helpers
 {
+template <typename T>
+concept PersistentColumn = std::is_same_v<typename T::persistent, std::true_type>;
+
 template <typename... Cs>
-std::array<arrow::ChunkedArray*, sizeof...(Cs)> getArrowColumns(arrow::Table* table, framework::pack<Cs...>)
+std::array<arrow::ChunkedArray*, sizeof...(Cs)> getArrowColumns(arrow::Table* table, framework::pack<Cs...>) requires(PersistentColumn<Cs> && ...)
 {
-  static_assert(std::conjunction_v<typename Cs::persistent...>, "Arrow columns: only persistent columns accepted (not dynamic and not index ones");
   return std::array<arrow::ChunkedArray*, sizeof...(Cs)>{o2::soa::getIndexFromLabel(table, Cs::columnLabel())...};
 }
 
 template <typename... Cs>
-std::array<std::shared_ptr<arrow::Array>, sizeof...(Cs)> getChunks(arrow::Table* table, framework::pack<Cs...>, uint64_t ci)
+std::array<std::shared_ptr<arrow::Array>, sizeof...(Cs)> getChunks(arrow::Table* table, framework::pack<Cs...>, uint64_t ci) requires(PersistentColumn<Cs> && ...)
 {
-  static_assert(std::conjunction_v<typename Cs::persistent...>, "Arrow chunks: only persistent columns accepted (not dynamic and not index ones");
   return std::array<std::shared_ptr<arrow::Array>, sizeof...(Cs)>{o2::soa::getIndexFromLabel(table, Cs::columnLabel())->chunk(ci)...};
 }
 

From 2489a281e9f1d8d00fa9bce8de2798dc43b87163 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 2 Feb 2024 12:54:37 +0100
Subject: [PATCH 303/726] DPL Analysis: introduce PersistentColumn concept

This way the compiler does not need to instanciate the conjuction type
and it will point to exactly which column is not persistent, when needed.
---
 Framework/Core/include/Framework/ASoA.h | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h
index 4d07fd124a8fe..4be6455ad3b5e 100644
--- a/Framework/Core/include/Framework/ASoA.h
+++ b/Framework/Core/include/Framework/ASoA.h
@@ -1775,17 +1775,18 @@ void notBoundTable(const char* tableName);
 
 namespace row_helpers
 {
+template <typename T>
+concept PersistentColumn = std::is_same_v<typename T::persistent, std::true_type>;
+
 template <typename... Cs>
-std::array<arrow::ChunkedArray*, sizeof...(Cs)> getArrowColumns(arrow::Table* table, framework::pack<Cs...>)
+std::array<arrow::ChunkedArray*, sizeof...(Cs)> getArrowColumns(arrow::Table* table, framework::pack<Cs...>) requires(PersistentColumn<Cs> && ...)
 {
-  static_assert(std::conjunction_v<typename Cs::persistent...>, "Arrow columns: only persistent columns accepted (not dynamic and not index ones");
   return std::array<arrow::ChunkedArray*, sizeof...(Cs)>{o2::soa::getIndexFromLabel(table, Cs::columnLabel())...};
 }
 
 template <typename... Cs>
-std::array<std::shared_ptr<arrow::Array>, sizeof...(Cs)> getChunks(arrow::Table* table, framework::pack<Cs...>, uint64_t ci)
+std::array<std::shared_ptr<arrow::Array>, sizeof...(Cs)> getChunks(arrow::Table* table, framework::pack<Cs...>, uint64_t ci) requires(PersistentColumn<Cs> && ...)
 {
-  static_assert(std::conjunction_v<typename Cs::persistent...>, "Arrow chunks: only persistent columns accepted (not dynamic and not index ones");
   return std::array<std::shared_ptr<arrow::Array>, sizeof...(Cs)>{o2::soa::getIndexFromLabel(table, Cs::columnLabel())->chunk(ci)...};
 }
 

From 29e513b5f284a9986d6fab4ed23618fc4200bb55 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 2 Feb 2024 13:06:51 +0100
Subject: [PATCH 304/726] DPL Analysis: introduce PersistentColumn concept

This way the compiler does not need to instanciate the conjuction type
and it will point to exactly which column is not persistent, when needed.
---
 Framework/Core/include/Framework/ASoA.h | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h
index 4d07fd124a8fe..07493c6db091d 100644
--- a/Framework/Core/include/Framework/ASoA.h
+++ b/Framework/Core/include/Framework/ASoA.h
@@ -1775,17 +1775,20 @@ void notBoundTable(const char* tableName);
 
 namespace row_helpers
 {
+template <typename T>
+concept PersistentColumn = std::is_same_v<typename T::persistent, std::true_type>;
+
 template <typename... Cs>
 std::array<arrow::ChunkedArray*, sizeof...(Cs)> getArrowColumns(arrow::Table* table, framework::pack<Cs...>)
+  requires(PersistentColumn<Cs> && ...)
 {
-  static_assert(std::conjunction_v<typename Cs::persistent...>, "Arrow columns: only persistent columns accepted (not dynamic and not index ones");
   return std::array<arrow::ChunkedArray*, sizeof...(Cs)>{o2::soa::getIndexFromLabel(table, Cs::columnLabel())...};
 }
 
 template <typename... Cs>
 std::array<std::shared_ptr<arrow::Array>, sizeof...(Cs)> getChunks(arrow::Table* table, framework::pack<Cs...>, uint64_t ci)
+  requires(PersistentColumn<Cs> && ...)
 {
-  static_assert(std::conjunction_v<typename Cs::persistent...>, "Arrow chunks: only persistent columns accepted (not dynamic and not index ones");
   return std::array<std::shared_ptr<arrow::Array>, sizeof...(Cs)>{o2::soa::getIndexFromLabel(table, Cs::columnLabel())->chunk(ci)...};
 }
 

From 78f4a8d7f5a7663ac3ffb305eacd3c9716a919ca Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 2 Feb 2024 13:15:24 +0100
Subject: [PATCH 305/726] DPL: add ability to disable inputs programmatically

Disabled inputs will not result in an actual route for the data, however it
will be stored in the configuration, so that analysis workflows will not need
to have the configuration available at every step.
---
 Framework/Core/include/Framework/InputSpec.h  |  3 ++
 Framework/Core/include/Framework/OutputSpec.h |  4 ++
 Framework/Core/src/DeviceSpecHelpers.cxx      | 15 +++++-
 Framework/Core/src/WorkflowHelpers.cxx        | 12 +++--
 .../Core/src/WorkflowSerializationHelpers.cxx | 32 ++++++++++++-
 .../Core/test/test_FrameworkDataFlowToDDS.cxx | 48 ++++++++++++-------
 .../Core/test/test_WorkflowSerialization.cxx  | 38 +++++++++++++++
 7 files changed, 130 insertions(+), 22 deletions(-)

diff --git a/Framework/Core/include/Framework/InputSpec.h b/Framework/Core/include/Framework/InputSpec.h
index 9be95e6853da2..7858aacb80664 100644
--- a/Framework/Core/include/Framework/InputSpec.h
+++ b/Framework/Core/include/Framework/InputSpec.h
@@ -74,6 +74,9 @@ struct InputSpec {
 
   /// A set of configurables which can be used to customise the InputSpec.
   std::vector<ConfigParamSpec> metadata;
+  /// Wether or not the input is to be considered enabled.
+  /// Useful to programmatically disable inputs e.g. for the ProcessorOptions.
+  bool enabled = true;
 
   friend std::ostream& operator<<(std::ostream& stream, InputSpec const& arg);
   bool operator==(InputSpec const& that) const;
diff --git a/Framework/Core/include/Framework/OutputSpec.h b/Framework/Core/include/Framework/OutputSpec.h
index e92792faa6b14..51527bce4e339 100644
--- a/Framework/Core/include/Framework/OutputSpec.h
+++ b/Framework/Core/include/Framework/OutputSpec.h
@@ -86,6 +86,10 @@ struct OutputSpec {
   /// A set of configurables which can be used to customise the InputSpec.
   std::vector<ConfigParamSpec> metadata;
 
+  /// Wether or not this output is enabled. This is useful to decide programmatically
+  /// wether or not to produce a given output.
+  bool enabled = true;
+
   friend std::ostream& operator<<(std::ostream& stream, OutputSpec const& arg);
 };
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index b1681a4779ceb..152ee24f1520e 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -679,19 +679,27 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
     assert(policyPtr != nullptr);
 
     if (edge.isForward == false) {
+      auto& matcher = outputsMatchers[edge.outputGlobalIndex];
+      if (matcher.enabled == false) {
+        throw runtime_error_f("Output %s is disabled but it was still used in topology", DataSpecUtils::describe(matcher).data());
+      }
       OutputRoute route{
         edge.timeIndex,
         consumer.maxInputTimeslices,
-        outputsMatchers[edge.outputGlobalIndex],
+        matcher,
         channel.name,
         policyPtr,
       };
       device.outputs.emplace_back(route);
     } else {
+      auto& matcher = workflow[edge.consumer].inputs[edge.consumerInputIndex];
+      if (matcher.enabled == false) {
+        throw runtime_error_f("Output %s is disabled but it was still used in topology", DataSpecUtils::describe(matcher).data());
+      }
       ForwardRoute route{
         edge.timeIndex,
         consumer.maxInputTimeslices,
-        workflow[edge.consumer].inputs[edge.consumerInputIndex],
+        matcher,
         channel.name};
       device.forwards.emplace_back(route);
     }
@@ -915,6 +923,9 @@ void DeviceSpecHelpers::processInEdgeActions(std::vector<DeviceSpec>& devices,
 
     auto const& inputSpec = consumer.inputs[edge.consumerInputIndex];
     auto const& sourceChannel = consumerDevice.inputChannels[ci].name;
+    if (inputSpec.enabled == false) {
+      throw runtime_error_f("Input %s is disabled but it was still used in topology", DataSpecUtils::describe(inputSpec).data());
+    }
 
     InputRoute route{
       inputSpec,
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 6b4ad35f826be..f07c556b32a9b 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -841,11 +841,13 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
     for (size_t wi = 0; wi < workflow.size(); ++wi) {
       auto& producer = workflow[wi];
 
-      for (size_t oi = 0; oi < producer.outputs.size(); ++oi) {
-        auto& out = producer.outputs[oi];
+      for (auto& output : producer.outputs) {
+        if (output.enabled == false) {
+          continue;
+        }
         auto uniqueOutputId = outputs.size();
         availableOutputsInfo.emplace_back(LogicalOutputInfo{wi, uniqueOutputId, false});
-        outputs.push_back(out);
+        outputs.push_back(output);
       }
     }
   };
@@ -879,6 +881,10 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
   std::vector<bool> matches(constOutputs.size());
   for (size_t consumer = 0; consumer < workflow.size(); ++consumer) {
     for (size_t input = 0; input < workflow[consumer].inputs.size(); ++input) {
+      // Skip disabled inputs.
+      if (workflow[consumer].inputs[input].enabled == false) {
+        continue;
+      }
       forwards.clear();
       for (size_t i = 0; i < constOutputs.size(); i++) {
         matches[i] = DataSpecUtils::match(workflow[consumer].inputs[input], constOutputs[i]);
diff --git a/Framework/Core/src/WorkflowSerializationHelpers.cxx b/Framework/Core/src/WorkflowSerializationHelpers.cxx
index e20e23f98c90b..56fd43df4ca54 100644
--- a/Framework/Core/src/WorkflowSerializationHelpers.cxx
+++ b/Framework/Core/src/WorkflowSerializationHelpers.cxx
@@ -61,6 +61,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
     IN_INPUT_ORIGIN,
     IN_INPUT_DESCRIPTION,
     IN_INPUT_SUBSPEC,
+    IN_INPUT_ENABLED,
     IN_INPUT_ORIGIN_REF,
     IN_INPUT_DESCRIPTION_REF,
     IN_INPUT_SUBSPEC_REF,
@@ -75,6 +76,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
     IN_OUTPUT_BINDING,
     IN_OUTPUT_ORIGIN,
     IN_OUTPUT_DESCRIPTION,
+    IN_OUTPUT_ENABLED,
     IN_OUTPUT_SUBSPEC,
     IN_OUTPUT_LIFETIME,
     IN_OUTPUT_OPTIONS,
@@ -167,6 +169,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       case State::IN_INPUT_SUBSPEC:
         s << "IN_INPUT_SUBSPEC";
         break;
+      case State::IN_INPUT_ENABLED:
+        s << "IN_INPUT_ENABLED";
+        break;
       case State::IN_INPUT_ORIGIN_REF:
         s << "IN_INPUT_ORIGIN_REF";
         break;
@@ -218,6 +223,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       case State::IN_OUTPUT_OPTIONS:
         s << "IN_OUTPUT_OPTIONS";
         break;
+      case WorkflowImporter::State::IN_OUTPUT_ENABLED:
+        s << "IN_OUTPUT_ENABLED";
+        break;
       case State::IN_OPTION:
         s << "IN_OPTION";
         break;
@@ -302,6 +310,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       dataProcessors.push_back(DataProcessorSpec{});
     } else if (in(State::IN_INPUTS)) {
       push(State::IN_INPUT);
+      enabled = true;
       inputMatcherNodes.clear();
     } else if (in(State::IN_INPUT_MATCHER)) {
       // start a new embedded matcher
@@ -313,6 +322,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       // will be merged into the parent matcher
     } else if (in(State::IN_OUTPUTS)) {
       push(State::IN_OUTPUT);
+      enabled = true;
       outputHasSubSpec = false;
     } else if (in(State::IN_OPTIONS)) {
       push(State::IN_OPTION);
@@ -370,9 +380,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       } else {
         dataProcessors.back().inputs.push_back(InputSpec({binding}, std::move(*matcher), lifetime, inputOptions));
       }
+      dataProcessors.back().inputs.back().enabled = enabled;
       inputMatcherNodes.clear();
       inputOptions.clear();
-
     } else if (in(State::IN_INPUT_MATCHER) && inputMatcherNodes.size() > 1) {
       data_matcher::Node child = std::move(inputMatcherNodes.back());
       inputMatcherNodes.pop_back();
@@ -433,6 +443,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       } else {
         dataProcessors.back().outputs.push_back(OutputSpec({binding}, {origin, description}, lifetime));
       }
+      dataProcessors.back().outputs.back().enabled = enabled;
       outputHasSubSpec = false;
     } else if (in(State::IN_OPTION)) {
       std::unique_ptr<ConfigParamSpec> opt{nullptr};
@@ -545,12 +556,14 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       push(State::IN_DATAPROCESSORS);
     } else if (in(State::IN_INPUTS)) {
       push(State::IN_INPUT);
+      enabled = true;
     } else if (in(State::IN_INPUT_OPTIONS)) {
       push(State::IN_OPTION);
     } else if (in(State::IN_OUTPUT_OPTIONS)) {
       push(State::IN_OPTION);
     } else if (in(State::IN_OUTPUTS)) {
       push(State::IN_OUTPUT);
+      enabled = true;
       outputHasSubSpec = false;
     } else if (in(State::IN_OPTIONS)) {
       push(State::IN_OPTION);
@@ -602,6 +615,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       push(State::IN_INPUT_DESCRIPTION_REF);
     } else if (in(State::IN_INPUT) && strncmp(str, "subspecRef", length) == 0) {
       push(State::IN_INPUT_SUBSPEC_REF);
+    } else if (in(State::IN_INPUT) && strncmp(str, "enabled", length) == 0) {
+      push(State::IN_INPUT_ENABLED);
     } else if (in(State::IN_INPUT) && strncmp(str, "matcher", length) == 0) {
       // the outermost matcher is starting here
       // we create a placeholder which is being updated later
@@ -664,6 +679,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       push(State::IN_OUTPUT_LIFETIME);
     } else if (in(State::IN_OUTPUT) && strncmp(str, "metadata", length) == 0) {
       push(State::IN_OUTPUT_OPTIONS);
+    } else if (in(State::IN_OUTPUT) && strncmp(str, "enabled", length) == 0) {
+      push(State::IN_OUTPUT_ENABLED);
     } else if (in(State::IN_DATAPROCESSOR) && strncmp(str, "name", length) == 0) {
       push(State::IN_DATAPROCESSOR_NAME);
     } else if (in(State::IN_DATAPROCESSOR) && strncmp(str, "ranks", length) == 0) {
@@ -734,6 +751,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       metadata.back().executable = s;
     } else if (in(State::IN_INPUT_BINDING)) {
       binding = s;
+    } else if (in(State::IN_INPUT_ENABLED)) {
+      enabled = (s == "true");
     } else if (in(State::IN_INPUT_ORIGIN)) {
       origin.runtimeInit(s.c_str(), std::min(s.size(), 4UL));
       std::string v(s.c_str(), std::min(s.size(), 4UL));
@@ -841,6 +860,10 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
       dataProcessors.back().inputTimeSliceId = i;
     } else if (in(State::IN_DATAPROCESSOR_MAX_TIMESLICES)) {
       dataProcessors.back().maxInputTimeslices = i;
+    } else if (in(State::IN_INPUT_ENABLED)) {
+      enabled = (i == 1);
+    } else if (in(State::IN_OUTPUT_ENABLED)) {
+      enabled = (i == 1);
     }
     pop();
     return true;
@@ -915,6 +938,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
   header::DataDescription description;
   size_t subspec;
   size_t ref;
+  // Keep track of the enabled state of the input/output
+  // Unless specified, inputs are enabled by default.
+  bool enabled = true;
   Lifetime lifetime;
   std::string metadatumKey;
   std::string metadatumValue;
@@ -1103,6 +1129,8 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
       }
       w.Key("lifetime");
       w.Uint((int)input.lifetime);
+      w.Key("enabled");
+      w.Uint((int)input.enabled);
       if (input.metadata.empty() == false) {
         w.Key("metadata");
         w.StartArray();
@@ -1152,6 +1180,8 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
       }
       w.Key("lifetime");
       w.Uint((int)output.lifetime);
+      w.Key("enabled");
+      w.Uint((int)output.enabled);
       if (output.metadata.empty() == false) {
         w.Key("metadata");
         w.StartArray();
diff --git a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
index 49420e4bd99ad..1a2fed675536a 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
@@ -169,14 +169,16 @@ TEST_CASE("TestDDS")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;A1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 },
                 {
                     &quot;binding&quot;: &quot;TST/A2/0&quot;,
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;A2&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;options&quot;: [],
@@ -195,7 +197,8 @@ TEST_CASE("TestDDS")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;A1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;outputs&quot;: [
@@ -204,7 +207,8 @@ TEST_CASE("TestDDS")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;B1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;options&quot;: [],
@@ -223,7 +227,8 @@ TEST_CASE("TestDDS")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;A2&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;outputs&quot;: [
@@ -232,7 +237,8 @@ TEST_CASE("TestDDS")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;C1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;options&quot;: [],
@@ -251,14 +257,16 @@ TEST_CASE("TestDDS")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;B1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 },
                 {
                     &quot;binding&quot;: &quot;y&quot;,
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;C1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;outputs&quot;: [],
@@ -433,14 +441,16 @@ TEST_CASE("TestDDSExpendable")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;A1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 },
                 {
                     &quot;binding&quot;: &quot;TST/A2/0&quot;,
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;A2&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;options&quot;: [],
@@ -459,7 +469,8 @@ TEST_CASE("TestDDSExpendable")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;A1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;outputs&quot;: [
@@ -468,7 +479,8 @@ TEST_CASE("TestDDSExpendable")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;B1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;options&quot;: [],
@@ -487,7 +499,8 @@ TEST_CASE("TestDDSExpendable")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;A2&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;outputs&quot;: [
@@ -496,7 +509,8 @@ TEST_CASE("TestDDSExpendable")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;C1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;options&quot;: [],
@@ -515,14 +529,16 @@ TEST_CASE("TestDDSExpendable")
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;B1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 },
                 {
                     &quot;binding&quot;: &quot;y&quot;,
                     &quot;origin&quot;: &quot;TST&quot;,
                     &quot;description&quot;: &quot;C1&quot;,
                     &quot;subspec&quot;: 0,
-                    &quot;lifetime&quot;: 0
+                    &quot;lifetime&quot;: 0,
+                    &quot;enabled&quot;: 1
                 }
             ],
             &quot;outputs&quot;: [],
diff --git a/Framework/Core/test/test_WorkflowSerialization.cxx b/Framework/Core/test/test_WorkflowSerialization.cxx
index 6e541f7d22f07..e7f39e7e38cbd 100644
--- a/Framework/Core/test/test_WorkflowSerialization.cxx
+++ b/Framework/Core/test/test_WorkflowSerialization.cxx
@@ -120,3 +120,41 @@ TEST_CASE("TestVerifyWildcard")
   // also check if the conversion to ConcreteDataMatcher is working at import
   // REQUIRE(std::get_if<ConcreteDataTypeMatcher>(&w1[0].inputs[0].matcher) != nullptr);;
 }
+
+/// Verify that disabled inputs / outputs are serialised correctly
+TEST_CASE("TestDisabled")
+{
+  using namespace o2::framework;
+  WorkflowSpec w0{
+    DataProcessorSpec{
+      .name = "A",
+      .inputs = {{"clbPayload", "CLP"}, {"clbWrapper", "CLW"}},
+      .outputs = {{"CP2", "1"}, {"CL2", "2"}},
+    }};
+
+  w0[0].inputs[0].enabled = false;
+  w0[0].outputs[1].enabled = false;
+
+  std::vector<DataProcessorInfo> dataProcessorInfoOut{
+    {"A", "test_Framework_test_SerializationWorkflow", {}},
+  };
+
+  CommandInfo commandInfoOut{"o2-dpl-workflow -b --option 1 --option 2"};
+
+  std::vector<DataProcessorInfo> dataProcessorInfoIn{};
+  CommandInfo commandInfoIn;
+
+  std::ostringstream firstDump;
+  WorkflowSerializationHelpers::dump(firstDump, w0, dataProcessorInfoOut, commandInfoOut);
+  std::istringstream is;
+  is.str(firstDump.str());
+  WorkflowSpec w1;
+  WorkflowSerializationHelpers::import(is, w1, dataProcessorInfoIn, commandInfoIn);
+  REQUIRE(w1.size() == 1);
+  REQUIRE(w1[0].inputs.size() == 2);
+  REQUIRE(w1[0].inputs[0].enabled == false);
+  REQUIRE(w1[0].inputs[1].enabled == true);
+  REQUIRE(w1[0].outputs.size() == 2);
+  REQUIRE(w1[0].outputs[0].enabled == true);
+  REQUIRE(w1[0].outputs[1].enabled == false);
+}

From 0ebe458f7b5d5ca58439c06333bff523a8139328 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sun, 4 Feb 2024 15:46:21 +0100
Subject: [PATCH 306/726] DPL: move stuff out of line

---
 Framework/Core/CMakeLists.txt                 |  1 +
 .../Core/include/Framework/AlgorithmSpec.h    | 19 ++---------
 Framework/Core/src/AlgorithmSpec.cxx          | 32 +++++++++++++++++++
 3 files changed, 36 insertions(+), 16 deletions(-)
 create mode 100644 Framework/Core/src/AlgorithmSpec.cxx

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index 25f2e60f402c4..05bcb1bd53083 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -11,6 +11,7 @@
 
 o2_add_library(Framework
                SOURCES src/AODReaderHelpers.cxx
+                       src/AlgorithmSpec.cxx
                        src/ArrowSupport.cxx
                        src/ArrowTableSlicingCache.cxx
                        src/AnalysisDataModel.cxx
diff --git a/Framework/Core/include/Framework/AlgorithmSpec.h b/Framework/Core/include/Framework/AlgorithmSpec.h
index 544e6a4840548..98f7f18e0d24d 100644
--- a/Framework/Core/include/Framework/AlgorithmSpec.h
+++ b/Framework/Core/include/Framework/AlgorithmSpec.h
@@ -46,22 +46,9 @@ struct AlgorithmSpec {
   using ErrorCallback = std::function<void(ErrorContext&)>;
   using InitErrorCallback = std::function<void(InitErrorContext&)>;
 
-  static AlgorithmSpec dummyAlgorithm()
-  {
-    return AlgorithmSpec{ProcessCallback{nullptr}};
-  }
-
-  static ErrorCallback& emptyErrorCallback()
-  {
-    static ErrorCallback callback = nullptr;
-    return callback;
-  }
-
-  static InitErrorCallback& emptyInitErrorCallback()
-  {
-    static InitErrorCallback callback = nullptr;
-    return callback;
-  }
+  static AlgorithmSpec dummyAlgorithm();
+  static ErrorCallback& emptyErrorCallback();
+  static InitErrorCallback& emptyInitErrorCallback();
 
   AlgorithmSpec() = default;
 
diff --git a/Framework/Core/src/AlgorithmSpec.cxx b/Framework/Core/src/AlgorithmSpec.cxx
new file mode 100644
index 0000000000000..218083c80947e
--- /dev/null
+++ b/Framework/Core/src/AlgorithmSpec.cxx
@@ -0,0 +1,32 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "Framework/AlgorithmSpec.h"
+
+namespace o2::framework
+{
+AlgorithmSpec AlgorithmSpec::dummyAlgorithm()
+{
+  return AlgorithmSpec{ProcessCallback{nullptr}};
+}
+
+AlgorithmSpec::ErrorCallback& AlgorithmSpec::emptyErrorCallback()
+{
+  static ErrorCallback callback = nullptr;
+  return callback;
+}
+
+AlgorithmSpec::InitErrorCallback& AlgorithmSpec::emptyInitErrorCallback()
+{
+  static InitErrorCallback callback = nullptr;
+  return callback;
+}
+} // namespace o2::framework

From ec23d755b304618b35f3c48fe9094039ab2b9cfd Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sun, 4 Feb 2024 15:48:58 +0100
Subject: [PATCH 307/726] DPL Analysis: introduce PersistentColumn concept

This way the compiler does not need to instanciate the conjuction type
and it will point to exactly which column is not persistent, when needed.
---
 Framework/Core/include/Framework/ASoA.h | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h
index 4d07fd124a8fe..07493c6db091d 100644
--- a/Framework/Core/include/Framework/ASoA.h
+++ b/Framework/Core/include/Framework/ASoA.h
@@ -1775,17 +1775,20 @@ void notBoundTable(const char* tableName);
 
 namespace row_helpers
 {
+template <typename T>
+concept PersistentColumn = std::is_same_v<typename T::persistent, std::true_type>;
+
 template <typename... Cs>
 std::array<arrow::ChunkedArray*, sizeof...(Cs)> getArrowColumns(arrow::Table* table, framework::pack<Cs...>)
+  requires(PersistentColumn<Cs> && ...)
 {
-  static_assert(std::conjunction_v<typename Cs::persistent...>, "Arrow columns: only persistent columns accepted (not dynamic and not index ones");
   return std::array<arrow::ChunkedArray*, sizeof...(Cs)>{o2::soa::getIndexFromLabel(table, Cs::columnLabel())...};
 }
 
 template <typename... Cs>
 std::array<std::shared_ptr<arrow::Array>, sizeof...(Cs)> getChunks(arrow::Table* table, framework::pack<Cs...>, uint64_t ci)
+  requires(PersistentColumn<Cs> && ...)
 {
-  static_assert(std::conjunction_v<typename Cs::persistent...>, "Arrow chunks: only persistent columns accepted (not dynamic and not index ones");
   return std::array<std::shared_ptr<arrow::Array>, sizeof...(Cs)>{o2::soa::getIndexFromLabel(table, Cs::columnLabel())->chunk(ci)...};
 }
 

From 0da3c1ee8a93ffc83bf7917d9d96f2985d21e4f2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 5 Feb 2024 13:12:27 +0100
Subject: [PATCH 308/726] DPL: reduce helpers for makeHolder / makeHolderTypes

C++20 allows simplifying the enumeration.
---
 .../Core/include/Framework/TableBuilder.h     | 42 +++++--------------
 1 file changed, 11 insertions(+), 31 deletions(-)

diff --git a/Framework/Core/include/Framework/TableBuilder.h b/Framework/Core/include/Framework/TableBuilder.h
index 7be20edf2bd93..2b19b43785d70 100644
--- a/Framework/Core/include/Framework/TableBuilder.h
+++ b/Framework/Core/include/Framework/TableBuilder.h
@@ -641,44 +641,24 @@ auto constexpr to_tuple(T&& object) noexcept
   }
 }
 
-template <size_t I, typename... ARGS>
-constexpr auto makeHolderType(framework::pack<ARGS...>&&)
-{
-  return typename HolderTrait<I, framework::pack_element_t<I, framework::pack<ARGS...>>>::Holder{arrow::default_memory_pool()};
-}
-
-template <size_t I, typename... ARGS>
-auto makeHolder(arrow::MemoryPool* pool, size_t nRows, framework::pack<ARGS...>&&)
-{
-  return typename HolderTrait<I, framework::pack_element_t<I, framework::pack<ARGS...>>>::Holder(pool, nRows);
-}
-
-template <size_t... Is, typename... ARGS>
-constexpr auto makeHolderTypesImpl(std::index_sequence<Is...>, framework::pack<ARGS...>&& pack)
-{
-  return std::tuple(makeHolderType<Is>(std::forward<framework::pack<ARGS...>>(pack))...);
-}
-
-template <size_t... Is, typename... ARGS>
-auto makeHoldersImpl(arrow::MemoryPool* pool, size_t nRows, std::index_sequence<Is...>, framework::pack<ARGS...>&& pack)
-{
-  return new std::tuple(makeHolder<Is>(pool, nRows, std::forward<framework::pack<ARGS...>>(pack))...);
-}
-
 template <typename... ARGS>
-constexpr auto makeHolderTypes(framework::pack<ARGS...>&& pack)
+constexpr auto makeHolderTypes()
 {
-  return makeHolderTypesImpl(std::make_index_sequence<sizeof...(ARGS)>{}, std::forward<framework::pack<ARGS...>>(pack));
+  return []<std::size_t... Is>(std::index_sequence<Is...>) {
+    return std::tuple(typename HolderTrait<Is, ARGS>::Holder(arrow::default_memory_pool())...);
+  }(std::make_index_sequence<sizeof...(ARGS)>{});
 }
 
 template <typename... ARGS>
-auto makeHolders(arrow::MemoryPool* pool, size_t nRows, framework::pack<ARGS...>&& pack)
+auto makeHolders(arrow::MemoryPool* pool, size_t nRows)
 {
-  return makeHoldersImpl(pool, nRows, std::make_index_sequence<sizeof...(ARGS)>{}, std::forward<framework::pack<ARGS...>>(pack));
+  return [pool, nRows]<std::size_t... Is>(std::index_sequence<Is...>) {
+    return new std::tuple(typename HolderTrait<Is, ARGS>::Holder(pool, nRows)...);
+  }(std::make_index_sequence<sizeof...(ARGS)>{});
 }
 
 template <typename... ARGS>
-using IndexedHoldersTuple = decltype(makeHolderTypes(framework::pack<ARGS...>{}));
+using IndexedHoldersTuple = decltype(makeHolderTypes<ARGS...>());
 
 /// Helper class which creates a lambda suitable for building
 /// an arrow table from a tuple. This can be used, for example
@@ -691,7 +671,7 @@ class TableBuilder
   using HoldersTuple = typename std::tuple<typename HolderTrait<0, ARGS>::Holder...>;
 
   template <typename... ARGS>
-  using HoldersTupleIndexed = decltype(makeHolderTypes(framework::pack<ARGS...>{}));
+  using HoldersTupleIndexed = decltype(makeHolderTypes<ARGS...>());
 
   /// Get the builders, assumning they were created with a given pack
   ///  of basic types
@@ -708,7 +688,7 @@ class TableBuilder
   {
     mSchema = std::make_shared<arrow::Schema>(TableBuilderHelpers::makeFields<ARGS...>(columnNames));
 
-    mHolders = makeHolders(mMemoryPool, nRows, framework::pack<ARGS...>{});
+    mHolders = makeHolders<ARGS...>(mMemoryPool, nRows);
     mFinalizer = [](std::vector<std::shared_ptr<arrow::Array>>& arrays, void* holders) -> bool {
       return TableBuilderHelpers::finalize(arrays, *(HoldersTupleIndexed<ARGS...>*)holders);
     };

From 4623a2f7f3b1544acd5bc1de060ccbcafa280f58 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 7 Feb 2024 08:54:50 +0100
Subject: [PATCH 309/726] DPL Analysis: avoid extra cursorHelper methods

---
 .../Core/include/Framework/TableBuilder.h     | 23 +++++--------------
 1 file changed, 6 insertions(+), 17 deletions(-)

diff --git a/Framework/Core/include/Framework/TableBuilder.h b/Framework/Core/include/Framework/TableBuilder.h
index 2b19b43785d70..7de45f58d2e62 100644
--- a/Framework/Core/include/Framework/TableBuilder.h
+++ b/Framework/Core/include/Framework/TableBuilder.h
@@ -795,13 +795,17 @@ class TableBuilder
   template <typename T>
   auto cursor()
   {
-    return cursorHelper(typename T::table_t::persistent_columns_t{});
+    return [this]<typename... Cs>(pack<Cs...>) {
+      return this->template persist<typename Cs::type...>({Cs::columnLabel()...});
+    }(typename T::table_t::persistent_columns_t{});
   }
 
   template <typename T, typename E>
   auto cursor()
   {
-    return cursorHelper2<E>(typename T::table_t::persistent_columns_t{});
+    return [this]<typename... Cs>(pack<Cs...>) {
+      return this->template persist<E>({Cs::columnLabel()...});
+    }(typename T::table_t::persistent_columns_t{});
   }
 
   template <typename... ARGS, size_t NCOLUMNS = sizeof...(ARGS)>
@@ -861,21 +865,6 @@ class TableBuilder
   std::shared_ptr<arrow::Table> finalize();
 
  private:
-  /// Helper which actually creates the insertion cursor. Notice that the
-  /// template argument T is a o2::soa::Table which contains only the
-  /// persistent columns.
-  template <typename... Cs>
-  auto cursorHelper(framework::pack<Cs...>)
-  {
-    return this->template persist<typename Cs::type...>({Cs::columnLabel()...});
-  }
-
-  template <typename E, typename... Cs>
-  auto cursorHelper2(framework::pack<Cs...>)
-  {
-    return this->template persist<E>({Cs::columnLabel()...});
-  }
-
   bool (*mFinalizer)(std::vector<std::shared_ptr<arrow::Array>>& arrays, void* holders);
   void (*mDestructor)(void* holders);
   void* mHolders;

From 34bb338e82d72b0ccab05ba5faecda72f06bb220 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 8 Feb 2024 13:40:53 +0100
Subject: [PATCH 310/726] DPL: use the SendingPolicy for every kind of message
 sending

---
 .../Core/include/Framework/ChannelInfo.h      |  3 ++
 .../include/Framework/DataProcessingHelpers.h | 22 +++++-----
 .../Core/include/Framework/ForwardRoute.h     |  9 ++--
 Framework/Core/src/CommonServices.cxx         | 10 ++---
 Framework/Core/src/DataProcessingDevice.cxx   | 22 +++-------
 Framework/Core/src/DataProcessingHelpers.cxx  | 41 ++++++++-----------
 Framework/Core/src/DeviceSpecHelpers.cxx      | 20 +++++----
 Framework/Core/src/FairMQDeviceProxy.cxx      |  3 +-
 .../src/DataInspectorService.cxx              |  2 +-
 Utilities/DataSampling/src/Dispatcher.cxx     |  3 +-
 10 files changed, 63 insertions(+), 72 deletions(-)

diff --git a/Framework/Core/include/Framework/ChannelInfo.h b/Framework/Core/include/Framework/ChannelInfo.h
index 0ebc4a8506301..582ef72fec75d 100644
--- a/Framework/Core/include/Framework/ChannelInfo.h
+++ b/Framework/Core/include/Framework/ChannelInfo.h
@@ -74,6 +74,7 @@ struct OutputChannelInfo {
   ChannelAccountingType channelType = ChannelAccountingType::DPL;
   fair::mq::Channel& channel;
   SendingPolicy const* policy;
+  ChannelIndex index = {-1};
 };
 
 struct OutputChannelState {
@@ -89,6 +90,8 @@ struct ForwardChannelInfo {
   /// Wether or not it's a DPL internal channel.
   ChannelAccountingType channelType = ChannelAccountingType::DPL;
   fair::mq::Channel& channel;
+  SendingPolicy const* policy;
+  ChannelIndex index = {-1};
 };
 
 struct ForwardChannelState {
diff --git a/Framework/Core/include/Framework/DataProcessingHelpers.h b/Framework/Core/include/Framework/DataProcessingHelpers.h
index 35f78f12cc405..7a05ddf25c46c 100644
--- a/Framework/Core/include/Framework/DataProcessingHelpers.h
+++ b/Framework/Core/include/Framework/DataProcessingHelpers.h
@@ -11,14 +11,16 @@
 #ifndef O2_FRAMEWORK_DATAPROCESSINGHELPERS_H_
 #define O2_FRAMEWORK_DATAPROCESSINGHELPERS_H_
 
-#include "Framework/TimesliceIndex.h"
-#include <fairmq/FwdDecls.h>
+#include <cstddef>
 
 namespace o2::framework
 {
-
+struct ServiceRegistryRef;
+struct ForwardChannelInfo;
+struct ForwardChannelState;
+struct OutputChannelInfo;
 struct OutputChannelSpec;
-class FairMQDeviceProxy;
+struct OutputChannelState;
 
 /// Generic helpers for DataProcessing releated functions.
 struct DataProcessingHelpers {
@@ -26,15 +28,13 @@ struct DataProcessingHelpers {
   /// @param device the fair::mq::Device which needs to send the EndOfStream message
   /// @param channel the OutputChannelSpec of the channel which needs to be signaled
   ///        for EndOfStream
-  static void sendEndOfStream(fair::mq::Device& device, OutputChannelSpec const& channel);
+  static void sendEndOfStream(ServiceRegistryRef const& ref, OutputChannelSpec const& channel);
   /// @returns true if we did send the oldest possible timeslice message, false otherwise.
-  static bool sendOldestPossibleTimeframe(ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice);
+  static bool sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice);
   /// @returns true if we did send the oldest possible timeslice message, false otherwise.
-  static bool sendOldestPossibleTimeframe(OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice);
-  static void broadcastOldestPossibleTimeslice(FairMQDeviceProxy& proxy, size_t timeslice);
-
- private:
-  static void sendOldestPossibleTimeframe(fair::mq::Channel& channel, size_t timeslice);
+  static bool sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice);
+  /// Broadcast the oldest possible timeslice to all channels in output
+  static void broadcastOldestPossibleTimeslice(ServiceRegistryRef const& ref, size_t timeslice);
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/include/Framework/ForwardRoute.h b/Framework/Core/include/Framework/ForwardRoute.h
index bae2eaacf1a44..40a76e3a9cce5 100644
--- a/Framework/Core/include/Framework/ForwardRoute.h
+++ b/Framework/Core/include/Framework/ForwardRoute.h
@@ -15,11 +15,11 @@
 #include <cstddef>
 #include <string>
 
-namespace o2
-{
-namespace framework
+namespace o2::framework
 {
 
+struct SendingPolicy;
+
 /// This uniquely identifies a route to be forwarded by the device if
 /// the InputSpec @a matcher matches an input which should also go to
 /// @a channel
@@ -28,8 +28,9 @@ struct ForwardRoute {
   size_t maxTimeslices;
   InputSpec matcher;
   std::string channel;
+  // The policy to use to send to on this route.
+  SendingPolicy const* policy;
 };
 
-} // namespace framework
 } // namespace o2
 #endif // FRAMEWORK_FORWARDROUTE_H
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index b7fc3410a1931..a0f7fcdc88201 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -588,7 +588,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           timesliceIndex.rescan();
         }
       }
-      DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, oldestPossibleOutput.timeslice.value);
+      DataProcessingHelpers::broadcastOldestPossibleTimeslice(ctx.services(), oldestPossibleOutput.timeslice.value);
 
       for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
         auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
@@ -598,7 +598,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
           continue;
         }
-        if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestPossibleOutput.timeslice.value)) {
+        if (DataProcessingHelpers::sendOldestPossibleTimeframe(ctx.services(), info, state, oldestPossibleOutput.timeslice.value)) {
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice",
                                  "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
                                  info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
@@ -646,7 +646,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
       O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Queueing oldest possible timeslice %" PRIu64 " propagation for execution.",
                              (uint64_t)oldestPossibleOutput.timeslice.value);
       AsyncQueueHelpers::post(
-        queue, decongestion.oldestPossibleTimesliceTask, [oldestPossibleOutput, &decongestion, &proxy, &spec, device, &timesliceIndex]() {
+        queue, decongestion.oldestPossibleTimesliceTask, [ref = services, oldestPossibleOutput, &decongestion, &proxy, &spec, device, &timesliceIndex]() {
           O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, &decongestion);
           if (decongestion.lastTimeslice >= oldestPossibleOutput.timeslice.value) {
             O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Not sending already sent value: %" PRIu64 "> %" PRIu64,
@@ -655,7 +655,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           }
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Running oldest possible timeslice %" PRIu64 " propagation.",
                                  (uint64_t)oldestPossibleOutput.timeslice.value);
-          DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, oldestPossibleOutput.timeslice.value);
+          DataProcessingHelpers::broadcastOldestPossibleTimeslice(ref, oldestPossibleOutput.timeslice.value);
 
           for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
             auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
@@ -665,7 +665,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
               O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
               continue;
             }
-            if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestPossibleOutput.timeslice.value)) {
+            if (DataProcessingHelpers::sendOldestPossibleTimeframe(ref, info, state, oldestPossibleOutput.timeslice.value)) {
               O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice",
                                      "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
                                      info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 8c914e201a479..941da7bc968c4 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -680,19 +680,11 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
     if (forwardedParts[fi].Size() == 0) {
       continue;
     }
-    auto channel = proxy.getForwardChannel(ChannelIndex{fi});
-    LOG(debug) << "Forwarding to " << channel->GetName() << " " << fi;
+    ForwardChannelInfo info = proxy.getForwardChannelInfo(ChannelIndex{fi});
+    LOG(debug) << "Forwarding to " << info.name << " " << fi;
     // in DPL we are using subchannel 0 only
     auto& parts = forwardedParts[fi];
-    int timeout = 30000;
-    auto res = channel->Send(parts, timeout);
-    if (res == (size_t)fair::mq::TransferCode::timeout) {
-      LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on {}.", timeout / 1000, channel->GetName());
-      channel->Send(parts);
-      LOGP(info, "Downstream backpressure on {} recovered.", channel->GetName());
-    } else if (res == (size_t)fair::mq::TransferCode::error) {
-      LOGP(fatal, "Error while sending on channel {}", channel->GetName());
-    }
+    info.policy->send(parts, ChannelIndex{fi}, registry); 
   }
 
   auto& asyncQueue = registry.get<AsyncQueue>();
@@ -713,7 +705,7 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
           LOG(debug) << "Skipping channel";
           continue;
         }
-        if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestTimeslice.timeslice.value)) {
+        if (DataProcessingHelpers::sendOldestPossibleTimeframe(registry, info, state, oldestTimeslice.timeslice.value)) {
           LOGP(debug, "Forwarding to channel {} oldest possible timeslice {}, prio 20", info.name, oldestTimeslice.timeslice.value);
         }
       }
@@ -1678,8 +1670,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
 
     for (auto& channel : spec.outputChannels) {
       LOGP(detail, "Sending end of stream to {}", channel.name);
-      auto& rawDevice = ref.get<RawDeviceService>();
-      DataProcessingHelpers::sendEndOfStream(*rawDevice.device(), channel);
+      DataProcessingHelpers::sendEndOfStream(ref, channel);
     }
     // This is needed because the transport is deleted before the device.
     relayer.clear();
@@ -2460,8 +2451,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   if (state.streaming == StreamingState::EndOfStreaming) {
     LOGP(detail, "Broadcasting end of stream");
     for (auto& channel : spec.outputChannels) {
-      auto& rawDevice = ref.get<RawDeviceService>();
-      DataProcessingHelpers::sendEndOfStream(*rawDevice.device(), channel);
+      DataProcessingHelpers::sendEndOfStream(ref, channel);
     }
     switchState(StreamingState::Idle);
   }
diff --git a/Framework/Core/src/DataProcessingHelpers.cxx b/Framework/Core/src/DataProcessingHelpers.cxx
index d424a53a52bd9..6dc3379d41cf8 100644
--- a/Framework/Core/src/DataProcessingHelpers.cxx
+++ b/Framework/Core/src/DataProcessingHelpers.cxx
@@ -18,79 +18,74 @@
 #include "Headers/DataHeader.h"
 #include "Headers/Stack.h"
 #include "Framework/Logger.h"
+#include "Framework/SendingPolicy.h"
+#include "Framework/RawDeviceService.h"
 
 #include <fairmq/Device.h>
 #include <fairmq/Channel.h>
 
 namespace o2::framework
 {
-void DataProcessingHelpers::sendEndOfStream(fair::mq::Device& device, OutputChannelSpec const& channel)
+void DataProcessingHelpers::sendEndOfStream(ServiceRegistryRef const& ref, OutputChannelSpec const& channel)
 {
+  fair::mq::Device* device = ref.get<RawDeviceService>().device();
   fair::mq::Parts parts;
-  fair::mq::MessagePtr payload(device.NewMessage());
+  fair::mq::MessagePtr payload(device->NewMessage());
   SourceInfoHeader sih;
   sih.state = InputChannelState::Completed;
-  auto channelAlloc = o2::pmr::getTransportAllocator(device.GetChannel(channel.name, 0).Transport());
+  auto channelAlloc = o2::pmr::getTransportAllocator(device->GetChannel(channel.name, 0).Transport());
   auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, sih});
   // sigh... See if we can avoid having it const by not
   // exposing it to the user in the first place.
   parts.AddPart(std::move(header));
   parts.AddPart(std::move(payload));
-  device.Send(parts, channel.name, 0);
+  device->Send(parts, channel.name, 0);
   LOGP(info, "Sending end-of-stream message to channel {}", channel.name);
 }
 
-void DataProcessingHelpers::sendOldestPossibleTimeframe(fair::mq::Channel& channel, size_t timeslice)
+void doSendOldestPossibleTimeframe(ServiceRegistryRef ref, fair::mq::TransportFactory *transport, ChannelIndex index, SendingPolicy::SendingCallback const& callback, size_t timeslice)
 {
   fair::mq::Parts parts;
-  fair::mq::MessagePtr payload(channel.Transport()->CreateMessage());
+  fair::mq::MessagePtr payload(transport->CreateMessage());
   o2::framework::DomainInfoHeader dih;
   dih.oldestPossibleTimeslice = timeslice;
-  auto channelAlloc = o2::pmr::getTransportAllocator(channel.Transport());
+  auto channelAlloc = o2::pmr::getTransportAllocator(transport);
   auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, dih});
   // sigh... See if we can avoid having it const by not
   // exposing it to the user in the first place.
   parts.AddPart(std::move(header));
   parts.AddPart(std::move(payload));
 
-  auto timeout = 1000;
-  auto res = channel.Send(parts, timeout);
-  if (res == (size_t)fair::mq::TransferCode::timeout) {
-    LOGP(warning, "Timed out sending oldest possible timeslice after {}s. Downstream backpressure detected on {}.", timeout / 1000, channel.GetName());
-    channel.Send(parts);
-    LOGP(info, "Downstream backpressure on {} recovered.", channel.GetName());
-  }
-  if (res < (size_t)fair::mq::TransferCode::success) {
-    LOGP(fatal, "Error sending oldest possible timeframe {} on channel {} (code {})", timeslice, channel.GetName(), res);
-  }
+  callback(parts, index, ref);
 }
 
-bool DataProcessingHelpers::sendOldestPossibleTimeframe(ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice)
+bool DataProcessingHelpers::sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice)
 {
   if (state.oldestForChannel.value >= timeslice) {
     return false;
   }
-  sendOldestPossibleTimeframe(info.channel, timeslice);
+  doSendOldestPossibleTimeframe(ref, info.channel.Transport(), info.index, info.policy->send, timeslice);
   state.oldestForChannel = {timeslice};
   return true;
 }
 
-bool DataProcessingHelpers::sendOldestPossibleTimeframe(OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice)
+bool DataProcessingHelpers::sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice)
 {
   if (state.oldestForChannel.value >= timeslice) {
     return false;
   }
-  sendOldestPossibleTimeframe(info.channel, timeslice);
+  doSendOldestPossibleTimeframe(ref, info.channel.Transport(), info.index, info.policy->send, timeslice);
   state.oldestForChannel = {timeslice};
   return true;
 }
 
-void DataProcessingHelpers::broadcastOldestPossibleTimeslice(FairMQDeviceProxy& proxy, size_t timeslice)
+void DataProcessingHelpers::broadcastOldestPossibleTimeslice(ServiceRegistryRef const& ref, size_t timeslice)
 {
+  auto &proxy = ref.get<FairMQDeviceProxy>();
   for (int ci = 0; ci < proxy.getNumOutputChannels(); ++ci) {
     auto& info = proxy.getOutputChannelInfo({ci});
     auto& state = proxy.getOutputChannelState({ci});
-    sendOldestPossibleTimeframe(info, state, timeslice);
+    sendOldestPossibleTimeframe(ref, info, state, timeslice);
   }
 }
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index b1681a4779ceb..90b4f8c0c5226 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -680,19 +680,21 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
 
     if (edge.isForward == false) {
       OutputRoute route{
-        edge.timeIndex,
-        consumer.maxInputTimeslices,
-        outputsMatchers[edge.outputGlobalIndex],
-        channel.name,
-        policyPtr,
+        .timeslice = edge.timeIndex,
+        .maxTimeslices = consumer.maxInputTimeslices,
+        .matcher = outputsMatchers[edge.outputGlobalIndex],
+        .channel = channel.name,
+        .policy = policyPtr,
       };
       device.outputs.emplace_back(route);
     } else {
       ForwardRoute route{
-        edge.timeIndex,
-        consumer.maxInputTimeslices,
-        workflow[edge.consumer].inputs[edge.consumerInputIndex],
-        channel.name};
+        .timeslice = edge.timeIndex,
+        .maxTimeslices = consumer.maxInputTimeslices,
+        .matcher = workflow[edge.consumer].inputs[edge.consumerInputIndex],
+        .channel = channel.name,
+        .policy = policyPtr,
+      };
       device.forwards.emplace_back(route);
     }
   };
diff --git a/Framework/Core/src/FairMQDeviceProxy.cxx b/Framework/Core/src/FairMQDeviceProxy.cxx
index c25a58e47ef2d..bdffddd5a4d1a 100644
--- a/Framework/Core/src/FairMQDeviceProxy.cxx
+++ b/Framework/Core/src/FairMQDeviceProxy.cxx
@@ -267,6 +267,7 @@ void FairMQDeviceProxy::bind(std::vector<OutputRoute> const& outputs, std::vecto
           .channelType = dplChannel,
           .channel = channel->second.at(0),
           .policy = route.policy,
+          .index = channelIndex,
         };
         mOutputChannelInfos.push_back(info);
         mOutputChannelStates.push_back({0});
@@ -345,7 +346,7 @@ void FairMQDeviceProxy::bind(std::vector<OutputRoute> const& outputs, std::vecto
           LOGP(fatal, "Expected channel {} not configured.", route.channel);
         }
         ChannelAccountingType dplChannel = (route.channel.rfind("from_", 0) == 0) ? ChannelAccountingType::DPL : ChannelAccountingType::RAWFMQ;
-        mForwardChannelInfos.push_back(ForwardChannelInfo{route.channel, dplChannel, channel->second.at(0)});
+        mForwardChannelInfos.push_back(ForwardChannelInfo{.name = route.channel, .channelType = dplChannel, .channel = channel->second.at(0), .policy = route.policy, .index = channelIndex});
         mForwardChannelStates.push_back(ForwardChannelState{0});
         channelNameToChannel[route.channel] = channelIndex;
         LOGP(detail, "Binding forward channel {} to channel index {}", route.channel, channelIndex.value);
diff --git a/Framework/DataInspector/src/DataInspectorService.cxx b/Framework/DataInspector/src/DataInspectorService.cxx
index 40399a366a303..712bf10b6fb2e 100644
--- a/Framework/DataInspector/src/DataInspectorService.cxx
+++ b/Framework/DataInspector/src/DataInspectorService.cxx
@@ -245,4 +245,4 @@ ServiceSpec* DIServicePlugin::create()
             } },
     .kind = ServiceKind::Global};
 };
-} // namespace o2::framework
\ No newline at end of file
+} // namespace o2::framework
diff --git a/Utilities/DataSampling/src/Dispatcher.cxx b/Utilities/DataSampling/src/Dispatcher.cxx
index de78993ff083e..28ff4d5568da9 100644
--- a/Utilities/DataSampling/src/Dispatcher.cxx
+++ b/Utilities/DataSampling/src/Dispatcher.cxx
@@ -125,10 +125,9 @@ void Dispatcher::run(ProcessingContext& ctx)
   if (ctx.inputs().isValid("timer-stats")) {
     reportStats(ctx.services().get<Monitoring>());
   }
-  auto& proxy = ctx.services().get<FairMQDeviceProxy>();
   auto& relayer = ctx.services().get<DataRelayer>();
   auto timeslice = relayer.getOldestPossibleOutput().timeslice.value;
-  DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, timeslice);
+  DataProcessingHelpers::broadcastOldestPossibleTimeslice(ctx.services(), timeslice);
 }
 
 void Dispatcher::reportStats(Monitoring& monitoring) const

From d723ff0539287817f1b0605557bd5dcc0a3fbd8d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 8 Feb 2024 13:40:53 +0100
Subject: [PATCH 311/726] DPL: add SendingPolicy for the case destination is
 expendable

---
 Framework/Core/src/SendingPolicy.cxx | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 17faef6dbd3d0..e8eb0f25914d5 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -103,6 +103,23 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               } else if (res == (size_t) fair::mq::TransferCode::error) {
                 LOGP(fatal, "Error while sending on channel {}", channel->GetName());
               } }},
+          SendingPolicy{
+            .name = "expendable",
+            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) { 
+              auto has_label = [](DataProcessorLabel const& label) {
+                return label.value == "expendable";
+              };
+              return std::find_if(dest.labels.begin(), dest.labels.end(), has_label) != dest.labels.end(); },
+            .send = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
+              auto &proxy = registry.get<FairMQDeviceProxy>();
+              auto *channel = proxy.getOutputChannel(channelIndex);
+              auto timeout = 1000;
+              auto res = channel->Send(parts, timeout);
+              if (res == (size_t)fair::mq::TransferCode::timeout) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
+              } else if (res == (size_t) fair::mq::TransferCode::error) {
+                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              } }},
           SendingPolicy{
             .name = "default",
             .matcher = [](DataProcessorSpec const&, DataProcessorSpec const&, ConfigContext const&) { return true; },

From 454be2b9ae2dc29e7447dd6380fe7ead4a86afc4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 8 Feb 2024 13:46:55 +0100
Subject: [PATCH 312/726] DPL: add SendingPolicy for the case destination is
 expendable

---
 Framework/Core/src/SendingPolicy.cxx | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 17faef6dbd3d0..e8eb0f25914d5 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -103,6 +103,23 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               } else if (res == (size_t) fair::mq::TransferCode::error) {
                 LOGP(fatal, "Error while sending on channel {}", channel->GetName());
               } }},
+          SendingPolicy{
+            .name = "expendable",
+            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) { 
+              auto has_label = [](DataProcessorLabel const& label) {
+                return label.value == "expendable";
+              };
+              return std::find_if(dest.labels.begin(), dest.labels.end(), has_label) != dest.labels.end(); },
+            .send = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
+              auto &proxy = registry.get<FairMQDeviceProxy>();
+              auto *channel = proxy.getOutputChannel(channelIndex);
+              auto timeout = 1000;
+              auto res = channel->Send(parts, timeout);
+              if (res == (size_t)fair::mq::TransferCode::timeout) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
+              } else if (res == (size_t) fair::mq::TransferCode::error) {
+                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              } }},
           SendingPolicy{
             .name = "default",
             .matcher = [](DataProcessorSpec const&, DataProcessorSpec const&, ConfigContext const&) { return true; },

From f76051457b59c5134945afa228927eb1e44867bc Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 8 Feb 2024 13:46:55 +0100
Subject: [PATCH 313/726] DPL: use the SendingPolicy for every kind of message
 sending

---
 .../Core/include/Framework/ChannelInfo.h      |  3 ++
 .../include/Framework/DataProcessingHelpers.h | 22 +++++-----
 .../Core/include/Framework/ForwardRoute.h     |  9 ++--
 Framework/Core/src/CommonServices.cxx         | 10 ++---
 Framework/Core/src/DataProcessingDevice.cxx   | 22 +++-------
 Framework/Core/src/DataProcessingHelpers.cxx  | 41 ++++++++-----------
 Framework/Core/src/DeviceSpecHelpers.cxx      | 20 +++++----
 Framework/Core/src/FairMQDeviceProxy.cxx      |  3 +-
 .../src/DataInspectorService.cxx              |  2 +-
 Utilities/DataSampling/src/Dispatcher.cxx     |  3 +-
 10 files changed, 63 insertions(+), 72 deletions(-)

diff --git a/Framework/Core/include/Framework/ChannelInfo.h b/Framework/Core/include/Framework/ChannelInfo.h
index 0ebc4a8506301..582ef72fec75d 100644
--- a/Framework/Core/include/Framework/ChannelInfo.h
+++ b/Framework/Core/include/Framework/ChannelInfo.h
@@ -74,6 +74,7 @@ struct OutputChannelInfo {
   ChannelAccountingType channelType = ChannelAccountingType::DPL;
   fair::mq::Channel& channel;
   SendingPolicy const* policy;
+  ChannelIndex index = {-1};
 };
 
 struct OutputChannelState {
@@ -89,6 +90,8 @@ struct ForwardChannelInfo {
   /// Wether or not it's a DPL internal channel.
   ChannelAccountingType channelType = ChannelAccountingType::DPL;
   fair::mq::Channel& channel;
+  SendingPolicy const* policy;
+  ChannelIndex index = {-1};
 };
 
 struct ForwardChannelState {
diff --git a/Framework/Core/include/Framework/DataProcessingHelpers.h b/Framework/Core/include/Framework/DataProcessingHelpers.h
index 35f78f12cc405..7a05ddf25c46c 100644
--- a/Framework/Core/include/Framework/DataProcessingHelpers.h
+++ b/Framework/Core/include/Framework/DataProcessingHelpers.h
@@ -11,14 +11,16 @@
 #ifndef O2_FRAMEWORK_DATAPROCESSINGHELPERS_H_
 #define O2_FRAMEWORK_DATAPROCESSINGHELPERS_H_
 
-#include "Framework/TimesliceIndex.h"
-#include <fairmq/FwdDecls.h>
+#include <cstddef>
 
 namespace o2::framework
 {
-
+struct ServiceRegistryRef;
+struct ForwardChannelInfo;
+struct ForwardChannelState;
+struct OutputChannelInfo;
 struct OutputChannelSpec;
-class FairMQDeviceProxy;
+struct OutputChannelState;
 
 /// Generic helpers for DataProcessing releated functions.
 struct DataProcessingHelpers {
@@ -26,15 +28,13 @@ struct DataProcessingHelpers {
   /// @param device the fair::mq::Device which needs to send the EndOfStream message
   /// @param channel the OutputChannelSpec of the channel which needs to be signaled
   ///        for EndOfStream
-  static void sendEndOfStream(fair::mq::Device& device, OutputChannelSpec const& channel);
+  static void sendEndOfStream(ServiceRegistryRef const& ref, OutputChannelSpec const& channel);
   /// @returns true if we did send the oldest possible timeslice message, false otherwise.
-  static bool sendOldestPossibleTimeframe(ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice);
+  static bool sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice);
   /// @returns true if we did send the oldest possible timeslice message, false otherwise.
-  static bool sendOldestPossibleTimeframe(OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice);
-  static void broadcastOldestPossibleTimeslice(FairMQDeviceProxy& proxy, size_t timeslice);
-
- private:
-  static void sendOldestPossibleTimeframe(fair::mq::Channel& channel, size_t timeslice);
+  static bool sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice);
+  /// Broadcast the oldest possible timeslice to all channels in output
+  static void broadcastOldestPossibleTimeslice(ServiceRegistryRef const& ref, size_t timeslice);
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/include/Framework/ForwardRoute.h b/Framework/Core/include/Framework/ForwardRoute.h
index bae2eaacf1a44..40a76e3a9cce5 100644
--- a/Framework/Core/include/Framework/ForwardRoute.h
+++ b/Framework/Core/include/Framework/ForwardRoute.h
@@ -15,11 +15,11 @@
 #include <cstddef>
 #include <string>
 
-namespace o2
-{
-namespace framework
+namespace o2::framework
 {
 
+struct SendingPolicy;
+
 /// This uniquely identifies a route to be forwarded by the device if
 /// the InputSpec @a matcher matches an input which should also go to
 /// @a channel
@@ -28,8 +28,9 @@ struct ForwardRoute {
   size_t maxTimeslices;
   InputSpec matcher;
   std::string channel;
+  // The policy to use to send to on this route.
+  SendingPolicy const* policy;
 };
 
-} // namespace framework
 } // namespace o2
 #endif // FRAMEWORK_FORWARDROUTE_H
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index b7fc3410a1931..a0f7fcdc88201 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -588,7 +588,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           timesliceIndex.rescan();
         }
       }
-      DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, oldestPossibleOutput.timeslice.value);
+      DataProcessingHelpers::broadcastOldestPossibleTimeslice(ctx.services(), oldestPossibleOutput.timeslice.value);
 
       for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
         auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
@@ -598,7 +598,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
           continue;
         }
-        if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestPossibleOutput.timeslice.value)) {
+        if (DataProcessingHelpers::sendOldestPossibleTimeframe(ctx.services(), info, state, oldestPossibleOutput.timeslice.value)) {
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice",
                                  "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
                                  info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
@@ -646,7 +646,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
       O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Queueing oldest possible timeslice %" PRIu64 " propagation for execution.",
                              (uint64_t)oldestPossibleOutput.timeslice.value);
       AsyncQueueHelpers::post(
-        queue, decongestion.oldestPossibleTimesliceTask, [oldestPossibleOutput, &decongestion, &proxy, &spec, device, &timesliceIndex]() {
+        queue, decongestion.oldestPossibleTimesliceTask, [ref = services, oldestPossibleOutput, &decongestion, &proxy, &spec, device, &timesliceIndex]() {
           O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, &decongestion);
           if (decongestion.lastTimeslice >= oldestPossibleOutput.timeslice.value) {
             O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Not sending already sent value: %" PRIu64 "> %" PRIu64,
@@ -655,7 +655,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           }
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Running oldest possible timeslice %" PRIu64 " propagation.",
                                  (uint64_t)oldestPossibleOutput.timeslice.value);
-          DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, oldestPossibleOutput.timeslice.value);
+          DataProcessingHelpers::broadcastOldestPossibleTimeslice(ref, oldestPossibleOutput.timeslice.value);
 
           for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
             auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
@@ -665,7 +665,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
               O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
               continue;
             }
-            if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestPossibleOutput.timeslice.value)) {
+            if (DataProcessingHelpers::sendOldestPossibleTimeframe(ref, info, state, oldestPossibleOutput.timeslice.value)) {
               O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice",
                                      "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
                                      info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 8c914e201a479..0305946116b9b 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -680,19 +680,11 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
     if (forwardedParts[fi].Size() == 0) {
       continue;
     }
-    auto channel = proxy.getForwardChannel(ChannelIndex{fi});
-    LOG(debug) << "Forwarding to " << channel->GetName() << " " << fi;
+    ForwardChannelInfo info = proxy.getForwardChannelInfo(ChannelIndex{fi});
+    LOG(debug) << "Forwarding to " << info.name << " " << fi;
     // in DPL we are using subchannel 0 only
     auto& parts = forwardedParts[fi];
-    int timeout = 30000;
-    auto res = channel->Send(parts, timeout);
-    if (res == (size_t)fair::mq::TransferCode::timeout) {
-      LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on {}.", timeout / 1000, channel->GetName());
-      channel->Send(parts);
-      LOGP(info, "Downstream backpressure on {} recovered.", channel->GetName());
-    } else if (res == (size_t)fair::mq::TransferCode::error) {
-      LOGP(fatal, "Error while sending on channel {}", channel->GetName());
-    }
+    info.policy->send(parts, ChannelIndex{fi}, registry);
   }
 
   auto& asyncQueue = registry.get<AsyncQueue>();
@@ -713,7 +705,7 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
           LOG(debug) << "Skipping channel";
           continue;
         }
-        if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestTimeslice.timeslice.value)) {
+        if (DataProcessingHelpers::sendOldestPossibleTimeframe(registry, info, state, oldestTimeslice.timeslice.value)) {
           LOGP(debug, "Forwarding to channel {} oldest possible timeslice {}, prio 20", info.name, oldestTimeslice.timeslice.value);
         }
       }
@@ -1678,8 +1670,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
 
     for (auto& channel : spec.outputChannels) {
       LOGP(detail, "Sending end of stream to {}", channel.name);
-      auto& rawDevice = ref.get<RawDeviceService>();
-      DataProcessingHelpers::sendEndOfStream(*rawDevice.device(), channel);
+      DataProcessingHelpers::sendEndOfStream(ref, channel);
     }
     // This is needed because the transport is deleted before the device.
     relayer.clear();
@@ -2460,8 +2451,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   if (state.streaming == StreamingState::EndOfStreaming) {
     LOGP(detail, "Broadcasting end of stream");
     for (auto& channel : spec.outputChannels) {
-      auto& rawDevice = ref.get<RawDeviceService>();
-      DataProcessingHelpers::sendEndOfStream(*rawDevice.device(), channel);
+      DataProcessingHelpers::sendEndOfStream(ref, channel);
     }
     switchState(StreamingState::Idle);
   }
diff --git a/Framework/Core/src/DataProcessingHelpers.cxx b/Framework/Core/src/DataProcessingHelpers.cxx
index d424a53a52bd9..6dc3379d41cf8 100644
--- a/Framework/Core/src/DataProcessingHelpers.cxx
+++ b/Framework/Core/src/DataProcessingHelpers.cxx
@@ -18,79 +18,74 @@
 #include "Headers/DataHeader.h"
 #include "Headers/Stack.h"
 #include "Framework/Logger.h"
+#include "Framework/SendingPolicy.h"
+#include "Framework/RawDeviceService.h"
 
 #include <fairmq/Device.h>
 #include <fairmq/Channel.h>
 
 namespace o2::framework
 {
-void DataProcessingHelpers::sendEndOfStream(fair::mq::Device& device, OutputChannelSpec const& channel)
+void DataProcessingHelpers::sendEndOfStream(ServiceRegistryRef const& ref, OutputChannelSpec const& channel)
 {
+  fair::mq::Device* device = ref.get<RawDeviceService>().device();
   fair::mq::Parts parts;
-  fair::mq::MessagePtr payload(device.NewMessage());
+  fair::mq::MessagePtr payload(device->NewMessage());
   SourceInfoHeader sih;
   sih.state = InputChannelState::Completed;
-  auto channelAlloc = o2::pmr::getTransportAllocator(device.GetChannel(channel.name, 0).Transport());
+  auto channelAlloc = o2::pmr::getTransportAllocator(device->GetChannel(channel.name, 0).Transport());
   auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, sih});
   // sigh... See if we can avoid having it const by not
   // exposing it to the user in the first place.
   parts.AddPart(std::move(header));
   parts.AddPart(std::move(payload));
-  device.Send(parts, channel.name, 0);
+  device->Send(parts, channel.name, 0);
   LOGP(info, "Sending end-of-stream message to channel {}", channel.name);
 }
 
-void DataProcessingHelpers::sendOldestPossibleTimeframe(fair::mq::Channel& channel, size_t timeslice)
+void doSendOldestPossibleTimeframe(ServiceRegistryRef ref, fair::mq::TransportFactory *transport, ChannelIndex index, SendingPolicy::SendingCallback const& callback, size_t timeslice)
 {
   fair::mq::Parts parts;
-  fair::mq::MessagePtr payload(channel.Transport()->CreateMessage());
+  fair::mq::MessagePtr payload(transport->CreateMessage());
   o2::framework::DomainInfoHeader dih;
   dih.oldestPossibleTimeslice = timeslice;
-  auto channelAlloc = o2::pmr::getTransportAllocator(channel.Transport());
+  auto channelAlloc = o2::pmr::getTransportAllocator(transport);
   auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, dih});
   // sigh... See if we can avoid having it const by not
   // exposing it to the user in the first place.
   parts.AddPart(std::move(header));
   parts.AddPart(std::move(payload));
 
-  auto timeout = 1000;
-  auto res = channel.Send(parts, timeout);
-  if (res == (size_t)fair::mq::TransferCode::timeout) {
-    LOGP(warning, "Timed out sending oldest possible timeslice after {}s. Downstream backpressure detected on {}.", timeout / 1000, channel.GetName());
-    channel.Send(parts);
-    LOGP(info, "Downstream backpressure on {} recovered.", channel.GetName());
-  }
-  if (res < (size_t)fair::mq::TransferCode::success) {
-    LOGP(fatal, "Error sending oldest possible timeframe {} on channel {} (code {})", timeslice, channel.GetName(), res);
-  }
+  callback(parts, index, ref);
 }
 
-bool DataProcessingHelpers::sendOldestPossibleTimeframe(ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice)
+bool DataProcessingHelpers::sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice)
 {
   if (state.oldestForChannel.value >= timeslice) {
     return false;
   }
-  sendOldestPossibleTimeframe(info.channel, timeslice);
+  doSendOldestPossibleTimeframe(ref, info.channel.Transport(), info.index, info.policy->send, timeslice);
   state.oldestForChannel = {timeslice};
   return true;
 }
 
-bool DataProcessingHelpers::sendOldestPossibleTimeframe(OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice)
+bool DataProcessingHelpers::sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice)
 {
   if (state.oldestForChannel.value >= timeslice) {
     return false;
   }
-  sendOldestPossibleTimeframe(info.channel, timeslice);
+  doSendOldestPossibleTimeframe(ref, info.channel.Transport(), info.index, info.policy->send, timeslice);
   state.oldestForChannel = {timeslice};
   return true;
 }
 
-void DataProcessingHelpers::broadcastOldestPossibleTimeslice(FairMQDeviceProxy& proxy, size_t timeslice)
+void DataProcessingHelpers::broadcastOldestPossibleTimeslice(ServiceRegistryRef const& ref, size_t timeslice)
 {
+  auto &proxy = ref.get<FairMQDeviceProxy>();
   for (int ci = 0; ci < proxy.getNumOutputChannels(); ++ci) {
     auto& info = proxy.getOutputChannelInfo({ci});
     auto& state = proxy.getOutputChannelState({ci});
-    sendOldestPossibleTimeframe(info, state, timeslice);
+    sendOldestPossibleTimeframe(ref, info, state, timeslice);
   }
 }
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index b1681a4779ceb..90b4f8c0c5226 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -680,19 +680,21 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
 
     if (edge.isForward == false) {
       OutputRoute route{
-        edge.timeIndex,
-        consumer.maxInputTimeslices,
-        outputsMatchers[edge.outputGlobalIndex],
-        channel.name,
-        policyPtr,
+        .timeslice = edge.timeIndex,
+        .maxTimeslices = consumer.maxInputTimeslices,
+        .matcher = outputsMatchers[edge.outputGlobalIndex],
+        .channel = channel.name,
+        .policy = policyPtr,
       };
       device.outputs.emplace_back(route);
     } else {
       ForwardRoute route{
-        edge.timeIndex,
-        consumer.maxInputTimeslices,
-        workflow[edge.consumer].inputs[edge.consumerInputIndex],
-        channel.name};
+        .timeslice = edge.timeIndex,
+        .maxTimeslices = consumer.maxInputTimeslices,
+        .matcher = workflow[edge.consumer].inputs[edge.consumerInputIndex],
+        .channel = channel.name,
+        .policy = policyPtr,
+      };
       device.forwards.emplace_back(route);
     }
   };
diff --git a/Framework/Core/src/FairMQDeviceProxy.cxx b/Framework/Core/src/FairMQDeviceProxy.cxx
index c25a58e47ef2d..bdffddd5a4d1a 100644
--- a/Framework/Core/src/FairMQDeviceProxy.cxx
+++ b/Framework/Core/src/FairMQDeviceProxy.cxx
@@ -267,6 +267,7 @@ void FairMQDeviceProxy::bind(std::vector<OutputRoute> const& outputs, std::vecto
           .channelType = dplChannel,
           .channel = channel->second.at(0),
           .policy = route.policy,
+          .index = channelIndex,
         };
         mOutputChannelInfos.push_back(info);
         mOutputChannelStates.push_back({0});
@@ -345,7 +346,7 @@ void FairMQDeviceProxy::bind(std::vector<OutputRoute> const& outputs, std::vecto
           LOGP(fatal, "Expected channel {} not configured.", route.channel);
         }
         ChannelAccountingType dplChannel = (route.channel.rfind("from_", 0) == 0) ? ChannelAccountingType::DPL : ChannelAccountingType::RAWFMQ;
-        mForwardChannelInfos.push_back(ForwardChannelInfo{route.channel, dplChannel, channel->second.at(0)});
+        mForwardChannelInfos.push_back(ForwardChannelInfo{.name = route.channel, .channelType = dplChannel, .channel = channel->second.at(0), .policy = route.policy, .index = channelIndex});
         mForwardChannelStates.push_back(ForwardChannelState{0});
         channelNameToChannel[route.channel] = channelIndex;
         LOGP(detail, "Binding forward channel {} to channel index {}", route.channel, channelIndex.value);
diff --git a/Framework/DataInspector/src/DataInspectorService.cxx b/Framework/DataInspector/src/DataInspectorService.cxx
index 40399a366a303..712bf10b6fb2e 100644
--- a/Framework/DataInspector/src/DataInspectorService.cxx
+++ b/Framework/DataInspector/src/DataInspectorService.cxx
@@ -245,4 +245,4 @@ ServiceSpec* DIServicePlugin::create()
             } },
     .kind = ServiceKind::Global};
 };
-} // namespace o2::framework
\ No newline at end of file
+} // namespace o2::framework
diff --git a/Utilities/DataSampling/src/Dispatcher.cxx b/Utilities/DataSampling/src/Dispatcher.cxx
index de78993ff083e..28ff4d5568da9 100644
--- a/Utilities/DataSampling/src/Dispatcher.cxx
+++ b/Utilities/DataSampling/src/Dispatcher.cxx
@@ -125,10 +125,9 @@ void Dispatcher::run(ProcessingContext& ctx)
   if (ctx.inputs().isValid("timer-stats")) {
     reportStats(ctx.services().get<Monitoring>());
   }
-  auto& proxy = ctx.services().get<FairMQDeviceProxy>();
   auto& relayer = ctx.services().get<DataRelayer>();
   auto timeslice = relayer.getOldestPossibleOutput().timeslice.value;
-  DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, timeslice);
+  DataProcessingHelpers::broadcastOldestPossibleTimeslice(ctx.services(), timeslice);
 }
 
 void Dispatcher::reportStats(Monitoring& monitoring) const

From 0e96e7b0b4f2323e1787307fe6db298a5733ba56 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 8 Feb 2024 13:49:06 +0100
Subject: [PATCH 314/726] DPL: add SendingPolicy for the case destination is
 expendable

---
 Framework/Core/src/SendingPolicy.cxx | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 17faef6dbd3d0..21e60dbcfa32e 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -29,7 +29,7 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
 {
   return {SendingPolicy{
             .name = "dispatcher",
-            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) { 
+            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) {
                 if (source.name == "Dispatcher") {
                   return true;
                 }
@@ -103,6 +103,23 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               } else if (res == (size_t) fair::mq::TransferCode::error) {
                 LOGP(fatal, "Error while sending on channel {}", channel->GetName());
               } }},
+          SendingPolicy{
+            .name = "expendable",
+            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) {
+              auto has_label = [](DataProcessorLabel const& label) {
+                return label.value == "expendable";
+              };
+              return std::find_if(dest.labels.begin(), dest.labels.end(), has_label) != dest.labels.end(); },
+            .send = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
+              auto &proxy = registry.get<FairMQDeviceProxy>();
+              auto *channel = proxy.getOutputChannel(channelIndex);
+              auto timeout = 1000;
+              auto res = channel->Send(parts, timeout);
+              if (res == (size_t)fair::mq::TransferCode::timeout) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
+              } else if (res == (size_t) fair::mq::TransferCode::error) {
+                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              } }},
           SendingPolicy{
             .name = "default",
             .matcher = [](DataProcessorSpec const&, DataProcessorSpec const&, ConfigContext const&) { return true; },

From 87ac28589a34ac8bb68ca86c7269d1b93ea9863f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 8 Feb 2024 13:57:26 +0100
Subject: [PATCH 315/726] DPL: use the SendingPolicy for every kind of message
 sending

---
 .../Core/include/Framework/ChannelInfo.h      |  3 ++
 .../include/Framework/DataProcessingHelpers.h | 22 +++++-----
 .../Core/include/Framework/ForwardRoute.h     |  9 ++--
 Framework/Core/src/CommonServices.cxx         | 10 ++---
 Framework/Core/src/DataProcessingDevice.cxx   | 22 +++-------
 Framework/Core/src/DataProcessingHelpers.cxx  | 41 ++++++++-----------
 Framework/Core/src/DeviceSpecHelpers.cxx      | 20 +++++----
 Framework/Core/src/FairMQDeviceProxy.cxx      |  3 +-
 .../src/DataInspectorService.cxx              |  2 +-
 Utilities/DataSampling/src/Dispatcher.cxx     |  3 +-
 10 files changed, 63 insertions(+), 72 deletions(-)

diff --git a/Framework/Core/include/Framework/ChannelInfo.h b/Framework/Core/include/Framework/ChannelInfo.h
index 0ebc4a8506301..582ef72fec75d 100644
--- a/Framework/Core/include/Framework/ChannelInfo.h
+++ b/Framework/Core/include/Framework/ChannelInfo.h
@@ -74,6 +74,7 @@ struct OutputChannelInfo {
   ChannelAccountingType channelType = ChannelAccountingType::DPL;
   fair::mq::Channel& channel;
   SendingPolicy const* policy;
+  ChannelIndex index = {-1};
 };
 
 struct OutputChannelState {
@@ -89,6 +90,8 @@ struct ForwardChannelInfo {
   /// Wether or not it's a DPL internal channel.
   ChannelAccountingType channelType = ChannelAccountingType::DPL;
   fair::mq::Channel& channel;
+  SendingPolicy const* policy;
+  ChannelIndex index = {-1};
 };
 
 struct ForwardChannelState {
diff --git a/Framework/Core/include/Framework/DataProcessingHelpers.h b/Framework/Core/include/Framework/DataProcessingHelpers.h
index 35f78f12cc405..7a05ddf25c46c 100644
--- a/Framework/Core/include/Framework/DataProcessingHelpers.h
+++ b/Framework/Core/include/Framework/DataProcessingHelpers.h
@@ -11,14 +11,16 @@
 #ifndef O2_FRAMEWORK_DATAPROCESSINGHELPERS_H_
 #define O2_FRAMEWORK_DATAPROCESSINGHELPERS_H_
 
-#include "Framework/TimesliceIndex.h"
-#include <fairmq/FwdDecls.h>
+#include <cstddef>
 
 namespace o2::framework
 {
-
+struct ServiceRegistryRef;
+struct ForwardChannelInfo;
+struct ForwardChannelState;
+struct OutputChannelInfo;
 struct OutputChannelSpec;
-class FairMQDeviceProxy;
+struct OutputChannelState;
 
 /// Generic helpers for DataProcessing releated functions.
 struct DataProcessingHelpers {
@@ -26,15 +28,13 @@ struct DataProcessingHelpers {
   /// @param device the fair::mq::Device which needs to send the EndOfStream message
   /// @param channel the OutputChannelSpec of the channel which needs to be signaled
   ///        for EndOfStream
-  static void sendEndOfStream(fair::mq::Device& device, OutputChannelSpec const& channel);
+  static void sendEndOfStream(ServiceRegistryRef const& ref, OutputChannelSpec const& channel);
   /// @returns true if we did send the oldest possible timeslice message, false otherwise.
-  static bool sendOldestPossibleTimeframe(ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice);
+  static bool sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice);
   /// @returns true if we did send the oldest possible timeslice message, false otherwise.
-  static bool sendOldestPossibleTimeframe(OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice);
-  static void broadcastOldestPossibleTimeslice(FairMQDeviceProxy& proxy, size_t timeslice);
-
- private:
-  static void sendOldestPossibleTimeframe(fair::mq::Channel& channel, size_t timeslice);
+  static bool sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice);
+  /// Broadcast the oldest possible timeslice to all channels in output
+  static void broadcastOldestPossibleTimeslice(ServiceRegistryRef const& ref, size_t timeslice);
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/include/Framework/ForwardRoute.h b/Framework/Core/include/Framework/ForwardRoute.h
index bae2eaacf1a44..40a76e3a9cce5 100644
--- a/Framework/Core/include/Framework/ForwardRoute.h
+++ b/Framework/Core/include/Framework/ForwardRoute.h
@@ -15,11 +15,11 @@
 #include <cstddef>
 #include <string>
 
-namespace o2
-{
-namespace framework
+namespace o2::framework
 {
 
+struct SendingPolicy;
+
 /// This uniquely identifies a route to be forwarded by the device if
 /// the InputSpec @a matcher matches an input which should also go to
 /// @a channel
@@ -28,8 +28,9 @@ struct ForwardRoute {
   size_t maxTimeslices;
   InputSpec matcher;
   std::string channel;
+  // The policy to use to send to on this route.
+  SendingPolicy const* policy;
 };
 
-} // namespace framework
 } // namespace o2
 #endif // FRAMEWORK_FORWARDROUTE_H
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index b7fc3410a1931..a0f7fcdc88201 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -588,7 +588,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           timesliceIndex.rescan();
         }
       }
-      DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, oldestPossibleOutput.timeslice.value);
+      DataProcessingHelpers::broadcastOldestPossibleTimeslice(ctx.services(), oldestPossibleOutput.timeslice.value);
 
       for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
         auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
@@ -598,7 +598,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
           continue;
         }
-        if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestPossibleOutput.timeslice.value)) {
+        if (DataProcessingHelpers::sendOldestPossibleTimeframe(ctx.services(), info, state, oldestPossibleOutput.timeslice.value)) {
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice",
                                  "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
                                  info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
@@ -646,7 +646,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
       O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Queueing oldest possible timeslice %" PRIu64 " propagation for execution.",
                              (uint64_t)oldestPossibleOutput.timeslice.value);
       AsyncQueueHelpers::post(
-        queue, decongestion.oldestPossibleTimesliceTask, [oldestPossibleOutput, &decongestion, &proxy, &spec, device, &timesliceIndex]() {
+        queue, decongestion.oldestPossibleTimesliceTask, [ref = services, oldestPossibleOutput, &decongestion, &proxy, &spec, device, &timesliceIndex]() {
           O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, &decongestion);
           if (decongestion.lastTimeslice >= oldestPossibleOutput.timeslice.value) {
             O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Not sending already sent value: %" PRIu64 "> %" PRIu64,
@@ -655,7 +655,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           }
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Running oldest possible timeslice %" PRIu64 " propagation.",
                                  (uint64_t)oldestPossibleOutput.timeslice.value);
-          DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, oldestPossibleOutput.timeslice.value);
+          DataProcessingHelpers::broadcastOldestPossibleTimeslice(ref, oldestPossibleOutput.timeslice.value);
 
           for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
             auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
@@ -665,7 +665,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
               O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
               continue;
             }
-            if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestPossibleOutput.timeslice.value)) {
+            if (DataProcessingHelpers::sendOldestPossibleTimeframe(ref, info, state, oldestPossibleOutput.timeslice.value)) {
               O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice",
                                      "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
                                      info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 8c914e201a479..0305946116b9b 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -680,19 +680,11 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
     if (forwardedParts[fi].Size() == 0) {
       continue;
     }
-    auto channel = proxy.getForwardChannel(ChannelIndex{fi});
-    LOG(debug) << "Forwarding to " << channel->GetName() << " " << fi;
+    ForwardChannelInfo info = proxy.getForwardChannelInfo(ChannelIndex{fi});
+    LOG(debug) << "Forwarding to " << info.name << " " << fi;
     // in DPL we are using subchannel 0 only
     auto& parts = forwardedParts[fi];
-    int timeout = 30000;
-    auto res = channel->Send(parts, timeout);
-    if (res == (size_t)fair::mq::TransferCode::timeout) {
-      LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on {}.", timeout / 1000, channel->GetName());
-      channel->Send(parts);
-      LOGP(info, "Downstream backpressure on {} recovered.", channel->GetName());
-    } else if (res == (size_t)fair::mq::TransferCode::error) {
-      LOGP(fatal, "Error while sending on channel {}", channel->GetName());
-    }
+    info.policy->send(parts, ChannelIndex{fi}, registry);
   }
 
   auto& asyncQueue = registry.get<AsyncQueue>();
@@ -713,7 +705,7 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
           LOG(debug) << "Skipping channel";
           continue;
         }
-        if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestTimeslice.timeslice.value)) {
+        if (DataProcessingHelpers::sendOldestPossibleTimeframe(registry, info, state, oldestTimeslice.timeslice.value)) {
           LOGP(debug, "Forwarding to channel {} oldest possible timeslice {}, prio 20", info.name, oldestTimeslice.timeslice.value);
         }
       }
@@ -1678,8 +1670,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
 
     for (auto& channel : spec.outputChannels) {
       LOGP(detail, "Sending end of stream to {}", channel.name);
-      auto& rawDevice = ref.get<RawDeviceService>();
-      DataProcessingHelpers::sendEndOfStream(*rawDevice.device(), channel);
+      DataProcessingHelpers::sendEndOfStream(ref, channel);
     }
     // This is needed because the transport is deleted before the device.
     relayer.clear();
@@ -2460,8 +2451,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   if (state.streaming == StreamingState::EndOfStreaming) {
     LOGP(detail, "Broadcasting end of stream");
     for (auto& channel : spec.outputChannels) {
-      auto& rawDevice = ref.get<RawDeviceService>();
-      DataProcessingHelpers::sendEndOfStream(*rawDevice.device(), channel);
+      DataProcessingHelpers::sendEndOfStream(ref, channel);
     }
     switchState(StreamingState::Idle);
   }
diff --git a/Framework/Core/src/DataProcessingHelpers.cxx b/Framework/Core/src/DataProcessingHelpers.cxx
index d424a53a52bd9..d3b6f9d0a99fa 100644
--- a/Framework/Core/src/DataProcessingHelpers.cxx
+++ b/Framework/Core/src/DataProcessingHelpers.cxx
@@ -18,79 +18,74 @@
 #include "Headers/DataHeader.h"
 #include "Headers/Stack.h"
 #include "Framework/Logger.h"
+#include "Framework/SendingPolicy.h"
+#include "Framework/RawDeviceService.h"
 
 #include <fairmq/Device.h>
 #include <fairmq/Channel.h>
 
 namespace o2::framework
 {
-void DataProcessingHelpers::sendEndOfStream(fair::mq::Device& device, OutputChannelSpec const& channel)
+void DataProcessingHelpers::sendEndOfStream(ServiceRegistryRef const& ref, OutputChannelSpec const& channel)
 {
+  fair::mq::Device* device = ref.get<RawDeviceService>().device();
   fair::mq::Parts parts;
-  fair::mq::MessagePtr payload(device.NewMessage());
+  fair::mq::MessagePtr payload(device->NewMessage());
   SourceInfoHeader sih;
   sih.state = InputChannelState::Completed;
-  auto channelAlloc = o2::pmr::getTransportAllocator(device.GetChannel(channel.name, 0).Transport());
+  auto channelAlloc = o2::pmr::getTransportAllocator(device->GetChannel(channel.name, 0).Transport());
   auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, sih});
   // sigh... See if we can avoid having it const by not
   // exposing it to the user in the first place.
   parts.AddPart(std::move(header));
   parts.AddPart(std::move(payload));
-  device.Send(parts, channel.name, 0);
+  device->Send(parts, channel.name, 0);
   LOGP(info, "Sending end-of-stream message to channel {}", channel.name);
 }
 
-void DataProcessingHelpers::sendOldestPossibleTimeframe(fair::mq::Channel& channel, size_t timeslice)
+void doSendOldestPossibleTimeframe(ServiceRegistryRef ref, fair::mq::TransportFactory* transport, ChannelIndex index, SendingPolicy::SendingCallback const& callback, size_t timeslice)
 {
   fair::mq::Parts parts;
-  fair::mq::MessagePtr payload(channel.Transport()->CreateMessage());
+  fair::mq::MessagePtr payload(transport->CreateMessage());
   o2::framework::DomainInfoHeader dih;
   dih.oldestPossibleTimeslice = timeslice;
-  auto channelAlloc = o2::pmr::getTransportAllocator(channel.Transport());
+  auto channelAlloc = o2::pmr::getTransportAllocator(transport);
   auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, dih});
   // sigh... See if we can avoid having it const by not
   // exposing it to the user in the first place.
   parts.AddPart(std::move(header));
   parts.AddPart(std::move(payload));
 
-  auto timeout = 1000;
-  auto res = channel.Send(parts, timeout);
-  if (res == (size_t)fair::mq::TransferCode::timeout) {
-    LOGP(warning, "Timed out sending oldest possible timeslice after {}s. Downstream backpressure detected on {}.", timeout / 1000, channel.GetName());
-    channel.Send(parts);
-    LOGP(info, "Downstream backpressure on {} recovered.", channel.GetName());
-  }
-  if (res < (size_t)fair::mq::TransferCode::success) {
-    LOGP(fatal, "Error sending oldest possible timeframe {} on channel {} (code {})", timeslice, channel.GetName(), res);
-  }
+  callback(parts, index, ref);
 }
 
-bool DataProcessingHelpers::sendOldestPossibleTimeframe(ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice)
+bool DataProcessingHelpers::sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice)
 {
   if (state.oldestForChannel.value >= timeslice) {
     return false;
   }
-  sendOldestPossibleTimeframe(info.channel, timeslice);
+  doSendOldestPossibleTimeframe(ref, info.channel.Transport(), info.index, info.policy->send, timeslice);
   state.oldestForChannel = {timeslice};
   return true;
 }
 
-bool DataProcessingHelpers::sendOldestPossibleTimeframe(OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice)
+bool DataProcessingHelpers::sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice)
 {
   if (state.oldestForChannel.value >= timeslice) {
     return false;
   }
-  sendOldestPossibleTimeframe(info.channel, timeslice);
+  doSendOldestPossibleTimeframe(ref, info.channel.Transport(), info.index, info.policy->send, timeslice);
   state.oldestForChannel = {timeslice};
   return true;
 }
 
-void DataProcessingHelpers::broadcastOldestPossibleTimeslice(FairMQDeviceProxy& proxy, size_t timeslice)
+void DataProcessingHelpers::broadcastOldestPossibleTimeslice(ServiceRegistryRef const& ref, size_t timeslice)
 {
+  auto& proxy = ref.get<FairMQDeviceProxy>();
   for (int ci = 0; ci < proxy.getNumOutputChannels(); ++ci) {
     auto& info = proxy.getOutputChannelInfo({ci});
     auto& state = proxy.getOutputChannelState({ci});
-    sendOldestPossibleTimeframe(info, state, timeslice);
+    sendOldestPossibleTimeframe(ref, info, state, timeslice);
   }
 }
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index b1681a4779ceb..90b4f8c0c5226 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -680,19 +680,21 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
 
     if (edge.isForward == false) {
       OutputRoute route{
-        edge.timeIndex,
-        consumer.maxInputTimeslices,
-        outputsMatchers[edge.outputGlobalIndex],
-        channel.name,
-        policyPtr,
+        .timeslice = edge.timeIndex,
+        .maxTimeslices = consumer.maxInputTimeslices,
+        .matcher = outputsMatchers[edge.outputGlobalIndex],
+        .channel = channel.name,
+        .policy = policyPtr,
       };
       device.outputs.emplace_back(route);
     } else {
       ForwardRoute route{
-        edge.timeIndex,
-        consumer.maxInputTimeslices,
-        workflow[edge.consumer].inputs[edge.consumerInputIndex],
-        channel.name};
+        .timeslice = edge.timeIndex,
+        .maxTimeslices = consumer.maxInputTimeslices,
+        .matcher = workflow[edge.consumer].inputs[edge.consumerInputIndex],
+        .channel = channel.name,
+        .policy = policyPtr,
+      };
       device.forwards.emplace_back(route);
     }
   };
diff --git a/Framework/Core/src/FairMQDeviceProxy.cxx b/Framework/Core/src/FairMQDeviceProxy.cxx
index c25a58e47ef2d..bdffddd5a4d1a 100644
--- a/Framework/Core/src/FairMQDeviceProxy.cxx
+++ b/Framework/Core/src/FairMQDeviceProxy.cxx
@@ -267,6 +267,7 @@ void FairMQDeviceProxy::bind(std::vector<OutputRoute> const& outputs, std::vecto
           .channelType = dplChannel,
           .channel = channel->second.at(0),
           .policy = route.policy,
+          .index = channelIndex,
         };
         mOutputChannelInfos.push_back(info);
         mOutputChannelStates.push_back({0});
@@ -345,7 +346,7 @@ void FairMQDeviceProxy::bind(std::vector<OutputRoute> const& outputs, std::vecto
           LOGP(fatal, "Expected channel {} not configured.", route.channel);
         }
         ChannelAccountingType dplChannel = (route.channel.rfind("from_", 0) == 0) ? ChannelAccountingType::DPL : ChannelAccountingType::RAWFMQ;
-        mForwardChannelInfos.push_back(ForwardChannelInfo{route.channel, dplChannel, channel->second.at(0)});
+        mForwardChannelInfos.push_back(ForwardChannelInfo{.name = route.channel, .channelType = dplChannel, .channel = channel->second.at(0), .policy = route.policy, .index = channelIndex});
         mForwardChannelStates.push_back(ForwardChannelState{0});
         channelNameToChannel[route.channel] = channelIndex;
         LOGP(detail, "Binding forward channel {} to channel index {}", route.channel, channelIndex.value);
diff --git a/Framework/DataInspector/src/DataInspectorService.cxx b/Framework/DataInspector/src/DataInspectorService.cxx
index 40399a366a303..712bf10b6fb2e 100644
--- a/Framework/DataInspector/src/DataInspectorService.cxx
+++ b/Framework/DataInspector/src/DataInspectorService.cxx
@@ -245,4 +245,4 @@ ServiceSpec* DIServicePlugin::create()
             } },
     .kind = ServiceKind::Global};
 };
-} // namespace o2::framework
\ No newline at end of file
+} // namespace o2::framework
diff --git a/Utilities/DataSampling/src/Dispatcher.cxx b/Utilities/DataSampling/src/Dispatcher.cxx
index de78993ff083e..28ff4d5568da9 100644
--- a/Utilities/DataSampling/src/Dispatcher.cxx
+++ b/Utilities/DataSampling/src/Dispatcher.cxx
@@ -125,10 +125,9 @@ void Dispatcher::run(ProcessingContext& ctx)
   if (ctx.inputs().isValid("timer-stats")) {
     reportStats(ctx.services().get<Monitoring>());
   }
-  auto& proxy = ctx.services().get<FairMQDeviceProxy>();
   auto& relayer = ctx.services().get<DataRelayer>();
   auto timeslice = relayer.getOldestPossibleOutput().timeslice.value;
-  DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, timeslice);
+  DataProcessingHelpers::broadcastOldestPossibleTimeslice(ctx.services(), timeslice);
 }
 
 void Dispatcher::reportStats(Monitoring& monitoring) const

From c614f05a69948d22c9b14e938f60d717fe6b1e1b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 8 Feb 2024 13:57:26 +0100
Subject: [PATCH 316/726] DPL: add SendingPolicy for the case destination is
 expendable

---
 Framework/Core/src/SendingPolicy.cxx | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 17faef6dbd3d0..21e60dbcfa32e 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -29,7 +29,7 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
 {
   return {SendingPolicy{
             .name = "dispatcher",
-            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) { 
+            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) {
                 if (source.name == "Dispatcher") {
                   return true;
                 }
@@ -103,6 +103,23 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               } else if (res == (size_t) fair::mq::TransferCode::error) {
                 LOGP(fatal, "Error while sending on channel {}", channel->GetName());
               } }},
+          SendingPolicy{
+            .name = "expendable",
+            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) {
+              auto has_label = [](DataProcessorLabel const& label) {
+                return label.value == "expendable";
+              };
+              return std::find_if(dest.labels.begin(), dest.labels.end(), has_label) != dest.labels.end(); },
+            .send = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
+              auto &proxy = registry.get<FairMQDeviceProxy>();
+              auto *channel = proxy.getOutputChannel(channelIndex);
+              auto timeout = 1000;
+              auto res = channel->Send(parts, timeout);
+              if (res == (size_t)fair::mq::TransferCode::timeout) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
+              } else if (res == (size_t) fair::mq::TransferCode::error) {
+                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              } }},
           SendingPolicy{
             .name = "default",
             .matcher = [](DataProcessorSpec const&, DataProcessorSpec const&, ConfigContext const&) { return true; },

From b868bb65bc115e7cbcf2e12a2822568c686b1a47 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 9 Feb 2024 21:01:27 +0100
Subject: [PATCH 317/726] DPL: use the SendingPolicy for every kind of message
 sending

---
 .../Core/include/Framework/ChannelInfo.h      |  4 ++
 .../include/Framework/DataProcessingHelpers.h | 22 +++---
 Framework/Core/include/Framework/DriverInfo.h |  3 +
 .../Core/include/Framework/ForwardRoute.h     |  9 +--
 .../Core/include/Framework/SendingPolicy.h    |  8 +++
 Framework/Core/src/CommonServices.cxx         | 10 +--
 Framework/Core/src/DataProcessingDevice.cxx   | 22 ++----
 Framework/Core/src/DataProcessingHelpers.cxx  | 41 +++++------
 Framework/Core/src/DeviceSpecHelpers.cxx      | 35 ++++++---
 Framework/Core/src/DeviceSpecHelpers.h        |  5 +-
 Framework/Core/src/FairMQDeviceProxy.cxx      |  3 +-
 Framework/Core/src/SendingPolicy.cxx          | 72 +++++++++++++++++++
 Framework/Core/src/runDataProcessing.cxx      |  3 +
 Framework/Core/test/test_DeviceSpec.cxx       |  6 +-
 .../src/DataInspectorService.cxx              |  2 +-
 Utilities/DataSampling/src/Dispatcher.cxx     |  3 +-
 16 files changed, 171 insertions(+), 77 deletions(-)

diff --git a/Framework/Core/include/Framework/ChannelInfo.h b/Framework/Core/include/Framework/ChannelInfo.h
index 0ebc4a8506301..1f4bd8bc9e90e 100644
--- a/Framework/Core/include/Framework/ChannelInfo.h
+++ b/Framework/Core/include/Framework/ChannelInfo.h
@@ -67,6 +67,7 @@ struct InputChannelInfo {
 };
 
 struct SendingPolicy;
+struct ForwardingPolicy;
 
 /// Output channel information
 struct OutputChannelInfo {
@@ -74,6 +75,7 @@ struct OutputChannelInfo {
   ChannelAccountingType channelType = ChannelAccountingType::DPL;
   fair::mq::Channel& channel;
   SendingPolicy const* policy;
+  ChannelIndex index = {-1};
 };
 
 struct OutputChannelState {
@@ -89,6 +91,8 @@ struct ForwardChannelInfo {
   /// Wether or not it's a DPL internal channel.
   ChannelAccountingType channelType = ChannelAccountingType::DPL;
   fair::mq::Channel& channel;
+  ForwardingPolicy const* policy;
+  ChannelIndex index = {-1};
 };
 
 struct ForwardChannelState {
diff --git a/Framework/Core/include/Framework/DataProcessingHelpers.h b/Framework/Core/include/Framework/DataProcessingHelpers.h
index 35f78f12cc405..7a05ddf25c46c 100644
--- a/Framework/Core/include/Framework/DataProcessingHelpers.h
+++ b/Framework/Core/include/Framework/DataProcessingHelpers.h
@@ -11,14 +11,16 @@
 #ifndef O2_FRAMEWORK_DATAPROCESSINGHELPERS_H_
 #define O2_FRAMEWORK_DATAPROCESSINGHELPERS_H_
 
-#include "Framework/TimesliceIndex.h"
-#include <fairmq/FwdDecls.h>
+#include <cstddef>
 
 namespace o2::framework
 {
-
+struct ServiceRegistryRef;
+struct ForwardChannelInfo;
+struct ForwardChannelState;
+struct OutputChannelInfo;
 struct OutputChannelSpec;
-class FairMQDeviceProxy;
+struct OutputChannelState;
 
 /// Generic helpers for DataProcessing releated functions.
 struct DataProcessingHelpers {
@@ -26,15 +28,13 @@ struct DataProcessingHelpers {
   /// @param device the fair::mq::Device which needs to send the EndOfStream message
   /// @param channel the OutputChannelSpec of the channel which needs to be signaled
   ///        for EndOfStream
-  static void sendEndOfStream(fair::mq::Device& device, OutputChannelSpec const& channel);
+  static void sendEndOfStream(ServiceRegistryRef const& ref, OutputChannelSpec const& channel);
   /// @returns true if we did send the oldest possible timeslice message, false otherwise.
-  static bool sendOldestPossibleTimeframe(ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice);
+  static bool sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice);
   /// @returns true if we did send the oldest possible timeslice message, false otherwise.
-  static bool sendOldestPossibleTimeframe(OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice);
-  static void broadcastOldestPossibleTimeslice(FairMQDeviceProxy& proxy, size_t timeslice);
-
- private:
-  static void sendOldestPossibleTimeframe(fair::mq::Channel& channel, size_t timeslice);
+  static bool sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice);
+  /// Broadcast the oldest possible timeslice to all channels in output
+  static void broadcastOldestPossibleTimeslice(ServiceRegistryRef const& ref, size_t timeslice);
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/include/Framework/DriverInfo.h b/Framework/Core/include/Framework/DriverInfo.h
index 043a689087325..45c4c85a32d01 100644
--- a/Framework/Core/include/Framework/DriverInfo.h
+++ b/Framework/Core/include/Framework/DriverInfo.h
@@ -111,6 +111,9 @@ struct DriverInfo {
   /// These are the policies which can be applied to decide how
   /// we send data.
   std::vector<SendingPolicy> sendingPolicies;
+  /// These are the policies which can be applied to decide how
+  /// we forward data.
+  std::vector<ForwardingPolicy> forwardingPolicies;
   /// The argc with which the driver was started.
   int argc;
   /// The argv with which the driver was started.
diff --git a/Framework/Core/include/Framework/ForwardRoute.h b/Framework/Core/include/Framework/ForwardRoute.h
index bae2eaacf1a44..0e3d4d12da24a 100644
--- a/Framework/Core/include/Framework/ForwardRoute.h
+++ b/Framework/Core/include/Framework/ForwardRoute.h
@@ -15,11 +15,11 @@
 #include <cstddef>
 #include <string>
 
-namespace o2
-{
-namespace framework
+namespace o2::framework
 {
 
+struct ForwardingPolicy;
+
 /// This uniquely identifies a route to be forwarded by the device if
 /// the InputSpec @a matcher matches an input which should also go to
 /// @a channel
@@ -28,8 +28,9 @@ struct ForwardRoute {
   size_t maxTimeslices;
   InputSpec matcher;
   std::string channel;
+  // The policy to use to send to on this route.
+  ForwardingPolicy const* policy;
 };
 
-} // namespace framework
 } // namespace o2
 #endif // FRAMEWORK_FORWARDROUTE_H
diff --git a/Framework/Core/include/Framework/SendingPolicy.h b/Framework/Core/include/Framework/SendingPolicy.h
index 1d68412d46db8..2f9472d1d8607 100644
--- a/Framework/Core/include/Framework/SendingPolicy.h
+++ b/Framework/Core/include/Framework/SendingPolicy.h
@@ -32,6 +32,14 @@ struct SendingPolicy {
   static std::vector<SendingPolicy> createDefaultPolicies();
 };
 
+struct ForwardingPolicy {
+  using ForwardingCallback = std::function<void(fair::mq::Parts&, ChannelIndex channelIndex, ServiceRegistryRef registry)>;
+  std::string name = "invalid";
+  EdgeMatcher matcher = nullptr;
+  ForwardingCallback forward = nullptr;
+  static std::vector<ForwardingPolicy> createDefaultPolicies();
+};
+
 } // namespace o2::framework
 
 #endif // O2_FRAMEWORK_SENDINGPOLICY_H_
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index b7fc3410a1931..a0f7fcdc88201 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -588,7 +588,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           timesliceIndex.rescan();
         }
       }
-      DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, oldestPossibleOutput.timeslice.value);
+      DataProcessingHelpers::broadcastOldestPossibleTimeslice(ctx.services(), oldestPossibleOutput.timeslice.value);
 
       for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
         auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
@@ -598,7 +598,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
           continue;
         }
-        if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestPossibleOutput.timeslice.value)) {
+        if (DataProcessingHelpers::sendOldestPossibleTimeframe(ctx.services(), info, state, oldestPossibleOutput.timeslice.value)) {
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice",
                                  "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
                                  info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
@@ -646,7 +646,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
       O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Queueing oldest possible timeslice %" PRIu64 " propagation for execution.",
                              (uint64_t)oldestPossibleOutput.timeslice.value);
       AsyncQueueHelpers::post(
-        queue, decongestion.oldestPossibleTimesliceTask, [oldestPossibleOutput, &decongestion, &proxy, &spec, device, &timesliceIndex]() {
+        queue, decongestion.oldestPossibleTimesliceTask, [ref = services, oldestPossibleOutput, &decongestion, &proxy, &spec, device, &timesliceIndex]() {
           O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, &decongestion);
           if (decongestion.lastTimeslice >= oldestPossibleOutput.timeslice.value) {
             O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Not sending already sent value: %" PRIu64 "> %" PRIu64,
@@ -655,7 +655,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           }
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Running oldest possible timeslice %" PRIu64 " propagation.",
                                  (uint64_t)oldestPossibleOutput.timeslice.value);
-          DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, oldestPossibleOutput.timeslice.value);
+          DataProcessingHelpers::broadcastOldestPossibleTimeslice(ref, oldestPossibleOutput.timeslice.value);
 
           for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
             auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
@@ -665,7 +665,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
               O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
               continue;
             }
-            if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestPossibleOutput.timeslice.value)) {
+            if (DataProcessingHelpers::sendOldestPossibleTimeframe(ref, info, state, oldestPossibleOutput.timeslice.value)) {
               O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice",
                                      "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
                                      info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 8c914e201a479..3bcb8e4c8ef89 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -680,19 +680,11 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
     if (forwardedParts[fi].Size() == 0) {
       continue;
     }
-    auto channel = proxy.getForwardChannel(ChannelIndex{fi});
-    LOG(debug) << "Forwarding to " << channel->GetName() << " " << fi;
+    ForwardChannelInfo info = proxy.getForwardChannelInfo(ChannelIndex{fi});
+    LOG(debug) << "Forwarding to " << info.name << " " << fi;
     // in DPL we are using subchannel 0 only
     auto& parts = forwardedParts[fi];
-    int timeout = 30000;
-    auto res = channel->Send(parts, timeout);
-    if (res == (size_t)fair::mq::TransferCode::timeout) {
-      LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on {}.", timeout / 1000, channel->GetName());
-      channel->Send(parts);
-      LOGP(info, "Downstream backpressure on {} recovered.", channel->GetName());
-    } else if (res == (size_t)fair::mq::TransferCode::error) {
-      LOGP(fatal, "Error while sending on channel {}", channel->GetName());
-    }
+    info.policy->forward(parts, ChannelIndex{fi}, registry);
   }
 
   auto& asyncQueue = registry.get<AsyncQueue>();
@@ -713,7 +705,7 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
           LOG(debug) << "Skipping channel";
           continue;
         }
-        if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestTimeslice.timeslice.value)) {
+        if (DataProcessingHelpers::sendOldestPossibleTimeframe(registry, info, state, oldestTimeslice.timeslice.value)) {
           LOGP(debug, "Forwarding to channel {} oldest possible timeslice {}, prio 20", info.name, oldestTimeslice.timeslice.value);
         }
       }
@@ -1678,8 +1670,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
 
     for (auto& channel : spec.outputChannels) {
       LOGP(detail, "Sending end of stream to {}", channel.name);
-      auto& rawDevice = ref.get<RawDeviceService>();
-      DataProcessingHelpers::sendEndOfStream(*rawDevice.device(), channel);
+      DataProcessingHelpers::sendEndOfStream(ref, channel);
     }
     // This is needed because the transport is deleted before the device.
     relayer.clear();
@@ -2460,8 +2451,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   if (state.streaming == StreamingState::EndOfStreaming) {
     LOGP(detail, "Broadcasting end of stream");
     for (auto& channel : spec.outputChannels) {
-      auto& rawDevice = ref.get<RawDeviceService>();
-      DataProcessingHelpers::sendEndOfStream(*rawDevice.device(), channel);
+      DataProcessingHelpers::sendEndOfStream(ref, channel);
     }
     switchState(StreamingState::Idle);
   }
diff --git a/Framework/Core/src/DataProcessingHelpers.cxx b/Framework/Core/src/DataProcessingHelpers.cxx
index d424a53a52bd9..58fda5a13bae9 100644
--- a/Framework/Core/src/DataProcessingHelpers.cxx
+++ b/Framework/Core/src/DataProcessingHelpers.cxx
@@ -18,79 +18,74 @@
 #include "Headers/DataHeader.h"
 #include "Headers/Stack.h"
 #include "Framework/Logger.h"
+#include "Framework/SendingPolicy.h"
+#include "Framework/RawDeviceService.h"
 
 #include <fairmq/Device.h>
 #include <fairmq/Channel.h>
 
 namespace o2::framework
 {
-void DataProcessingHelpers::sendEndOfStream(fair::mq::Device& device, OutputChannelSpec const& channel)
+void DataProcessingHelpers::sendEndOfStream(ServiceRegistryRef const& ref, OutputChannelSpec const& channel)
 {
+  fair::mq::Device* device = ref.get<RawDeviceService>().device();
   fair::mq::Parts parts;
-  fair::mq::MessagePtr payload(device.NewMessage());
+  fair::mq::MessagePtr payload(device->NewMessage());
   SourceInfoHeader sih;
   sih.state = InputChannelState::Completed;
-  auto channelAlloc = o2::pmr::getTransportAllocator(device.GetChannel(channel.name, 0).Transport());
+  auto channelAlloc = o2::pmr::getTransportAllocator(device->GetChannel(channel.name, 0).Transport());
   auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, sih});
   // sigh... See if we can avoid having it const by not
   // exposing it to the user in the first place.
   parts.AddPart(std::move(header));
   parts.AddPart(std::move(payload));
-  device.Send(parts, channel.name, 0);
+  device->Send(parts, channel.name, 0);
   LOGP(info, "Sending end-of-stream message to channel {}", channel.name);
 }
 
-void DataProcessingHelpers::sendOldestPossibleTimeframe(fair::mq::Channel& channel, size_t timeslice)
+void doSendOldestPossibleTimeframe(ServiceRegistryRef ref, fair::mq::TransportFactory* transport, ChannelIndex index, SendingPolicy::SendingCallback const& callback, size_t timeslice)
 {
   fair::mq::Parts parts;
-  fair::mq::MessagePtr payload(channel.Transport()->CreateMessage());
+  fair::mq::MessagePtr payload(transport->CreateMessage());
   o2::framework::DomainInfoHeader dih;
   dih.oldestPossibleTimeslice = timeslice;
-  auto channelAlloc = o2::pmr::getTransportAllocator(channel.Transport());
+  auto channelAlloc = o2::pmr::getTransportAllocator(transport);
   auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, dih});
   // sigh... See if we can avoid having it const by not
   // exposing it to the user in the first place.
   parts.AddPart(std::move(header));
   parts.AddPart(std::move(payload));
 
-  auto timeout = 1000;
-  auto res = channel.Send(parts, timeout);
-  if (res == (size_t)fair::mq::TransferCode::timeout) {
-    LOGP(warning, "Timed out sending oldest possible timeslice after {}s. Downstream backpressure detected on {}.", timeout / 1000, channel.GetName());
-    channel.Send(parts);
-    LOGP(info, "Downstream backpressure on {} recovered.", channel.GetName());
-  }
-  if (res < (size_t)fair::mq::TransferCode::success) {
-    LOGP(fatal, "Error sending oldest possible timeframe {} on channel {} (code {})", timeslice, channel.GetName(), res);
-  }
+  callback(parts, index, ref);
 }
 
-bool DataProcessingHelpers::sendOldestPossibleTimeframe(ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice)
+bool DataProcessingHelpers::sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice)
 {
   if (state.oldestForChannel.value >= timeslice) {
     return false;
   }
-  sendOldestPossibleTimeframe(info.channel, timeslice);
+  doSendOldestPossibleTimeframe(ref, info.channel.Transport(), info.index, info.policy->forward, timeslice);
   state.oldestForChannel = {timeslice};
   return true;
 }
 
-bool DataProcessingHelpers::sendOldestPossibleTimeframe(OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice)
+bool DataProcessingHelpers::sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice)
 {
   if (state.oldestForChannel.value >= timeslice) {
     return false;
   }
-  sendOldestPossibleTimeframe(info.channel, timeslice);
+  doSendOldestPossibleTimeframe(ref, info.channel.Transport(), info.index, info.policy->send, timeslice);
   state.oldestForChannel = {timeslice};
   return true;
 }
 
-void DataProcessingHelpers::broadcastOldestPossibleTimeslice(FairMQDeviceProxy& proxy, size_t timeslice)
+void DataProcessingHelpers::broadcastOldestPossibleTimeslice(ServiceRegistryRef const& ref, size_t timeslice)
 {
+  auto& proxy = ref.get<FairMQDeviceProxy>();
   for (int ci = 0; ci < proxy.getNumOutputChannels(); ++ci) {
     auto& info = proxy.getOutputChannelInfo({ci});
     auto& state = proxy.getOutputChannelState({ci});
-    sendOldestPossibleTimeframe(info, state, timeslice);
+    sendOldestPossibleTimeframe(ref, info, state, timeslice);
   }
 }
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index b1681a4779ceb..e9df320fe8b95 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -493,6 +493,7 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
                                               const std::vector<OutputSpec>& outputsMatchers,
                                               const std::vector<ChannelConfigurationPolicy>& channelPolicies,
                                               const std::vector<SendingPolicy>& sendingPolicies,
+                                              const std::vector<ForwardingPolicy>& forwardingPolicies,
                                               std::string const& channelPrefix,
                                               ComputingOffer const& defaultOffer,
                                               OverrideServiceSpecs const& overrideServices)
@@ -656,7 +657,7 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
   // whether this is a real OutputRoute or if it's a forward from
   // a previous consumer device.
   // FIXME: where do I find the InputSpec for the forward?
-  auto appendOutputRouteToSourceDeviceChannel = [&outputsMatchers, &workflow, &devices, &logicalEdges, &sendingPolicies, &configContext](
+  auto appendOutputRouteToSourceDeviceChannel = [&outputsMatchers, &workflow, &devices, &logicalEdges, &sendingPolicies, &forwardingPolicies, &configContext](
                                                   size_t ei, size_t di, size_t ci) {
     assert(ei < logicalEdges.size());
     assert(di < devices.size());
@@ -670,29 +671,40 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
     assert(edge.outputGlobalIndex < outputsMatchers.size());
     // Iterate over all the policies and apply the first one that matches.
     SendingPolicy const* policyPtr = nullptr;
+    ForwardingPolicy const* forwardPolicyPtr = nullptr;
     for (auto& policy : sendingPolicies) {
       if (policy.matcher(producer, consumer, configContext)) {
         policyPtr = &policy;
         break;
       }
     }
+    assert(forwardingPolicies.empty() == false);
+    for (auto& policy : forwardingPolicies) {
+      if (policy.matcher(producer, consumer, configContext)) {
+        forwardPolicyPtr = &policy;
+        break;
+      }
+    }
     assert(policyPtr != nullptr);
+    assert(forwardPolicyPtr != nullptr);
 
     if (edge.isForward == false) {
       OutputRoute route{
-        edge.timeIndex,
-        consumer.maxInputTimeslices,
-        outputsMatchers[edge.outputGlobalIndex],
-        channel.name,
-        policyPtr,
+        .timeslice = edge.timeIndex,
+        .maxTimeslices = consumer.maxInputTimeslices,
+        .matcher = outputsMatchers[edge.outputGlobalIndex],
+        .channel = channel.name,
+        .policy = policyPtr,
       };
       device.outputs.emplace_back(route);
     } else {
       ForwardRoute route{
-        edge.timeIndex,
-        consumer.maxInputTimeslices,
-        workflow[edge.consumer].inputs[edge.consumerInputIndex],
-        channel.name};
+        .timeslice = edge.timeIndex,
+        .maxTimeslices = consumer.maxInputTimeslices,
+        .matcher = workflow[edge.consumer].inputs[edge.consumerInputIndex],
+        .channel = channel.name,
+        .policy = forwardPolicyPtr,
+      };
       device.forwards.emplace_back(route);
     }
   };
@@ -1051,6 +1063,7 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
                                                        std::vector<ResourcePolicy> const& resourcePolicies,
                                                        std::vector<CallbacksPolicy> const& callbacksPolicies,
                                                        std::vector<SendingPolicy> const& sendingPolicies,
+                                                       std::vector<ForwardingPolicy> const& forwardingPolicies,
                                                        std::vector<DeviceSpec>& devices,
                                                        ResourceManager& resourceManager,
                                                        std::string const& uniqueWorkflowId,
@@ -1111,7 +1124,7 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
   defaultOffer.memory /= deviceCount + 1;
 
   processOutEdgeActions(configContext, devices, deviceIndex, connections, resourceManager, outEdgeIndex, logicalEdges,
-                        outActions, workflow, outputs, channelPolicies, sendingPolicies, channelPrefix, defaultOffer, overrideServices);
+                        outActions, workflow, outputs, channelPolicies, sendingPolicies, forwardingPolicies, channelPrefix, defaultOffer, overrideServices);
 
   // FIXME: is this not the case???
   std::sort(connections.begin(), connections.end());
diff --git a/Framework/Core/src/DeviceSpecHelpers.h b/Framework/Core/src/DeviceSpecHelpers.h
index 1fb1866ae2dc3..17601c81342c1 100644
--- a/Framework/Core/src/DeviceSpecHelpers.h
+++ b/Framework/Core/src/DeviceSpecHelpers.h
@@ -53,6 +53,7 @@ struct DeviceSpecHelpers {
     std::vector<ResourcePolicy> const& resourcePolicies,
     std::vector<CallbacksPolicy> const& callbacksPolicies,
     std::vector<SendingPolicy> const& sendingPolicy,
+    std::vector<ForwardingPolicy> const& forwardingPolicies,
     std::vector<DeviceSpec>& devices,
     ResourceManager& resourceManager,
     std::string const& uniqueWorkflowId,
@@ -80,9 +81,10 @@ struct DeviceSpecHelpers {
     std::vector<DispatchPolicy> dispatchPolicies = DispatchPolicy::createDefaultPolicies();
     std::vector<ResourcePolicy> resourcePolicies = ResourcePolicy::createDefaultPolicies();
     std::vector<SendingPolicy> sendingPolicies = SendingPolicy::createDefaultPolicies();
+    std::vector<ForwardingPolicy> forwardingPolicies = ForwardingPolicy::createDefaultPolicies();
     dataProcessorSpecs2DeviceSpecs(workflow, channelPolicies, completionPolicies,
                                    dispatchPolicies, resourcePolicies, callbacksPolicies,
-                                   sendingPolicies, devices,
+                                   sendingPolicies, forwardingPolicies, devices,
                                    resourceManager, uniqueWorkflowId, configContext, optimizeTopology,
                                    resourcesMonitoringInterval, channelPrefix, overrideServices);
   }
@@ -150,6 +152,7 @@ struct DeviceSpecHelpers {
     const std::vector<OutputSpec>& outputs,
     std::vector<ChannelConfigurationPolicy> const& channelPolicies,
     std::vector<SendingPolicy> const& sendingPolicies,
+    std::vector<ForwardingPolicy> const& forwardingPolicies,
     std::string const& channelPrefix,
     ComputingOffer const& defaultOffer,
     OverrideServiceSpecs const& overrideServices = {});
diff --git a/Framework/Core/src/FairMQDeviceProxy.cxx b/Framework/Core/src/FairMQDeviceProxy.cxx
index c25a58e47ef2d..bdffddd5a4d1a 100644
--- a/Framework/Core/src/FairMQDeviceProxy.cxx
+++ b/Framework/Core/src/FairMQDeviceProxy.cxx
@@ -267,6 +267,7 @@ void FairMQDeviceProxy::bind(std::vector<OutputRoute> const& outputs, std::vecto
           .channelType = dplChannel,
           .channel = channel->second.at(0),
           .policy = route.policy,
+          .index = channelIndex,
         };
         mOutputChannelInfos.push_back(info);
         mOutputChannelStates.push_back({0});
@@ -345,7 +346,7 @@ void FairMQDeviceProxy::bind(std::vector<OutputRoute> const& outputs, std::vecto
           LOGP(fatal, "Expected channel {} not configured.", route.channel);
         }
         ChannelAccountingType dplChannel = (route.channel.rfind("from_", 0) == 0) ? ChannelAccountingType::DPL : ChannelAccountingType::RAWFMQ;
-        mForwardChannelInfos.push_back(ForwardChannelInfo{route.channel, dplChannel, channel->second.at(0)});
+        mForwardChannelInfos.push_back(ForwardChannelInfo{.name = route.channel, .channelType = dplChannel, .channel = channel->second.at(0), .policy = route.policy, .index = channelIndex});
         mForwardChannelStates.push_back(ForwardChannelState{0});
         channelNameToChannel[route.channel] = channelIndex;
         LOGP(detail, "Binding forward channel {} to channel index {}", route.channel, channelIndex.value);
diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 17faef6dbd3d0..7c3ea631d072c 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -119,4 +119,76 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
                 LOGP(fatal, "Error while sending on channel {}", channel->GetName());
               } }}};
 }
+
+std::vector<ForwardingPolicy> ForwardingPolicy::createDefaultPolicies()
+{
+  return {ForwardingPolicy{
+            .name = "profiling",
+            .matcher = [](DataProcessorSpec const&, DataProcessorSpec const&, ConfigContext const&) { return getenv("DPL_DEBUG_MESSAGE_SIZE"); },
+            .forward = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
+              auto &proxy = registry.get<FairMQDeviceProxy>();
+              auto *channel = proxy.getForwardChannel(channelIndex);
+              auto timeout = 1000;
+              int count = 0;
+              auto& relayer = registry.get<DataRelayer>();
+              for (auto& part : parts) {
+                auto* dh = o2::header::get<o2::header::DataHeader*>(part->GetData());
+                if (dh == nullptr) {
+                  // This is a payload.
+                  continue;
+                }
+                LOGP(info, "Sent {}/{}/{} for a total of {} bytes", dh->dataOrigin, dh->dataDescription, dh->subSpecification, dh->payloadSize);
+                count+= dh->payloadSize;
+                auto* dph = o2::header::get<o2::framework::DataProcessingHeader*>(part->GetData());
+                if (dph == nullptr) {
+                  // This is a payload.
+                  continue;
+                }
+                auto oldestPossibleOutput = relayer.getOldestPossibleOutput();
+                if ((size_t)dph->startTime < oldestPossibleOutput.timeslice.value) {
+                  LOGP(error, "Sent startTime {} while oldestPossibleOutput is {}. This should not be possible.", dph->startTime, oldestPossibleOutput.timeslice.value);
+                }
+              }
+              LOGP(info, "Sent {} parts for a total of {} bytes", parts.Size(), count);
+              auto res = channel->Send(parts, timeout);
+              if (res == (size_t)fair::mq::TransferCode::timeout) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on {}.", timeout/1000, channel->GetName());
+                channel->Send(parts);
+                LOGP(info, "Downstream backpressure on {} recovered.", channel->GetName());
+              } else if (res == (size_t) fair::mq::TransferCode::error) {
+                LOGP(fatal, "Error while sending on channel {}", channel->GetName());
+              } }},
+          ForwardingPolicy{
+            .name = "expendable",
+            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) {
+              auto has_label = [](DataProcessorLabel const& label) {
+                return label.value == "expendable";
+              };
+              return std::find_if(dest.labels.begin(), dest.labels.end(), has_label) != dest.labels.end(); },
+            .forward = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
+              auto &proxy = registry.get<FairMQDeviceProxy>();
+              auto *channel = proxy.getForwardChannel(channelIndex);
+              auto timeout = 1000;
+              auto res = channel->Send(parts, timeout);
+              if (res == (size_t)fair::mq::TransferCode::timeout) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
+              } else if (res == (size_t) fair::mq::TransferCode::error) {
+                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              } }},
+          ForwardingPolicy{
+            .name = "default",
+            .matcher = [](DataProcessorSpec const&, DataProcessorSpec const&, ConfigContext const&) { return true; },
+            .forward = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
+              auto &proxy = registry.get<FairMQDeviceProxy>();
+              auto *channel = proxy.getForwardChannel(channelIndex);
+              auto timeout = 1000;
+              auto res = channel->Send(parts, timeout);
+              if (res == (size_t)fair::mq::TransferCode::timeout) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on {}.", timeout/1000, channel->GetName());
+                channel->Send(parts);
+                LOGP(info, "Downstream backpressure on {} recovered.", channel->GetName());
+              } else if (res == (size_t) fair::mq::TransferCode::error) {
+                LOGP(fatal, "Error while sending on channel {}", channel->GetName());
+              } }}};
+}
 } // namespace o2::framework
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 9ca46e060b1a2..cfbb616b0d0a5 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1650,6 +1650,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
                                                             driverInfo.resourcePolicies,
                                                             driverInfo.callbacksPolicies,
                                                             driverInfo.sendingPolicies,
+                                                            driverInfo.forwardingPolicies,
                                                             runningWorkflow.devices,
                                                             *resourceManager,
                                                             driverInfo.uniqueWorkflowId,
@@ -2644,6 +2645,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
 {
   std::vector<std::string> currentArgs;
   std::vector<PluginInfo> plugins;
+  std::vector<ForwardingPolicy> forwardingPolicies = ForwardingPolicy::createDefaultPolicies();
 
   for (int ai = 1; ai < argc; ++ai) {
     currentArgs.emplace_back(argv[ai]);
@@ -3001,6 +3003,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
   };
   DriverInfo driverInfo{
     .sendingPolicies = sendingPolicies,
+    .forwardingPolicies = forwardingPolicies,
     .callbacksPolicies = callbacksPolicies};
   driverInfo.states.reserve(10);
   driverInfo.sigintRequested = false;
diff --git a/Framework/Core/test/test_DeviceSpec.cxx b/Framework/Core/test/test_DeviceSpec.cxx
index 99494d5f8ea4f..72b19da542a76 100644
--- a/Framework/Core/test/test_DeviceSpec.cxx
+++ b/Framework/Core/test/test_DeviceSpec.cxx
@@ -421,6 +421,7 @@ TEST_CASE("TestOutEdgeProcessingHelpers")
   auto configContext = makeEmptyConfigContext();
   auto channelPolicies = ChannelConfigurationPolicy::createDefaultPolicies(*configContext);
   auto sendingPolicies = SendingPolicy::createDefaultPolicies();
+  auto forwardingPolicies = ForwardingPolicy::createDefaultPolicies();
 
   std::vector<ComputingResource> resources{ComputingResourceHelpers::getLocalhostResource()};
   SimpleResourceManager rm(resources);
@@ -429,7 +430,7 @@ TEST_CASE("TestOutEdgeProcessingHelpers")
   defaultOffer.memory = 0.01;
 
   DeviceSpecHelpers::processOutEdgeActions(*configContext, devices, deviceIndex, connections, rm, edgeOutIndex, logicalEdges,
-                                           actions, workflow, globalOutputs, channelPolicies, sendingPolicies, "", defaultOffer);
+                                           actions, workflow, globalOutputs, channelPolicies, sendingPolicies, forwardingPolicies, "", defaultOffer);
 
   std::vector<DeviceId> expectedDeviceIndex = {{0, 0, 0}, {0, 0, 0}, {0, 0, 0}, {1, 0, 1}, {1, 0, 1}, {1, 1, 2}, {1, 1, 2}, {1, 2, 3}, {1, 2, 3}};
   REQUIRE(devices.size() == 4);
@@ -713,6 +714,7 @@ TEST_CASE("TestSimpleWildcard")
   auto configContext = makeEmptyConfigContext();
   auto channelPolicies = ChannelConfigurationPolicy::createDefaultPolicies(*configContext);
   auto sendingPolicies = SendingPolicy::createDefaultPolicies();
+  auto forwardingPolicies = ForwardingPolicy::createDefaultPolicies();
 
   std::vector<DeviceSpec> devices;
   std::vector<DeviceId> deviceIndex;
@@ -751,7 +753,7 @@ TEST_CASE("TestSimpleWildcard")
   defaultOffer.memory = 0.01;
 
   DeviceSpecHelpers::processOutEdgeActions(*configContext, devices, deviceIndex, connections, rm, edgeOutIndex, logicalEdges,
-                                           outActions, workflow, globalOutputs, channelPolicies, sendingPolicies, "", defaultOffer);
+                                           outActions, workflow, globalOutputs, channelPolicies, sendingPolicies, forwardingPolicies, "", defaultOffer);
 
   REQUIRE(devices.size() == 2);
   ; // Two devices have outputs: A and Timer
diff --git a/Framework/DataInspector/src/DataInspectorService.cxx b/Framework/DataInspector/src/DataInspectorService.cxx
index 40399a366a303..712bf10b6fb2e 100644
--- a/Framework/DataInspector/src/DataInspectorService.cxx
+++ b/Framework/DataInspector/src/DataInspectorService.cxx
@@ -245,4 +245,4 @@ ServiceSpec* DIServicePlugin::create()
             } },
     .kind = ServiceKind::Global};
 };
-} // namespace o2::framework
\ No newline at end of file
+} // namespace o2::framework
diff --git a/Utilities/DataSampling/src/Dispatcher.cxx b/Utilities/DataSampling/src/Dispatcher.cxx
index de78993ff083e..28ff4d5568da9 100644
--- a/Utilities/DataSampling/src/Dispatcher.cxx
+++ b/Utilities/DataSampling/src/Dispatcher.cxx
@@ -125,10 +125,9 @@ void Dispatcher::run(ProcessingContext& ctx)
   if (ctx.inputs().isValid("timer-stats")) {
     reportStats(ctx.services().get<Monitoring>());
   }
-  auto& proxy = ctx.services().get<FairMQDeviceProxy>();
   auto& relayer = ctx.services().get<DataRelayer>();
   auto timeslice = relayer.getOldestPossibleOutput().timeslice.value;
-  DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, timeslice);
+  DataProcessingHelpers::broadcastOldestPossibleTimeslice(ctx.services(), timeslice);
 }
 
 void Dispatcher::reportStats(Monitoring& monitoring) const

From 29ce029734b3e1d4096821e231628b0434bbbc21 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 9 Feb 2024 21:01:27 +0100
Subject: [PATCH 318/726] DPL: add SendingPolicy for the case destination is
 expendable

---
 Framework/Core/src/SendingPolicy.cxx | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 7c3ea631d072c..b572fe8b86911 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -29,7 +29,7 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
 {
   return {SendingPolicy{
             .name = "dispatcher",
-            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) { 
+            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) {
                 if (source.name == "Dispatcher") {
                   return true;
                 }
@@ -103,6 +103,23 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               } else if (res == (size_t) fair::mq::TransferCode::error) {
                 LOGP(fatal, "Error while sending on channel {}", channel->GetName());
               } }},
+          SendingPolicy{
+            .name = "expendable",
+            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) {
+              auto has_label = [](DataProcessorLabel const& label) {
+                return label.value == "expendable";
+              };
+              return std::find_if(dest.labels.begin(), dest.labels.end(), has_label) != dest.labels.end(); },
+            .send = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
+              auto &proxy = registry.get<FairMQDeviceProxy>();
+              auto *channel = proxy.getOutputChannel(channelIndex);
+              auto timeout = 1000;
+              auto res = channel->Send(parts, timeout);
+              if (res == (size_t)fair::mq::TransferCode::timeout) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
+              } else if (res == (size_t) fair::mq::TransferCode::error) {
+                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              } }},
           SendingPolicy{
             .name = "default",
             .matcher = [](DataProcessorSpec const&, DataProcessorSpec const&, ConfigContext const&) { return true; },

From eb9aadcf7ff44f8d0dd25c8f2192212cdbf9f3a7 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 10 Feb 2024 00:32:25 +0100
Subject: [PATCH 319/726] DPL: use the SendingPolicy for every kind of message
 sending

---
 .../Core/include/Framework/ChannelInfo.h      |  4 +
 .../include/Framework/DataProcessingHelpers.h | 22 +++---
 Framework/Core/include/Framework/DriverInfo.h |  3 +
 .../Core/include/Framework/ForwardRoute.h     |  9 ++-
 .../Core/include/Framework/SendingPolicy.h    |  9 +++
 Framework/Core/src/CommonServices.cxx         | 10 +--
 Framework/Core/src/DataProcessingDevice.cxx   | 24 +++---
 Framework/Core/src/DataProcessingHelpers.cxx  | 41 +++++-----
 Framework/Core/src/DeviceSpecHelpers.cxx      | 35 ++++++---
 Framework/Core/src/DeviceSpecHelpers.h        |  5 +-
 .../Core/src/ExternalFairMQDeviceProxy.cxx    |  9 ++-
 Framework/Core/src/FairMQDeviceProxy.cxx      |  3 +-
 Framework/Core/src/SendingPolicy.cxx          | 77 +++++++++++++++++++
 Framework/Core/src/runDataProcessing.cxx      |  3 +
 Framework/Core/test/test_DeviceSpec.cxx       |  6 +-
 .../src/DataInspectorService.cxx              |  2 +-
 Utilities/DataSampling/src/Dispatcher.cxx     |  3 +-
 17 files changed, 188 insertions(+), 77 deletions(-)

diff --git a/Framework/Core/include/Framework/ChannelInfo.h b/Framework/Core/include/Framework/ChannelInfo.h
index 0ebc4a8506301..1f4bd8bc9e90e 100644
--- a/Framework/Core/include/Framework/ChannelInfo.h
+++ b/Framework/Core/include/Framework/ChannelInfo.h
@@ -67,6 +67,7 @@ struct InputChannelInfo {
 };
 
 struct SendingPolicy;
+struct ForwardingPolicy;
 
 /// Output channel information
 struct OutputChannelInfo {
@@ -74,6 +75,7 @@ struct OutputChannelInfo {
   ChannelAccountingType channelType = ChannelAccountingType::DPL;
   fair::mq::Channel& channel;
   SendingPolicy const* policy;
+  ChannelIndex index = {-1};
 };
 
 struct OutputChannelState {
@@ -89,6 +91,8 @@ struct ForwardChannelInfo {
   /// Wether or not it's a DPL internal channel.
   ChannelAccountingType channelType = ChannelAccountingType::DPL;
   fair::mq::Channel& channel;
+  ForwardingPolicy const* policy;
+  ChannelIndex index = {-1};
 };
 
 struct ForwardChannelState {
diff --git a/Framework/Core/include/Framework/DataProcessingHelpers.h b/Framework/Core/include/Framework/DataProcessingHelpers.h
index 35f78f12cc405..7a05ddf25c46c 100644
--- a/Framework/Core/include/Framework/DataProcessingHelpers.h
+++ b/Framework/Core/include/Framework/DataProcessingHelpers.h
@@ -11,14 +11,16 @@
 #ifndef O2_FRAMEWORK_DATAPROCESSINGHELPERS_H_
 #define O2_FRAMEWORK_DATAPROCESSINGHELPERS_H_
 
-#include "Framework/TimesliceIndex.h"
-#include <fairmq/FwdDecls.h>
+#include <cstddef>
 
 namespace o2::framework
 {
-
+struct ServiceRegistryRef;
+struct ForwardChannelInfo;
+struct ForwardChannelState;
+struct OutputChannelInfo;
 struct OutputChannelSpec;
-class FairMQDeviceProxy;
+struct OutputChannelState;
 
 /// Generic helpers for DataProcessing releated functions.
 struct DataProcessingHelpers {
@@ -26,15 +28,13 @@ struct DataProcessingHelpers {
   /// @param device the fair::mq::Device which needs to send the EndOfStream message
   /// @param channel the OutputChannelSpec of the channel which needs to be signaled
   ///        for EndOfStream
-  static void sendEndOfStream(fair::mq::Device& device, OutputChannelSpec const& channel);
+  static void sendEndOfStream(ServiceRegistryRef const& ref, OutputChannelSpec const& channel);
   /// @returns true if we did send the oldest possible timeslice message, false otherwise.
-  static bool sendOldestPossibleTimeframe(ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice);
+  static bool sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice);
   /// @returns true if we did send the oldest possible timeslice message, false otherwise.
-  static bool sendOldestPossibleTimeframe(OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice);
-  static void broadcastOldestPossibleTimeslice(FairMQDeviceProxy& proxy, size_t timeslice);
-
- private:
-  static void sendOldestPossibleTimeframe(fair::mq::Channel& channel, size_t timeslice);
+  static bool sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice);
+  /// Broadcast the oldest possible timeslice to all channels in output
+  static void broadcastOldestPossibleTimeslice(ServiceRegistryRef const& ref, size_t timeslice);
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/include/Framework/DriverInfo.h b/Framework/Core/include/Framework/DriverInfo.h
index 043a689087325..45c4c85a32d01 100644
--- a/Framework/Core/include/Framework/DriverInfo.h
+++ b/Framework/Core/include/Framework/DriverInfo.h
@@ -111,6 +111,9 @@ struct DriverInfo {
   /// These are the policies which can be applied to decide how
   /// we send data.
   std::vector<SendingPolicy> sendingPolicies;
+  /// These are the policies which can be applied to decide how
+  /// we forward data.
+  std::vector<ForwardingPolicy> forwardingPolicies;
   /// The argc with which the driver was started.
   int argc;
   /// The argv with which the driver was started.
diff --git a/Framework/Core/include/Framework/ForwardRoute.h b/Framework/Core/include/Framework/ForwardRoute.h
index bae2eaacf1a44..0e3d4d12da24a 100644
--- a/Framework/Core/include/Framework/ForwardRoute.h
+++ b/Framework/Core/include/Framework/ForwardRoute.h
@@ -15,11 +15,11 @@
 #include <cstddef>
 #include <string>
 
-namespace o2
-{
-namespace framework
+namespace o2::framework
 {
 
+struct ForwardingPolicy;
+
 /// This uniquely identifies a route to be forwarded by the device if
 /// the InputSpec @a matcher matches an input which should also go to
 /// @a channel
@@ -28,8 +28,9 @@ struct ForwardRoute {
   size_t maxTimeslices;
   InputSpec matcher;
   std::string channel;
+  // The policy to use to send to on this route.
+  ForwardingPolicy const* policy;
 };
 
-} // namespace framework
 } // namespace o2
 #endif // FRAMEWORK_FORWARDROUTE_H
diff --git a/Framework/Core/include/Framework/SendingPolicy.h b/Framework/Core/include/Framework/SendingPolicy.h
index 1d68412d46db8..7cc5c16e2c923 100644
--- a/Framework/Core/include/Framework/SendingPolicy.h
+++ b/Framework/Core/include/Framework/SendingPolicy.h
@@ -32,6 +32,15 @@ struct SendingPolicy {
   static std::vector<SendingPolicy> createDefaultPolicies();
 };
 
+struct ForwardingPolicy {
+  using ForwardingCallback = std::function<void(fair::mq::Parts&, ChannelIndex channelIndex, ServiceRegistryRef registry)>;
+  std::string name = "invalid";
+  EdgeMatcher matcher = nullptr;
+  ForwardingCallback forward = nullptr;
+  static std::vector<ForwardingPolicy> createDefaultPolicies();
+  static ForwardingPolicy createDefaultForwardingPolicy();
+};
+
 } // namespace o2::framework
 
 #endif // O2_FRAMEWORK_SENDINGPOLICY_H_
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index b7fc3410a1931..a0f7fcdc88201 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -588,7 +588,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           timesliceIndex.rescan();
         }
       }
-      DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, oldestPossibleOutput.timeslice.value);
+      DataProcessingHelpers::broadcastOldestPossibleTimeslice(ctx.services(), oldestPossibleOutput.timeslice.value);
 
       for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
         auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
@@ -598,7 +598,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
           continue;
         }
-        if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestPossibleOutput.timeslice.value)) {
+        if (DataProcessingHelpers::sendOldestPossibleTimeframe(ctx.services(), info, state, oldestPossibleOutput.timeslice.value)) {
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice",
                                  "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
                                  info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
@@ -646,7 +646,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
       O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Queueing oldest possible timeslice %" PRIu64 " propagation for execution.",
                              (uint64_t)oldestPossibleOutput.timeslice.value);
       AsyncQueueHelpers::post(
-        queue, decongestion.oldestPossibleTimesliceTask, [oldestPossibleOutput, &decongestion, &proxy, &spec, device, &timesliceIndex]() {
+        queue, decongestion.oldestPossibleTimesliceTask, [ref = services, oldestPossibleOutput, &decongestion, &proxy, &spec, device, &timesliceIndex]() {
           O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, &decongestion);
           if (decongestion.lastTimeslice >= oldestPossibleOutput.timeslice.value) {
             O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Not sending already sent value: %" PRIu64 "> %" PRIu64,
@@ -655,7 +655,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           }
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Running oldest possible timeslice %" PRIu64 " propagation.",
                                  (uint64_t)oldestPossibleOutput.timeslice.value);
-          DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, oldestPossibleOutput.timeslice.value);
+          DataProcessingHelpers::broadcastOldestPossibleTimeslice(ref, oldestPossibleOutput.timeslice.value);
 
           for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
             auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
@@ -665,7 +665,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
               O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
               continue;
             }
-            if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestPossibleOutput.timeslice.value)) {
+            if (DataProcessingHelpers::sendOldestPossibleTimeframe(ref, info, state, oldestPossibleOutput.timeslice.value)) {
               O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice",
                                      "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
                                      info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 8c914e201a479..50934bd59a729 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -680,19 +680,15 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
     if (forwardedParts[fi].Size() == 0) {
       continue;
     }
-    auto channel = proxy.getForwardChannel(ChannelIndex{fi});
-    LOG(debug) << "Forwarding to " << channel->GetName() << " " << fi;
+    ForwardChannelInfo info = proxy.getForwardChannelInfo(ChannelIndex{fi});
+    LOG(debug) << "Forwarding to " << info.name << " " << fi;
     // in DPL we are using subchannel 0 only
     auto& parts = forwardedParts[fi];
-    int timeout = 30000;
-    auto res = channel->Send(parts, timeout);
-    if (res == (size_t)fair::mq::TransferCode::timeout) {
-      LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on {}.", timeout / 1000, channel->GetName());
-      channel->Send(parts);
-      LOGP(info, "Downstream backpressure on {} recovered.", channel->GetName());
-    } else if (res == (size_t)fair::mq::TransferCode::error) {
-      LOGP(fatal, "Error while sending on channel {}", channel->GetName());
+    if (info.policy == nullptr) {
+      LOG(error) << "Forwarding to " << info.name << " " << fi << " has no policy";
+      continue;
     }
+    info.policy->forward(parts, ChannelIndex{fi}, registry);
   }
 
   auto& asyncQueue = registry.get<AsyncQueue>();
@@ -713,7 +709,7 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
           LOG(debug) << "Skipping channel";
           continue;
         }
-        if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestTimeslice.timeslice.value)) {
+        if (DataProcessingHelpers::sendOldestPossibleTimeframe(registry, info, state, oldestTimeslice.timeslice.value)) {
           LOGP(debug, "Forwarding to channel {} oldest possible timeslice {}, prio 20", info.name, oldestTimeslice.timeslice.value);
         }
       }
@@ -1678,8 +1674,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
 
     for (auto& channel : spec.outputChannels) {
       LOGP(detail, "Sending end of stream to {}", channel.name);
-      auto& rawDevice = ref.get<RawDeviceService>();
-      DataProcessingHelpers::sendEndOfStream(*rawDevice.device(), channel);
+      DataProcessingHelpers::sendEndOfStream(ref, channel);
     }
     // This is needed because the transport is deleted before the device.
     relayer.clear();
@@ -2460,8 +2455,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   if (state.streaming == StreamingState::EndOfStreaming) {
     LOGP(detail, "Broadcasting end of stream");
     for (auto& channel : spec.outputChannels) {
-      auto& rawDevice = ref.get<RawDeviceService>();
-      DataProcessingHelpers::sendEndOfStream(*rawDevice.device(), channel);
+      DataProcessingHelpers::sendEndOfStream(ref, channel);
     }
     switchState(StreamingState::Idle);
   }
diff --git a/Framework/Core/src/DataProcessingHelpers.cxx b/Framework/Core/src/DataProcessingHelpers.cxx
index d424a53a52bd9..58fda5a13bae9 100644
--- a/Framework/Core/src/DataProcessingHelpers.cxx
+++ b/Framework/Core/src/DataProcessingHelpers.cxx
@@ -18,79 +18,74 @@
 #include "Headers/DataHeader.h"
 #include "Headers/Stack.h"
 #include "Framework/Logger.h"
+#include "Framework/SendingPolicy.h"
+#include "Framework/RawDeviceService.h"
 
 #include <fairmq/Device.h>
 #include <fairmq/Channel.h>
 
 namespace o2::framework
 {
-void DataProcessingHelpers::sendEndOfStream(fair::mq::Device& device, OutputChannelSpec const& channel)
+void DataProcessingHelpers::sendEndOfStream(ServiceRegistryRef const& ref, OutputChannelSpec const& channel)
 {
+  fair::mq::Device* device = ref.get<RawDeviceService>().device();
   fair::mq::Parts parts;
-  fair::mq::MessagePtr payload(device.NewMessage());
+  fair::mq::MessagePtr payload(device->NewMessage());
   SourceInfoHeader sih;
   sih.state = InputChannelState::Completed;
-  auto channelAlloc = o2::pmr::getTransportAllocator(device.GetChannel(channel.name, 0).Transport());
+  auto channelAlloc = o2::pmr::getTransportAllocator(device->GetChannel(channel.name, 0).Transport());
   auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, sih});
   // sigh... See if we can avoid having it const by not
   // exposing it to the user in the first place.
   parts.AddPart(std::move(header));
   parts.AddPart(std::move(payload));
-  device.Send(parts, channel.name, 0);
+  device->Send(parts, channel.name, 0);
   LOGP(info, "Sending end-of-stream message to channel {}", channel.name);
 }
 
-void DataProcessingHelpers::sendOldestPossibleTimeframe(fair::mq::Channel& channel, size_t timeslice)
+void doSendOldestPossibleTimeframe(ServiceRegistryRef ref, fair::mq::TransportFactory* transport, ChannelIndex index, SendingPolicy::SendingCallback const& callback, size_t timeslice)
 {
   fair::mq::Parts parts;
-  fair::mq::MessagePtr payload(channel.Transport()->CreateMessage());
+  fair::mq::MessagePtr payload(transport->CreateMessage());
   o2::framework::DomainInfoHeader dih;
   dih.oldestPossibleTimeslice = timeslice;
-  auto channelAlloc = o2::pmr::getTransportAllocator(channel.Transport());
+  auto channelAlloc = o2::pmr::getTransportAllocator(transport);
   auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, dih});
   // sigh... See if we can avoid having it const by not
   // exposing it to the user in the first place.
   parts.AddPart(std::move(header));
   parts.AddPart(std::move(payload));
 
-  auto timeout = 1000;
-  auto res = channel.Send(parts, timeout);
-  if (res == (size_t)fair::mq::TransferCode::timeout) {
-    LOGP(warning, "Timed out sending oldest possible timeslice after {}s. Downstream backpressure detected on {}.", timeout / 1000, channel.GetName());
-    channel.Send(parts);
-    LOGP(info, "Downstream backpressure on {} recovered.", channel.GetName());
-  }
-  if (res < (size_t)fair::mq::TransferCode::success) {
-    LOGP(fatal, "Error sending oldest possible timeframe {} on channel {} (code {})", timeslice, channel.GetName(), res);
-  }
+  callback(parts, index, ref);
 }
 
-bool DataProcessingHelpers::sendOldestPossibleTimeframe(ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice)
+bool DataProcessingHelpers::sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice)
 {
   if (state.oldestForChannel.value >= timeslice) {
     return false;
   }
-  sendOldestPossibleTimeframe(info.channel, timeslice);
+  doSendOldestPossibleTimeframe(ref, info.channel.Transport(), info.index, info.policy->forward, timeslice);
   state.oldestForChannel = {timeslice};
   return true;
 }
 
-bool DataProcessingHelpers::sendOldestPossibleTimeframe(OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice)
+bool DataProcessingHelpers::sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice)
 {
   if (state.oldestForChannel.value >= timeslice) {
     return false;
   }
-  sendOldestPossibleTimeframe(info.channel, timeslice);
+  doSendOldestPossibleTimeframe(ref, info.channel.Transport(), info.index, info.policy->send, timeslice);
   state.oldestForChannel = {timeslice};
   return true;
 }
 
-void DataProcessingHelpers::broadcastOldestPossibleTimeslice(FairMQDeviceProxy& proxy, size_t timeslice)
+void DataProcessingHelpers::broadcastOldestPossibleTimeslice(ServiceRegistryRef const& ref, size_t timeslice)
 {
+  auto& proxy = ref.get<FairMQDeviceProxy>();
   for (int ci = 0; ci < proxy.getNumOutputChannels(); ++ci) {
     auto& info = proxy.getOutputChannelInfo({ci});
     auto& state = proxy.getOutputChannelState({ci});
-    sendOldestPossibleTimeframe(info, state, timeslice);
+    sendOldestPossibleTimeframe(ref, info, state, timeslice);
   }
 }
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index b1681a4779ceb..e9df320fe8b95 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -493,6 +493,7 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
                                               const std::vector<OutputSpec>& outputsMatchers,
                                               const std::vector<ChannelConfigurationPolicy>& channelPolicies,
                                               const std::vector<SendingPolicy>& sendingPolicies,
+                                              const std::vector<ForwardingPolicy>& forwardingPolicies,
                                               std::string const& channelPrefix,
                                               ComputingOffer const& defaultOffer,
                                               OverrideServiceSpecs const& overrideServices)
@@ -656,7 +657,7 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
   // whether this is a real OutputRoute or if it's a forward from
   // a previous consumer device.
   // FIXME: where do I find the InputSpec for the forward?
-  auto appendOutputRouteToSourceDeviceChannel = [&outputsMatchers, &workflow, &devices, &logicalEdges, &sendingPolicies, &configContext](
+  auto appendOutputRouteToSourceDeviceChannel = [&outputsMatchers, &workflow, &devices, &logicalEdges, &sendingPolicies, &forwardingPolicies, &configContext](
                                                   size_t ei, size_t di, size_t ci) {
     assert(ei < logicalEdges.size());
     assert(di < devices.size());
@@ -670,29 +671,40 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
     assert(edge.outputGlobalIndex < outputsMatchers.size());
     // Iterate over all the policies and apply the first one that matches.
     SendingPolicy const* policyPtr = nullptr;
+    ForwardingPolicy const* forwardPolicyPtr = nullptr;
     for (auto& policy : sendingPolicies) {
       if (policy.matcher(producer, consumer, configContext)) {
         policyPtr = &policy;
         break;
       }
     }
+    assert(forwardingPolicies.empty() == false);
+    for (auto& policy : forwardingPolicies) {
+      if (policy.matcher(producer, consumer, configContext)) {
+        forwardPolicyPtr = &policy;
+        break;
+      }
+    }
     assert(policyPtr != nullptr);
+    assert(forwardPolicyPtr != nullptr);
 
     if (edge.isForward == false) {
       OutputRoute route{
-        edge.timeIndex,
-        consumer.maxInputTimeslices,
-        outputsMatchers[edge.outputGlobalIndex],
-        channel.name,
-        policyPtr,
+        .timeslice = edge.timeIndex,
+        .maxTimeslices = consumer.maxInputTimeslices,
+        .matcher = outputsMatchers[edge.outputGlobalIndex],
+        .channel = channel.name,
+        .policy = policyPtr,
       };
       device.outputs.emplace_back(route);
     } else {
       ForwardRoute route{
-        edge.timeIndex,
-        consumer.maxInputTimeslices,
-        workflow[edge.consumer].inputs[edge.consumerInputIndex],
-        channel.name};
+        .timeslice = edge.timeIndex,
+        .maxTimeslices = consumer.maxInputTimeslices,
+        .matcher = workflow[edge.consumer].inputs[edge.consumerInputIndex],
+        .channel = channel.name,
+        .policy = forwardPolicyPtr,
+      };
       device.forwards.emplace_back(route);
     }
   };
@@ -1051,6 +1063,7 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
                                                        std::vector<ResourcePolicy> const& resourcePolicies,
                                                        std::vector<CallbacksPolicy> const& callbacksPolicies,
                                                        std::vector<SendingPolicy> const& sendingPolicies,
+                                                       std::vector<ForwardingPolicy> const& forwardingPolicies,
                                                        std::vector<DeviceSpec>& devices,
                                                        ResourceManager& resourceManager,
                                                        std::string const& uniqueWorkflowId,
@@ -1111,7 +1124,7 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
   defaultOffer.memory /= deviceCount + 1;
 
   processOutEdgeActions(configContext, devices, deviceIndex, connections, resourceManager, outEdgeIndex, logicalEdges,
-                        outActions, workflow, outputs, channelPolicies, sendingPolicies, channelPrefix, defaultOffer, overrideServices);
+                        outActions, workflow, outputs, channelPolicies, sendingPolicies, forwardingPolicies, channelPrefix, defaultOffer, overrideServices);
 
   // FIXME: is this not the case???
   std::sort(connections.begin(), connections.end());
diff --git a/Framework/Core/src/DeviceSpecHelpers.h b/Framework/Core/src/DeviceSpecHelpers.h
index 1fb1866ae2dc3..17601c81342c1 100644
--- a/Framework/Core/src/DeviceSpecHelpers.h
+++ b/Framework/Core/src/DeviceSpecHelpers.h
@@ -53,6 +53,7 @@ struct DeviceSpecHelpers {
     std::vector<ResourcePolicy> const& resourcePolicies,
     std::vector<CallbacksPolicy> const& callbacksPolicies,
     std::vector<SendingPolicy> const& sendingPolicy,
+    std::vector<ForwardingPolicy> const& forwardingPolicies,
     std::vector<DeviceSpec>& devices,
     ResourceManager& resourceManager,
     std::string const& uniqueWorkflowId,
@@ -80,9 +81,10 @@ struct DeviceSpecHelpers {
     std::vector<DispatchPolicy> dispatchPolicies = DispatchPolicy::createDefaultPolicies();
     std::vector<ResourcePolicy> resourcePolicies = ResourcePolicy::createDefaultPolicies();
     std::vector<SendingPolicy> sendingPolicies = SendingPolicy::createDefaultPolicies();
+    std::vector<ForwardingPolicy> forwardingPolicies = ForwardingPolicy::createDefaultPolicies();
     dataProcessorSpecs2DeviceSpecs(workflow, channelPolicies, completionPolicies,
                                    dispatchPolicies, resourcePolicies, callbacksPolicies,
-                                   sendingPolicies, devices,
+                                   sendingPolicies, forwardingPolicies, devices,
                                    resourceManager, uniqueWorkflowId, configContext, optimizeTopology,
                                    resourcesMonitoringInterval, channelPrefix, overrideServices);
   }
@@ -150,6 +152,7 @@ struct DeviceSpecHelpers {
     const std::vector<OutputSpec>& outputs,
     std::vector<ChannelConfigurationPolicy> const& channelPolicies,
     std::vector<SendingPolicy> const& sendingPolicies,
+    std::vector<ForwardingPolicy> const& forwardingPolicies,
     std::string const& channelPrefix,
     ComputingOffer const& defaultOffer,
     OverrideServiceSpecs const& overrideServices = {});
diff --git a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
index 0f769ae2dc850..b2957b8f3953c 100644
--- a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
+++ b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
@@ -29,6 +29,7 @@
 #include "Framework/TimingInfo.h"
 #include "Framework/DeviceState.h"
 #include "Framework/Monitoring.h"
+#include "Framework/SendingPolicy.h"
 #include "Headers/DataHeader.h"
 #include "Headers/Stack.h"
 #include "DecongestionService.h"
@@ -1010,7 +1011,13 @@ DataProcessorSpec specifyFairMQDeviceMultiOutputProxy(char const* name,
         if (device->GetChannels().count(channel) == 0) {
           throw std::runtime_error("no corresponding output channel found for input '" + channel + "'");
         }
-        ForwardRoute route{0, 1, spec, channel};
+        static auto policy = ForwardingPolicy::createDefaultForwardingPolicy();
+        ForwardRoute route{
+          .timeslice = 0,
+          .maxTimeslices = 1,
+          .matcher = spec,
+          .channel = channel,
+          .policy = &policy};
         // this we will try to fix on the framework level, there will be an API to
         // set external routes. Basically, this has to be added while setting up the
         // workflow. After that, the actual spec provided by the service is supposed
diff --git a/Framework/Core/src/FairMQDeviceProxy.cxx b/Framework/Core/src/FairMQDeviceProxy.cxx
index c25a58e47ef2d..bdffddd5a4d1a 100644
--- a/Framework/Core/src/FairMQDeviceProxy.cxx
+++ b/Framework/Core/src/FairMQDeviceProxy.cxx
@@ -267,6 +267,7 @@ void FairMQDeviceProxy::bind(std::vector<OutputRoute> const& outputs, std::vecto
           .channelType = dplChannel,
           .channel = channel->second.at(0),
           .policy = route.policy,
+          .index = channelIndex,
         };
         mOutputChannelInfos.push_back(info);
         mOutputChannelStates.push_back({0});
@@ -345,7 +346,7 @@ void FairMQDeviceProxy::bind(std::vector<OutputRoute> const& outputs, std::vecto
           LOGP(fatal, "Expected channel {} not configured.", route.channel);
         }
         ChannelAccountingType dplChannel = (route.channel.rfind("from_", 0) == 0) ? ChannelAccountingType::DPL : ChannelAccountingType::RAWFMQ;
-        mForwardChannelInfos.push_back(ForwardChannelInfo{route.channel, dplChannel, channel->second.at(0)});
+        mForwardChannelInfos.push_back(ForwardChannelInfo{.name = route.channel, .channelType = dplChannel, .channel = channel->second.at(0), .policy = route.policy, .index = channelIndex});
         mForwardChannelStates.push_back(ForwardChannelState{0});
         channelNameToChannel[route.channel] = channelIndex;
         LOGP(detail, "Binding forward channel {} to channel index {}", route.channel, channelIndex.value);
diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 17faef6dbd3d0..3d65975919676 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -119,4 +119,81 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
                 LOGP(fatal, "Error while sending on channel {}", channel->GetName());
               } }}};
 }
+
+ForwardingPolicy ForwardingPolicy::createDefaultForwardingPolicy()
+{
+  return ForwardingPolicy{
+    .name = "default",
+    .matcher = [](DataProcessorSpec const&, DataProcessorSpec const&, ConfigContext const&) { return true; },
+    .forward = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
+              auto &proxy = registry.get<FairMQDeviceProxy>();
+              auto *channel = proxy.getForwardChannel(channelIndex);
+              auto timeout = 1000;
+              auto res = channel->Send(parts, timeout);
+              if (res == (size_t)fair::mq::TransferCode::timeout) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on {}.", timeout/1000, channel->GetName());
+                channel->Send(parts);
+                LOGP(info, "Downstream backpressure on {} recovered.", channel->GetName());
+              } else if (res == (size_t) fair::mq::TransferCode::error) {
+                LOGP(fatal, "Error while sending on channel {}", channel->GetName());
+              } }};
+}
+
+std::vector<ForwardingPolicy> ForwardingPolicy::createDefaultPolicies()
+{
+  return {ForwardingPolicy{
+            .name = "profiling",
+            .matcher = [](DataProcessorSpec const&, DataProcessorSpec const&, ConfigContext const&) { return getenv("DPL_DEBUG_MESSAGE_SIZE"); },
+            .forward = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
+              auto &proxy = registry.get<FairMQDeviceProxy>();
+              auto *channel = proxy.getForwardChannel(channelIndex);
+              auto timeout = 1000;
+              int count = 0;
+              auto& relayer = registry.get<DataRelayer>();
+              for (auto& part : parts) {
+                auto* dh = o2::header::get<o2::header::DataHeader*>(part->GetData());
+                if (dh == nullptr) {
+                  // This is a payload.
+                  continue;
+                }
+                LOGP(info, "Sent {}/{}/{} for a total of {} bytes", dh->dataOrigin, dh->dataDescription, dh->subSpecification, dh->payloadSize);
+                count+= dh->payloadSize;
+                auto* dph = o2::header::get<o2::framework::DataProcessingHeader*>(part->GetData());
+                if (dph == nullptr) {
+                  // This is a payload.
+                  continue;
+                }
+                auto oldestPossibleOutput = relayer.getOldestPossibleOutput();
+                if ((size_t)dph->startTime < oldestPossibleOutput.timeslice.value) {
+                  LOGP(error, "Sent startTime {} while oldestPossibleOutput is {}. This should not be possible.", dph->startTime, oldestPossibleOutput.timeslice.value);
+                }
+              }
+              LOGP(info, "Sent {} parts for a total of {} bytes", parts.Size(), count);
+              auto res = channel->Send(parts, timeout);
+              if (res == (size_t)fair::mq::TransferCode::timeout) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on {}.", timeout/1000, channel->GetName());
+                channel->Send(parts);
+                LOGP(info, "Downstream backpressure on {} recovered.", channel->GetName());
+              } else if (res == (size_t) fair::mq::TransferCode::error) {
+                LOGP(fatal, "Error while sending on channel {}", channel->GetName());
+              } }},
+          ForwardingPolicy{
+            .name = "expendable",
+            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) {
+              auto has_label = [](DataProcessorLabel const& label) {
+                return label.value == "expendable";
+              };
+              return std::find_if(dest.labels.begin(), dest.labels.end(), has_label) != dest.labels.end(); },
+            .forward = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
+              auto &proxy = registry.get<FairMQDeviceProxy>();
+              auto *channel = proxy.getForwardChannel(channelIndex);
+              auto timeout = 1000;
+              auto res = channel->Send(parts, timeout);
+              if (res == (size_t)fair::mq::TransferCode::timeout) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
+              } else if (res == (size_t) fair::mq::TransferCode::error) {
+                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              } }},
+          createDefaultForwardingPolicy()};
+}
 } // namespace o2::framework
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 9ca46e060b1a2..cfbb616b0d0a5 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1650,6 +1650,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
                                                             driverInfo.resourcePolicies,
                                                             driverInfo.callbacksPolicies,
                                                             driverInfo.sendingPolicies,
+                                                            driverInfo.forwardingPolicies,
                                                             runningWorkflow.devices,
                                                             *resourceManager,
                                                             driverInfo.uniqueWorkflowId,
@@ -2644,6 +2645,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
 {
   std::vector<std::string> currentArgs;
   std::vector<PluginInfo> plugins;
+  std::vector<ForwardingPolicy> forwardingPolicies = ForwardingPolicy::createDefaultPolicies();
 
   for (int ai = 1; ai < argc; ++ai) {
     currentArgs.emplace_back(argv[ai]);
@@ -3001,6 +3003,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
   };
   DriverInfo driverInfo{
     .sendingPolicies = sendingPolicies,
+    .forwardingPolicies = forwardingPolicies,
     .callbacksPolicies = callbacksPolicies};
   driverInfo.states.reserve(10);
   driverInfo.sigintRequested = false;
diff --git a/Framework/Core/test/test_DeviceSpec.cxx b/Framework/Core/test/test_DeviceSpec.cxx
index 99494d5f8ea4f..72b19da542a76 100644
--- a/Framework/Core/test/test_DeviceSpec.cxx
+++ b/Framework/Core/test/test_DeviceSpec.cxx
@@ -421,6 +421,7 @@ TEST_CASE("TestOutEdgeProcessingHelpers")
   auto configContext = makeEmptyConfigContext();
   auto channelPolicies = ChannelConfigurationPolicy::createDefaultPolicies(*configContext);
   auto sendingPolicies = SendingPolicy::createDefaultPolicies();
+  auto forwardingPolicies = ForwardingPolicy::createDefaultPolicies();
 
   std::vector<ComputingResource> resources{ComputingResourceHelpers::getLocalhostResource()};
   SimpleResourceManager rm(resources);
@@ -429,7 +430,7 @@ TEST_CASE("TestOutEdgeProcessingHelpers")
   defaultOffer.memory = 0.01;
 
   DeviceSpecHelpers::processOutEdgeActions(*configContext, devices, deviceIndex, connections, rm, edgeOutIndex, logicalEdges,
-                                           actions, workflow, globalOutputs, channelPolicies, sendingPolicies, "", defaultOffer);
+                                           actions, workflow, globalOutputs, channelPolicies, sendingPolicies, forwardingPolicies, "", defaultOffer);
 
   std::vector<DeviceId> expectedDeviceIndex = {{0, 0, 0}, {0, 0, 0}, {0, 0, 0}, {1, 0, 1}, {1, 0, 1}, {1, 1, 2}, {1, 1, 2}, {1, 2, 3}, {1, 2, 3}};
   REQUIRE(devices.size() == 4);
@@ -713,6 +714,7 @@ TEST_CASE("TestSimpleWildcard")
   auto configContext = makeEmptyConfigContext();
   auto channelPolicies = ChannelConfigurationPolicy::createDefaultPolicies(*configContext);
   auto sendingPolicies = SendingPolicy::createDefaultPolicies();
+  auto forwardingPolicies = ForwardingPolicy::createDefaultPolicies();
 
   std::vector<DeviceSpec> devices;
   std::vector<DeviceId> deviceIndex;
@@ -751,7 +753,7 @@ TEST_CASE("TestSimpleWildcard")
   defaultOffer.memory = 0.01;
 
   DeviceSpecHelpers::processOutEdgeActions(*configContext, devices, deviceIndex, connections, rm, edgeOutIndex, logicalEdges,
-                                           outActions, workflow, globalOutputs, channelPolicies, sendingPolicies, "", defaultOffer);
+                                           outActions, workflow, globalOutputs, channelPolicies, sendingPolicies, forwardingPolicies, "", defaultOffer);
 
   REQUIRE(devices.size() == 2);
   ; // Two devices have outputs: A and Timer
diff --git a/Framework/DataInspector/src/DataInspectorService.cxx b/Framework/DataInspector/src/DataInspectorService.cxx
index 40399a366a303..712bf10b6fb2e 100644
--- a/Framework/DataInspector/src/DataInspectorService.cxx
+++ b/Framework/DataInspector/src/DataInspectorService.cxx
@@ -245,4 +245,4 @@ ServiceSpec* DIServicePlugin::create()
             } },
     .kind = ServiceKind::Global};
 };
-} // namespace o2::framework
\ No newline at end of file
+} // namespace o2::framework
diff --git a/Utilities/DataSampling/src/Dispatcher.cxx b/Utilities/DataSampling/src/Dispatcher.cxx
index de78993ff083e..28ff4d5568da9 100644
--- a/Utilities/DataSampling/src/Dispatcher.cxx
+++ b/Utilities/DataSampling/src/Dispatcher.cxx
@@ -125,10 +125,9 @@ void Dispatcher::run(ProcessingContext& ctx)
   if (ctx.inputs().isValid("timer-stats")) {
     reportStats(ctx.services().get<Monitoring>());
   }
-  auto& proxy = ctx.services().get<FairMQDeviceProxy>();
   auto& relayer = ctx.services().get<DataRelayer>();
   auto timeslice = relayer.getOldestPossibleOutput().timeslice.value;
-  DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, timeslice);
+  DataProcessingHelpers::broadcastOldestPossibleTimeslice(ctx.services(), timeslice);
 }
 
 void Dispatcher::reportStats(Monitoring& monitoring) const

From 7897c3742059165a7cf6be3264f5f2641206753e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 10 Feb 2024 00:32:25 +0100
Subject: [PATCH 320/726] DPL: add SendingPolicy for the case destination is
 expendable

---
 Framework/Core/src/SendingPolicy.cxx | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 3d65975919676..3aeccb680c150 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -29,7 +29,7 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
 {
   return {SendingPolicy{
             .name = "dispatcher",
-            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) { 
+            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) {
                 if (source.name == "Dispatcher") {
                   return true;
                 }
@@ -103,6 +103,23 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               } else if (res == (size_t) fair::mq::TransferCode::error) {
                 LOGP(fatal, "Error while sending on channel {}", channel->GetName());
               } }},
+          SendingPolicy{
+            .name = "expendable",
+            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) {
+              auto has_label = [](DataProcessorLabel const& label) {
+                return label.value == "expendable";
+              };
+              return std::find_if(dest.labels.begin(), dest.labels.end(), has_label) != dest.labels.end(); },
+            .send = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
+              auto &proxy = registry.get<FairMQDeviceProxy>();
+              auto *channel = proxy.getOutputChannel(channelIndex);
+              auto timeout = 1000;
+              auto res = channel->Send(parts, timeout);
+              if (res == (size_t)fair::mq::TransferCode::timeout) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
+              } else if (res == (size_t) fair::mq::TransferCode::error) {
+                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              } }},
           SendingPolicy{
             .name = "default",
             .matcher = [](DataProcessorSpec const&, DataProcessorSpec const&, ConfigContext const&) { return true; },

From f3fcdd4dbff0adc793cf82ac84e6cfa9c20adf39 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 10 Feb 2024 01:13:36 +0100
Subject: [PATCH 321/726] DPL: keep code checker happy

---
 Framework/DataInspector/src/DataInspector.cxx | 4 ++--
 Framework/DataInspector/src/DataInspector.h   | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Framework/DataInspector/src/DataInspector.cxx b/Framework/DataInspector/src/DataInspector.cxx
index a38b173cdd22b..b948d969f4d10 100644
--- a/Framework/DataInspector/src/DataInspector.cxx
+++ b/Framework/DataInspector/src/DataInspector.cxx
@@ -44,7 +44,7 @@
 
 using namespace rapidjson;
 
-namespace o2::framework::DataInspector
+namespace o2::framework::data_inspector
 {
 #if BOOST_ENDIAN_BIG_BYTE
 static const auto endianness = "BIG";
@@ -147,4 +147,4 @@ std::vector<DIMessage> serializeO2Messages(const std::vector<DataRef>& refs, con
 
   return messages;
 }
-} // namespace o2::framework::DataInspector
+} // namespace o2::framework::data_inspector
diff --git a/Framework/DataInspector/src/DataInspector.h b/Framework/DataInspector/src/DataInspector.h
index 27c9987a45081..5e28f6bb3853b 100644
--- a/Framework/DataInspector/src/DataInspector.h
+++ b/Framework/DataInspector/src/DataInspector.h
@@ -20,7 +20,7 @@
 
 #include <cstring>
 
-namespace o2::framework::DataInspector
+namespace o2::framework::data_inspector
 {
 inline bool isNonInternalDevice(const DeviceSpec& spec)
 {
@@ -28,6 +28,6 @@ inline bool isNonInternalDevice(const DeviceSpec& spec)
 }
 
 std::vector<DIMessage> serializeO2Messages(const std::vector<DataRef>& refs, const std::string& deviceName);
-} // namespace o2::framework::DataInspector
+} // namespace o2::framework::data_inspector
 
 #endif // O2_DATAINSPECTOR_H

From d37ae9a6cfc3e605747f82f2b10bc43b5234d686 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 10 Feb 2024 01:17:27 +0100
Subject: [PATCH 322/726] DPL: keep code checker happy

---
 Framework/DataInspector/src/DataInspector.cxx        | 4 ++--
 Framework/DataInspector/src/DataInspector.h          | 4 ++--
 Framework/DataInspector/src/DataInspectorService.cxx | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/Framework/DataInspector/src/DataInspector.cxx b/Framework/DataInspector/src/DataInspector.cxx
index a38b173cdd22b..b948d969f4d10 100644
--- a/Framework/DataInspector/src/DataInspector.cxx
+++ b/Framework/DataInspector/src/DataInspector.cxx
@@ -44,7 +44,7 @@
 
 using namespace rapidjson;
 
-namespace o2::framework::DataInspector
+namespace o2::framework::data_inspector
 {
 #if BOOST_ENDIAN_BIG_BYTE
 static const auto endianness = "BIG";
@@ -147,4 +147,4 @@ std::vector<DIMessage> serializeO2Messages(const std::vector<DataRef>& refs, con
 
   return messages;
 }
-} // namespace o2::framework::DataInspector
+} // namespace o2::framework::data_inspector
diff --git a/Framework/DataInspector/src/DataInspector.h b/Framework/DataInspector/src/DataInspector.h
index 27c9987a45081..5e28f6bb3853b 100644
--- a/Framework/DataInspector/src/DataInspector.h
+++ b/Framework/DataInspector/src/DataInspector.h
@@ -20,7 +20,7 @@
 
 #include <cstring>
 
-namespace o2::framework::DataInspector
+namespace o2::framework::data_inspector
 {
 inline bool isNonInternalDevice(const DeviceSpec& spec)
 {
@@ -28,6 +28,6 @@ inline bool isNonInternalDevice(const DeviceSpec& spec)
 }
 
 std::vector<DIMessage> serializeO2Messages(const std::vector<DataRef>& refs, const std::string& deviceName);
-} // namespace o2::framework::DataInspector
+} // namespace o2::framework::data_inspector
 
 #endif // O2_DATAINSPECTOR_H
diff --git a/Framework/DataInspector/src/DataInspectorService.cxx b/Framework/DataInspector/src/DataInspectorService.cxx
index 40399a366a303..b24abd27d3c6a 100644
--- a/Framework/DataInspector/src/DataInspectorService.cxx
+++ b/Framework/DataInspector/src/DataInspectorService.cxx
@@ -205,7 +205,7 @@ ServiceSpec* DIServicePlugin::create()
       std::string runId = std::getenv("O2_DATAINSPECTOR_ID");
 
       const auto& spec = services.get<const DeviceSpec>();
-      if (DataInspector::isNonInternalDevice(spec)) {
+      if (data_inspector::isNonInternalDevice(spec)) {
         auto* diService = new DataInspectorProxyService(services, spec, proxyAddress, proxyPort, runId);
         return ServiceHandle{TypeIdHelpers::uniqueId<DataInspectorProxyService>(), diService};
       } else {
@@ -238,11 +238,11 @@ ServiceSpec* DIServicePlugin::create()
               }
 
               // Send copy to proxy
-              auto proxyMessages = DataInspector::serializeO2Messages(refs, registry.get < DeviceSpec const>().name);
+              auto proxyMessages = data_inspector::serializeO2Messages(refs, registry.get < DeviceSpec const>().name);
               for (auto &proxyMessage: proxyMessages) {
                 diService.send(std::move(proxyMessage));
               }
             } },
     .kind = ServiceKind::Global};
 };
-} // namespace o2::framework
\ No newline at end of file
+} // namespace o2::framework

From bd6037f96dd90aa6901a3c73933892277cd04da1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 10 Feb 2024 01:17:29 +0100
Subject: [PATCH 323/726] DPL: use the SendingPolicy for every kind of message
 sending

---
 .../Core/include/Framework/ChannelInfo.h      |  4 +
 .../include/Framework/DataProcessingHelpers.h | 22 +++---
 Framework/Core/include/Framework/DriverInfo.h |  3 +
 .../Core/include/Framework/ForwardRoute.h     |  9 ++-
 .../Core/include/Framework/SendingPolicy.h    |  9 +++
 Framework/Core/src/CommonServices.cxx         | 10 +--
 Framework/Core/src/DataProcessingDevice.cxx   | 24 +++---
 Framework/Core/src/DataProcessingHelpers.cxx  | 41 +++++-----
 Framework/Core/src/DeviceSpecHelpers.cxx      | 35 ++++++---
 Framework/Core/src/DeviceSpecHelpers.h        |  5 +-
 .../Core/src/ExternalFairMQDeviceProxy.cxx    | 17 +++-
 Framework/Core/src/FairMQDeviceProxy.cxx      |  3 +-
 Framework/Core/src/SendingPolicy.cxx          | 77 +++++++++++++++++++
 Framework/Core/src/runDataProcessing.cxx      |  3 +
 Framework/Core/test/test_DeviceSpec.cxx       |  6 +-
 Utilities/DataSampling/src/Dispatcher.cxx     |  3 +-
 16 files changed, 194 insertions(+), 77 deletions(-)

diff --git a/Framework/Core/include/Framework/ChannelInfo.h b/Framework/Core/include/Framework/ChannelInfo.h
index 0ebc4a8506301..1f4bd8bc9e90e 100644
--- a/Framework/Core/include/Framework/ChannelInfo.h
+++ b/Framework/Core/include/Framework/ChannelInfo.h
@@ -67,6 +67,7 @@ struct InputChannelInfo {
 };
 
 struct SendingPolicy;
+struct ForwardingPolicy;
 
 /// Output channel information
 struct OutputChannelInfo {
@@ -74,6 +75,7 @@ struct OutputChannelInfo {
   ChannelAccountingType channelType = ChannelAccountingType::DPL;
   fair::mq::Channel& channel;
   SendingPolicy const* policy;
+  ChannelIndex index = {-1};
 };
 
 struct OutputChannelState {
@@ -89,6 +91,8 @@ struct ForwardChannelInfo {
   /// Wether or not it's a DPL internal channel.
   ChannelAccountingType channelType = ChannelAccountingType::DPL;
   fair::mq::Channel& channel;
+  ForwardingPolicy const* policy;
+  ChannelIndex index = {-1};
 };
 
 struct ForwardChannelState {
diff --git a/Framework/Core/include/Framework/DataProcessingHelpers.h b/Framework/Core/include/Framework/DataProcessingHelpers.h
index 35f78f12cc405..7a05ddf25c46c 100644
--- a/Framework/Core/include/Framework/DataProcessingHelpers.h
+++ b/Framework/Core/include/Framework/DataProcessingHelpers.h
@@ -11,14 +11,16 @@
 #ifndef O2_FRAMEWORK_DATAPROCESSINGHELPERS_H_
 #define O2_FRAMEWORK_DATAPROCESSINGHELPERS_H_
 
-#include "Framework/TimesliceIndex.h"
-#include <fairmq/FwdDecls.h>
+#include <cstddef>
 
 namespace o2::framework
 {
-
+struct ServiceRegistryRef;
+struct ForwardChannelInfo;
+struct ForwardChannelState;
+struct OutputChannelInfo;
 struct OutputChannelSpec;
-class FairMQDeviceProxy;
+struct OutputChannelState;
 
 /// Generic helpers for DataProcessing releated functions.
 struct DataProcessingHelpers {
@@ -26,15 +28,13 @@ struct DataProcessingHelpers {
   /// @param device the fair::mq::Device which needs to send the EndOfStream message
   /// @param channel the OutputChannelSpec of the channel which needs to be signaled
   ///        for EndOfStream
-  static void sendEndOfStream(fair::mq::Device& device, OutputChannelSpec const& channel);
+  static void sendEndOfStream(ServiceRegistryRef const& ref, OutputChannelSpec const& channel);
   /// @returns true if we did send the oldest possible timeslice message, false otherwise.
-  static bool sendOldestPossibleTimeframe(ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice);
+  static bool sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice);
   /// @returns true if we did send the oldest possible timeslice message, false otherwise.
-  static bool sendOldestPossibleTimeframe(OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice);
-  static void broadcastOldestPossibleTimeslice(FairMQDeviceProxy& proxy, size_t timeslice);
-
- private:
-  static void sendOldestPossibleTimeframe(fair::mq::Channel& channel, size_t timeslice);
+  static bool sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice);
+  /// Broadcast the oldest possible timeslice to all channels in output
+  static void broadcastOldestPossibleTimeslice(ServiceRegistryRef const& ref, size_t timeslice);
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/include/Framework/DriverInfo.h b/Framework/Core/include/Framework/DriverInfo.h
index 043a689087325..45c4c85a32d01 100644
--- a/Framework/Core/include/Framework/DriverInfo.h
+++ b/Framework/Core/include/Framework/DriverInfo.h
@@ -111,6 +111,9 @@ struct DriverInfo {
   /// These are the policies which can be applied to decide how
   /// we send data.
   std::vector<SendingPolicy> sendingPolicies;
+  /// These are the policies which can be applied to decide how
+  /// we forward data.
+  std::vector<ForwardingPolicy> forwardingPolicies;
   /// The argc with which the driver was started.
   int argc;
   /// The argv with which the driver was started.
diff --git a/Framework/Core/include/Framework/ForwardRoute.h b/Framework/Core/include/Framework/ForwardRoute.h
index bae2eaacf1a44..0e3d4d12da24a 100644
--- a/Framework/Core/include/Framework/ForwardRoute.h
+++ b/Framework/Core/include/Framework/ForwardRoute.h
@@ -15,11 +15,11 @@
 #include <cstddef>
 #include <string>
 
-namespace o2
-{
-namespace framework
+namespace o2::framework
 {
 
+struct ForwardingPolicy;
+
 /// This uniquely identifies a route to be forwarded by the device if
 /// the InputSpec @a matcher matches an input which should also go to
 /// @a channel
@@ -28,8 +28,9 @@ struct ForwardRoute {
   size_t maxTimeslices;
   InputSpec matcher;
   std::string channel;
+  // The policy to use to send to on this route.
+  ForwardingPolicy const* policy;
 };
 
-} // namespace framework
 } // namespace o2
 #endif // FRAMEWORK_FORWARDROUTE_H
diff --git a/Framework/Core/include/Framework/SendingPolicy.h b/Framework/Core/include/Framework/SendingPolicy.h
index 1d68412d46db8..7cc5c16e2c923 100644
--- a/Framework/Core/include/Framework/SendingPolicy.h
+++ b/Framework/Core/include/Framework/SendingPolicy.h
@@ -32,6 +32,15 @@ struct SendingPolicy {
   static std::vector<SendingPolicy> createDefaultPolicies();
 };
 
+struct ForwardingPolicy {
+  using ForwardingCallback = std::function<void(fair::mq::Parts&, ChannelIndex channelIndex, ServiceRegistryRef registry)>;
+  std::string name = "invalid";
+  EdgeMatcher matcher = nullptr;
+  ForwardingCallback forward = nullptr;
+  static std::vector<ForwardingPolicy> createDefaultPolicies();
+  static ForwardingPolicy createDefaultForwardingPolicy();
+};
+
 } // namespace o2::framework
 
 #endif // O2_FRAMEWORK_SENDINGPOLICY_H_
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index b7fc3410a1931..a0f7fcdc88201 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -588,7 +588,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           timesliceIndex.rescan();
         }
       }
-      DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, oldestPossibleOutput.timeslice.value);
+      DataProcessingHelpers::broadcastOldestPossibleTimeslice(ctx.services(), oldestPossibleOutput.timeslice.value);
 
       for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
         auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
@@ -598,7 +598,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
           continue;
         }
-        if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestPossibleOutput.timeslice.value)) {
+        if (DataProcessingHelpers::sendOldestPossibleTimeframe(ctx.services(), info, state, oldestPossibleOutput.timeslice.value)) {
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice",
                                  "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
                                  info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
@@ -646,7 +646,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
       O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Queueing oldest possible timeslice %" PRIu64 " propagation for execution.",
                              (uint64_t)oldestPossibleOutput.timeslice.value);
       AsyncQueueHelpers::post(
-        queue, decongestion.oldestPossibleTimesliceTask, [oldestPossibleOutput, &decongestion, &proxy, &spec, device, &timesliceIndex]() {
+        queue, decongestion.oldestPossibleTimesliceTask, [ref = services, oldestPossibleOutput, &decongestion, &proxy, &spec, device, &timesliceIndex]() {
           O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, &decongestion);
           if (decongestion.lastTimeslice >= oldestPossibleOutput.timeslice.value) {
             O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Not sending already sent value: %" PRIu64 "> %" PRIu64,
@@ -655,7 +655,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
           }
           O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Running oldest possible timeslice %" PRIu64 " propagation.",
                                  (uint64_t)oldestPossibleOutput.timeslice.value);
-          DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, oldestPossibleOutput.timeslice.value);
+          DataProcessingHelpers::broadcastOldestPossibleTimeslice(ref, oldestPossibleOutput.timeslice.value);
 
           for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
             auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
@@ -665,7 +665,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
               O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
               continue;
             }
-            if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestPossibleOutput.timeslice.value)) {
+            if (DataProcessingHelpers::sendOldestPossibleTimeframe(ref, info, state, oldestPossibleOutput.timeslice.value)) {
               O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice",
                                      "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
                                      info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 8c914e201a479..50934bd59a729 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -680,19 +680,15 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
     if (forwardedParts[fi].Size() == 0) {
       continue;
     }
-    auto channel = proxy.getForwardChannel(ChannelIndex{fi});
-    LOG(debug) << "Forwarding to " << channel->GetName() << " " << fi;
+    ForwardChannelInfo info = proxy.getForwardChannelInfo(ChannelIndex{fi});
+    LOG(debug) << "Forwarding to " << info.name << " " << fi;
     // in DPL we are using subchannel 0 only
     auto& parts = forwardedParts[fi];
-    int timeout = 30000;
-    auto res = channel->Send(parts, timeout);
-    if (res == (size_t)fair::mq::TransferCode::timeout) {
-      LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on {}.", timeout / 1000, channel->GetName());
-      channel->Send(parts);
-      LOGP(info, "Downstream backpressure on {} recovered.", channel->GetName());
-    } else if (res == (size_t)fair::mq::TransferCode::error) {
-      LOGP(fatal, "Error while sending on channel {}", channel->GetName());
+    if (info.policy == nullptr) {
+      LOG(error) << "Forwarding to " << info.name << " " << fi << " has no policy";
+      continue;
     }
+    info.policy->forward(parts, ChannelIndex{fi}, registry);
   }
 
   auto& asyncQueue = registry.get<AsyncQueue>();
@@ -713,7 +709,7 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
           LOG(debug) << "Skipping channel";
           continue;
         }
-        if (DataProcessingHelpers::sendOldestPossibleTimeframe(info, state, oldestTimeslice.timeslice.value)) {
+        if (DataProcessingHelpers::sendOldestPossibleTimeframe(registry, info, state, oldestTimeslice.timeslice.value)) {
           LOGP(debug, "Forwarding to channel {} oldest possible timeslice {}, prio 20", info.name, oldestTimeslice.timeslice.value);
         }
       }
@@ -1678,8 +1674,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
 
     for (auto& channel : spec.outputChannels) {
       LOGP(detail, "Sending end of stream to {}", channel.name);
-      auto& rawDevice = ref.get<RawDeviceService>();
-      DataProcessingHelpers::sendEndOfStream(*rawDevice.device(), channel);
+      DataProcessingHelpers::sendEndOfStream(ref, channel);
     }
     // This is needed because the transport is deleted before the device.
     relayer.clear();
@@ -2460,8 +2455,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   if (state.streaming == StreamingState::EndOfStreaming) {
     LOGP(detail, "Broadcasting end of stream");
     for (auto& channel : spec.outputChannels) {
-      auto& rawDevice = ref.get<RawDeviceService>();
-      DataProcessingHelpers::sendEndOfStream(*rawDevice.device(), channel);
+      DataProcessingHelpers::sendEndOfStream(ref, channel);
     }
     switchState(StreamingState::Idle);
   }
diff --git a/Framework/Core/src/DataProcessingHelpers.cxx b/Framework/Core/src/DataProcessingHelpers.cxx
index d424a53a52bd9..58fda5a13bae9 100644
--- a/Framework/Core/src/DataProcessingHelpers.cxx
+++ b/Framework/Core/src/DataProcessingHelpers.cxx
@@ -18,79 +18,74 @@
 #include "Headers/DataHeader.h"
 #include "Headers/Stack.h"
 #include "Framework/Logger.h"
+#include "Framework/SendingPolicy.h"
+#include "Framework/RawDeviceService.h"
 
 #include <fairmq/Device.h>
 #include <fairmq/Channel.h>
 
 namespace o2::framework
 {
-void DataProcessingHelpers::sendEndOfStream(fair::mq::Device& device, OutputChannelSpec const& channel)
+void DataProcessingHelpers::sendEndOfStream(ServiceRegistryRef const& ref, OutputChannelSpec const& channel)
 {
+  fair::mq::Device* device = ref.get<RawDeviceService>().device();
   fair::mq::Parts parts;
-  fair::mq::MessagePtr payload(device.NewMessage());
+  fair::mq::MessagePtr payload(device->NewMessage());
   SourceInfoHeader sih;
   sih.state = InputChannelState::Completed;
-  auto channelAlloc = o2::pmr::getTransportAllocator(device.GetChannel(channel.name, 0).Transport());
+  auto channelAlloc = o2::pmr::getTransportAllocator(device->GetChannel(channel.name, 0).Transport());
   auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, sih});
   // sigh... See if we can avoid having it const by not
   // exposing it to the user in the first place.
   parts.AddPart(std::move(header));
   parts.AddPart(std::move(payload));
-  device.Send(parts, channel.name, 0);
+  device->Send(parts, channel.name, 0);
   LOGP(info, "Sending end-of-stream message to channel {}", channel.name);
 }
 
-void DataProcessingHelpers::sendOldestPossibleTimeframe(fair::mq::Channel& channel, size_t timeslice)
+void doSendOldestPossibleTimeframe(ServiceRegistryRef ref, fair::mq::TransportFactory* transport, ChannelIndex index, SendingPolicy::SendingCallback const& callback, size_t timeslice)
 {
   fair::mq::Parts parts;
-  fair::mq::MessagePtr payload(channel.Transport()->CreateMessage());
+  fair::mq::MessagePtr payload(transport->CreateMessage());
   o2::framework::DomainInfoHeader dih;
   dih.oldestPossibleTimeslice = timeslice;
-  auto channelAlloc = o2::pmr::getTransportAllocator(channel.Transport());
+  auto channelAlloc = o2::pmr::getTransportAllocator(transport);
   auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, dih});
   // sigh... See if we can avoid having it const by not
   // exposing it to the user in the first place.
   parts.AddPart(std::move(header));
   parts.AddPart(std::move(payload));
 
-  auto timeout = 1000;
-  auto res = channel.Send(parts, timeout);
-  if (res == (size_t)fair::mq::TransferCode::timeout) {
-    LOGP(warning, "Timed out sending oldest possible timeslice after {}s. Downstream backpressure detected on {}.", timeout / 1000, channel.GetName());
-    channel.Send(parts);
-    LOGP(info, "Downstream backpressure on {} recovered.", channel.GetName());
-  }
-  if (res < (size_t)fair::mq::TransferCode::success) {
-    LOGP(fatal, "Error sending oldest possible timeframe {} on channel {} (code {})", timeslice, channel.GetName(), res);
-  }
+  callback(parts, index, ref);
 }
 
-bool DataProcessingHelpers::sendOldestPossibleTimeframe(ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice)
+bool DataProcessingHelpers::sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, ForwardChannelInfo const& info, ForwardChannelState& state, size_t timeslice)
 {
   if (state.oldestForChannel.value >= timeslice) {
     return false;
   }
-  sendOldestPossibleTimeframe(info.channel, timeslice);
+  doSendOldestPossibleTimeframe(ref, info.channel.Transport(), info.index, info.policy->forward, timeslice);
   state.oldestForChannel = {timeslice};
   return true;
 }
 
-bool DataProcessingHelpers::sendOldestPossibleTimeframe(OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice)
+bool DataProcessingHelpers::sendOldestPossibleTimeframe(ServiceRegistryRef const& ref, OutputChannelInfo const& info, OutputChannelState& state, size_t timeslice)
 {
   if (state.oldestForChannel.value >= timeslice) {
     return false;
   }
-  sendOldestPossibleTimeframe(info.channel, timeslice);
+  doSendOldestPossibleTimeframe(ref, info.channel.Transport(), info.index, info.policy->send, timeslice);
   state.oldestForChannel = {timeslice};
   return true;
 }
 
-void DataProcessingHelpers::broadcastOldestPossibleTimeslice(FairMQDeviceProxy& proxy, size_t timeslice)
+void DataProcessingHelpers::broadcastOldestPossibleTimeslice(ServiceRegistryRef const& ref, size_t timeslice)
 {
+  auto& proxy = ref.get<FairMQDeviceProxy>();
   for (int ci = 0; ci < proxy.getNumOutputChannels(); ++ci) {
     auto& info = proxy.getOutputChannelInfo({ci});
     auto& state = proxy.getOutputChannelState({ci});
-    sendOldestPossibleTimeframe(info, state, timeslice);
+    sendOldestPossibleTimeframe(ref, info, state, timeslice);
   }
 }
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index b1681a4779ceb..e9df320fe8b95 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -493,6 +493,7 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
                                               const std::vector<OutputSpec>& outputsMatchers,
                                               const std::vector<ChannelConfigurationPolicy>& channelPolicies,
                                               const std::vector<SendingPolicy>& sendingPolicies,
+                                              const std::vector<ForwardingPolicy>& forwardingPolicies,
                                               std::string const& channelPrefix,
                                               ComputingOffer const& defaultOffer,
                                               OverrideServiceSpecs const& overrideServices)
@@ -656,7 +657,7 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
   // whether this is a real OutputRoute or if it's a forward from
   // a previous consumer device.
   // FIXME: where do I find the InputSpec for the forward?
-  auto appendOutputRouteToSourceDeviceChannel = [&outputsMatchers, &workflow, &devices, &logicalEdges, &sendingPolicies, &configContext](
+  auto appendOutputRouteToSourceDeviceChannel = [&outputsMatchers, &workflow, &devices, &logicalEdges, &sendingPolicies, &forwardingPolicies, &configContext](
                                                   size_t ei, size_t di, size_t ci) {
     assert(ei < logicalEdges.size());
     assert(di < devices.size());
@@ -670,29 +671,40 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
     assert(edge.outputGlobalIndex < outputsMatchers.size());
     // Iterate over all the policies and apply the first one that matches.
     SendingPolicy const* policyPtr = nullptr;
+    ForwardingPolicy const* forwardPolicyPtr = nullptr;
     for (auto& policy : sendingPolicies) {
       if (policy.matcher(producer, consumer, configContext)) {
         policyPtr = &policy;
         break;
       }
     }
+    assert(forwardingPolicies.empty() == false);
+    for (auto& policy : forwardingPolicies) {
+      if (policy.matcher(producer, consumer, configContext)) {
+        forwardPolicyPtr = &policy;
+        break;
+      }
+    }
     assert(policyPtr != nullptr);
+    assert(forwardPolicyPtr != nullptr);
 
     if (edge.isForward == false) {
       OutputRoute route{
-        edge.timeIndex,
-        consumer.maxInputTimeslices,
-        outputsMatchers[edge.outputGlobalIndex],
-        channel.name,
-        policyPtr,
+        .timeslice = edge.timeIndex,
+        .maxTimeslices = consumer.maxInputTimeslices,
+        .matcher = outputsMatchers[edge.outputGlobalIndex],
+        .channel = channel.name,
+        .policy = policyPtr,
       };
       device.outputs.emplace_back(route);
     } else {
       ForwardRoute route{
-        edge.timeIndex,
-        consumer.maxInputTimeslices,
-        workflow[edge.consumer].inputs[edge.consumerInputIndex],
-        channel.name};
+        .timeslice = edge.timeIndex,
+        .maxTimeslices = consumer.maxInputTimeslices,
+        .matcher = workflow[edge.consumer].inputs[edge.consumerInputIndex],
+        .channel = channel.name,
+        .policy = forwardPolicyPtr,
+      };
       device.forwards.emplace_back(route);
     }
   };
@@ -1051,6 +1063,7 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
                                                        std::vector<ResourcePolicy> const& resourcePolicies,
                                                        std::vector<CallbacksPolicy> const& callbacksPolicies,
                                                        std::vector<SendingPolicy> const& sendingPolicies,
+                                                       std::vector<ForwardingPolicy> const& forwardingPolicies,
                                                        std::vector<DeviceSpec>& devices,
                                                        ResourceManager& resourceManager,
                                                        std::string const& uniqueWorkflowId,
@@ -1111,7 +1124,7 @@ void DeviceSpecHelpers::dataProcessorSpecs2DeviceSpecs(const WorkflowSpec& workf
   defaultOffer.memory /= deviceCount + 1;
 
   processOutEdgeActions(configContext, devices, deviceIndex, connections, resourceManager, outEdgeIndex, logicalEdges,
-                        outActions, workflow, outputs, channelPolicies, sendingPolicies, channelPrefix, defaultOffer, overrideServices);
+                        outActions, workflow, outputs, channelPolicies, sendingPolicies, forwardingPolicies, channelPrefix, defaultOffer, overrideServices);
 
   // FIXME: is this not the case???
   std::sort(connections.begin(), connections.end());
diff --git a/Framework/Core/src/DeviceSpecHelpers.h b/Framework/Core/src/DeviceSpecHelpers.h
index 1fb1866ae2dc3..17601c81342c1 100644
--- a/Framework/Core/src/DeviceSpecHelpers.h
+++ b/Framework/Core/src/DeviceSpecHelpers.h
@@ -53,6 +53,7 @@ struct DeviceSpecHelpers {
     std::vector<ResourcePolicy> const& resourcePolicies,
     std::vector<CallbacksPolicy> const& callbacksPolicies,
     std::vector<SendingPolicy> const& sendingPolicy,
+    std::vector<ForwardingPolicy> const& forwardingPolicies,
     std::vector<DeviceSpec>& devices,
     ResourceManager& resourceManager,
     std::string const& uniqueWorkflowId,
@@ -80,9 +81,10 @@ struct DeviceSpecHelpers {
     std::vector<DispatchPolicy> dispatchPolicies = DispatchPolicy::createDefaultPolicies();
     std::vector<ResourcePolicy> resourcePolicies = ResourcePolicy::createDefaultPolicies();
     std::vector<SendingPolicy> sendingPolicies = SendingPolicy::createDefaultPolicies();
+    std::vector<ForwardingPolicy> forwardingPolicies = ForwardingPolicy::createDefaultPolicies();
     dataProcessorSpecs2DeviceSpecs(workflow, channelPolicies, completionPolicies,
                                    dispatchPolicies, resourcePolicies, callbacksPolicies,
-                                   sendingPolicies, devices,
+                                   sendingPolicies, forwardingPolicies, devices,
                                    resourceManager, uniqueWorkflowId, configContext, optimizeTopology,
                                    resourcesMonitoringInterval, channelPrefix, overrideServices);
   }
@@ -150,6 +152,7 @@ struct DeviceSpecHelpers {
     const std::vector<OutputSpec>& outputs,
     std::vector<ChannelConfigurationPolicy> const& channelPolicies,
     std::vector<SendingPolicy> const& sendingPolicies,
+    std::vector<ForwardingPolicy> const& forwardingPolicies,
     std::string const& channelPrefix,
     ComputingOffer const& defaultOffer,
     OverrideServiceSpecs const& overrideServices = {});
diff --git a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
index 0f769ae2dc850..4846363efd93b 100644
--- a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
+++ b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
@@ -29,6 +29,7 @@
 #include "Framework/TimingInfo.h"
 #include "Framework/DeviceState.h"
 #include "Framework/Monitoring.h"
+#include "Framework/SendingPolicy.h"
 #include "Headers/DataHeader.h"
 #include "Headers/Stack.h"
 #include "DecongestionService.h"
@@ -923,10 +924,16 @@ DataProcessorSpec specifyFairMQDeviceOutputProxy(char const* name,
     auto lastDataProcessingHeader = std::make_shared<DataProcessingHeader>(0, 0);
 
     auto& spec = const_cast<DeviceSpec&>(deviceSpec);
+    static auto policy = ForwardingPolicy::createDefaultForwardingPolicy();
     for (auto const& inputSpec : inputSpecs) {
       // this is a prototype, in principle we want to have all spec objects const
       // and so only the const object can be retrieved from service registry
-      ForwardRoute route{0, 1, inputSpec, outputChannelName};
+      ForwardRoute route{
+        .timeslice = 0,
+        .maxTimeslices = 1,
+        .matcher = inputSpec,
+        .channel = outputChannelName,
+        .policy = &policy};
       spec.forwards.emplace_back(route);
     }
 
@@ -1010,7 +1017,13 @@ DataProcessorSpec specifyFairMQDeviceMultiOutputProxy(char const* name,
         if (device->GetChannels().count(channel) == 0) {
           throw std::runtime_error("no corresponding output channel found for input '" + channel + "'");
         }
-        ForwardRoute route{0, 1, spec, channel};
+        static auto policy = ForwardingPolicy::createDefaultForwardingPolicy();
+        ForwardRoute route{
+          .timeslice = 0,
+          .maxTimeslices = 1,
+          .matcher = spec,
+          .channel = channel,
+          .policy = &policy};
         // this we will try to fix on the framework level, there will be an API to
         // set external routes. Basically, this has to be added while setting up the
         // workflow. After that, the actual spec provided by the service is supposed
diff --git a/Framework/Core/src/FairMQDeviceProxy.cxx b/Framework/Core/src/FairMQDeviceProxy.cxx
index c25a58e47ef2d..bdffddd5a4d1a 100644
--- a/Framework/Core/src/FairMQDeviceProxy.cxx
+++ b/Framework/Core/src/FairMQDeviceProxy.cxx
@@ -267,6 +267,7 @@ void FairMQDeviceProxy::bind(std::vector<OutputRoute> const& outputs, std::vecto
           .channelType = dplChannel,
           .channel = channel->second.at(0),
           .policy = route.policy,
+          .index = channelIndex,
         };
         mOutputChannelInfos.push_back(info);
         mOutputChannelStates.push_back({0});
@@ -345,7 +346,7 @@ void FairMQDeviceProxy::bind(std::vector<OutputRoute> const& outputs, std::vecto
           LOGP(fatal, "Expected channel {} not configured.", route.channel);
         }
         ChannelAccountingType dplChannel = (route.channel.rfind("from_", 0) == 0) ? ChannelAccountingType::DPL : ChannelAccountingType::RAWFMQ;
-        mForwardChannelInfos.push_back(ForwardChannelInfo{route.channel, dplChannel, channel->second.at(0)});
+        mForwardChannelInfos.push_back(ForwardChannelInfo{.name = route.channel, .channelType = dplChannel, .channel = channel->second.at(0), .policy = route.policy, .index = channelIndex});
         mForwardChannelStates.push_back(ForwardChannelState{0});
         channelNameToChannel[route.channel] = channelIndex;
         LOGP(detail, "Binding forward channel {} to channel index {}", route.channel, channelIndex.value);
diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 17faef6dbd3d0..3d65975919676 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -119,4 +119,81 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
                 LOGP(fatal, "Error while sending on channel {}", channel->GetName());
               } }}};
 }
+
+ForwardingPolicy ForwardingPolicy::createDefaultForwardingPolicy()
+{
+  return ForwardingPolicy{
+    .name = "default",
+    .matcher = [](DataProcessorSpec const&, DataProcessorSpec const&, ConfigContext const&) { return true; },
+    .forward = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
+              auto &proxy = registry.get<FairMQDeviceProxy>();
+              auto *channel = proxy.getForwardChannel(channelIndex);
+              auto timeout = 1000;
+              auto res = channel->Send(parts, timeout);
+              if (res == (size_t)fair::mq::TransferCode::timeout) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on {}.", timeout/1000, channel->GetName());
+                channel->Send(parts);
+                LOGP(info, "Downstream backpressure on {} recovered.", channel->GetName());
+              } else if (res == (size_t) fair::mq::TransferCode::error) {
+                LOGP(fatal, "Error while sending on channel {}", channel->GetName());
+              } }};
+}
+
+std::vector<ForwardingPolicy> ForwardingPolicy::createDefaultPolicies()
+{
+  return {ForwardingPolicy{
+            .name = "profiling",
+            .matcher = [](DataProcessorSpec const&, DataProcessorSpec const&, ConfigContext const&) { return getenv("DPL_DEBUG_MESSAGE_SIZE"); },
+            .forward = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
+              auto &proxy = registry.get<FairMQDeviceProxy>();
+              auto *channel = proxy.getForwardChannel(channelIndex);
+              auto timeout = 1000;
+              int count = 0;
+              auto& relayer = registry.get<DataRelayer>();
+              for (auto& part : parts) {
+                auto* dh = o2::header::get<o2::header::DataHeader*>(part->GetData());
+                if (dh == nullptr) {
+                  // This is a payload.
+                  continue;
+                }
+                LOGP(info, "Sent {}/{}/{} for a total of {} bytes", dh->dataOrigin, dh->dataDescription, dh->subSpecification, dh->payloadSize);
+                count+= dh->payloadSize;
+                auto* dph = o2::header::get<o2::framework::DataProcessingHeader*>(part->GetData());
+                if (dph == nullptr) {
+                  // This is a payload.
+                  continue;
+                }
+                auto oldestPossibleOutput = relayer.getOldestPossibleOutput();
+                if ((size_t)dph->startTime < oldestPossibleOutput.timeslice.value) {
+                  LOGP(error, "Sent startTime {} while oldestPossibleOutput is {}. This should not be possible.", dph->startTime, oldestPossibleOutput.timeslice.value);
+                }
+              }
+              LOGP(info, "Sent {} parts for a total of {} bytes", parts.Size(), count);
+              auto res = channel->Send(parts, timeout);
+              if (res == (size_t)fair::mq::TransferCode::timeout) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on {}.", timeout/1000, channel->GetName());
+                channel->Send(parts);
+                LOGP(info, "Downstream backpressure on {} recovered.", channel->GetName());
+              } else if (res == (size_t) fair::mq::TransferCode::error) {
+                LOGP(fatal, "Error while sending on channel {}", channel->GetName());
+              } }},
+          ForwardingPolicy{
+            .name = "expendable",
+            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) {
+              auto has_label = [](DataProcessorLabel const& label) {
+                return label.value == "expendable";
+              };
+              return std::find_if(dest.labels.begin(), dest.labels.end(), has_label) != dest.labels.end(); },
+            .forward = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
+              auto &proxy = registry.get<FairMQDeviceProxy>();
+              auto *channel = proxy.getForwardChannel(channelIndex);
+              auto timeout = 1000;
+              auto res = channel->Send(parts, timeout);
+              if (res == (size_t)fair::mq::TransferCode::timeout) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
+              } else if (res == (size_t) fair::mq::TransferCode::error) {
+                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              } }},
+          createDefaultForwardingPolicy()};
+}
 } // namespace o2::framework
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 9ca46e060b1a2..cfbb616b0d0a5 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1650,6 +1650,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
                                                             driverInfo.resourcePolicies,
                                                             driverInfo.callbacksPolicies,
                                                             driverInfo.sendingPolicies,
+                                                            driverInfo.forwardingPolicies,
                                                             runningWorkflow.devices,
                                                             *resourceManager,
                                                             driverInfo.uniqueWorkflowId,
@@ -2644,6 +2645,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
 {
   std::vector<std::string> currentArgs;
   std::vector<PluginInfo> plugins;
+  std::vector<ForwardingPolicy> forwardingPolicies = ForwardingPolicy::createDefaultPolicies();
 
   for (int ai = 1; ai < argc; ++ai) {
     currentArgs.emplace_back(argv[ai]);
@@ -3001,6 +3003,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
   };
   DriverInfo driverInfo{
     .sendingPolicies = sendingPolicies,
+    .forwardingPolicies = forwardingPolicies,
     .callbacksPolicies = callbacksPolicies};
   driverInfo.states.reserve(10);
   driverInfo.sigintRequested = false;
diff --git a/Framework/Core/test/test_DeviceSpec.cxx b/Framework/Core/test/test_DeviceSpec.cxx
index 99494d5f8ea4f..72b19da542a76 100644
--- a/Framework/Core/test/test_DeviceSpec.cxx
+++ b/Framework/Core/test/test_DeviceSpec.cxx
@@ -421,6 +421,7 @@ TEST_CASE("TestOutEdgeProcessingHelpers")
   auto configContext = makeEmptyConfigContext();
   auto channelPolicies = ChannelConfigurationPolicy::createDefaultPolicies(*configContext);
   auto sendingPolicies = SendingPolicy::createDefaultPolicies();
+  auto forwardingPolicies = ForwardingPolicy::createDefaultPolicies();
 
   std::vector<ComputingResource> resources{ComputingResourceHelpers::getLocalhostResource()};
   SimpleResourceManager rm(resources);
@@ -429,7 +430,7 @@ TEST_CASE("TestOutEdgeProcessingHelpers")
   defaultOffer.memory = 0.01;
 
   DeviceSpecHelpers::processOutEdgeActions(*configContext, devices, deviceIndex, connections, rm, edgeOutIndex, logicalEdges,
-                                           actions, workflow, globalOutputs, channelPolicies, sendingPolicies, "", defaultOffer);
+                                           actions, workflow, globalOutputs, channelPolicies, sendingPolicies, forwardingPolicies, "", defaultOffer);
 
   std::vector<DeviceId> expectedDeviceIndex = {{0, 0, 0}, {0, 0, 0}, {0, 0, 0}, {1, 0, 1}, {1, 0, 1}, {1, 1, 2}, {1, 1, 2}, {1, 2, 3}, {1, 2, 3}};
   REQUIRE(devices.size() == 4);
@@ -713,6 +714,7 @@ TEST_CASE("TestSimpleWildcard")
   auto configContext = makeEmptyConfigContext();
   auto channelPolicies = ChannelConfigurationPolicy::createDefaultPolicies(*configContext);
   auto sendingPolicies = SendingPolicy::createDefaultPolicies();
+  auto forwardingPolicies = ForwardingPolicy::createDefaultPolicies();
 
   std::vector<DeviceSpec> devices;
   std::vector<DeviceId> deviceIndex;
@@ -751,7 +753,7 @@ TEST_CASE("TestSimpleWildcard")
   defaultOffer.memory = 0.01;
 
   DeviceSpecHelpers::processOutEdgeActions(*configContext, devices, deviceIndex, connections, rm, edgeOutIndex, logicalEdges,
-                                           outActions, workflow, globalOutputs, channelPolicies, sendingPolicies, "", defaultOffer);
+                                           outActions, workflow, globalOutputs, channelPolicies, sendingPolicies, forwardingPolicies, "", defaultOffer);
 
   REQUIRE(devices.size() == 2);
   ; // Two devices have outputs: A and Timer
diff --git a/Utilities/DataSampling/src/Dispatcher.cxx b/Utilities/DataSampling/src/Dispatcher.cxx
index de78993ff083e..28ff4d5568da9 100644
--- a/Utilities/DataSampling/src/Dispatcher.cxx
+++ b/Utilities/DataSampling/src/Dispatcher.cxx
@@ -125,10 +125,9 @@ void Dispatcher::run(ProcessingContext& ctx)
   if (ctx.inputs().isValid("timer-stats")) {
     reportStats(ctx.services().get<Monitoring>());
   }
-  auto& proxy = ctx.services().get<FairMQDeviceProxy>();
   auto& relayer = ctx.services().get<DataRelayer>();
   auto timeslice = relayer.getOldestPossibleOutput().timeslice.value;
-  DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, timeslice);
+  DataProcessingHelpers::broadcastOldestPossibleTimeslice(ctx.services(), timeslice);
 }
 
 void Dispatcher::reportStats(Monitoring& monitoring) const

From 37b4c4302e6c90e4b01413d3896a150426e57481 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 10 Feb 2024 01:17:29 +0100
Subject: [PATCH 324/726] DPL: add SendingPolicy for the case destination is
 expendable

---
 Framework/Core/src/SendingPolicy.cxx | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 3d65975919676..3aeccb680c150 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -29,7 +29,7 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
 {
   return {SendingPolicy{
             .name = "dispatcher",
-            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) { 
+            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) {
                 if (source.name == "Dispatcher") {
                   return true;
                 }
@@ -103,6 +103,23 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               } else if (res == (size_t) fair::mq::TransferCode::error) {
                 LOGP(fatal, "Error while sending on channel {}", channel->GetName());
               } }},
+          SendingPolicy{
+            .name = "expendable",
+            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) {
+              auto has_label = [](DataProcessorLabel const& label) {
+                return label.value == "expendable";
+              };
+              return std::find_if(dest.labels.begin(), dest.labels.end(), has_label) != dest.labels.end(); },
+            .send = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
+              auto &proxy = registry.get<FairMQDeviceProxy>();
+              auto *channel = proxy.getOutputChannel(channelIndex);
+              auto timeout = 1000;
+              auto res = channel->Send(parts, timeout);
+              if (res == (size_t)fair::mq::TransferCode::timeout) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
+              } else if (res == (size_t) fair::mq::TransferCode::error) {
+                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              } }},
           SendingPolicy{
             .name = "default",
             .matcher = [](DataProcessorSpec const&, DataProcessorSpec const&, ConfigContext const&) { return true; },

From dca1c3af1f24062b13444cccf781b4102efe1e9b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 10 Feb 2024 09:25:41 +0100
Subject: [PATCH 325/726] DPL: keep code checker happy

---
 Framework/DataInspector/src/DataInspector.cxx        | 4 ++--
 Framework/DataInspector/src/DataInspector.h          | 4 ++--
 Framework/DataInspector/src/DataInspectorService.cxx | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/Framework/DataInspector/src/DataInspector.cxx b/Framework/DataInspector/src/DataInspector.cxx
index a38b173cdd22b..b948d969f4d10 100644
--- a/Framework/DataInspector/src/DataInspector.cxx
+++ b/Framework/DataInspector/src/DataInspector.cxx
@@ -44,7 +44,7 @@
 
 using namespace rapidjson;
 
-namespace o2::framework::DataInspector
+namespace o2::framework::data_inspector
 {
 #if BOOST_ENDIAN_BIG_BYTE
 static const auto endianness = "BIG";
@@ -147,4 +147,4 @@ std::vector<DIMessage> serializeO2Messages(const std::vector<DataRef>& refs, con
 
   return messages;
 }
-} // namespace o2::framework::DataInspector
+} // namespace o2::framework::data_inspector
diff --git a/Framework/DataInspector/src/DataInspector.h b/Framework/DataInspector/src/DataInspector.h
index 27c9987a45081..5e28f6bb3853b 100644
--- a/Framework/DataInspector/src/DataInspector.h
+++ b/Framework/DataInspector/src/DataInspector.h
@@ -20,7 +20,7 @@
 
 #include <cstring>
 
-namespace o2::framework::DataInspector
+namespace o2::framework::data_inspector
 {
 inline bool isNonInternalDevice(const DeviceSpec& spec)
 {
@@ -28,6 +28,6 @@ inline bool isNonInternalDevice(const DeviceSpec& spec)
 }
 
 std::vector<DIMessage> serializeO2Messages(const std::vector<DataRef>& refs, const std::string& deviceName);
-} // namespace o2::framework::DataInspector
+} // namespace o2::framework::data_inspector
 
 #endif // O2_DATAINSPECTOR_H
diff --git a/Framework/DataInspector/src/DataInspectorService.cxx b/Framework/DataInspector/src/DataInspectorService.cxx
index 40399a366a303..661065da35ff6 100644
--- a/Framework/DataInspector/src/DataInspectorService.cxx
+++ b/Framework/DataInspector/src/DataInspectorService.cxx
@@ -205,7 +205,7 @@ ServiceSpec* DIServicePlugin::create()
       std::string runId = std::getenv("O2_DATAINSPECTOR_ID");
 
       const auto& spec = services.get<const DeviceSpec>();
-      if (DataInspector::isNonInternalDevice(spec)) {
+      if (data_inspector::isNonInternalDevice(spec)) {
         auto* diService = new DataInspectorProxyService(services, spec, proxyAddress, proxyPort, runId);
         return ServiceHandle{TypeIdHelpers::uniqueId<DataInspectorProxyService>(), diService};
       } else {
@@ -238,11 +238,11 @@ ServiceSpec* DIServicePlugin::create()
               }
 
               // Send copy to proxy
-              auto proxyMessages = DataInspector::serializeO2Messages(refs, registry.get < DeviceSpec const>().name);
+              auto proxyMessages = data_inspector::serializeO2Messages(refs, registry.get<DeviceSpec const>().name);
               for (auto &proxyMessage: proxyMessages) {
                 diService.send(std::move(proxyMessage));
               }
             } },
     .kind = ServiceKind::Global};
 };
-} // namespace o2::framework
\ No newline at end of file
+} // namespace o2::framework

From eb0e5147a4fafcb0298736f3f5117ab38c24997d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 12 Feb 2024 10:44:18 +0100
Subject: [PATCH 326/726] DPL: hide exception handling from header

---
 .../include/Framework/runDataProcessing.h     | 43 ++-----------------
 Framework/Core/src/runDataProcessing.cxx      | 39 +++++++++++++++++
 2 files changed, 42 insertions(+), 40 deletions(-)

diff --git a/Framework/Core/include/Framework/runDataProcessing.h b/Framework/Core/include/Framework/runDataProcessing.h
index bfda47d07971c..186d76499c1d6 100644
--- a/Framework/Core/include/Framework/runDataProcessing.h
+++ b/Framework/Core/include/Framework/runDataProcessing.h
@@ -25,7 +25,6 @@
 #include "Framework/CustomWorkflowTerminationHook.h"
 #include "Framework/CommonServices.h"
 #include "Framework/WorkflowCustomizationHelpers.h"
-#include "Framework/RuntimeError.h"
 #include "Framework/ResourcePolicyHelpers.h"
 #include "Framework/Logger.h"
 #include "Framework/CheckTypes.h"
@@ -33,12 +32,6 @@
 
 #include <vector>
 #include <cstring>
-#include <exception>
-
-namespace boost
-{
-class exception;
-}
 
 namespace o2::framework
 {
@@ -147,9 +140,6 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& specs,
            std::vector<o2::framework::ConfigParamSpec> const& workflowOptions,
            o2::framework::ConfigContext& configContext);
 
-void doBoostException(boost::exception& e, const char*);
-void doDPLException(o2::framework::RuntimeErrorRef& ref, char const*);
-void doUnknownException(std::string const& s, char const*);
 void doDefaultWorkflowTerminationHook();
 
 template <typename T>
@@ -220,41 +210,14 @@ int mainNoCatch(int argc, char** argv)
                 resourcePolicies, callbacksPolicies, sendingPolicies, workflowOptions, configContext);
 }
 
+int callMain(int argc, char** argv, int (*)(int, char**));
+
 int main(int argc, char** argv)
 {
   using namespace o2::framework;
   using namespace boost::program_options;
 
-  static bool noCatch = getenv("O2_NO_CATCHALL_EXCEPTIONS") && strcmp(getenv("O2_NO_CATCHALL_EXCEPTIONS"), "0");
-  int result = 1;
-  if (noCatch) {
-    try {
-      result = mainNoCatch(argc, argv);
-    } catch (o2::framework::RuntimeErrorRef& ref) {
-      doDPLException(ref, argv[0]);
-      throw;
-    }
-  } else {
-    try {
-      // The 0 here is an int, therefore having the template matching in the
-      // SFINAE expression above fit better the version which invokes user code over
-      // the default one.
-      // The default policy is a catch all pub/sub setup to be consistent with the past.
-      result = mainNoCatch(argc, argv);
-    } catch (boost::exception& e) {
-      doBoostException(e, argv[0]);
-      throw;
-    } catch (std::exception const& error) {
-      doUnknownException(error.what(), argv[0]);
-      throw;
-    } catch (o2::framework::RuntimeErrorRef& ref) {
-      doDPLException(ref, argv[0]);
-      throw;
-    } catch (...) {
-      doUnknownException("", argv[0]);
-      throw;
-    }
-  }
+  int result = callMain(argc, argv, mainNoCatch);
 
   char* idstring = nullptr;
   for (int argi = 0; argi < argc; argi++) {
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index cfbb616b0d0a5..64c034ebb8533 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -177,6 +177,45 @@ bpo::options_description gHiddenDeviceOptions("Hidden child options");
 
 O2_DECLARE_DYNAMIC_LOG(driver);
 
+void doBoostException(boost::exception& e, const char*);
+void doDPLException(o2::framework::RuntimeErrorRef& ref, char const*);
+void doUnknownException(std::string const& s, char const*);
+
+int callMain(int argc, char** argv, int (*mainNoCatch)(int, char**))
+{
+  static bool noCatch = getenv("O2_NO_CATCHALL_EXCEPTIONS") && strcmp(getenv("O2_NO_CATCHALL_EXCEPTIONS"), "0");
+  int result = 1;
+  if (noCatch) {
+    try {
+      result = mainNoCatch(argc, argv);
+    } catch (o2::framework::RuntimeErrorRef& ref) {
+      doDPLException(ref, argv[0]);
+      throw;
+    }
+  } else {
+    try {
+      // The 0 here is an int, therefore having the template matching in the
+      // SFINAE expression above fit better the version which invokes user code over
+      // the default one.
+      // The default policy is a catch all pub/sub setup to be consistent with the past.
+      result = mainNoCatch(argc, argv);
+    } catch (boost::exception& e) {
+      doBoostException(e, argv[0]);
+      throw;
+    } catch (std::exception const& error) {
+      doUnknownException(error.what(), argv[0]);
+      throw;
+    } catch (o2::framework::RuntimeErrorRef& ref) {
+      doDPLException(ref, argv[0]);
+      throw;
+    } catch (...) {
+      doUnknownException("", argv[0]);
+      throw;
+    }
+  }
+  return result;
+}
+
 // Read from a given fd and print it.
 // return true if we can still read from it,
 // return false if we need to close the input pipe.

From fe24d077b30ebd5d5acd8fbbcdc100013b5ef1e3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 12 Feb 2024 17:42:23 +0100
Subject: [PATCH 327/726] DPL: apparently we also need non-critical

---
 Framework/Core/src/SendingPolicy.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 3aeccb680c150..4b98a87052225 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -107,7 +107,7 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
             .name = "expendable",
             .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) {
               auto has_label = [](DataProcessorLabel const& label) {
-                return label.value == "expendable";
+                return label.value == "expendable" || label.value == "non-critical";
               };
               return std::find_if(dest.labels.begin(), dest.labels.end(), has_label) != dest.labels.end(); },
             .send = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {

From e4a6f20db541626565dce37f66a8ba0e0cabb4ab Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 13 Feb 2024 11:03:52 +0100
Subject: [PATCH 328/726] DPL: fix non lossy dispatcher

---
 Framework/Core/src/SendingPolicy.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 3aeccb680c150..b24761dd728ff 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -61,7 +61,7 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               if (state.droppedMessages > 10) {
                 timeout = 0;
               }
-              size_t result = info.channel.Send(parts, timeout);
+              int64_t result = info.channel.Send(parts, timeout);
               if (result > 0) {
                 state.droppedMessages = 0;
               } else if (state.droppedMessages < std::numeric_limits<decltype(state.droppedMessages)>::max()) {

From f264f26638bda1bb537e1337d9a9914ef414f842 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 13 Feb 2024 11:42:41 +0100
Subject: [PATCH 329/726] DPL: fix non lossy dispatcher

---
 Framework/Core/src/SendingPolicy.cxx | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 3aeccb680c150..28fb1a7524da3 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -61,7 +61,7 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               if (state.droppedMessages > 10) {
                 timeout = 0;
               }
-              size_t result = info.channel.Send(parts, timeout);
+              int64_t result = info.channel.Send(parts, timeout);
               if (result > 0) {
                 state.droppedMessages = 0;
               } else if (state.droppedMessages < std::numeric_limits<decltype(state.droppedMessages)>::max()) {
@@ -113,12 +113,18 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
             .send = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
               auto &proxy = registry.get<FairMQDeviceProxy>();
               auto *channel = proxy.getOutputChannel(channelIndex);
+              OutputChannelState& state = proxy.getOutputChannelState(channelIndex);
               auto timeout = 1000;
+              if (state.droppedMessages > 0) {
+                timeout = 0;
+              }
               auto res = channel->Send(parts, timeout);
               if (res == (size_t)fair::mq::TransferCode::timeout) {
-                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}. Switching to dropping mode.", timeout/1000, channel->GetName());
+                state.droppedMessages++;
               } else if (res == (size_t) fair::mq::TransferCode::error) {
-                LOGP(info, "Error while sending on channel {}", channel->GetName());
+                LOGP(warning, "Error while sending on expendable channel {}. Switching to lossy mode for that channel.", channel->GetName());
+                state.droppedMessages++;
               } }},
           SendingPolicy{
             .name = "default",
@@ -204,12 +210,18 @@ std::vector<ForwardingPolicy> ForwardingPolicy::createDefaultPolicies()
             .forward = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
               auto &proxy = registry.get<FairMQDeviceProxy>();
               auto *channel = proxy.getForwardChannel(channelIndex);
+              OutputChannelState& state = proxy.getOutputChannelState(channelIndex);
               auto timeout = 1000;
+              if (state.droppedMessages > 0) {
+                timeout = 0;
+              }
               auto res = channel->Send(parts, timeout);
               if (res == (size_t)fair::mq::TransferCode::timeout) {
-                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}. Switching to dropping mode.", timeout/1000, channel->GetName());
+                state.droppedMessages++;
               } else if (res == (size_t) fair::mq::TransferCode::error) {
-                LOGP(info, "Error while sending on channel {}", channel->GetName());
+                LOGP(warning, "Error sending after {}s. Downstream backpressure detected on expendable channel {}. Switching to dropping mode.", timeout/1000, channel->GetName());
+                state.droppedMessages++;
               } }},
           createDefaultForwardingPolicy()};
 }

From 6a985cea11eae89a3a977fa815fa273eb0da3d6d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 13 Feb 2024 13:40:00 +0100
Subject: [PATCH 330/726] DPL: fix non lossy dispatcher

---
 Framework/Core/src/SendingPolicy.cxx | 49 ++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 3aeccb680c150..2c99382873789 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -56,13 +56,16 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               // non-blocking approach.
               int64_t timeout = 10;
               if (state.droppedMessages == 10 + 1) {
-                LOG(warning) << "Failed to send 10 messages with 10ms timeout in a row, switching to completely non-blocking mode";
+                LOG(warning) << "Failed to send 10 messages with 10ms timeout in a row, switching to completely non-blocking mode.";
+              }
+              if (state.droppedMessages == 0) {
+                timeout = 10;
               }
               if (state.droppedMessages > 10) {
                 timeout = 0;
               }
-              size_t result = info.channel.Send(parts, timeout);
-              if (result > 0) {
+              int64_t result = info.channel.Send(parts, timeout);
+              if (result >= 0) {
                 state.droppedMessages = 0;
               } else if (state.droppedMessages < std::numeric_limits<decltype(state.droppedMessages)>::max()) {
                 state.droppedMessages++;
@@ -113,12 +116,22 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
             .send = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
               auto &proxy = registry.get<FairMQDeviceProxy>();
               auto *channel = proxy.getOutputChannel(channelIndex);
+              OutputChannelState& state = proxy.getOutputChannelState(channelIndex);
               auto timeout = 1000;
-              auto res = channel->Send(parts, timeout);
-              if (res == (size_t)fair::mq::TransferCode::timeout) {
-                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
-              } else if (res == (size_t) fair::mq::TransferCode::error) {
-                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              if (state.droppedMessages > 0) {
+                timeout = 0;
+              }
+              if (state.droppedMessages == 1) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}. Switching to dropping mode.", timeout/1000, channel->GetName());
+              }
+              if (state.droppedMessages == 0) {
+                timeout = 1000;
+              }
+              int64_t res = channel->Send(parts, timeout);
+              if (res >= 0) {
+                state.droppedMessages = 0;
+              } else {
+                state.droppedMessages++;
               } }},
           SendingPolicy{
             .name = "default",
@@ -204,12 +217,22 @@ std::vector<ForwardingPolicy> ForwardingPolicy::createDefaultPolicies()
             .forward = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
               auto &proxy = registry.get<FairMQDeviceProxy>();
               auto *channel = proxy.getForwardChannel(channelIndex);
+              OutputChannelState& state = proxy.getOutputChannelState(channelIndex);
               auto timeout = 1000;
-              auto res = channel->Send(parts, timeout);
-              if (res == (size_t)fair::mq::TransferCode::timeout) {
-                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
-              } else if (res == (size_t) fair::mq::TransferCode::error) {
-                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              if (state.droppedMessages > 0) {
+                timeout = 0;
+              }
+              if (state.droppedMessages == 1) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}. Switching to dropping mode.", timeout/1000, channel->GetName());
+              }
+              if (state.droppedMessages == 0) {
+                timeout = 1000;
+              }
+              int64_t res = channel->Send(parts, timeout);
+              if (res >= 0) {
+                state.droppedMessages = 0;
+              } else {
+                state.droppedMessages++;
               } }},
           createDefaultForwardingPolicy()};
 }

From 0a5cde1678491a4994867cb53faa0c9fb69d04b4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 13 Feb 2024 13:43:36 +0100
Subject: [PATCH 331/726] DPL: fix lossy sending / forwarding policies

---
 Framework/Core/src/SendingPolicy.cxx | 49 ++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 3aeccb680c150..2c99382873789 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -56,13 +56,16 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               // non-blocking approach.
               int64_t timeout = 10;
               if (state.droppedMessages == 10 + 1) {
-                LOG(warning) << "Failed to send 10 messages with 10ms timeout in a row, switching to completely non-blocking mode";
+                LOG(warning) << "Failed to send 10 messages with 10ms timeout in a row, switching to completely non-blocking mode.";
+              }
+              if (state.droppedMessages == 0) {
+                timeout = 10;
               }
               if (state.droppedMessages > 10) {
                 timeout = 0;
               }
-              size_t result = info.channel.Send(parts, timeout);
-              if (result > 0) {
+              int64_t result = info.channel.Send(parts, timeout);
+              if (result >= 0) {
                 state.droppedMessages = 0;
               } else if (state.droppedMessages < std::numeric_limits<decltype(state.droppedMessages)>::max()) {
                 state.droppedMessages++;
@@ -113,12 +116,22 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
             .send = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
               auto &proxy = registry.get<FairMQDeviceProxy>();
               auto *channel = proxy.getOutputChannel(channelIndex);
+              OutputChannelState& state = proxy.getOutputChannelState(channelIndex);
               auto timeout = 1000;
-              auto res = channel->Send(parts, timeout);
-              if (res == (size_t)fair::mq::TransferCode::timeout) {
-                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
-              } else if (res == (size_t) fair::mq::TransferCode::error) {
-                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              if (state.droppedMessages > 0) {
+                timeout = 0;
+              }
+              if (state.droppedMessages == 1) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}. Switching to dropping mode.", timeout/1000, channel->GetName());
+              }
+              if (state.droppedMessages == 0) {
+                timeout = 1000;
+              }
+              int64_t res = channel->Send(parts, timeout);
+              if (res >= 0) {
+                state.droppedMessages = 0;
+              } else {
+                state.droppedMessages++;
               } }},
           SendingPolicy{
             .name = "default",
@@ -204,12 +217,22 @@ std::vector<ForwardingPolicy> ForwardingPolicy::createDefaultPolicies()
             .forward = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
               auto &proxy = registry.get<FairMQDeviceProxy>();
               auto *channel = proxy.getForwardChannel(channelIndex);
+              OutputChannelState& state = proxy.getOutputChannelState(channelIndex);
               auto timeout = 1000;
-              auto res = channel->Send(parts, timeout);
-              if (res == (size_t)fair::mq::TransferCode::timeout) {
-                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
-              } else if (res == (size_t) fair::mq::TransferCode::error) {
-                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              if (state.droppedMessages > 0) {
+                timeout = 0;
+              }
+              if (state.droppedMessages == 1) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}. Switching to dropping mode.", timeout/1000, channel->GetName());
+              }
+              if (state.droppedMessages == 0) {
+                timeout = 1000;
+              }
+              int64_t res = channel->Send(parts, timeout);
+              if (res >= 0) {
+                state.droppedMessages = 0;
+              } else {
+                state.droppedMessages++;
               } }},
           createDefaultForwardingPolicy()};
 }

From 280a47a1d763a5a07fe64529e0388eb070dd3334 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 13 Feb 2024 14:39:06 +0100
Subject: [PATCH 332/726] DPL: fix lossy sending / forwarding policies

---
 Framework/Core/src/SendingPolicy.cxx | 53 ++++++++++++++++++++--------
 1 file changed, 38 insertions(+), 15 deletions(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 3aeccb680c150..da83b644efe60 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -56,13 +56,16 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               // non-blocking approach.
               int64_t timeout = 10;
               if (state.droppedMessages == 10 + 1) {
-                LOG(warning) << "Failed to send 10 messages with 10ms timeout in a row, switching to completely non-blocking mode";
+                LOG(warning) << "Failed to send 10 messages with 10ms timeout in a row, switching to completely non-blocking mode.";
+              }
+              if (state.droppedMessages == 0) {
+                timeout = 10;
               }
               if (state.droppedMessages > 10) {
                 timeout = 0;
               }
-              size_t result = info.channel.Send(parts, timeout);
-              if (result > 0) {
+              int64_t result = info.channel.Send(parts, timeout);
+              if (result >= 0) {
                 state.droppedMessages = 0;
               } else if (state.droppedMessages < std::numeric_limits<decltype(state.droppedMessages)>::max()) {
                 state.droppedMessages++;
@@ -97,10 +100,10 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               LOGP(info, "Sent {} parts for a total of {} bytes", parts.Size(), count);
               auto res = channel->Send(parts, timeout);
               if (res == (size_t)fair::mq::TransferCode::timeout) {
-                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on {}.", timeout/1000, channel->GetName());
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on {}.", timeout / 1000, channel->GetName());
                 channel->Send(parts);
                 LOGP(info, "Downstream backpressure on {} recovered.", channel->GetName());
-              } else if (res == (size_t) fair::mq::TransferCode::error) {
+              } else if (res == (size_t)fair::mq::TransferCode::error) {
                 LOGP(fatal, "Error while sending on channel {}", channel->GetName());
               } }},
           SendingPolicy{
@@ -113,12 +116,22 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
             .send = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
               auto &proxy = registry.get<FairMQDeviceProxy>();
               auto *channel = proxy.getOutputChannel(channelIndex);
+              OutputChannelState& state = proxy.getOutputChannelState(channelIndex);
               auto timeout = 1000;
-              auto res = channel->Send(parts, timeout);
-              if (res == (size_t)fair::mq::TransferCode::timeout) {
-                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
-              } else if (res == (size_t) fair::mq::TransferCode::error) {
-                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              if (state.droppedMessages > 0) {
+                timeout = 0;
+              }
+              if (state.droppedMessages == 1) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}. Switching to dropping mode.", timeout / 1000, channel->GetName());
+              }
+              if (state.droppedMessages == 0) {
+                timeout = 1000;
+              }
+              int64_t res = channel->Send(parts, timeout);
+              if (res >= 0) {
+                state.droppedMessages = 0;
+              } else {
+                state.droppedMessages++;
               } }},
           SendingPolicy{
             .name = "default",
@@ -204,12 +217,22 @@ std::vector<ForwardingPolicy> ForwardingPolicy::createDefaultPolicies()
             .forward = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
               auto &proxy = registry.get<FairMQDeviceProxy>();
               auto *channel = proxy.getForwardChannel(channelIndex);
+              OutputChannelState& state = proxy.getOutputChannelState(channelIndex);
               auto timeout = 1000;
-              auto res = channel->Send(parts, timeout);
-              if (res == (size_t)fair::mq::TransferCode::timeout) {
-                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
-              } else if (res == (size_t) fair::mq::TransferCode::error) {
-                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              if (state.droppedMessages > 0) {
+                timeout = 0;
+              }
+              if (state.droppedMessages == 1) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}. Switching to dropping mode.", timeout/1000, channel->GetName());
+              }
+              if (state.droppedMessages == 0) {
+                timeout = 1000;
+              }
+              int64_t res = channel->Send(parts, timeout);
+              if (res >= 0) {
+                state.droppedMessages = 0;
+              } else {
+                state.droppedMessages++;
               } }},
           createDefaultForwardingPolicy()};
 }

From 0026e429e25a006f1884bd16c1470e940fb29da5 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 13 Feb 2024 14:39:58 +0100
Subject: [PATCH 333/726] DPL: drop reference to the now obsolete non-critical
 label

---
 Framework/Core/src/SendingPolicy.cxx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index da83b644efe60..3956ff0e99899 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -39,9 +39,9 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
                     return true;
                   }
                 }
-                // Check if any of the destination's labels is "expendable" or "non-critical"
+                // Check if any of the destination's labels is "expendable"
                 for (auto const& label : dest.labels) {
-                  if (label.value == "expendable" || label.value == "non-critical") {
+                  if (label.value == "expendable") {
                     return true;
                   }
                 }

From 623613eed9ebd9c3db3f4d9d0e97f39b7ec1e292 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 13 Feb 2024 14:42:07 +0100
Subject: [PATCH 334/726] DPL: fix lossy sending / forwarding policies

---
 Framework/Core/src/SendingPolicy.cxx | 53 ++++++++++++++++++++--------
 1 file changed, 38 insertions(+), 15 deletions(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 3aeccb680c150..52123378e1e46 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -56,13 +56,16 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               // non-blocking approach.
               int64_t timeout = 10;
               if (state.droppedMessages == 10 + 1) {
-                LOG(warning) << "Failed to send 10 messages with 10ms timeout in a row, switching to completely non-blocking mode";
+                LOG(warning) << "Failed to send 10 messages with 10ms timeout in a row, switching to completely non-blocking mode.";
+              }
+              if (state.droppedMessages == 0) {
+                timeout = 10;
               }
               if (state.droppedMessages > 10) {
                 timeout = 0;
               }
-              size_t result = info.channel.Send(parts, timeout);
-              if (result > 0) {
+              int64_t result = info.channel.Send(parts, timeout);
+              if (result >= 0) {
                 state.droppedMessages = 0;
               } else if (state.droppedMessages < std::numeric_limits<decltype(state.droppedMessages)>::max()) {
                 state.droppedMessages++;
@@ -97,10 +100,10 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               LOGP(info, "Sent {} parts for a total of {} bytes", parts.Size(), count);
               auto res = channel->Send(parts, timeout);
               if (res == (size_t)fair::mq::TransferCode::timeout) {
-                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on {}.", timeout/1000, channel->GetName());
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on {}.", timeout / 1000, channel->GetName());
                 channel->Send(parts);
                 LOGP(info, "Downstream backpressure on {} recovered.", channel->GetName());
-              } else if (res == (size_t) fair::mq::TransferCode::error) {
+              } else if (res == (size_t)fair::mq::TransferCode::error) {
                 LOGP(fatal, "Error while sending on channel {}", channel->GetName());
               } }},
           SendingPolicy{
@@ -113,12 +116,22 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
             .send = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
               auto &proxy = registry.get<FairMQDeviceProxy>();
               auto *channel = proxy.getOutputChannel(channelIndex);
+              OutputChannelState& state = proxy.getOutputChannelState(channelIndex);
               auto timeout = 1000;
-              auto res = channel->Send(parts, timeout);
-              if (res == (size_t)fair::mq::TransferCode::timeout) {
-                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
-              } else if (res == (size_t) fair::mq::TransferCode::error) {
-                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              if (state.droppedMessages > 0) {
+                timeout = 0;
+              }
+              if (state.droppedMessages == 1) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}. Switching to dropping mode.", timeout / 1000, channel->GetName());
+              }
+              if (state.droppedMessages == 0) {
+                timeout = 1000;
+              }
+              int64_t res = channel->Send(parts, timeout);
+              if (res >= 0) {
+                state.droppedMessages = 0;
+              } else {
+                state.droppedMessages++;
               } }},
           SendingPolicy{
             .name = "default",
@@ -204,12 +217,22 @@ std::vector<ForwardingPolicy> ForwardingPolicy::createDefaultPolicies()
             .forward = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
               auto &proxy = registry.get<FairMQDeviceProxy>();
               auto *channel = proxy.getForwardChannel(channelIndex);
+              OutputChannelState& state = proxy.getOutputChannelState(channelIndex);
               auto timeout = 1000;
-              auto res = channel->Send(parts, timeout);
-              if (res == (size_t)fair::mq::TransferCode::timeout) {
-                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}.", timeout/1000, channel->GetName());
-              } else if (res == (size_t) fair::mq::TransferCode::error) {
-                LOGP(info, "Error while sending on channel {}", channel->GetName());
+              if (state.droppedMessages > 0) {
+                timeout = 0;
+              }
+              if (state.droppedMessages == 1) {
+                LOGP(warning, "Timed out sending after {}s. Downstream backpressure detected on expendable channel {}. Switching to dropping mode.", timeout / 1000, channel->GetName());
+              }
+              if (state.droppedMessages == 0) {
+                timeout = 1000;
+              }
+              int64_t res = channel->Send(parts, timeout);
+              if (res >= 0) {
+                state.droppedMessages = 0;
+              } else {
+                state.droppedMessages++;
               } }},
           createDefaultForwardingPolicy()};
 }

From 466f501fa1bfa1520ed92a075aaf60ccbb6914cb Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 13 Feb 2024 14:42:07 +0100
Subject: [PATCH 335/726] DPL: drop reference to the now obsolete non-critical
 label

---
 Framework/Core/src/SendingPolicy.cxx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 52123378e1e46..3dbbc451915b0 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -39,9 +39,9 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
                     return true;
                   }
                 }
-                // Check if any of the destination's labels is "expendable" or "non-critical"
+                // Check if any of the destination's labels is "expendable"
                 for (auto const& label : dest.labels) {
-                  if (label.value == "expendable" || label.value == "non-critical") {
+                  if (label.value == "expendable") {
                     return true;
                   }
                 }

From 6f6ec56bc0a79cb82525033ff2bab9a3a1b1313f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 13 Feb 2024 16:17:20 +0100
Subject: [PATCH 336/726] DPL: make sure homogeneous_apply_ref complains when
 we are missing an explicit constructor

---
 Framework/Foundation/include/Framework/StructToTuple.h | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Framework/Foundation/include/Framework/StructToTuple.h b/Framework/Foundation/include/Framework/StructToTuple.h
index d94c0e3aaae03..dbfccbcb95acc 100644
--- a/Framework/Foundation/include/Framework/StructToTuple.h
+++ b/Framework/Foundation/include/Framework/StructToTuple.h
@@ -135,10 +135,13 @@ struct UniversalType {
 template <typename T>
 consteval auto brace_constructible_size(auto... Members)
 {
-  if constexpr (requires { T{Members...}; } == false)
+  if constexpr (requires { T{Members...}; } == false) {
+    static_assert(sizeof...(Members) != 0, "You need to make sure that you have implicit constructors or that you call the explicit constructor correctly.");
     return sizeof...(Members) - 1;
-  else
+  }
+  else {
     return brace_constructible_size<T>(Members..., UniversalType{});
+  }
 }
 #else
 template <typename T>
@@ -282,6 +285,7 @@ auto homogeneous_apply_refs(L l, T&& object)
   using type = std::decay_t<T>;
   constexpr int nesting = B ? 1 : 0;
   constexpr unsigned long numElements = brace_constructible_size<type>() - nesting;
+  static_assert(numElements > 99, "Too many elements in the struct");
   // clang-format off
   if DPL_HOMOGENEOUS_APPLY_ENTRY (9, 9)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (9, 8)

From a643e97a4ac25c13523acd1b901a33158606c82f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 13 Feb 2024 16:31:57 +0100
Subject: [PATCH 337/726] DPL: make sure homogeneous_apply_ref complains when
 we are missing an explicit constructor

---
 Framework/Foundation/include/Framework/StructToTuple.h | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Framework/Foundation/include/Framework/StructToTuple.h b/Framework/Foundation/include/Framework/StructToTuple.h
index d94c0e3aaae03..b8b0c0d5800fe 100644
--- a/Framework/Foundation/include/Framework/StructToTuple.h
+++ b/Framework/Foundation/include/Framework/StructToTuple.h
@@ -135,10 +135,13 @@ struct UniversalType {
 template <typename T>
 consteval auto brace_constructible_size(auto... Members)
 {
-  if constexpr (requires { T{Members...}; } == false)
+  if constexpr (requires { T{Members...}; } == false) {
+    static_assert(sizeof...(Members) != 0, "You need to make sure that you have implicit constructors or that you call the explicit constructor correctly.");
     return sizeof...(Members) - 1;
-  else
+  }
+  else {
     return brace_constructible_size<T>(Members..., UniversalType{});
+  }
 }
 #else
 template <typename T>
@@ -282,6 +285,7 @@ auto homogeneous_apply_refs(L l, T&& object)
   using type = std::decay_t<T>;
   constexpr int nesting = B ? 1 : 0;
   constexpr unsigned long numElements = brace_constructible_size<type>() - nesting;
+  static_assert(numElements < 99, "Too many elements in the struct");
   // clang-format off
   if DPL_HOMOGENEOUS_APPLY_ENTRY (9, 9)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (9, 8)

From 025c3aebd0e1ae92e83271a38706d66d0e2ce62a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 13 Feb 2024 17:19:21 +0100
Subject: [PATCH 338/726] DPL: make sure homogeneous_apply_ref complains when
 we are missing an explicit constructor

---
 Framework/Foundation/include/Framework/StructToTuple.h | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Framework/Foundation/include/Framework/StructToTuple.h b/Framework/Foundation/include/Framework/StructToTuple.h
index d94c0e3aaae03..0ef3be4efcb3c 100644
--- a/Framework/Foundation/include/Framework/StructToTuple.h
+++ b/Framework/Foundation/include/Framework/StructToTuple.h
@@ -135,10 +135,12 @@ struct UniversalType {
 template <typename T>
 consteval auto brace_constructible_size(auto... Members)
 {
-  if constexpr (requires { T{Members...}; } == false)
+  if constexpr (requires { T{Members...}; } == false) {
+    static_assert(sizeof...(Members) != 0, "You need to make sure that you have implicit constructors or that you call the explicit constructor correctly.");
     return sizeof...(Members) - 1;
-  else
+  } else {
     return brace_constructible_size<T>(Members..., UniversalType{});
+  }
 }
 #else
 template <typename T>
@@ -282,6 +284,7 @@ auto homogeneous_apply_refs(L l, T&& object)
   using type = std::decay_t<T>;
   constexpr int nesting = B ? 1 : 0;
   constexpr unsigned long numElements = brace_constructible_size<type>() - nesting;
+  static_assert(numElements < 99, "Too many elements in the struct");
   // clang-format off
   if DPL_HOMOGENEOUS_APPLY_ENTRY (9, 9)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (9, 8)

From 7940f6c9c315c908147a2e280a19670b8aa12290 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 14 Feb 2024 08:50:56 +0100
Subject: [PATCH 339/726] DPL: make sure homogeneous_apply_ref complains when
 we are missing an explicit constructor

---
 Framework/Foundation/include/Framework/StructToTuple.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Framework/Foundation/include/Framework/StructToTuple.h b/Framework/Foundation/include/Framework/StructToTuple.h
index d94c0e3aaae03..6d9a6a2ef305a 100644
--- a/Framework/Foundation/include/Framework/StructToTuple.h
+++ b/Framework/Foundation/include/Framework/StructToTuple.h
@@ -135,10 +135,12 @@ struct UniversalType {
 template <typename T>
 consteval auto brace_constructible_size(auto... Members)
 {
-  if constexpr (requires { T{Members...}; } == false)
+  if constexpr (requires { T{Members...}; } == false) {
+    static_assert(sizeof...(Members) != 0, "You need to make sure that you have implicit constructors or that you call the explicit constructor correctly.");
     return sizeof...(Members) - 1;
-  else
+  } else {
     return brace_constructible_size<T>(Members..., UniversalType{});
+  }
 }
 #else
 template <typename T>

From 983f5e4a130027a838ef1e02c06cf4c1f2dc03e6 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 14 Feb 2024 08:50:56 +0100
Subject: [PATCH 340/726] DPL: check for the maximum number of members
 supported for tasks

---
 Framework/Foundation/include/Framework/StructToTuple.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Framework/Foundation/include/Framework/StructToTuple.h b/Framework/Foundation/include/Framework/StructToTuple.h
index 6d9a6a2ef305a..0ef3be4efcb3c 100644
--- a/Framework/Foundation/include/Framework/StructToTuple.h
+++ b/Framework/Foundation/include/Framework/StructToTuple.h
@@ -284,6 +284,7 @@ auto homogeneous_apply_refs(L l, T&& object)
   using type = std::decay_t<T>;
   constexpr int nesting = B ? 1 : 0;
   constexpr unsigned long numElements = brace_constructible_size<type>() - nesting;
+  static_assert(numElements < 99, "Too many elements in the struct");
   // clang-format off
   if DPL_HOMOGENEOUS_APPLY_ENTRY (9, 9)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (9, 8)

From fc2a17a575f2551f163cb60f605dca6a46ed4fd2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 14 Feb 2024 16:40:15 +0100
Subject: [PATCH 341/726] DPL: allow extracting data from processing context
 directly using arguments

---
 .../Core/include/Framework/AlgorithmSpec.h    | 48 +++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/Framework/Core/include/Framework/AlgorithmSpec.h b/Framework/Core/include/Framework/AlgorithmSpec.h
index 98f7f18e0d24d..e98a3abefbce1 100644
--- a/Framework/Core/include/Framework/AlgorithmSpec.h
+++ b/Framework/Core/include/Framework/AlgorithmSpec.h
@@ -84,6 +84,33 @@ struct AlgorithmPlugin {
   virtual AlgorithmSpec create() = 0;
 };
 
+template <size_t N>
+struct StringLiteral {
+  constexpr StringLiteral(const char (&str)[N])
+  {
+    std::copy_n(str, N, value);
+  }
+
+  char value[N];
+};
+
+// Allow fetching inputs from the context using a string literal.
+template <StringLiteral lit, typename T>
+struct Input {
+  // The size of the string is available as a constant expression.
+  static constexpr auto size = sizeof(lit.value);
+  // and so is the string's content.
+  static constexpr auto contents = lit.value;
+  ProcessingContext& ctx;
+  Input(ProcessingContext& c) : ctx(c)
+  {
+  }
+  operator T const&() const
+  {
+    return ctx.inputs().template get<T>(lit.value);
+  }
+};
+
 template <typename T, typename S = std::void_t<>>
 struct ContextElementTraits {
   static decltype(auto) get(ProcessingContext& ctx)
@@ -137,6 +164,22 @@ struct ContextElementTraits<ProcessingContext> {
   }
 };
 
+template <StringLiteral L, typename T>
+struct ContextElementTraits<Input<L, T> const> {
+  static Input<L, T> get(ProcessingContext& ctx)
+  {
+    return Input<L, T>{ctx};
+  }
+};
+
+template <StringLiteral L, typename T>
+struct ContextElementTraits<Input<L, T>> {
+  static Input<L, T> get(ProcessingContext& ctx)
+  {
+    static_assert(always_static_assert_v<Input<L, T>>, "Should be Input<L, T> const&");
+  }
+};
+
 template <typename... CONTEXTELEMENT>
 AlgorithmSpec::ProcessCallback adaptStatelessF(std::function<void(CONTEXTELEMENT&...)> callback)
 {
@@ -177,6 +220,11 @@ AlgorithmSpec::ProcessCallback adaptStatelessP(R (*callback)(ARGS...))
 ///   inputs.get<int>("someInt");
 /// }}
 ///
+/// and if you have C++20 enabled you can also do:
+///
+/// AlgorithmSpec{[](Input<"someInt", int> someInt){
+///  someInt.value; // do something with the inputs
+/// }
 /// Notice you can specify in any order any of InputRecord, DataAllocator,
 /// ConfigParamRegistry or any of the services which are usually hanging
 /// from the ServiceRegistry, e.g. ControlService.

From abeda3ba165b0bbc0df6daeea73d3886a9087c22 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 14 Feb 2024 16:53:51 +0100
Subject: [PATCH 342/726] DPL: calibration workflow example

---
 Framework/TestWorkflows/CMakeLists.txt        |  4 +
 .../src/o2DummyCalibrationWorkflow.cxx        | 73 +++++++++++++++++++
 2 files changed, 77 insertions(+)
 create mode 100644 Framework/TestWorkflows/src/o2DummyCalibrationWorkflow.cxx

diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index c2c89e19b894e..848337200a4d2 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -33,6 +33,10 @@ o2_add_dpl_workflow(diamond-workflow
                   SOURCES src/o2DiamondWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
 
+o2_add_dpl_workflow(dummy-calibration-workflow
+                  SOURCES src/o2DummyCalibrationWorkflow.cxx
+                  COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(diamond-workflow-leaky
                   SOURCES src/o2DiamondWorkflowLeaky.cxx
                   COMPONENT_NAME TestWorkflows)
diff --git a/Framework/TestWorkflows/src/o2DummyCalibrationWorkflow.cxx b/Framework/TestWorkflows/src/o2DummyCalibrationWorkflow.cxx
new file mode 100644
index 0000000000000..94916cfa36984
--- /dev/null
+++ b/Framework/TestWorkflows/src/o2DummyCalibrationWorkflow.cxx
@@ -0,0 +1,73 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/DataTakingContext.h"
+#include "Framework/CompletionPolicyHelpers.h"
+#include "Framework/DeviceSpec.h"
+#include "Framework/ControlService.h"
+#include "Framework/Configurable.h"
+#include "Framework/RunningWorkflowInfo.h"
+#include "Framework/CallbackService.h"
+#include "Framework/EndOfStreamContext.h"
+#include <fairmq/Device.h>
+
+#include <iostream>
+#include <vector>
+
+using namespace o2::framework;
+
+#include "Framework/runDataProcessing.h"
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& specs)
+{
+  DataProcessorSpec a{
+    .name = "counter",
+    .outputs = {OutputSpec{{"counter"}, "TST", "A1"}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, ProcessingContext& pcx) {
+        static int counter = 0;
+        auto& aData = outputs.make<int>(OutputRef{"counter"});
+        aData = counter++;
+        if (counter == 10) {
+          pcx.services().get<ControlService>().endOfStream();
+        }
+      })},
+  };
+
+  DataProcessorSpec b{
+    .name = "aggregator",
+    .inputs = {InputSpec{"x", "TST", "A1", Lifetime::Timeframe}},
+    .outputs = {OutputSpec{{"average"}, "TST", "B1", Lifetime::Sporadic}},
+    .algorithm = adaptStateful([](CallbackService& callbacks) {
+        static int sum = 0;
+        auto eosCallback = [](EndOfStreamContext &ctx) {
+          auto& aData = ctx.outputs().make<int>(OutputRef{"average"});
+          aData = sum;
+          ctx.services().get<ControlService>().endOfStream();
+        };
+        callbacks.set<CallbackService::Id::EndOfStream>(eosCallback);
+        return adaptStateless([](Input<"x", int> const& x)
+          {
+            sum += x;
+            std::cout << "Sum: " << sum << std::endl;
+        }); })};
+
+  DataProcessorSpec c{.name = "publisher",
+                      .inputs = {InputSpec{"average", "TST", "B1", Lifetime::Sporadic}},
+                      .algorithm = adaptStateless([](Input<"average", int> const& counter) {
+                        std::cout << "Counter to publish: " << counter << std::endl;
+                      })};
+
+  return workflow::concat(WorkflowSpec{a},
+                          WorkflowSpec{b},
+                          WorkflowSpec{c});
+}

From a29393538806fdfa497239168460da9876bd0f07 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 15 Feb 2024 10:44:11 +0100
Subject: [PATCH 343/726] DPL: allow extracting data from processing context
 directly using arguments

---
 .../TPC/workflow/src/FileReaderWorkflow.cxx   | 20 ++++----
 .../Core/include/Framework/AlgorithmSpec.h    | 48 +++++++++++++++++++
 2 files changed, 58 insertions(+), 10 deletions(-)

diff --git a/Detectors/TPC/workflow/src/FileReaderWorkflow.cxx b/Detectors/TPC/workflow/src/FileReaderWorkflow.cxx
index c50b5685cdcd0..091674b447105 100644
--- a/Detectors/TPC/workflow/src/FileReaderWorkflow.cxx
+++ b/Detectors/TPC/workflow/src/FileReaderWorkflow.cxx
@@ -38,13 +38,13 @@ void customize(std::vector<o2::framework::ConfigParamSpec>& workflowOptions)
 
 using namespace o2::framework;
 
-enum struct Input { Clusters,
-                    Tracks
+enum struct InputType { Clusters,
+                        Tracks
 };
 
-const std::unordered_map<std::string, Input> InputMap{
-  {"clusters", Input::Clusters},
-  {"tracks", Input::Tracks}};
+const std::unordered_map<std::string, InputType> InputMap{
+  {"clusters", InputType::Clusters},
+  {"tracks", InputType::Tracks}};
 
 /// MC info is processed by default, disabled by using command line option `--disable-mc`
 ///
@@ -56,24 +56,24 @@ WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
   auto inputType = cfgc.options().get<std::string>("input-type");
   bool doMC = not cfgc.options().get<bool>("disable-mc");
 
-  std::vector<Input> inputTypes;
+  std::vector<InputType> inputTypes;
   try {
-    inputTypes = o2::RangeTokenizer::tokenize<Input>(inputType, [](std::string const& token) { return InputMap.at(token); });
+    inputTypes = o2::RangeTokenizer::tokenize<InputType>(inputType, [](std::string const& token) { return InputMap.at(token); });
   } catch (std::out_of_range&) {
     throw std::invalid_argument(std::string("invalid input type: ") + inputType);
   }
-  auto isEnabled = [&inputTypes](Input type) {
+  auto isEnabled = [&inputTypes](InputType type) {
     return std::find(inputTypes.begin(), inputTypes.end(), type) != inputTypes.end();
   };
 
-  if (isEnabled(Input::Clusters)) {
+  if (isEnabled(InputType::Clusters)) {
     specs.emplace_back(o2::tpc::getClusterReaderSpec(doMC));
     if (!getenv("DPL_DISABLE_TPC_TRIGGER_READER") || atoi(getenv("DPL_DISABLE_TPC_TRIGGER_READER")) != 1) {
       specs.emplace_back(o2::tpc::getTPCTriggerReaderSpec());
     }
   }
 
-  if (isEnabled(Input::Tracks)) {
+  if (isEnabled(InputType::Tracks)) {
 
     specs.push_back(o2::tpc::getTPCTrackReaderSpec(doMC));
   }
diff --git a/Framework/Core/include/Framework/AlgorithmSpec.h b/Framework/Core/include/Framework/AlgorithmSpec.h
index 98f7f18e0d24d..e98a3abefbce1 100644
--- a/Framework/Core/include/Framework/AlgorithmSpec.h
+++ b/Framework/Core/include/Framework/AlgorithmSpec.h
@@ -84,6 +84,33 @@ struct AlgorithmPlugin {
   virtual AlgorithmSpec create() = 0;
 };
 
+template <size_t N>
+struct StringLiteral {
+  constexpr StringLiteral(const char (&str)[N])
+  {
+    std::copy_n(str, N, value);
+  }
+
+  char value[N];
+};
+
+// Allow fetching inputs from the context using a string literal.
+template <StringLiteral lit, typename T>
+struct Input {
+  // The size of the string is available as a constant expression.
+  static constexpr auto size = sizeof(lit.value);
+  // and so is the string's content.
+  static constexpr auto contents = lit.value;
+  ProcessingContext& ctx;
+  Input(ProcessingContext& c) : ctx(c)
+  {
+  }
+  operator T const&() const
+  {
+    return ctx.inputs().template get<T>(lit.value);
+  }
+};
+
 template <typename T, typename S = std::void_t<>>
 struct ContextElementTraits {
   static decltype(auto) get(ProcessingContext& ctx)
@@ -137,6 +164,22 @@ struct ContextElementTraits<ProcessingContext> {
   }
 };
 
+template <StringLiteral L, typename T>
+struct ContextElementTraits<Input<L, T> const> {
+  static Input<L, T> get(ProcessingContext& ctx)
+  {
+    return Input<L, T>{ctx};
+  }
+};
+
+template <StringLiteral L, typename T>
+struct ContextElementTraits<Input<L, T>> {
+  static Input<L, T> get(ProcessingContext& ctx)
+  {
+    static_assert(always_static_assert_v<Input<L, T>>, "Should be Input<L, T> const&");
+  }
+};
+
 template <typename... CONTEXTELEMENT>
 AlgorithmSpec::ProcessCallback adaptStatelessF(std::function<void(CONTEXTELEMENT&...)> callback)
 {
@@ -177,6 +220,11 @@ AlgorithmSpec::ProcessCallback adaptStatelessP(R (*callback)(ARGS...))
 ///   inputs.get<int>("someInt");
 /// }}
 ///
+/// and if you have C++20 enabled you can also do:
+///
+/// AlgorithmSpec{[](Input<"someInt", int> someInt){
+///  someInt.value; // do something with the inputs
+/// }
 /// Notice you can specify in any order any of InputRecord, DataAllocator,
 /// ConfigParamRegistry or any of the services which are usually hanging
 /// from the ServiceRegistry, e.g. ControlService.

From 0266b45c904e4169b28cf56b197bd8a7278576a3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 15 Feb 2024 10:44:11 +0100
Subject: [PATCH 344/726] DPL: calibration workflow example

---
 Framework/TestWorkflows/CMakeLists.txt        |  4 +
 .../src/o2DummyCalibrationWorkflow.cxx        | 73 +++++++++++++++++++
 2 files changed, 77 insertions(+)
 create mode 100644 Framework/TestWorkflows/src/o2DummyCalibrationWorkflow.cxx

diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index c2c89e19b894e..848337200a4d2 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -33,6 +33,10 @@ o2_add_dpl_workflow(diamond-workflow
                   SOURCES src/o2DiamondWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
 
+o2_add_dpl_workflow(dummy-calibration-workflow
+                  SOURCES src/o2DummyCalibrationWorkflow.cxx
+                  COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(diamond-workflow-leaky
                   SOURCES src/o2DiamondWorkflowLeaky.cxx
                   COMPONENT_NAME TestWorkflows)
diff --git a/Framework/TestWorkflows/src/o2DummyCalibrationWorkflow.cxx b/Framework/TestWorkflows/src/o2DummyCalibrationWorkflow.cxx
new file mode 100644
index 0000000000000..94916cfa36984
--- /dev/null
+++ b/Framework/TestWorkflows/src/o2DummyCalibrationWorkflow.cxx
@@ -0,0 +1,73 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/DataTakingContext.h"
+#include "Framework/CompletionPolicyHelpers.h"
+#include "Framework/DeviceSpec.h"
+#include "Framework/ControlService.h"
+#include "Framework/Configurable.h"
+#include "Framework/RunningWorkflowInfo.h"
+#include "Framework/CallbackService.h"
+#include "Framework/EndOfStreamContext.h"
+#include <fairmq/Device.h>
+
+#include <iostream>
+#include <vector>
+
+using namespace o2::framework;
+
+#include "Framework/runDataProcessing.h"
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& specs)
+{
+  DataProcessorSpec a{
+    .name = "counter",
+    .outputs = {OutputSpec{{"counter"}, "TST", "A1"}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, ProcessingContext& pcx) {
+        static int counter = 0;
+        auto& aData = outputs.make<int>(OutputRef{"counter"});
+        aData = counter++;
+        if (counter == 10) {
+          pcx.services().get<ControlService>().endOfStream();
+        }
+      })},
+  };
+
+  DataProcessorSpec b{
+    .name = "aggregator",
+    .inputs = {InputSpec{"x", "TST", "A1", Lifetime::Timeframe}},
+    .outputs = {OutputSpec{{"average"}, "TST", "B1", Lifetime::Sporadic}},
+    .algorithm = adaptStateful([](CallbackService& callbacks) {
+        static int sum = 0;
+        auto eosCallback = [](EndOfStreamContext &ctx) {
+          auto& aData = ctx.outputs().make<int>(OutputRef{"average"});
+          aData = sum;
+          ctx.services().get<ControlService>().endOfStream();
+        };
+        callbacks.set<CallbackService::Id::EndOfStream>(eosCallback);
+        return adaptStateless([](Input<"x", int> const& x)
+          {
+            sum += x;
+            std::cout << "Sum: " << sum << std::endl;
+        }); })};
+
+  DataProcessorSpec c{.name = "publisher",
+                      .inputs = {InputSpec{"average", "TST", "B1", Lifetime::Sporadic}},
+                      .algorithm = adaptStateless([](Input<"average", int> const& counter) {
+                        std::cout << "Counter to publish: " << counter << std::endl;
+                      })};
+
+  return workflow::concat(WorkflowSpec{a},
+                          WorkflowSpec{b},
+                          WorkflowSpec{c});
+}

From ac00ed6b19021bc9d5e765aa02559cda30cf1515 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 16 Feb 2024 12:28:17 +0100
Subject: [PATCH 345/726] DPL: allow Signposts to work as LOGF(level)
 replacement

This introduces a specialised API to emit signposts which can
be used in place of LOG(info), LOG(warning) , LOG(error).

Using it, errors and warning messages will appear as signposts as well (matching
the correct interval and so on), while the info messages will appear as signposts
if the signpost is enabled, without duplication, simplifying grepping.
---
 .../Foundation/include/Framework/Signpost.h   | 47 +++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index 6dad1ed9f6341..7750687088a1d 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -114,6 +114,21 @@ o2_log_handle_t* o2_walk_logs(bool (*callback)(char const* name, void* log, void
 #define O2_LOG_ENABLED(log) false
 #endif
 
+#if !defined(O2_LOG_MACRO_RAW) && __has_include("Framework/Logger.h")
+#include "Framework/Logger.h"
+#define O2_LOG_MACRO_RAW(level, ...) LOGF(level, __VA_ARGS__)
+#elif !defined(O2_LOG_MACRO_RAW)
+// If we do not have the fairlogger, we simply print out the signposts to the console.
+// This is useful for things like the tests, which this way do not need to depend on the FairLogger.
+#define O2_LOG_MACRO_RAW(level, format, ...) \
+  do {                                       \
+    printf(#level ":" #format, __VA_ARGS__); \
+    printf("\n");                            \
+  } while (0)
+#else
+#define O2_LOG_MACRO_RAW(...)
+#endif // O2_LOG_MACRO_RAW
+
 #if !defined(O2_LOG_MACRO) && __has_include("Framework/Logger.h")
 #include "Framework/Logger.h"
 #define O2_LOG_MACRO(...) LOGF(info, __VA_ARGS__)
@@ -480,6 +495,38 @@ void o2_debug_log_set_stacktrace(_o2_log_t* log, int stacktrace)
     _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
   }                                                                                                                 \
 })
+
+// Similar to the above, however it will print a normal info message if the signpost is not enabled.
+#define O2_SIGNPOST_EVENT_EMIT_INFO(log, id, name, format, ...) __extension__({                                     \
+  if (O2_BUILTIN_UNLIKELY(O2_SIGNPOST_ENABLED_MAC(log))) {                                                          \
+    O2_SIGNPOST_EVENT_EMIT_MAC(log, id, name, format, ##__VA_ARGS__);                                               \
+  } else if (O2_BUILTIN_UNLIKELY(private_o2_log_##log->stacktrace)) {                                               \
+    _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
+  } else {                                                                                                          \
+    O2_LOG_MACRO_RAW(info, format, ##__VA_ARGS__);                                                                  \
+  }                                                                                                                 \
+})
+
+// Similar to the above, however it will always print a normal error message regardless of the signpost being enabled or not.
+#define O2_SIGNPOST_EVENT_EMIT_ERROR(log, id, name, format, ...) __extension__({                                    \
+  if (O2_BUILTIN_UNLIKELY(O2_SIGNPOST_ENABLED_MAC(log))) {                                                          \
+    O2_SIGNPOST_EVENT_EMIT_MAC(log, id, name, format, ##__VA_ARGS__);                                               \
+  } else if (O2_BUILTIN_UNLIKELY(private_o2_log_##log->stacktrace)) {                                               \
+    _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
+  }                                                                                                                 \
+  O2_LOG_MACRO_RAW(error, format, ##__VA_ARGS__);                                                                   \
+})
+
+// Similar to the above, however it will also print a normal warning message regardless of the signpost being enabled or not.
+#define O2_SIGNPOST_EVENT_EMIT_WARN(log, id, name, format, ...) __extension__({                                     \
+  if (O2_BUILTIN_UNLIKELY(O2_SIGNPOST_ENABLED_MAC(log))) {                                                          \
+    O2_SIGNPOST_EVENT_EMIT_MAC(log, id, name, format, ##__VA_ARGS__);                                               \
+  } else if (O2_BUILTIN_UNLIKELY(private_o2_log_##log->stacktrace)) {                                               \
+    _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
+  }                                                                                                                 \
+  O2_RAW_LOG_RAW(warn, ##__VA_ARGS__);                                                                              \
+})
+
 #define O2_SIGNPOST_START(log, id, name, format, ...)                                                                   \
   if (O2_BUILTIN_UNLIKELY(O2_SIGNPOST_ENABLED_MAC(log))) {                                                              \
     O2_SIGNPOST_START_MAC(log, id, name, format, ##__VA_ARGS__);                                                        \

From eeb81b848f0cdb610be7994585d2e7af445787a1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 16 Feb 2024 12:28:17 +0100
Subject: [PATCH 346/726] DPL: allow to enable signposts for a given data
 processor

---
 Framework/Core/src/DeviceSpecHelpers.cxx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index e9df320fe8b95..43ba0f2d250e5 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1497,6 +1497,7 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
         realOdesc.add_options()("network-interface", bpo::value<std::string>());
         realOdesc.add_options()("early-forward-policy", bpo::value<std::string>());
         realOdesc.add_options()("session", bpo::value<std::string>());
+        realOdesc.add_options()("signposts", bpo::value<std::string>());
         filterArgsFct(expansions.we_wordc, expansions.we_wordv, realOdesc);
         wordfree(&expansions);
         return;

From fba4259e31e071c81df3d4f3adbf594b426097a1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 16 Feb 2024 12:35:24 +0100
Subject: [PATCH 347/726] DPL: improve Signposts documentation

---
 Framework/Core/COOKBOOK.md | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/Framework/Core/COOKBOOK.md b/Framework/Core/COOKBOOK.md
index 1a27079c3dc13..c24bb1bdbf542 100644
--- a/Framework/Core/COOKBOOK.md
+++ b/Framework/Core/COOKBOOK.md
@@ -543,12 +543,11 @@ and then you can either upload it to https://www.speedscope.app or use chrome://
 
 Debug log entries for several DPL components are now provided via the Signpost API.
 
-Such streams can be enabled using the "Signpost" part of the Device Inspector GUI.
+Such streams can be enabled using the "Signpost" part of the Device Inspector GUI or alternatively 
+from the command line, by specifying the `--signposts <log name>,...` option or exporting the variable `DPL_SIGNPOSTS=<log name>,...`).
 
-Alternatively for the case were the GUI is not available, you can attach to a process via GDB / LLDB and invoke the `o2_debug_log_set_stacktrace` function, passing it the log you want to enable and the amount of stacktrace you want to see. E.g. to enable a log created with `O2_DECLARE_DYNAMIC_LOG(completion)` you can use:
+Streams can be explicitly enabled or disabled in code using the `O2_SIGNPOST_ENABLE()` and `O2_SIGNPOST_DISABLE` macros.
 
-```
-p o2_debug_log_set_stacktrace(private_o2_log_completion, 1)
-```
+If a process is already running and you wish to enable one or more of its signposts logs, you can do so using the `o2-log` utility, passing the address of the log to enable and the PID of the running process. E.g. `o2-log -p <PID> -a <hook address of the signpost>`.
 
-If you are on a mac, you can also use Instruments to visualise your Signpost, just like any other macOS application. In order to do so you need to enable the "Signpost" instrument, making sure you add `ch.cern.aliceo2.completion` to the list of loggers to watch.
+Finally, on macOS, you can also use Instruments to visualise your Signpost, just like any other macOS application. In order to do so you need to enable the "Signpost" instrument, making sure you add `ch.cern.aliceo2.completion` to the list of loggers to watch.

From 679a07c24f3d9b98d4bb0aebf3de7f454300d817 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 16 Feb 2024 12:36:18 +0100
Subject: [PATCH 348/726] DPL: Improve Signposts

---
 Framework/CHANGELOG.md | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/Framework/CHANGELOG.md b/Framework/CHANGELOG.md
index 2973174f81d28..1134cfd0ee83d 100644
--- a/Framework/CHANGELOG.md
+++ b/Framework/CHANGELOG.md
@@ -1,4 +1,14 @@
-* Improved C++20 support. Most of the macros which were failing when C++20
-  support is enabled now seem to work fine. The issue seems to be related to
-  some forward declaration logic which seems to be not working correctly in
-  ROOT 6.30.01. The issue is discussed in <https://github.com/root-project/root/issues/14230> and it seems to be not trivial to fix with the current ROOT version.
+# 2024-02-16: Improved Signposts.
+
+In particular:
+
+* New API so that Signposts can now act as a replacement of LOGF(info), LOGF(error), LOGF(warn).
+* Improved documentation, including some hints about how to use `o2-log`.
+* Bug fix to get `--signposts` work on a per device basis.
+
+# 2024-01-10: Improved C++20 support. 
+
+Most of the macros which were failing when C++20
+support is enabled now seem to work fine. The issue seems to be related to
+some forward declaration logic which seems to be not working correctly in
+ROOT 6.30.01. The issue is discussed in <https://github.com/root-project/root/issues/14230> and it seems to be not trivial to fix with the current ROOT version.

From 4e66c5407b5a9a43d5647d892bca634fa1b2cd4c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 16 Feb 2024 12:48:45 +0100
Subject: [PATCH 349/726] DPL: improve Signposts documentation

---
 Framework/Core/COOKBOOK.md | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/Framework/Core/COOKBOOK.md b/Framework/Core/COOKBOOK.md
index 1a27079c3dc13..d92006e63d650 100644
--- a/Framework/Core/COOKBOOK.md
+++ b/Framework/Core/COOKBOOK.md
@@ -543,12 +543,10 @@ and then you can either upload it to https://www.speedscope.app or use chrome://
 
 Debug log entries for several DPL components are now provided via the Signpost API.
 
-Such streams can be enabled using the "Signpost" part of the Device Inspector GUI.
+Such streams can be enabled using the "Signpost" part of the Device Inspector GUI or alternatively from the command line, by specifying the `--signposts <log name>,...` option or exporting the variable `DPL_SIGNPOSTS=<log name>,...`).
 
-Alternatively for the case were the GUI is not available, you can attach to a process via GDB / LLDB and invoke the `o2_debug_log_set_stacktrace` function, passing it the log you want to enable and the amount of stacktrace you want to see. E.g. to enable a log created with `O2_DECLARE_DYNAMIC_LOG(completion)` you can use:
+Streams can be explicitly enabled or disabled in code using the `O2_SIGNPOST_ENABLE()` and `O2_SIGNPOST_DISABLE` macros.
 
-```
-p o2_debug_log_set_stacktrace(private_o2_log_completion, 1)
-```
+If a process is already running and you wish to enable one or more of its signposts logs, you can do so using the `o2-log` utility, passing the address of the log to enable and the PID of the running process. E.g. `o2-log -p <PID> -a <hook address of the signpost>`.
 
-If you are on a mac, you can also use Instruments to visualise your Signpost, just like any other macOS application. In order to do so you need to enable the "Signpost" instrument, making sure you add `ch.cern.aliceo2.completion` to the list of loggers to watch.
+Finally, on macOS, you can also use Instruments to visualise your Signpost, just like any other macOS application. In order to do so you need to enable the "Signpost" instrument, making sure you add `ch.cern.aliceo2.completion` to the list of loggers to watch.

From 1d21b4b9df67042f908d1f277a5cb3d105b0ecfe Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 16 Feb 2024 12:48:45 +0100
Subject: [PATCH 350/726] DPL: Improve Signposts

---
 Framework/CHANGELOG.md | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/Framework/CHANGELOG.md b/Framework/CHANGELOG.md
index 2973174f81d28..e7278733de246 100644
--- a/Framework/CHANGELOG.md
+++ b/Framework/CHANGELOG.md
@@ -1,4 +1,13 @@
-* Improved C++20 support. Most of the macros which were failing when C++20
-  support is enabled now seem to work fine. The issue seems to be related to
-  some forward declaration logic which seems to be not working correctly in
-  ROOT 6.30.01. The issue is discussed in <https://github.com/root-project/root/issues/14230> and it seems to be not trivial to fix with the current ROOT version.
+# 2024-02-16: Improved Signposts.
+
+In particular:
+
+* New API so that Signposts can now act as a replacement of LOGF(info), LOGF(error), LOGF(warn).
+* Improved documentation, including some hints about how to use `o2-log`.
+* Bug fix to get `--signposts` work on a per device basis.
+
+# 2024-01-10: Improved C++20 support.
+
+Most of the macros which were failing when C++20 support is enabled now seem to work fine. The issue seems to be related to
+some forward declaration logic which seems to be not working correctly in
+ROOT 6.30.01. The issue is discussed in <https://github.com/root-project/root/issues/14230> and it seems to be not trivial to fix with the current ROOT version.

From 671e9dddbeb7faea12c76d8fe3aee640dfb1eea0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 16 Feb 2024 12:48:45 +0100
Subject: [PATCH 351/726] DPL: introduce flag to signal that data should not be
 dropped at EoS

---
 .../Headers/include/Headers/DataHeader.h      |  5 +++--
 .../include/Framework/DataProcessingHeader.h  |  3 +++
 .../Core/include/Framework/DeviceContext.h    | 10 ++++++++++
 .../Core/include/Framework/DeviceStateEnums.h |  2 ++
 Framework/Core/include/Framework/TimingInfo.h |  4 +++-
 Framework/Core/src/DataAllocator.cxx          |  1 +
 Framework/Core/src/DataProcessingDevice.cxx   | 20 ++++++++++++++-----
 Framework/Core/src/DeviceSpecHelpers.cxx      |  4 +++-
 Framework/Core/src/O2ControlHelpers.cxx       |  2 ++
 Framework/Core/src/runDataProcessing.cxx      |  2 ++
 10 files changed, 44 insertions(+), 9 deletions(-)

diff --git a/DataFormats/Headers/include/Headers/DataHeader.h b/DataFormats/Headers/include/Headers/DataHeader.h
index c37eff9b34f20..e4ddaded20aba 100644
--- a/DataFormats/Headers/include/Headers/DataHeader.h
+++ b/DataFormats/Headers/include/Headers/DataHeader.h
@@ -372,8 +372,9 @@ struct BaseHeader {
   union {
     uint32_t flags;
     struct {
-      uint32_t flagsNextHeader : 1, // do we have a next header after this one?
-        flagsUnused : 31;           // currently unused
+      uint32_t flagsNextHeader : 1,        // do we have a next header after this one?
+               flagsReserved : 15,         // reserved for future use
+               flagsDerivedHeader : 16;    // reserved for usage by the derived header
     };
   };
 
diff --git a/Framework/Core/include/Framework/DataProcessingHeader.h b/Framework/Core/include/Framework/DataProcessingHeader.h
index 5c068b4e4179a..484dbb9d51a8e 100644
--- a/Framework/Core/include/Framework/DataProcessingHeader.h
+++ b/Framework/Core/include/Framework/DataProcessingHeader.h
@@ -42,6 +42,9 @@ namespace o2::framework
 /// @ingroup aliceo2_dataformats_dataheader
 struct DataProcessingHeader : public header::BaseHeader {
   static constexpr uint64_t DUMMY_CREATION_TIME_OFFSET = 0x8000000000000000;
+  // The following flags are used to indicate the behavior of the data processing
+  static constexpr int32_t KEEP_AT_EOS_FLAG = 1;
+
   /// We return some number of milliseconds, offsetting int by 0x8000000000000000
   /// to make sure we can understand when the dummy constructor of DataProcessingHeader was
   /// used without overriding it with an actual real time from epoch.
diff --git a/Framework/Core/include/Framework/DeviceContext.h b/Framework/Core/include/Framework/DeviceContext.h
index 3777e7f608b75..04cf663d5e276 100644
--- a/Framework/Core/include/Framework/DeviceContext.h
+++ b/Framework/Core/include/Framework/DeviceContext.h
@@ -28,8 +28,18 @@ struct ComputingQuotaStats;
 struct DeviceContext {
   ComputingQuotaStats* quotaStats = nullptr;
   uv_timer_t* gracePeriodTimer = nullptr;
+  uv_timer_t* dataProcessingGracePeriodTimer = nullptr;
   uv_signal_t* sigusr1Handle = nullptr;
   int expectedRegionCallbacks = 0;
+  // The timeout for the data processing to stop on this device.
+  // After this is reached, incoming data not marked to be kept will
+  // be dropped and the data processing will be stopped. However the 
+  // calibrations will still be done and objects resulting from calibrations
+  // will be marked to be kept.
+  int dataProcessingTimeout = 0;
+  // The timeout for the whole processing to stop on this device.
+  // This includes the grace period for processing and the time 
+  // for the calibrations to be done.
   int exitTransitionTimeout = 0;
 };
 
diff --git a/Framework/Core/include/Framework/DeviceStateEnums.h b/Framework/Core/include/Framework/DeviceStateEnums.h
index 291faac0ac982..a4c02c70c2bf6 100644
--- a/Framework/Core/include/Framework/DeviceStateEnums.h
+++ b/Framework/Core/include/Framework/DeviceStateEnums.h
@@ -30,6 +30,8 @@ enum struct TransitionHandlingState {
   NoTransition,
   /// A transition was notified to be requested
   Requested,
+  /// Only calibrations can be done
+  DataProcessingExpired,
   /// A transition needs to be fullfilled ASAP
   Expired
 };
diff --git a/Framework/Core/include/Framework/TimingInfo.h b/Framework/Core/include/Framework/TimingInfo.h
index 84f3971ad3a4a..1734da8dd3941 100644
--- a/Framework/Core/include/Framework/TimingInfo.h
+++ b/Framework/Core/include/Framework/TimingInfo.h
@@ -36,9 +36,11 @@ struct TimingInfo {
   /// from a new run, as being processed by the current stream.
   /// FIXME: for now this is the same as the above.
   bool streamRunNumberChanged = false;
+  /// Wether this kind of data should be flushed during end of stream.
+  bool keepAtEndOfStream = false;
 
   static bool timesliceIsTimer(size_t timeslice) { return timeslice > 1652945069870351; }
-  bool isTimer() const { return timesliceIsTimer(timeslice); };
+  [[nodiscard]] bool isTimer() const { return timesliceIsTimer(timeslice); };
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index de500cd6d6c4e..157e530392d4e 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -118,6 +118,7 @@ fair::mq::MessagePtr DataAllocator::headerMessageFromOutput(Output const& spec,
   dh.runNumber = timingInfo.runNumber;
 
   DataProcessingHeader dph{timingInfo.timeslice, 1, timingInfo.creation};
+  static_cast<o2::header::BaseHeader&>(dph).flagsDerivedHeader |= timingInfo.keepAtEndOfStream ? DataProcessingHeader::KEEP_AT_EOS_FLAG : 0;
   auto& proxy = mRegistry.get<FairMQDeviceProxy>();
   auto* transport = proxy.getOutputTransport(routeIndex);
 
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 50934bd59a729..6225001aba828 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -969,6 +969,7 @@ void DataProcessingDevice::InitTask()
   }
 
   deviceContext.expectedRegionCallbacks = std::stoi(fConfig->GetValue<std::string>("expected-region-callbacks"));
+  deviceContext.dataProcessingTimeout = std::stoi(fConfig->GetValue<std::string>("data-processing-timeout"));
   deviceContext.exitTransitionTimeout = std::stoi(fConfig->GetValue<std::string>("exit-transition-timeout"));
 
   for (auto& channel : GetChannels()) {
@@ -1611,9 +1612,13 @@ void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
 void DataProcessingDevice::doRun(ServiceRegistryRef ref)
 {
   auto& context = ref.get<DataProcessorContext>();
+  O2_SIGNPOST_ID_FROM_POINTER(dpid, device, &context);
   auto switchState = [ref](StreamingState newState) {
     auto& state = ref.get<DeviceState>();
-    LOG(detail) << "New state " << (int)newState << " old state " << (int)state.streaming;
+    auto& context = ref.get<DataProcessorContext>();
+    O2_SIGNPOST_ID_FROM_POINTER(dpid, device, &context);
+    O2_SIGNPOST_END(device, dpid, "state", "End of processing state %d", (int)state.streaming);
+    O2_SIGNPOST_START(device, dpid, "state", "Starting processing state %d", (int)newState);
     state.streaming = newState;
     ref.get<ControlService>().notifyStreamingState(state.streaming);
   };
@@ -1652,7 +1657,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
   }
 
   if (state.streaming == StreamingState::EndOfStreaming) {
-    LOGP(detail, "We are in EndOfStreaming. Flushing queues.");
+    O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "Flushing queues.");
     // We keep processing data until we are Idle.
     // FIXME: not sure this is the correct way to drain the queues, but
     // I guess we will see.
@@ -1663,6 +1668,11 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
     while (DataProcessingDevice::tryDispatchComputation(ref, context.completed) && hasOnlyGenerated == false) {
       relayer.processDanglingInputs(context.expirationHandlers, *context.registry, false);
     }
+
+    auto &timingInfo = ref.get<TimingInfo>();
+    // We should keep the data generated at end of stream only for those
+    // which are not sources.
+    timingInfo.keepAtEndOfStream = (hasOnlyGenerated == false);
     EndOfStreamContext eosContext{*context.registry, ref.get<DataAllocator>()};
 
     context.preEOSCallbacks(eosContext);
@@ -1673,7 +1683,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
     context.postEOSCallbacks(eosContext);
 
     for (auto& channel : spec.outputChannels) {
-      LOGP(detail, "Sending end of stream to {}", channel.name);
+      O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "Sending end of stream to %s.", channel.name.c_str());
       DataProcessingHelpers::sendEndOfStream(ref, channel);
     }
     // This is needed because the transport is deleted before the device.
@@ -1685,7 +1695,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
       *context.wasActive = true;
     }
     // On end of stream we shut down all output pollers.
-    LOGP(detail, "Shutting down output pollers");
+    O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "Shutting down output pollers.");
     for (auto& poller : state.activeOutputPollers) {
       uv_poll_stop(poller);
     }
@@ -1694,7 +1704,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
 
   if (state.streaming == StreamingState::Idle) {
     // On end of stream we shut down all output pollers.
-    LOGP(detail, "We are in Idle. Shutting down output pollers.");
+    O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "Shutting down output pollers.");
     for (auto& poller : state.activeOutputPollers) {
       uv_poll_stop(poller);
     }
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 43ba0f2d250e5..526719907519f 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1475,6 +1475,7 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
         realOdesc.add_options()("child-driver", bpo::value<std::string>());
         realOdesc.add_options()("rate", bpo::value<std::string>());
         realOdesc.add_options()("exit-transition-timeout", bpo::value<std::string>());
+        realOdesc.add_options()("data-processing-timeout", bpo::value<std::string>());
         realOdesc.add_options()("expected-region-callbacks", bpo::value<std::string>());
         realOdesc.add_options()("timeframes-rate-limit", bpo::value<std::string>());
         realOdesc.add_options()("environment", bpo::value<std::string>());
@@ -1659,6 +1660,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
     ("control-port", bpo::value<std::string>(), "Utility port to be used by O2 Control")                                                                             //
     ("rate", bpo::value<std::string>(), "rate for a data source device (Hz)")                                                                                        //
     ("exit-transition-timeout", bpo::value<std::string>(), "timeout before switching to READY state")                                                                //
+    ("data-processing-timeout", bpo::value<std::string>(), "timeout before switching to calibration processing mode")                                                //
     ("expected-region-callbacks", bpo::value<std::string>(), "region callbacks to expect before starting")                                                           //
     ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframes can be in fly")                                                    //
     ("shm-monitor", bpo::value<std::string>(), "whether to use the shared memory monitor")                                                                           //
@@ -1689,7 +1691,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
     ("infologger-mode", bpo::value<std::string>(), "O2_INFOLOGGER_MODE override")                                                                                    //
     ("infologger-severity", bpo::value<std::string>(), "minimun FairLogger severity which goes to info logger")                                                      //
     ("dpl-tracing-flags", bpo::value<std::string>(), "pipe separated list of events to trace")                                                                       //
-    ("signposts", bpo::value<std::string>(),                                                                                                                         //
+    ("signposts", bpo::value<std::string>()->default_value(defaultSignposts),                                                                                        //
      "comma separated list of signposts to enable (any of `completion`, `data_processor_context`, `stream_context`, `device`, `monitoring_service`)")                //
     ("child-driver", bpo::value<std::string>(), "external driver to start childs with (e.g. valgrind)");                                                             //
 
diff --git a/Framework/Core/src/O2ControlHelpers.cxx b/Framework/Core/src/O2ControlHelpers.cxx
index 88132572a6210..e19e4a995d505 100644
--- a/Framework/Core/src/O2ControlHelpers.cxx
+++ b/Framework/Core/src/O2ControlHelpers.cxx
@@ -261,6 +261,8 @@ void dumpCommand(std::ostream& dumpOut, const DeviceExecution& execution, std::s
   dumpOut << indLevel << indScheme << "- \"-b\"\n";
   dumpOut << indLevel << indScheme << "- \"--exit-transition-timeout\"\n";
   dumpOut << indLevel << indScheme << "- \"'{{ exit_transition_timeout }}'\"\n";
+  dumpOut << indLevel << indScheme << "- \"--data-processing-timeout\"\n";
+  dumpOut << indLevel << indScheme << "- \"'{{ data_processing_timeout }}'\"\n";
   dumpOut << indLevel << indScheme << "- \"--monitoring-backend\"\n";
   dumpOut << indLevel << indScheme << "- \"'{{ monitoring_dpl_url }}'\"\n";
   dumpOut << indLevel << indScheme << "- \"--session\"\n";
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 64c034ebb8533..0ef7aa8bffb50 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1006,6 +1006,7 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
   // declared in the workflow definition are allowed.
   runner.AddHook<fair::mq::hooks::SetCustomCmdLineOptions>([&spec, driverConfig, defaultDriverClient](fair::mq::DeviceRunner& r) {
     std::string defaultExitTransitionTimeout = "0";
+    std::string defaultDataProcessingTimeout = "0";
     std::string defaultInfologgerMode = "";
     o2::framework::DeploymentMode deploymentMode = o2::framework::DefaultsHelpers::deploymentMode();
     if (deploymentMode == o2::framework::DeploymentMode::OnlineDDS) {
@@ -1024,6 +1025,7 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
       ("signposts", bpo::value<std::string>()->default_value(defaultSignposts ? defaultSignposts : ""), "comma separated list of signposts to enable")                                     //
       ("expected-region-callbacks", bpo::value<std::string>()->default_value("0"), "how many region callbacks we are expecting")                                                           //
       ("exit-transition-timeout", bpo::value<std::string>()->default_value(defaultExitTransitionTimeout), "how many second to wait before switching from RUN to READY")                    //
+      ("data-processing-timeout", bpo::value<std::string>()->default_value(defaultDataProcessingTimeout), "how many second to wait before switching from RUN to CALIBRATION")              //
       ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframe can be in fly at the same moment (0 disables)")                                         //
       ("configuration,cfg", bpo::value<std::string>()->default_value("command-line"), "configuration backend")                                                                             //
       ("infologger-mode", bpo::value<std::string>()->default_value(defaultInfologgerMode), "O2_INFOLOGGER_MODE override");

From aca46d026ad3649cfdd67ed69404432dcf0fdb03 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 16 Feb 2024 12:48:45 +0100
Subject: [PATCH 352/726] DPL: attempt to enable calibration mode

---
 Framework/Core/src/DataProcessingDevice.cxx | 84 ++++++++++++++-------
 1 file changed, 55 insertions(+), 29 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 6225001aba828..0710a6e4232fe 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -120,12 +120,21 @@ void on_idle_timer(uv_timer_t* handle)
 void on_transition_requested_expired(uv_timer_t* handle)
 {
   ZoneScopedN("Transition expired");
-  auto* state = (DeviceState*)handle->data;
+  auto *state = (DeviceState*)handle->data;
   state->loopReason |= DeviceState::TIMER_EXPIRED;
-  LOGP(info, "Timer expired. Forcing transition to READY");
+  LOGP(info, "Grace period for Data Processing & calibration expired.");
   state->transitionHandling = TransitionHandlingState::Expired;
 }
 
+void on_data_processing_grace_expired(uv_timer_t* handle)
+{
+  ZoneScopedN("Transition expired");
+  auto *state = (DeviceState*)handle->data;
+  state->loopReason |= DeviceState::TIMER_EXPIRED;
+  LOGP(info, "Grace period for Data Processing Expired. Waiting for calibration.");
+  state->transitionHandling = TransitionHandlingState::DataProcessingExpired;
+}
+
 void on_communication_requested(uv_async_t* s)
 {
   auto* state = (DeviceState*)s->data;
@@ -219,12 +228,15 @@ DataProcessingDevice::DataProcessingDevice(RunningDeviceRef running, ServiceRegi
 // one with the thread id. For the moment we simply use the first one.
 void run_callback(uv_work_t* handle)
 {
-  ZoneScopedN("run_callback");
   auto* task = (TaskStreamInfo*)handle->data;
   auto ref = ServiceRegistryRef{*task->registry, ServiceRegistry::globalStreamSalt(task->id.index + 1)};
+  // We create a new signpost interval for this specific data processor. Same id, same data processor.
+  auto& dataProcessorContext = ref.get<DataProcessorContext>();
+  O2_SIGNPOST_ID_FROM_POINTER(sid, device, &dataProcessorContext);
+  O2_SIGNPOST_START(device, sid, "run_callback", "Starting run callback on stream %d", task->id.index);
   DataProcessingDevice::doPrepare(ref);
   DataProcessingDevice::doRun(ref);
-  //  FrameMark;
+  O2_SIGNPOST_END(device, sid, "run_callback", "Done processing data for stream %d", task->id.index);
 }
 
 // Once the processing in a thread is done, this is executed on the main thread.
@@ -1228,6 +1240,8 @@ void DataProcessingDevice::Run()
   auto& state = ref.get<DeviceState>();
   state.loopReason = DeviceState::LoopReason::FIRST_LOOP;
   bool firstLoop = true;
+  O2_SIGNPOST_ID_FROM_POINTER(lid, device, state.loop);
+  O2_SIGNPOST_START(device, lid, "device_state", "First iteration of the device loop");
   while (state.transitionHandling != TransitionHandlingState::Expired) {
     if (state.nextFairMQState.empty() == false) {
       (void)this->ChangeState(state.nextFairMQState.back());
@@ -1259,13 +1273,13 @@ void DataProcessingDevice::Run()
         state.loopReason |= DeviceState::LoopReason::PREVIOUSLY_ACTIVE;
       }
       if (NewStatePending()) {
+        O2_SIGNPOST_EVENT_EMIT(device, lid, "run_loop", "New state pending. Waiting for it to be handled.");
         shouldNotWait = true;
         state.loopReason |= DeviceState::LoopReason::NEW_STATE_PENDING;
       }
       if (state.transitionHandling == TransitionHandlingState::NoTransition && NewStatePending()) {
         state.transitionHandling = TransitionHandlingState::Requested;
         auto& deviceContext = ref.get<DeviceContext>();
-        auto timeout = deviceContext.exitTransitionTimeout;
         // Check if we only have timers
         bool onlyTimers = true;
         auto& spec = ref.get<DeviceSpec const>();
@@ -1278,31 +1292,40 @@ void DataProcessingDevice::Run()
         if (onlyTimers) {
           state.streaming = StreamingState::EndOfStreaming;
         }
-        if (timeout != 0 && state.streaming != StreamingState::Idle) {
+        
+        if (deviceContext.exitTransitionTimeout != 0 && state.streaming != StreamingState::Idle) {
           state.transitionHandling = TransitionHandlingState::Requested;
           ref.get<CallbackService>().call<CallbackService::Id::ExitRequested>(ServiceRegistryRef{ref});
           uv_update_time(state.loop);
-          uv_timer_start(deviceContext.gracePeriodTimer, on_transition_requested_expired, timeout * 1000, 0);
+          uv_timer_start(deviceContext.gracePeriodTimer, on_transition_requested_expired, deviceContext.exitTransitionTimeout * 1000, 0);
+          // In case we have a calibration grace period it will always be longer than the data processing timeout
+          if (deviceContext.dataProcessingTimeout != deviceContext.exitTransitionTimeout) {
+            uv_timer_start(deviceContext.dataProcessingGracePeriodTimer, on_data_processing_grace_expired, deviceContext.dataProcessingTimeout * 1000, 0);
+          } else {
+            deviceContext.dataProcessingGracePeriodTimer = nullptr;
+          }
           if (mProcessingPolicies.termination == TerminationPolicy::QUIT) {
-            LOGP(info, "New state requested. Waiting for {} seconds before quitting.", timeout);
+            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state requested. Waiting for %d seconds before quitting.", (int)deviceContext.exitTransitionTimeout);
           } else {
-            LOGP(info, "New state requested. Waiting for {} seconds before switching to READY state.", timeout);
+            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state requested. Waiting for %d seconds before switching to READY state.", (int)deviceContext.exitTransitionTimeout);
           }
         } else {
           state.transitionHandling = TransitionHandlingState::Expired;
-          if (timeout == 0 && mProcessingPolicies.termination == TerminationPolicy::QUIT) {
-            LOGP(info, "New state requested. No timeout set, quitting immediately as per --completion-policy");
-          } else if (timeout == 0 && mProcessingPolicies.termination != TerminationPolicy::QUIT) {
-            LOGP(info, "New state requested. No timeout set, switching to READY state immediately");
+          if (deviceContext.exitTransitionTimeout == 0 && mProcessingPolicies.termination == TerminationPolicy::QUIT) {
+            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state requested. No timeout set, quitting immediately as per --completion-policy");
+          } else if (deviceContext.exitTransitionTimeout == 0 && mProcessingPolicies.termination != TerminationPolicy::QUIT) {
+            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state requested. No timeout set, switching to READY state immediately");
           } else if (mProcessingPolicies.termination == TerminationPolicy::QUIT) {
-            LOGP(info, "New state pending and we are already idle, quitting immediately as per --completion-policy");
+            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state pending and we are already idle, quitting immediately as per --completion-policy");
           } else {
-            LOGP(info, "New state pending and we are already idle, switching to READY immediately.");
+            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "runb_loop", "New state pending and we are already idle, switching to READY immediately.");
           }
         }
       }
-      // If we are Idle, we can then consider the transition to be expired.
-      if (state.transitionHandling == TransitionHandlingState::Requested && state.streaming == StreamingState::Idle) {
+      // If we are Idle, we can then consider the transition to be expired when it was requested or when the data processing timeout expired.
+      if ((state.transitionHandling == TransitionHandlingState::Requested || state.transitionHandling == TransitionHandlingState::DataProcessingExpired) 
+          && state.streaming == StreamingState::Idle) {
+        O2_SIGNPOST_EVENT_EMIT(device, lid, "run_loop", "State transition requested and we are now in Idle. We can consider it to be completed.");
         state.transitionHandling = TransitionHandlingState::Expired;
       }
       TracyPlot("shouldNotWait", (int)shouldNotWait);
@@ -1323,8 +1346,8 @@ void DataProcessingDevice::Run()
       // - we can trigger further events from the queue
       // - we can guarantee this is the last thing we do in the loop (
       //   assuming no one else is adding to the queue before this point).
-      auto onDrop = [&registry = mServiceRegistry](TimesliceSlot slot, std::vector<MessageSet>& dropped, TimesliceIndex::OldestOutputInfo oldestOutputInfo) {
-        LOGP(debug, "Dropping message from slot {}. Forwarding as needed.", slot.index);
+      auto onDrop = [&registry = mServiceRegistry, lid](TimesliceSlot slot, std::vector<MessageSet>& dropped, TimesliceIndex::OldestOutputInfo oldestOutputInfo) {
+        O2_SIGNPOST_START(device, lid, "run_loop", "Dropping message from slot %" PRIu64 ". Forwarding as needed.", (uint64_t)slot.index);
         ServiceRegistryRef ref{registry};
         ref.get<AsyncQueue>();
         ref.get<DecongestionService>();
@@ -1343,7 +1366,9 @@ void DataProcessingDevice::Run()
         auto& dpContext = ref.get<DataProcessorContext>();
         dpContext.preLoopCallbacks(ref);
       }
+      O2_SIGNPOST_END(device, lid, "run_loop", "Run loop completed. %{}s", shouldNotWait ? "Will immediately schedule a new one" : "Waiting for next event.");
       uv_run(state.loop, shouldNotWait ? UV_RUN_NOWAIT : UV_RUN_ONCE);
+      O2_SIGNPOST_START(device, lid, "run_loop", "Run loop started. Loop reason %d.", state.loopReason);
       if ((state.loopReason & state.tracingFlags) != 0) {
         state.severityStack.push_back((int)fair::Logger::GetConsoleSeverity());
         fair::Logger::SetConsoleSeverity(fair::Severity::trace);
@@ -1351,17 +1376,15 @@ void DataProcessingDevice::Run()
         fair::Logger::SetConsoleSeverity((fair::Severity)state.severityStack.back());
         state.severityStack.pop_back();
       }
-      TracyPlot("loopReason", (int64_t)(uint64_t)state.loopReason);
-      LOGP(debug, "Loop reason mask {:b} & {:b} = {:b}",
-           state.loopReason, state.tracingFlags,
-           state.loopReason & state.tracingFlags);
+      O2_SIGNPOST_EVENT_EMIT(device, lid, "run_loop", "Loop reason mask %x & %x = %x", state.loopReason, state.tracingFlags, state.loopReason & state.tracingFlags);
 
       if ((state.loopReason & DeviceState::LoopReason::OOB_ACTIVITY) != 0) {
-        LOGP(debug, "We were awakened by a OOB event. Rescanning everything.");
+        O2_SIGNPOST_EVENT_EMIT(device, lid, "run_loop", "Out of band activity detected. Rescanning everything.");
         relayer.rescan();
       }
 
       if (!state.pendingOffers.empty()) {
+        O2_SIGNPOST_EVENT_EMIT(device, lid, "run_loop", "Pending %" PRIu64 " offers. updating the ComputingQuotaEvaluator.", (uint64_t)state.pendingOffers.size());
         ref.get<ComputingQuotaEvaluator>().updateOffers(state.pendingOffers, uv_now(state.loop));
       }
     }
@@ -1435,8 +1458,9 @@ void DataProcessingDevice::Run()
     } else {
       mWasActive = false;
     }
-    FrameMark;
   }
+
+  O2_SIGNPOST_END(device, lid, "run_loop", "Run loop completed. Transition handling state %d.", state.transitionHandling);
   auto& spec = ref.get<DeviceSpec const>();
   /// Cleanup messages which are still pending on exit.
   for (size_t ci = 0; ci < spec.inputChannels.size(); ++ci) {
@@ -1450,8 +1474,9 @@ void DataProcessingDevice::Run()
 /// non-data triggers like those which are time based.
 void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
 {
-  ZoneScopedN("DataProcessingDevice::doPrepare");
   auto& context = ref.get<DataProcessorContext>();
+  O2_SIGNPOST_ID_FROM_POINTER(dpid, device, &context);
+  O2_SIGNPOST_START(device, dpid, "do_prepare", "Starting DataProcessorContext::doPrepare.");
 
   *context.wasActive = false;
   {
@@ -1479,7 +1504,7 @@ void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
   });
 
   // Whether or not all the channels are completed
-  LOGP(debug, "Processing {} input channels.", spec.inputChannels.size());
+  O2_SIGNPOST_EVENT_EMIT(device, dpid, "do_prepare", "Processing %zu input channels.", spec.inputChannels.size());
   /// Sort channels by oldest possible timeframe and
   /// process them in such order.
   static std::vector<int> pollOrder;
@@ -1491,13 +1516,14 @@ void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
 
   // Nothing to poll...
   if (pollOrder.empty()) {
+    O2_SIGNPOST_END(device, dpid, "do_prepare", "Nothing to poll. Waiting for next iteration.");
     return;
   }
   auto currentOldest = state.inputChannelInfos[pollOrder.front()].oldestForChannel;
   auto currentNewest = state.inputChannelInfos[pollOrder.back()].oldestForChannel;
   auto delta = currentNewest.value - currentOldest.value;
-  LOGP(debug, "oldest possible timeframe range {}, {} => {} delta", currentOldest.value, currentNewest.value,
-       delta);
+  O2_SIGNPOST_EVENT_EMIT(device, dpid, "do_prepare", "Oldest possible timeframe range %" PRIu64 " => %" PRIu64 " delta %" PRIu64,
+                        (int64_t)currentOldest.value, (int64_t)currentNewest.value, (int64_t)delta);
   auto& infos = state.inputChannelInfos;
 
   if (context.balancingInputs) {

From 7d0324bb31589f2133b0275ff7c937b3f2dea049 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 16 Feb 2024 12:50:35 +0100
Subject: [PATCH 353/726] [DRAFT] DPL: Introduce calibration mode

---
 Framework/CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Framework/CHANGELOG.md b/Framework/CHANGELOG.md
index e7278733de246..f220e02b87dc7 100644
--- a/Framework/CHANGELOG.md
+++ b/Framework/CHANGELOG.md
@@ -1,3 +1,8 @@
+# 2024-02-17: Introduce calibration mode
+
+This introduces a new --data-processing-timeout `<timeout>` option which can be used to specify
+that data processing should finish after `<timeout>` seconds. After it, only the messages which are produced on EndOfStream will be send.
+
 # 2024-02-16: Improved Signposts.
 
 In particular:

From c03d18f398e6812c1b17267bb4f318c94ada7189 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 16 Feb 2024 20:41:35 +0100
Subject: [PATCH 354/726] DPL: make sure O2_SIGNPOST_ID_FROM_POINTER allows for
 const pointers

---
 Framework/Foundation/include/Framework/Signpost.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index 7750687088a1d..9ea8aea1192a9 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -202,8 +202,6 @@ struct _o2_log_t {
 
 bool _o2_lock_free_stack_push(_o2_lock_free_stack& stack, const int& value, bool spin = false);
 bool _o2_lock_free_stack_pop(_o2_lock_free_stack& stack, int& value, bool spin = false);
-//_o2_signpost_id_t _o2_signpost_id_generate_local(_o2_log_t* log);
-//_o2_signpost_id_t _o2_signpost_id_make_with_pointer(_o2_log_t* log, void* pointer);
 void* _o2_log_create(char const* name, int stacktrace);
 void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...);
 void _o2_signpost_interval_begin(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...);
@@ -224,7 +222,7 @@ inline _o2_signpost_id_t _o2_signpost_id_generate_local(_o2_log_t* log)
 
 // Generate a unique id for a signpost. Do not use this directly, use O2_SIGNPOST_ID_FROM_POINTER instead.
 // Notice that this will fail for pointers to bytes as it might overlap with the id above.
-inline _o2_signpost_id_t _o2_signpost_id_make_with_pointer(_o2_log_t* log, void* pointer)
+inline _o2_signpost_id_t _o2_signpost_id_make_with_pointer(_o2_log_t* log, void const* pointer)
 {
   assert(((int64_t)pointer & 1) != 1);
   _o2_signpost_id_t uniqueId{(int64_t)pointer};

From fcf7b808eb1aaf7ab94f0df9ca0e48c50a7dfdd4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:10:16 +0100
Subject: [PATCH 355/726] DPL: replace Tracy allocation profiler with Signposts

---
 Framework/Core/src/DataAllocator.cxx        |  7 ++++
 Framework/Core/src/DataProcessingDevice.cxx | 42 +++++++++++++++------
 Framework/Core/src/LifetimeHelpers.cxx      |  8 +++-
 3 files changed, 44 insertions(+), 13 deletions(-)

diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index de500cd6d6c4e..fe38283d5e2de 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -35,6 +35,7 @@
 #include <utility>
 
 O2_DECLARE_DYNAMIC_LOG(stream_context);
+O2_DECLARE_DYNAMIC_LOG(parts);
 
 namespace o2::framework
 {
@@ -129,6 +130,8 @@ void DataAllocator::addPartToContext(RouteIndex routeIndex, fair::mq::MessagePtr
                                      o2::header::SerializationMethod serializationMethod)
 {
   auto headerMessage = headerMessageFromOutput(spec, routeIndex, serializationMethod, 0);
+  O2_SIGNPOST_ID_FROM_POINTER(pid, parts, headerMessage->GetData());
+  O2_SIGNPOST_START(parts, pid, "parts", "addPartToContext %p", headerMessage->GetData());
 
   // FIXME: this is kind of ugly, we know that we can change the content of the
   // header message because we have just created it, but the API declares it const
@@ -150,6 +153,8 @@ void DataAllocator::adopt(const Output& spec, std::string* ptr)
   // the correct payload size is set later when sending the
   // StringContext, see DataProcessor::doSend
   auto header = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, 0);
+  O2_SIGNPOST_ID_FROM_POINTER(pid, parts, header->GetData());
+  O2_SIGNPOST_START(parts, pid, "parts", "addPartToContext %p", header->GetData());
   mRegistry.get<StringContext>().addString(std::move(header), std::move(payload), routeIndex);
   assert(payload.get() == nullptr);
 }
@@ -206,6 +211,8 @@ void DataAllocator::adopt(const Output& spec, LifetimeHolder<TableBuilder>& tb)
   auto& timingInfo = mRegistry.get<TimingInfo>();
   RouteIndex routeIndex = matchDataHeader(spec, timingInfo.timeslice);
   auto header = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodArrow, 0);
+  O2_SIGNPOST_ID_FROM_POINTER(pid, parts, header->GetData());
+  O2_SIGNPOST_START(parts, pid, "parts", "adopt %p", header->GetData());
   auto& context = mRegistry.get<ArrowContext>();
 
   auto creator = [transport = context.proxy().getOutputTransport(routeIndex)](size_t s) -> std::unique_ptr<fair::mq::Message> {
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 50934bd59a729..1fd35471c8fc1 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -92,7 +92,10 @@ struct formatter<o2::framework::CompletionPolicy::CompletionOp> : ostream_format
 };
 } // namespace fmt
 
+// A log to use for general device logging
 O2_DECLARE_DYNAMIC_LOG(device);
+// Special log to keep track of the lifetime of the parts
+O2_DECLARE_DYNAMIC_LOG(parts);
 
 using namespace o2::framework;
 using ConfigurationInterface = o2::configuration::ConfigurationInterface;
@@ -1804,9 +1807,14 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
         LOGP(error, "DataHeader payloadSize mismatch");
         continue;
       }
-      TracyPlot("payload size", (int64_t)dh->payloadSize);
       auto dph = o2::header::get<DataProcessingHeader*>(headerData);
-      TracyAlloc(parts.At(pi + 1)->GetData(), parts.At(pi + 1)->GetSize());
+      // We only deal with the tracking of parts if the log is enabled.
+      // This is because in principle we should track the size of each of 
+      // the parts and sum it up. Not for now.
+      if (O2_LOG_ENABLED(parts) == true) {
+        O2_SIGNPOST_ID_FROM_POINTER(pid, parts, headerData);
+        O2_SIGNPOST_START(parts, pid, "parts", "Processing DataHeader with splitPayloadParts %d and splitPayloadIndex %d", dh->splitPayloadParts, dh->splitPayloadIndex);
+      }
       if (!dph) {
         insertInputInfo(pi, 2, InputType::Invalid);
         LOGP(error, "Header stack does not contain DataProcessingHeader");
@@ -2180,10 +2188,12 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   // Function to cleanup record. For the moment we
   // simply use it to keep track of input messages
   // which are not needed, to display them in the GUI.
-#ifdef TRACY_ENABLE
   auto cleanupRecord = [](InputRecord& record) {
-    for (size_t ii = 0, ie = record.size(); ii < ie; ++ii) {
-      DataRef input = record.getByPos(ii);
+    if (O2_LOG_ENABLED(parts) == false) {
+      return;
+    }
+    for (size_t pi = 0, pe = record.size(); pi < pe; ++pi) {
+      DataRef input = record.getByPos(pi);
       if (input.header == nullptr) {
         continue;
       }
@@ -2196,10 +2206,21 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       if (!dh) {
         continue;
       }
-      TracyFree(input.payload);
+      // We use the address of the first header of a split payload
+      // to identify the interval.
+      O2_SIGNPOST_ID_FROM_POINTER(pid, parts, dh);
+      O2_SIGNPOST_END(parts, pid, "parts", "Cleaning up parts associated to %p", dh);
+
+      // No split parts, we simply skip the payload
+      if (dh->splitPayloadParts > 0 && dh->splitPayloadParts == dh->splitPayloadIndex) {
+        // this is indicating a sequence of payloads following the header
+        // FIXME: we will probably also set the DataHeader version
+        pi += dh->splitPayloadParts - 1;
+      } else {
+        size_t pi = pi + (dh->splitPayloadParts > 0 ? dh->splitPayloadParts : 1) * 2;
+      }
     }
   };
-#endif
 
   auto switchState = [ref](StreamingState newState) {
     auto& control = ref.get<ControlService>();
@@ -2432,6 +2453,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
     // We forward inputs only when we consume them. If we simply Process them,
     // we keep them for next message arriving.
     if (action.op == CompletionPolicy::CompletionOp::Consume) {
+      cleanupRecord(record);
       context.postDispatchingCallbacks(processContext);
       ref.get<CallbackService>().call<CallbackService::Id::DataConsumed>(o2::framework::ServiceRegistryRef{ref});
     }
@@ -2441,11 +2463,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       forwardInputs(ref, action.slot, currentSetOfInputs, timesliceIndex.getOldestPossibleOutput(), false, action.op == CompletionPolicy::CompletionOp::Consume);
     }
     context.postForwardingCallbacks(processContext);
-    if (action.op == CompletionPolicy::CompletionOp::Consume) {
-#ifdef TRACY_ENABLE
-      cleanupRecord(record);
-#endif
-    } else if (action.op == CompletionPolicy::CompletionOp::Process) {
+    if (action.op == CompletionPolicy::CompletionOp::Process) {
       cleanTimers(action.slot, record);
     }
   }
diff --git a/Framework/Core/src/LifetimeHelpers.cxx b/Framework/Core/src/LifetimeHelpers.cxx
index 1aa53fa0493ca..8c8605d41849c 100644
--- a/Framework/Core/src/LifetimeHelpers.cxx
+++ b/Framework/Core/src/LifetimeHelpers.cxx
@@ -25,6 +25,7 @@
 #include "Framework/FairMQDeviceProxy.h"
 #include "Framework/Formatters.h"
 #include "Framework/DeviceState.h"
+#include "Framework/Signpost.h"
 
 #include "Headers/DataHeader.h"
 #include "Headers/DataHeaderHelpers.h"
@@ -44,6 +45,8 @@
 using namespace o2::header;
 using namespace fair;
 
+O2_DECLARE_DYNAMIC_LOG(parts);
+
 namespace o2::framework
 {
 
@@ -411,7 +414,6 @@ ExpirationHandler::Handler LifetimeHelpers::enumerate(ConcreteDataMatcher const&
     assert(!ref.payload);
 
     auto timestamp = VariableContextHelpers::getTimeslice(variables).value;
-    LOGP(debug, "Enumerating record");
     DataHeader dh;
     dh.dataOrigin = matcher.origin;
     dh.dataDescription = matcher.description;
@@ -432,6 +434,8 @@ ExpirationHandler::Handler LifetimeHelpers::enumerate(ConcreteDataMatcher const&
     auto&& transport = deviceProxy.getInputChannel(channelIndex)->Transport();
     auto channelAlloc = o2::pmr::getTransportAllocator(transport);
     auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, dh, dph});
+    O2_SIGNPOST_ID_FROM_POINTER(hid, parts, header->GetData());
+    O2_SIGNPOST_START(parts, hid, "parts", "Enumerating part %p with timestamp %zu", header->GetData(), timestamp);
     ref.header = std::move(header);
 
     auto payload = transport->CreateMessage(sizeof(counter_t));
@@ -486,6 +490,8 @@ ExpirationHandler::Handler LifetimeHelpers::dummy(ConcreteDataMatcher const& mat
     auto channelAlloc = o2::pmr::getTransportAllocator(transport);
     auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, dh, dph});
     ref.header = std::move(header);
+    O2_SIGNPOST_ID_FROM_POINTER(hid, parts, header->GetData());
+    O2_SIGNPOST_START(parts, hid, "parts", "Enumerating part %p with timestamp %zu", header->GetData(), timestamp);
     auto payload = transport->CreateMessage(0);
     ref.payload = std::move(payload);
   };

From 29bae99542b44a261a6e60a1620dfddb8b967f50 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:10:16 +0100
Subject: [PATCH 356/726] DPL: use signposts for OOB callback

---
 Framework/Core/src/DataProcessingDevice.cxx | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 1fd35471c8fc1..ce58903553524 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -318,7 +318,7 @@ void on_socket_polled(uv_poll_t* poller, int status, int events)
       O2_SIGNPOST_END(device, sid, "socket_state", "Socket disconnected in context %{public}s", context->name);
     } break;
     case UV_PRIORITIZED: {
-      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "Data pending on socket for context %{public}s", context->name);
+      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "Socket prioritized for context %{public}s", context->name);
     } break;
   }
   // We do nothing, all the logic for now stays in DataProcessingDevice::doRun()
@@ -326,6 +326,7 @@ void on_socket_polled(uv_poll_t* poller, int status, int events)
 
 void on_out_of_band_polled(uv_poll_t* poller, int status, int events)
 {
+  O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
   auto* context = (PollerContext*)poller->data;
   context->state->loopReason |= DeviceState::OOB_ACTIVITY;
   if (status < 0) {
@@ -334,32 +335,27 @@ void on_out_of_band_polled(uv_poll_t* poller, int status, int events)
   }
   switch (events) {
     case UV_READABLE: {
-      ZoneScopedN("socket readable event");
+      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "Data pending on socket for channel %{public}s", context->name);
       context->state->loopReason |= DeviceState::DATA_INCOMING;
       assert(context->channelInfo);
-      LOGP(debug, "oob socket {} polled UV_READABLE.",
-           context->name,
-           context->channelInfo->hasPendingEvents);
       context->channelInfo->readPolled = true;
     } break;
     case UV_WRITABLE: {
-      ZoneScopedN("socket writeable");
+      O2_SIGNPOST_END(device, sid, "socket_state", "OOB socket connected for channel %{public}s", context->name);
       if (context->read) {
-        LOG(debug) << "socket polled UV_CONNECT" << context->name;
+        O2_SIGNPOST_START(device, sid, "socket_state", "OOB socket connected for read in context %{public}s", context->name);
         uv_poll_start(poller, UV_READABLE | UV_DISCONNECT | UV_PRIORITIZED, &on_out_of_band_polled);
       } else {
-        LOG(debug) << "socket polled UV_WRITABLE" << context->name;
+        O2_SIGNPOST_START(device, sid, "socket_state", "OOB socket connected for write for channel %{public}s", context->name);
         context->state->loopReason |= DeviceState::DATA_OUTGOING;
       }
     } break;
     case UV_DISCONNECT: {
-      ZoneScopedN("socket disconnect");
-      LOG(debug) << "socket polled UV_DISCONNECT";
+      O2_SIGNPOST_END(device, sid, "socket_state", "OOB socket disconnected in context %{public}s", context->name);
       uv_poll_start(poller, UV_WRITABLE, &on_out_of_band_polled);
     } break;
     case UV_PRIORITIZED: {
-      ZoneScopedN("socket prioritized");
-      LOG(debug) << "socket polled UV_PRIORITIZED";
+      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "OOB socket prioritized for context %{public}s", context->name);
     } break;
   }
   // We do nothing, all the logic for now stays in DataProcessingDevice::doRun()

From e482dad4508fc99b3be2a4d681c015bf522af1eb Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:10:16 +0100
Subject: [PATCH 357/726] DPL: rename TracyLockable to O2_LOCKABLE

---
 .../Core/include/Framework/DataRelayer.h      |  2 +-
 Framework/Core/include/Framework/DataSender.h |  2 +-
 Framework/Core/src/DataRelayer.cxx            | 32 +++++++++----------
 Framework/Core/src/DataSender.cxx             |  2 +-
 Framework/Core/src/ServiceRegistry.cxx        |  6 ++--
 .../Foundation/include/Framework/Tracing.h    |  4 +--
 6 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/Framework/Core/include/Framework/DataRelayer.h b/Framework/Core/include/Framework/DataRelayer.h
index 8fdedbdd1119e..1ebcf549d6a29 100644
--- a/Framework/Core/include/Framework/DataRelayer.h
+++ b/Framework/Core/include/Framework/DataRelayer.h
@@ -194,7 +194,7 @@ class DataRelayer
   std::vector<PruneOp> mPruneOps;
   size_t mMaxLanes;
 
-  TracyLockableN(std::recursive_mutex, mMutex, "data relayer mutex");
+  O2_LOCKABLE_NAMED(std::recursive_mutex, mMutex, "data relayer mutex");
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/include/Framework/DataSender.h b/Framework/Core/include/Framework/DataSender.h
index 7cdc0ea537bea..2937bd80f78f8 100644
--- a/Framework/Core/include/Framework/DataSender.h
+++ b/Framework/Core/include/Framework/DataSender.h
@@ -61,7 +61,7 @@ class DataSender
   std::vector<bool> mPresent;
   std::vector<bool> mPresentDefaults;
 
-  TracyLockableN(std::recursive_mutex, mMutex, "data relayer mutex");
+  O2_LOCKABLE_NAMED(std::recursive_mutex, mMutex, "data relayer mutex");
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index 0b459dd155a21..1daf4bbd2a20b 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -70,7 +70,7 @@ DataRelayer::DataRelayer(const CompletionPolicy& policy,
     mInputMatchers{DataRelayerHelpers::createInputMatchers(routes)},
     mMaxLanes{InputRouteHelpers::maxLanes(routes)}
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   if (policy.configureRelayer == nullptr) {
     static int pipelineLength = DefaultsHelpers::pipelineLength();
@@ -100,7 +100,7 @@ DataRelayer::DataRelayer(const CompletionPolicy& policy,
 
 TimesliceId DataRelayer::getTimesliceForSlot(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   auto& variables = mTimesliceIndex.getVariablesForSlot(slot);
   return VariableContextHelpers::getTimeslice(variables);
 }
@@ -109,7 +109,7 @@ DataRelayer::ActivityStats DataRelayer::processDanglingInputs(std::vector<Expira
                                                               ServiceRegistryRef services, bool createNew)
 {
   LOGP(debug, "DataRelayer::processDanglingInputs");
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   auto& deviceProxy = services.get<FairMQDeviceProxy>();
 
   ActivityStats activity;
@@ -394,7 +394,7 @@ DataRelayer::RelayChoice
                      size_t nPayloads,
                      std::function<void(TimesliceSlot, std::vector<MessageSet>&, TimesliceIndex::OldestOutputInfo)> onDrop)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   DataProcessingHeader const* dph = o2::header::get<DataProcessingHeader*>(rawHeader);
   // IMPLEMENTATION DETAILS
   //
@@ -616,7 +616,7 @@ DataRelayer::RelayChoice
 void DataRelayer::getReadyToProcess(std::vector<DataRelayer::RecordAction>& completed)
 {
   LOGP(debug, "DataRelayer::getReadyToProcess");
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   // THE STATE
   const auto& cache = mCache;
@@ -749,7 +749,7 @@ void DataRelayer::getReadyToProcess(std::vector<DataRelayer::RecordAction>& comp
 
 void DataRelayer::updateCacheStatus(TimesliceSlot slot, CacheEntryStatus oldStatus, CacheEntryStatus newStatus)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   const auto numInputTypes = mDistinctRoutesIndex.size();
 
   auto markInputDone = [&cachedStateMetrics = mCachedStateMetrics,
@@ -767,7 +767,7 @@ void DataRelayer::updateCacheStatus(TimesliceSlot slot, CacheEntryStatus oldStat
 
 std::vector<o2::framework::MessageSet> DataRelayer::consumeAllInputsForTimeslice(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   const auto numInputTypes = mDistinctRoutesIndex.size();
   // State of the computation
@@ -821,7 +821,7 @@ std::vector<o2::framework::MessageSet> DataRelayer::consumeAllInputsForTimeslice
 
 std::vector<o2::framework::MessageSet> DataRelayer::consumeExistingInputsForTimeslice(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   const auto numInputTypes = mDistinctRoutesIndex.size();
   // State of the computation
@@ -864,7 +864,7 @@ std::vector<o2::framework::MessageSet> DataRelayer::consumeExistingInputsForTime
 
 void DataRelayer::clear()
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   for (auto& cache : mCache) {
     cache.clear();
@@ -886,7 +886,7 @@ size_t
 /// the time pipelining.
 void DataRelayer::setPipelineLength(size_t s)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   mTimesliceIndex.resize(s);
   mVariableContextes.resize(s);
@@ -895,7 +895,7 @@ void DataRelayer::setPipelineLength(size_t s)
 
 void DataRelayer::publishMetrics()
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   auto numInputTypes = mDistinctRoutesIndex.size();
   // FIXME: many of the DataRelayer function rely on allocated cache, so its
@@ -932,31 +932,31 @@ void DataRelayer::publishMetrics()
 
 uint32_t DataRelayer::getFirstTFOrbitForSlot(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   return VariableContextHelpers::getFirstTFOrbit(mTimesliceIndex.getVariablesForSlot(slot));
 }
 
 uint32_t DataRelayer::getFirstTFCounterForSlot(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   return VariableContextHelpers::getFirstTFCounter(mTimesliceIndex.getVariablesForSlot(slot));
 }
 
 uint32_t DataRelayer::getRunNumberForSlot(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   return VariableContextHelpers::getRunNumber(mTimesliceIndex.getVariablesForSlot(slot));
 }
 
 uint64_t DataRelayer::getCreationTimeForSlot(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   return VariableContextHelpers::getCreationTime(mTimesliceIndex.getVariablesForSlot(slot));
 }
 
 void DataRelayer::sendContextState()
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   auto& states = mContext.get<DataProcessingStates>();
   for (size_t ci = 0; ci < mTimesliceIndex.size(); ++ci) {
     auto slot = TimesliceSlot{ci};
diff --git a/Framework/Core/src/DataSender.cxx b/Framework/Core/src/DataSender.cxx
index f0753102e2d40..3acce1c4b9704 100644
--- a/Framework/Core/src/DataSender.cxx
+++ b/Framework/Core/src/DataSender.cxx
@@ -51,7 +51,7 @@ DataSender::DataSender(ServiceRegistryRef registry)
     mSpec{registry.get<DeviceSpec const>()},
     mDistinctRoutesIndex{createDistinctOutputRouteIndex(mSpec.outputs)}
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   auto numInputTypes = mDistinctRoutesIndex.size();
   auto& routes = mSpec.outputs;
diff --git a/Framework/Core/src/ServiceRegistry.cxx b/Framework/Core/src/ServiceRegistry.cxx
index 7b77c521fe6a7..09921f6b4a7d4 100644
--- a/Framework/Core/src/ServiceRegistry.cxx
+++ b/Framework/Core/src/ServiceRegistry.cxx
@@ -271,7 +271,7 @@ void ServiceRegistry::postRenderGUICallbacks(ServiceRegistryRef ref)
 
 void ServiceRegistry::bindService(ServiceRegistry::Salt salt, ServiceSpec const& spec, void* service) const
 {
-  static TracyLockableN(std::mutex, bindMutex, "bind mutex");
+  static O2_LOCKABLE_NAMED(std::mutex, bindMutex, "bind mutex");
   // Stream services need to store their callbacks in the stream context.
   // This is to make sure we invoke the correct callback only once per
   // stream, since they could bind multiple times.
@@ -280,12 +280,12 @@ void ServiceRegistry::bindService(ServiceRegistry::Salt salt, ServiceSpec const&
   if (spec.kind == ServiceKind::Stream) {
     ServiceRegistryRef ref{const_cast<ServiceRegistry&>(*this), salt};
     auto& streamContext = ref.get<StreamContext>();
-    std::scoped_lock<LockableBase(std::mutex)> lock(bindMutex);
+    std::scoped_lock<O2_LOCKABLE(std::mutex)> lock(bindMutex);
     auto& dataProcessorContext = ref.get<DataProcessorContext>();
     ContextHelpers::bindStreamService(dataProcessorContext, streamContext, spec, service);
   } else {
     ServiceRegistryRef ref{const_cast<ServiceRegistry&>(*this), salt};
-    std::scoped_lock<LockableBase(std::mutex)> lock(bindMutex);
+    std::scoped_lock<O2_LOCKABLE(std::mutex)> lock(bindMutex);
     if (ref.active<DataProcessorContext>()) {
       auto& dataProcessorContext = ref.get<DataProcessorContext>();
       ContextHelpers::bindProcessorService(dataProcessorContext, spec, service);
diff --git a/Framework/Foundation/include/Framework/Tracing.h b/Framework/Foundation/include/Framework/Tracing.h
index f5bcecd3889d7..6c9c65f4423e5 100644
--- a/Framework/Foundation/include/Framework/Tracing.h
+++ b/Framework/Foundation/include/Framework/Tracing.h
@@ -39,8 +39,8 @@
 #define TracyAppInfo(...) \
   while (false) {         \
   }
-#define TracyLockableN(T, V, N) T V
-#define LockableBase(T) T
+#define O2_LOCKABLE_NAMED(T, V, N) T V
+#define O2_LOCKABLE(T) T
 #endif
 
 #endif // O2_FRAMEWORK_TRACING_H_

From 56a82107e0c74b8ef8ed7c37fcd6bd1c0f80c27d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:10:16 +0100
Subject: [PATCH 358/726] DPL: do not use Tracy anymore

Tracy usage never took off, so I am removing all the explicit calls to it and
I will replace them with Signpost usage, where it still makes sense.
---
 .../Core/src/CommonMessageBackendsHelpers.h   |  1 -
 Framework/Core/src/CommonServices.cxx         |  1 -
 Framework/Core/src/DataProcessingDevice.cxx   | 29 -------------------
 3 files changed, 31 deletions(-)

diff --git a/Framework/Core/src/CommonMessageBackendsHelpers.h b/Framework/Core/src/CommonMessageBackendsHelpers.h
index 10d8a9c5d347a..a3684906ff838 100644
--- a/Framework/Core/src/CommonMessageBackendsHelpers.h
+++ b/Framework/Core/src/CommonMessageBackendsHelpers.h
@@ -38,7 +38,6 @@ struct CommonMessageBackendsHelpers {
   static ServiceProcessingCallback sendCallback()
   {
     return [](ProcessingContext& ctx, void* service) {
-      ZoneScopedN("send message callback");
       T* context = reinterpret_cast<T*>(service);
       DataProcessor::doSend(ctx.services().get<DataSender>(), *context, ctx.services());
     };
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index a0f7fcdc88201..0d0828729ef42 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -741,7 +741,6 @@ auto sendRelayerMetrics(ServiceRegistryRef registry, DataProcessingStats& stats)
     stats.updateStats({static_cast<unsigned short>(static_cast<int>(ProcessingStatsId::AVAILABLE_MANAGED_SHM_BASE) + (runningWorkflow.shmSegmentId % 512)), DataProcessingStats::Op::SetIfPositive, freeMemory});
   }
 
-  ZoneScopedN("send metrics");
   auto device = registry.get<RawDeviceService>().device();
 
   int64_t totalBytesIn = 0;
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index ce58903553524..0a887f40efdcf 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -115,14 +115,12 @@ struct ServiceKindExtractor<ConfigurationInterface> {
 /// Watching stdin for commands probably a better approach.
 void on_idle_timer(uv_timer_t* handle)
 {
-  ZoneScopedN("Idle timer");
   auto* state = (DeviceState*)handle->data;
   state->loopReason |= DeviceState::TIMER_EXPIRED;
 }
 
 void on_transition_requested_expired(uv_timer_t* handle)
 {
-  ZoneScopedN("Transition expired");
   auto* state = (DeviceState*)handle->data;
   state->loopReason |= DeviceState::TIMER_EXPIRED;
   LOGP(info, "Timer expired. Forcing transition to READY");
@@ -222,12 +220,10 @@ DataProcessingDevice::DataProcessingDevice(RunningDeviceRef running, ServiceRegi
 // one with the thread id. For the moment we simply use the first one.
 void run_callback(uv_work_t* handle)
 {
-  ZoneScopedN("run_callback");
   auto* task = (TaskStreamInfo*)handle->data;
   auto ref = ServiceRegistryRef{*task->registry, ServiceRegistry::globalStreamSalt(task->id.index + 1)};
   DataProcessingDevice::doPrepare(ref);
   DataProcessingDevice::doRun(ref);
-  //  FrameMark;
 }
 
 // Once the processing in a thread is done, this is executed on the main thread.
@@ -268,7 +264,6 @@ void run_completion(uv_work_t* handle, int status)
   quotaEvaluator.handleExpired(reportExpiredOffer);
   quotaEvaluator.dispose(task->id.index);
   task->running = false;
-  ZoneScopedN("run_completion");
 }
 
 // Context for polling
@@ -377,8 +372,6 @@ void DataProcessingDevice::Init()
   context.statefulProcess = nullptr;
   context.error = spec.algorithm.onError;
   context.initError = spec.algorithm.onInitError;
-  TracyAppInfo(spec.name.data(), spec.name.size());
-  ZoneScopedN("DataProcessingDevice::Init");
 
   auto configStore = DeviceConfigurationHelpers::getConfiguration(mServiceRegistry, spec.name.c_str(), spec.options);
   if (configStore == nullptr) {
@@ -412,7 +405,6 @@ void DataProcessingDevice::Init()
   if (context.initError) {
     context.initErrorHandling = [&errorCallback = context.initError,
                                  &serviceRegistry = mServiceRegistry](RuntimeErrorRef e) {
-      ZoneScopedN("Error handling");
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
       ServiceRegistryRef ref{serviceRegistry, ServiceRegistry::globalDeviceSalt()};
@@ -426,7 +418,6 @@ void DataProcessingDevice::Init()
     };
   } else {
     context.initErrorHandling = [&serviceRegistry = mServiceRegistry](RuntimeErrorRef e) {
-      ZoneScopedN("Error handling");
       auto& err = error_from_ref(e);
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
@@ -449,7 +440,6 @@ void DataProcessingDevice::Init()
       try {
         context.statefulProcess = context.init(initContext);
       } catch (o2::framework::RuntimeErrorRef e) {
-        ZoneScopedN("error handling");
         if (context.initErrorHandling) {
           (context.initErrorHandling)(e);
         }
@@ -458,14 +448,12 @@ void DataProcessingDevice::Init()
       try {
         context.statefulProcess = context.init(initContext);
       } catch (std::exception& ex) {
-        ZoneScopedN("error handling");
         /// Convert a standard exception to a RuntimeErrorRef
         /// Notice how this will lose the backtrace information
         /// and report the exception coming from here.
         auto e = runtime_error(ex.what());
         (context.initErrorHandling)(e);
       } catch (o2::framework::RuntimeErrorRef e) {
-        ZoneScopedN("error handling");
         (context.initErrorHandling)(e);
       }
     }
@@ -621,7 +609,6 @@ static auto toBeforwardedMessageSet = [](std::vector<ChannelIndex>& cachedForwar
 // FIXME: do it in a smarter way than O(N^2)
 static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot, std::vector<MessageSet>& currentSetOfInputs,
                                TimesliceIndex::OldestOutputInfo oldestTimeslice, bool copy, bool consume = true) {
-  ZoneScopedN("forward inputs");
   auto& proxy = registry.get<FairMQDeviceProxy>();
   // we collect all messages per forward in a map and send them together
   std::vector<fair::mq::Parts> forwardedParts;
@@ -1078,7 +1065,6 @@ void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceCont
   if (context.error != nullptr) {
     context.errorHandling = [&errorCallback = context.error,
                              &serviceRegistry = mServiceRegistry](RuntimeErrorRef e, InputRecord& record) {
-      ZoneScopedN("Error handling");
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
       ServiceRegistryRef ref{serviceRegistry, ServiceRegistry::globalDeviceSalt()};
@@ -1093,7 +1079,6 @@ void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceCont
   } else {
     context.errorHandling = [&errorPolicy = mProcessingPolicies.error,
                              &serviceRegistry = mServiceRegistry](RuntimeErrorRef e, InputRecord& record) {
-      ZoneScopedN("Error handling");
       auto& err = error_from_ref(e);
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
@@ -1242,8 +1227,6 @@ void DataProcessingDevice::Run()
     // so that devices which do not have a timer can still start an
     // enumeration.
     {
-      ZoneScopedN("uv idle");
-      TracyPlot("past activity", (int64_t)mWasActive);
       ServiceRegistryRef ref{mServiceRegistry};
       ref.get<DriverClient>().flushPending(mServiceRegistry);
       auto shouldNotWait = (mWasActive &&
@@ -1303,7 +1286,6 @@ void DataProcessingDevice::Run()
       if (state.transitionHandling == TransitionHandlingState::Requested && state.streaming == StreamingState::Idle) {
         state.transitionHandling = TransitionHandlingState::Expired;
       }
-      TracyPlot("shouldNotWait", (int)shouldNotWait);
       if (state.severityStack.empty() == false) {
         fair::Logger::SetConsoleSeverity((fair::Severity)state.severityStack.back());
         state.severityStack.pop_back();
@@ -1349,7 +1331,6 @@ void DataProcessingDevice::Run()
         fair::Logger::SetConsoleSeverity((fair::Severity)state.severityStack.back());
         state.severityStack.pop_back();
       }
-      TracyPlot("loopReason", (int64_t)(uint64_t)state.loopReason);
       LOGP(debug, "Loop reason mask {:b} & {:b} = {:b}",
            state.loopReason, state.tracingFlags,
            state.loopReason & state.tracingFlags);
@@ -1433,7 +1414,6 @@ void DataProcessingDevice::Run()
     } else {
       mWasActive = false;
     }
-    FrameMark;
   }
   auto& spec = ref.get<DeviceSpec const>();
   /// Cleanup messages which are still pending on exit.
@@ -1448,12 +1428,10 @@ void DataProcessingDevice::Run()
 /// non-data triggers like those which are time based.
 void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
 {
-  ZoneScopedN("DataProcessingDevice::doPrepare");
   auto& context = ref.get<DataProcessorContext>();
 
   *context.wasActive = false;
   {
-    ZoneScopedN("CallbackService::Id::ClockTick");
     ref.get<CallbackService>().call<CallbackService::Id::ClockTick>();
   }
   // Whether or not we had something to do.
@@ -1734,7 +1712,6 @@ struct WaitBackpressurePolicy {
 void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo& info)
 {
   auto& context = ref.get<DataProcessorContext>();
-  ZoneScopedN("DataProcessingDevice::handleData");
 
   enum struct InputType : int {
     Invalid = 0,
@@ -1763,7 +1740,6 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
     auto& parts = info.parts;
     stats.updateStats({(int)ProcessingStatsId::TOTAL_INPUTS, DataProcessingStats::Op::Set, (int64_t)parts.Size()});
 
-    TracyPlot("messages received", (int64_t)parts.Size());
     std::vector<InputInfo> results;
     // we can reserve the upper limit
     results.reserve(parts.Size() / 2);
@@ -2060,7 +2036,6 @@ void update_maximum(std::atomic<T>& maximum_value, T const& value) noexcept
 bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::vector<DataRelayer::RecordAction>& completed)
 {
   auto& context = ref.get<DataProcessorContext>();
-  ZoneScopedN("DataProcessingDevice::tryDispatchComputation");
   LOGP(debug, "DataProcessingDevice::tryDispatchComputation");
   // This is the actual hidden state for the outer loop. In case we decide we
   // want to support multithreaded dispatching of operations, I can simply
@@ -2135,7 +2110,6 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   auto prepareAllocatorForCurrentTimeSlice = [ref](TimesliceSlot i) -> void {
     auto& relayer = ref.get<DataRelayer>();
     auto& timingInfo = ref.get<TimingInfo>();
-    ZoneScopedN("DataProcessingDevice::prepareForCurrentTimeslice");
     auto timeslice = relayer.getTimesliceForSlot(i);
 
     timingInfo.timeslice = timeslice.value;
@@ -2422,21 +2396,18 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       try {
         runNoCatch(action);
       } catch (o2::framework::RuntimeErrorRef e) {
-        ZoneScopedN("error handling");
         (context.errorHandling)(e, record);
       }
     } else {
       try {
         runNoCatch(action);
       } catch (std::exception& ex) {
-        ZoneScopedN("error handling");
         /// Convert a standard exception to a RuntimeErrorRef
         /// Notice how this will lose the backtrace information
         /// and report the exception coming from here.
         auto e = runtime_error(ex.what());
         (context.errorHandling)(e, record);
       } catch (o2::framework::RuntimeErrorRef e) {
-        ZoneScopedN("error handling");
         (context.errorHandling)(e, record);
       }
     }

From b5d41a03d914d482eeee4776b5ff012a06765493 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:10:16 +0100
Subject: [PATCH 359/726] DPL: drop unused Tracy APIs

ZoneScoped*, TracyAppInfo, FrameMark, TracyAlloc, TracyPlot were all removed
and will be replaced by Signposts as needed.
---
 .../Foundation/include/Framework/Tracing.h    | 24 -------------------
 1 file changed, 24 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Tracing.h b/Framework/Foundation/include/Framework/Tracing.h
index 6c9c65f4423e5..91c3b99eb8dd0 100644
--- a/Framework/Foundation/include/Framework/Tracing.h
+++ b/Framework/Foundation/include/Framework/Tracing.h
@@ -15,30 +15,6 @@
 #define DPL_HAS_TRACING
 #include <tracy/Tracy.hpp>
 #else
-#define ZoneScoped \
-  while (false) {  \
-  }
-#define FrameMark \
-  while (false) { \
-  }
-#define TracyPlot(...) \
-  while (false) {      \
-  }
-#define ZoneScopedN(...) \
-  while (false) {        \
-  }
-#define ZoneScopedNS(...) \
-  while (false) {         \
-  }
-#define TracyAlloc(...) \
-  while (false) {       \
-  }
-#define TracyFree(...) \
-  while (false) {      \
-  }
-#define TracyAppInfo(...) \
-  while (false) {         \
-  }
 #define O2_LOCKABLE_NAMED(T, V, N) T V
 #define O2_LOCKABLE(T) T
 #endif

From 73e5911bda21e3f7079c3c759d748373dea11232 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:10:16 +0100
Subject: [PATCH 360/726] DPL: drop tracy support from the GUI

No one is using it and Signposts are probably a better alternative.
---
 .../GUISupport/src/FrameworkGUIDeviceInspector.cxx  | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
index eb62fcc09d338..3ef8b99ed26bb 100644
--- a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
@@ -261,9 +261,6 @@ void displayDeviceInspector(DeviceSpec const& spec,
     ImGui::Text("Pid: %d (exit status: %d)", info.pid, info.exitStatus);
   }
   ImGui::Text("Device state: %s", info.deviceState.data());
-#ifdef DPL_ENABLE_TRACING
-  ImGui::Text("Tracy Port: %d", info.tracyPort);
-#endif
   ImGui::Text("Rank: %zu/%zu%%%zu/%zu", spec.rank, spec.nSlots, spec.inputTimesliceId, spec.maxInputTimeslices);
 
   if (ImGui::Button(ICON_FA_BUG "Attach debugger")) {
@@ -324,16 +321,6 @@ void displayDeviceInspector(DeviceSpec const& spec,
   }
 #endif
 
-#if DPL_ENABLE_TRACING
-  ImGui::SameLine();
-  if (ImGui::Button("Tracy")) {
-    std::string tracyPort = std::to_string(info.tracyPort);
-    auto cmd = fmt::format("tracy-profiler -p {} -a 127.0.0.1 &", info.tracyPort);
-    LOG(debug) << cmd;
-    int retVal = system(cmd.c_str());
-    (void)retVal;
-  }
-#endif
   if (control.controller) {
     if (ImGui::Button("Offer SHM")) {
       control.controller->write("/shm-offer 1000", strlen("/shm-offer 1000"));

From 653ecf58f80efd07ff7a2c040e869be8d6a27716 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:10:16 +0100
Subject: [PATCH 361/726] DPL: drop tracy support for good

---
 Framework/Core/src/DataProcessingDevice.cxx      | 4 ----
 Framework/Foundation/include/Framework/Tracing.h | 7 ++++---
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 0a887f40efdcf..c175e4e47e0c9 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -8,10 +8,6 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
-#ifdef DPL_ENABLE_TRACING
-#define TRACY_ENABLE
-#include <tracy/TracyClient.cpp>
-#endif
 #include "Framework/AsyncQueue.h"
 #include "Framework/DataProcessingDevice.h"
 #include "Framework/ChannelMatching.h"
diff --git a/Framework/Foundation/include/Framework/Tracing.h b/Framework/Foundation/include/Framework/Tracing.h
index 91c3b99eb8dd0..72c052ca6c161 100644
--- a/Framework/Foundation/include/Framework/Tracing.h
+++ b/Framework/Foundation/include/Framework/Tracing.h
@@ -11,9 +11,10 @@
 #ifndef O2_FRAMEWORK_TRACING_H_
 #define O2_FRAMEWORK_TRACING_H_
 
-#if DPL_ENABLE_TRACING && __has_include(<tracy/Tracy.hpp>)
-#define DPL_HAS_TRACING
-#include <tracy/Tracy.hpp>
+#if DPL_ENABLE_TRACING
+// FIXME: not implemented yet in terms of Signposts
+#define O2_LOCKABLE_NAMED(T, V, N) T V
+#define O2_LOCKABLE(T) T
 #else
 #define O2_LOCKABLE_NAMED(T, V, N) T V
 #define O2_LOCKABLE(T) T

From a75ef55efc126b0afecbf4cbbea7acd3ce605ddf Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:10:16 +0100
Subject: [PATCH 362/726] [FEAT] Drop support for Tracy

---
 Framework/CHANGELOG.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/Framework/CHANGELOG.md b/Framework/CHANGELOG.md
index e7278733de246..044274d9515c4 100644
--- a/Framework/CHANGELOG.md
+++ b/Framework/CHANGELOG.md
@@ -1,3 +1,12 @@
+# 2024-02-22: Drop Tracy support
+
+Tracy support never took off, so I am dropping it. This was mostly because people do not know about it and having a per process profile GUI was way unpractical. Moreover, needing an extra compile time flag meant one most likely did not have the support compiled in when needed.
+
+I have therefore decided to replace it with signposts, which hopefully will see better adoption thanks
+to the integration with Instruments on mac and the easy way they can be enabled dynamically.
+
+We could then reintroduce Tracy support as a hook on top of signposts, if really needed.
+
 # 2024-02-16: Improved Signposts.
 
 In particular:

From eba163e2b62d29bd153eaaab9c30dff02286aeea Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:19:01 +0100
Subject: [PATCH 363/726] DPL: replace Tracy allocation profiler with Signposts

---
 Framework/Core/src/DataAllocator.cxx        |  7 ++++
 Framework/Core/src/DataProcessingDevice.cxx | 42 +++++++++++++++------
 Framework/Core/src/LifetimeHelpers.cxx      |  8 +++-
 3 files changed, 44 insertions(+), 13 deletions(-)

diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index de500cd6d6c4e..fe38283d5e2de 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -35,6 +35,7 @@
 #include <utility>
 
 O2_DECLARE_DYNAMIC_LOG(stream_context);
+O2_DECLARE_DYNAMIC_LOG(parts);
 
 namespace o2::framework
 {
@@ -129,6 +130,8 @@ void DataAllocator::addPartToContext(RouteIndex routeIndex, fair::mq::MessagePtr
                                      o2::header::SerializationMethod serializationMethod)
 {
   auto headerMessage = headerMessageFromOutput(spec, routeIndex, serializationMethod, 0);
+  O2_SIGNPOST_ID_FROM_POINTER(pid, parts, headerMessage->GetData());
+  O2_SIGNPOST_START(parts, pid, "parts", "addPartToContext %p", headerMessage->GetData());
 
   // FIXME: this is kind of ugly, we know that we can change the content of the
   // header message because we have just created it, but the API declares it const
@@ -150,6 +153,8 @@ void DataAllocator::adopt(const Output& spec, std::string* ptr)
   // the correct payload size is set later when sending the
   // StringContext, see DataProcessor::doSend
   auto header = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, 0);
+  O2_SIGNPOST_ID_FROM_POINTER(pid, parts, header->GetData());
+  O2_SIGNPOST_START(parts, pid, "parts", "addPartToContext %p", header->GetData());
   mRegistry.get<StringContext>().addString(std::move(header), std::move(payload), routeIndex);
   assert(payload.get() == nullptr);
 }
@@ -206,6 +211,8 @@ void DataAllocator::adopt(const Output& spec, LifetimeHolder<TableBuilder>& tb)
   auto& timingInfo = mRegistry.get<TimingInfo>();
   RouteIndex routeIndex = matchDataHeader(spec, timingInfo.timeslice);
   auto header = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodArrow, 0);
+  O2_SIGNPOST_ID_FROM_POINTER(pid, parts, header->GetData());
+  O2_SIGNPOST_START(parts, pid, "parts", "adopt %p", header->GetData());
   auto& context = mRegistry.get<ArrowContext>();
 
   auto creator = [transport = context.proxy().getOutputTransport(routeIndex)](size_t s) -> std::unique_ptr<fair::mq::Message> {
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 50934bd59a729..387a9ab753575 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -92,7 +92,10 @@ struct formatter<o2::framework::CompletionPolicy::CompletionOp> : ostream_format
 };
 } // namespace fmt
 
+// A log to use for general device logging
 O2_DECLARE_DYNAMIC_LOG(device);
+// Special log to keep track of the lifetime of the parts
+O2_DECLARE_DYNAMIC_LOG(parts);
 
 using namespace o2::framework;
 using ConfigurationInterface = o2::configuration::ConfigurationInterface;
@@ -1804,9 +1807,14 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
         LOGP(error, "DataHeader payloadSize mismatch");
         continue;
       }
-      TracyPlot("payload size", (int64_t)dh->payloadSize);
       auto dph = o2::header::get<DataProcessingHeader*>(headerData);
-      TracyAlloc(parts.At(pi + 1)->GetData(), parts.At(pi + 1)->GetSize());
+      // We only deal with the tracking of parts if the log is enabled.
+      // This is because in principle we should track the size of each of
+      // the parts and sum it up. Not for now.
+      if (O2_LOG_ENABLED(parts) == true) {
+        O2_SIGNPOST_ID_FROM_POINTER(pid, parts, headerData);
+        O2_SIGNPOST_START(parts, pid, "parts", "Processing DataHeader with splitPayloadParts %d and splitPayloadIndex %d", dh->splitPayloadParts, dh->splitPayloadIndex);
+      }
       if (!dph) {
         insertInputInfo(pi, 2, InputType::Invalid);
         LOGP(error, "Header stack does not contain DataProcessingHeader");
@@ -2180,10 +2188,12 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   // Function to cleanup record. For the moment we
   // simply use it to keep track of input messages
   // which are not needed, to display them in the GUI.
-#ifdef TRACY_ENABLE
   auto cleanupRecord = [](InputRecord& record) {
-    for (size_t ii = 0, ie = record.size(); ii < ie; ++ii) {
-      DataRef input = record.getByPos(ii);
+    if (O2_LOG_ENABLED(parts) == false) {
+      return;
+    }
+    for (size_t pi = 0, pe = record.size(); pi < pe; ++pi) {
+      DataRef input = record.getByPos(pi);
       if (input.header == nullptr) {
         continue;
       }
@@ -2196,10 +2206,21 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       if (!dh) {
         continue;
       }
-      TracyFree(input.payload);
+      // We use the address of the first header of a split payload
+      // to identify the interval.
+      O2_SIGNPOST_ID_FROM_POINTER(pid, parts, dh);
+      O2_SIGNPOST_END(parts, pid, "parts", "Cleaning up parts associated to %p", dh);
+
+      // No split parts, we simply skip the payload
+      if (dh->splitPayloadParts > 0 && dh->splitPayloadParts == dh->splitPayloadIndex) {
+        // this is indicating a sequence of payloads following the header
+        // FIXME: we will probably also set the DataHeader version
+        pi += dh->splitPayloadParts - 1;
+      } else {
+        size_t pi = pi + (dh->splitPayloadParts > 0 ? dh->splitPayloadParts : 1) * 2;
+      }
     }
   };
-#endif
 
   auto switchState = [ref](StreamingState newState) {
     auto& control = ref.get<ControlService>();
@@ -2432,6 +2453,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
     // We forward inputs only when we consume them. If we simply Process them,
     // we keep them for next message arriving.
     if (action.op == CompletionPolicy::CompletionOp::Consume) {
+      cleanupRecord(record);
       context.postDispatchingCallbacks(processContext);
       ref.get<CallbackService>().call<CallbackService::Id::DataConsumed>(o2::framework::ServiceRegistryRef{ref});
     }
@@ -2441,11 +2463,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       forwardInputs(ref, action.slot, currentSetOfInputs, timesliceIndex.getOldestPossibleOutput(), false, action.op == CompletionPolicy::CompletionOp::Consume);
     }
     context.postForwardingCallbacks(processContext);
-    if (action.op == CompletionPolicy::CompletionOp::Consume) {
-#ifdef TRACY_ENABLE
-      cleanupRecord(record);
-#endif
-    } else if (action.op == CompletionPolicy::CompletionOp::Process) {
+    if (action.op == CompletionPolicy::CompletionOp::Process) {
       cleanTimers(action.slot, record);
     }
   }
diff --git a/Framework/Core/src/LifetimeHelpers.cxx b/Framework/Core/src/LifetimeHelpers.cxx
index 1aa53fa0493ca..8c8605d41849c 100644
--- a/Framework/Core/src/LifetimeHelpers.cxx
+++ b/Framework/Core/src/LifetimeHelpers.cxx
@@ -25,6 +25,7 @@
 #include "Framework/FairMQDeviceProxy.h"
 #include "Framework/Formatters.h"
 #include "Framework/DeviceState.h"
+#include "Framework/Signpost.h"
 
 #include "Headers/DataHeader.h"
 #include "Headers/DataHeaderHelpers.h"
@@ -44,6 +45,8 @@
 using namespace o2::header;
 using namespace fair;
 
+O2_DECLARE_DYNAMIC_LOG(parts);
+
 namespace o2::framework
 {
 
@@ -411,7 +414,6 @@ ExpirationHandler::Handler LifetimeHelpers::enumerate(ConcreteDataMatcher const&
     assert(!ref.payload);
 
     auto timestamp = VariableContextHelpers::getTimeslice(variables).value;
-    LOGP(debug, "Enumerating record");
     DataHeader dh;
     dh.dataOrigin = matcher.origin;
     dh.dataDescription = matcher.description;
@@ -432,6 +434,8 @@ ExpirationHandler::Handler LifetimeHelpers::enumerate(ConcreteDataMatcher const&
     auto&& transport = deviceProxy.getInputChannel(channelIndex)->Transport();
     auto channelAlloc = o2::pmr::getTransportAllocator(transport);
     auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, dh, dph});
+    O2_SIGNPOST_ID_FROM_POINTER(hid, parts, header->GetData());
+    O2_SIGNPOST_START(parts, hid, "parts", "Enumerating part %p with timestamp %zu", header->GetData(), timestamp);
     ref.header = std::move(header);
 
     auto payload = transport->CreateMessage(sizeof(counter_t));
@@ -486,6 +490,8 @@ ExpirationHandler::Handler LifetimeHelpers::dummy(ConcreteDataMatcher const& mat
     auto channelAlloc = o2::pmr::getTransportAllocator(transport);
     auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, dh, dph});
     ref.header = std::move(header);
+    O2_SIGNPOST_ID_FROM_POINTER(hid, parts, header->GetData());
+    O2_SIGNPOST_START(parts, hid, "parts", "Enumerating part %p with timestamp %zu", header->GetData(), timestamp);
     auto payload = transport->CreateMessage(0);
     ref.payload = std::move(payload);
   };

From 02954611aaf64aeb8d2619c28cc7c40ccb0adc44 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:19:01 +0100
Subject: [PATCH 364/726] DPL: use signposts for OOB callback

---
 Framework/Core/src/DataProcessingDevice.cxx | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 387a9ab753575..175fc8ab50f6e 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -318,7 +318,7 @@ void on_socket_polled(uv_poll_t* poller, int status, int events)
       O2_SIGNPOST_END(device, sid, "socket_state", "Socket disconnected in context %{public}s", context->name);
     } break;
     case UV_PRIORITIZED: {
-      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "Data pending on socket for context %{public}s", context->name);
+      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "Socket prioritized for context %{public}s", context->name);
     } break;
   }
   // We do nothing, all the logic for now stays in DataProcessingDevice::doRun()
@@ -326,6 +326,7 @@ void on_socket_polled(uv_poll_t* poller, int status, int events)
 
 void on_out_of_band_polled(uv_poll_t* poller, int status, int events)
 {
+  O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
   auto* context = (PollerContext*)poller->data;
   context->state->loopReason |= DeviceState::OOB_ACTIVITY;
   if (status < 0) {
@@ -334,32 +335,27 @@ void on_out_of_band_polled(uv_poll_t* poller, int status, int events)
   }
   switch (events) {
     case UV_READABLE: {
-      ZoneScopedN("socket readable event");
+      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "Data pending on socket for channel %{public}s", context->name);
       context->state->loopReason |= DeviceState::DATA_INCOMING;
       assert(context->channelInfo);
-      LOGP(debug, "oob socket {} polled UV_READABLE.",
-           context->name,
-           context->channelInfo->hasPendingEvents);
       context->channelInfo->readPolled = true;
     } break;
     case UV_WRITABLE: {
-      ZoneScopedN("socket writeable");
+      O2_SIGNPOST_END(device, sid, "socket_state", "OOB socket connected for channel %{public}s", context->name);
       if (context->read) {
-        LOG(debug) << "socket polled UV_CONNECT" << context->name;
+        O2_SIGNPOST_START(device, sid, "socket_state", "OOB socket connected for read in context %{public}s", context->name);
         uv_poll_start(poller, UV_READABLE | UV_DISCONNECT | UV_PRIORITIZED, &on_out_of_band_polled);
       } else {
-        LOG(debug) << "socket polled UV_WRITABLE" << context->name;
+        O2_SIGNPOST_START(device, sid, "socket_state", "OOB socket connected for write for channel %{public}s", context->name);
         context->state->loopReason |= DeviceState::DATA_OUTGOING;
       }
     } break;
     case UV_DISCONNECT: {
-      ZoneScopedN("socket disconnect");
-      LOG(debug) << "socket polled UV_DISCONNECT";
+      O2_SIGNPOST_END(device, sid, "socket_state", "OOB socket disconnected in context %{public}s", context->name);
       uv_poll_start(poller, UV_WRITABLE, &on_out_of_band_polled);
     } break;
     case UV_PRIORITIZED: {
-      ZoneScopedN("socket prioritized");
-      LOG(debug) << "socket polled UV_PRIORITIZED";
+      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "OOB socket prioritized for context %{public}s", context->name);
     } break;
   }
   // We do nothing, all the logic for now stays in DataProcessingDevice::doRun()

From a83b94be6c9d2d1ea14aab7537418995228ab793 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:19:01 +0100
Subject: [PATCH 365/726] DPL: rename TracyLockable to O2_LOCKABLE

---
 .../Core/include/Framework/DataRelayer.h      |  2 +-
 Framework/Core/include/Framework/DataSender.h |  2 +-
 Framework/Core/src/DataRelayer.cxx            | 32 +++++++++----------
 Framework/Core/src/DataSender.cxx             |  2 +-
 Framework/Core/src/ServiceRegistry.cxx        |  6 ++--
 .../Foundation/include/Framework/Tracing.h    |  4 +--
 6 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/Framework/Core/include/Framework/DataRelayer.h b/Framework/Core/include/Framework/DataRelayer.h
index 8fdedbdd1119e..1ebcf549d6a29 100644
--- a/Framework/Core/include/Framework/DataRelayer.h
+++ b/Framework/Core/include/Framework/DataRelayer.h
@@ -194,7 +194,7 @@ class DataRelayer
   std::vector<PruneOp> mPruneOps;
   size_t mMaxLanes;
 
-  TracyLockableN(std::recursive_mutex, mMutex, "data relayer mutex");
+  O2_LOCKABLE_NAMED(std::recursive_mutex, mMutex, "data relayer mutex");
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/include/Framework/DataSender.h b/Framework/Core/include/Framework/DataSender.h
index 7cdc0ea537bea..2937bd80f78f8 100644
--- a/Framework/Core/include/Framework/DataSender.h
+++ b/Framework/Core/include/Framework/DataSender.h
@@ -61,7 +61,7 @@ class DataSender
   std::vector<bool> mPresent;
   std::vector<bool> mPresentDefaults;
 
-  TracyLockableN(std::recursive_mutex, mMutex, "data relayer mutex");
+  O2_LOCKABLE_NAMED(std::recursive_mutex, mMutex, "data relayer mutex");
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index 0b459dd155a21..1daf4bbd2a20b 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -70,7 +70,7 @@ DataRelayer::DataRelayer(const CompletionPolicy& policy,
     mInputMatchers{DataRelayerHelpers::createInputMatchers(routes)},
     mMaxLanes{InputRouteHelpers::maxLanes(routes)}
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   if (policy.configureRelayer == nullptr) {
     static int pipelineLength = DefaultsHelpers::pipelineLength();
@@ -100,7 +100,7 @@ DataRelayer::DataRelayer(const CompletionPolicy& policy,
 
 TimesliceId DataRelayer::getTimesliceForSlot(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   auto& variables = mTimesliceIndex.getVariablesForSlot(slot);
   return VariableContextHelpers::getTimeslice(variables);
 }
@@ -109,7 +109,7 @@ DataRelayer::ActivityStats DataRelayer::processDanglingInputs(std::vector<Expira
                                                               ServiceRegistryRef services, bool createNew)
 {
   LOGP(debug, "DataRelayer::processDanglingInputs");
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   auto& deviceProxy = services.get<FairMQDeviceProxy>();
 
   ActivityStats activity;
@@ -394,7 +394,7 @@ DataRelayer::RelayChoice
                      size_t nPayloads,
                      std::function<void(TimesliceSlot, std::vector<MessageSet>&, TimesliceIndex::OldestOutputInfo)> onDrop)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   DataProcessingHeader const* dph = o2::header::get<DataProcessingHeader*>(rawHeader);
   // IMPLEMENTATION DETAILS
   //
@@ -616,7 +616,7 @@ DataRelayer::RelayChoice
 void DataRelayer::getReadyToProcess(std::vector<DataRelayer::RecordAction>& completed)
 {
   LOGP(debug, "DataRelayer::getReadyToProcess");
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   // THE STATE
   const auto& cache = mCache;
@@ -749,7 +749,7 @@ void DataRelayer::getReadyToProcess(std::vector<DataRelayer::RecordAction>& comp
 
 void DataRelayer::updateCacheStatus(TimesliceSlot slot, CacheEntryStatus oldStatus, CacheEntryStatus newStatus)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   const auto numInputTypes = mDistinctRoutesIndex.size();
 
   auto markInputDone = [&cachedStateMetrics = mCachedStateMetrics,
@@ -767,7 +767,7 @@ void DataRelayer::updateCacheStatus(TimesliceSlot slot, CacheEntryStatus oldStat
 
 std::vector<o2::framework::MessageSet> DataRelayer::consumeAllInputsForTimeslice(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   const auto numInputTypes = mDistinctRoutesIndex.size();
   // State of the computation
@@ -821,7 +821,7 @@ std::vector<o2::framework::MessageSet> DataRelayer::consumeAllInputsForTimeslice
 
 std::vector<o2::framework::MessageSet> DataRelayer::consumeExistingInputsForTimeslice(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   const auto numInputTypes = mDistinctRoutesIndex.size();
   // State of the computation
@@ -864,7 +864,7 @@ std::vector<o2::framework::MessageSet> DataRelayer::consumeExistingInputsForTime
 
 void DataRelayer::clear()
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   for (auto& cache : mCache) {
     cache.clear();
@@ -886,7 +886,7 @@ size_t
 /// the time pipelining.
 void DataRelayer::setPipelineLength(size_t s)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   mTimesliceIndex.resize(s);
   mVariableContextes.resize(s);
@@ -895,7 +895,7 @@ void DataRelayer::setPipelineLength(size_t s)
 
 void DataRelayer::publishMetrics()
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   auto numInputTypes = mDistinctRoutesIndex.size();
   // FIXME: many of the DataRelayer function rely on allocated cache, so its
@@ -932,31 +932,31 @@ void DataRelayer::publishMetrics()
 
 uint32_t DataRelayer::getFirstTFOrbitForSlot(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   return VariableContextHelpers::getFirstTFOrbit(mTimesliceIndex.getVariablesForSlot(slot));
 }
 
 uint32_t DataRelayer::getFirstTFCounterForSlot(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   return VariableContextHelpers::getFirstTFCounter(mTimesliceIndex.getVariablesForSlot(slot));
 }
 
 uint32_t DataRelayer::getRunNumberForSlot(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   return VariableContextHelpers::getRunNumber(mTimesliceIndex.getVariablesForSlot(slot));
 }
 
 uint64_t DataRelayer::getCreationTimeForSlot(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   return VariableContextHelpers::getCreationTime(mTimesliceIndex.getVariablesForSlot(slot));
 }
 
 void DataRelayer::sendContextState()
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   auto& states = mContext.get<DataProcessingStates>();
   for (size_t ci = 0; ci < mTimesliceIndex.size(); ++ci) {
     auto slot = TimesliceSlot{ci};
diff --git a/Framework/Core/src/DataSender.cxx b/Framework/Core/src/DataSender.cxx
index f0753102e2d40..3acce1c4b9704 100644
--- a/Framework/Core/src/DataSender.cxx
+++ b/Framework/Core/src/DataSender.cxx
@@ -51,7 +51,7 @@ DataSender::DataSender(ServiceRegistryRef registry)
     mSpec{registry.get<DeviceSpec const>()},
     mDistinctRoutesIndex{createDistinctOutputRouteIndex(mSpec.outputs)}
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   auto numInputTypes = mDistinctRoutesIndex.size();
   auto& routes = mSpec.outputs;
diff --git a/Framework/Core/src/ServiceRegistry.cxx b/Framework/Core/src/ServiceRegistry.cxx
index 7b77c521fe6a7..09921f6b4a7d4 100644
--- a/Framework/Core/src/ServiceRegistry.cxx
+++ b/Framework/Core/src/ServiceRegistry.cxx
@@ -271,7 +271,7 @@ void ServiceRegistry::postRenderGUICallbacks(ServiceRegistryRef ref)
 
 void ServiceRegistry::bindService(ServiceRegistry::Salt salt, ServiceSpec const& spec, void* service) const
 {
-  static TracyLockableN(std::mutex, bindMutex, "bind mutex");
+  static O2_LOCKABLE_NAMED(std::mutex, bindMutex, "bind mutex");
   // Stream services need to store their callbacks in the stream context.
   // This is to make sure we invoke the correct callback only once per
   // stream, since they could bind multiple times.
@@ -280,12 +280,12 @@ void ServiceRegistry::bindService(ServiceRegistry::Salt salt, ServiceSpec const&
   if (spec.kind == ServiceKind::Stream) {
     ServiceRegistryRef ref{const_cast<ServiceRegistry&>(*this), salt};
     auto& streamContext = ref.get<StreamContext>();
-    std::scoped_lock<LockableBase(std::mutex)> lock(bindMutex);
+    std::scoped_lock<O2_LOCKABLE(std::mutex)> lock(bindMutex);
     auto& dataProcessorContext = ref.get<DataProcessorContext>();
     ContextHelpers::bindStreamService(dataProcessorContext, streamContext, spec, service);
   } else {
     ServiceRegistryRef ref{const_cast<ServiceRegistry&>(*this), salt};
-    std::scoped_lock<LockableBase(std::mutex)> lock(bindMutex);
+    std::scoped_lock<O2_LOCKABLE(std::mutex)> lock(bindMutex);
     if (ref.active<DataProcessorContext>()) {
       auto& dataProcessorContext = ref.get<DataProcessorContext>();
       ContextHelpers::bindProcessorService(dataProcessorContext, spec, service);
diff --git a/Framework/Foundation/include/Framework/Tracing.h b/Framework/Foundation/include/Framework/Tracing.h
index f5bcecd3889d7..6c9c65f4423e5 100644
--- a/Framework/Foundation/include/Framework/Tracing.h
+++ b/Framework/Foundation/include/Framework/Tracing.h
@@ -39,8 +39,8 @@
 #define TracyAppInfo(...) \
   while (false) {         \
   }
-#define TracyLockableN(T, V, N) T V
-#define LockableBase(T) T
+#define O2_LOCKABLE_NAMED(T, V, N) T V
+#define O2_LOCKABLE(T) T
 #endif
 
 #endif // O2_FRAMEWORK_TRACING_H_

From c5f7aaf11e502e2bf70e0ea74f79936e771848ac Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:19:01 +0100
Subject: [PATCH 366/726] DPL: do not use Tracy anymore

Tracy usage never took off, so I am removing all the explicit calls to it and
I will replace them with Signpost usage, where it still makes sense.
---
 .../Core/src/CommonMessageBackendsHelpers.h   |  1 -
 Framework/Core/src/CommonServices.cxx         |  1 -
 Framework/Core/src/DataProcessingDevice.cxx   | 29 -------------------
 3 files changed, 31 deletions(-)

diff --git a/Framework/Core/src/CommonMessageBackendsHelpers.h b/Framework/Core/src/CommonMessageBackendsHelpers.h
index 10d8a9c5d347a..a3684906ff838 100644
--- a/Framework/Core/src/CommonMessageBackendsHelpers.h
+++ b/Framework/Core/src/CommonMessageBackendsHelpers.h
@@ -38,7 +38,6 @@ struct CommonMessageBackendsHelpers {
   static ServiceProcessingCallback sendCallback()
   {
     return [](ProcessingContext& ctx, void* service) {
-      ZoneScopedN("send message callback");
       T* context = reinterpret_cast<T*>(service);
       DataProcessor::doSend(ctx.services().get<DataSender>(), *context, ctx.services());
     };
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index a0f7fcdc88201..0d0828729ef42 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -741,7 +741,6 @@ auto sendRelayerMetrics(ServiceRegistryRef registry, DataProcessingStats& stats)
     stats.updateStats({static_cast<unsigned short>(static_cast<int>(ProcessingStatsId::AVAILABLE_MANAGED_SHM_BASE) + (runningWorkflow.shmSegmentId % 512)), DataProcessingStats::Op::SetIfPositive, freeMemory});
   }
 
-  ZoneScopedN("send metrics");
   auto device = registry.get<RawDeviceService>().device();
 
   int64_t totalBytesIn = 0;
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 175fc8ab50f6e..75ff3df50c3ea 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -115,14 +115,12 @@ struct ServiceKindExtractor<ConfigurationInterface> {
 /// Watching stdin for commands probably a better approach.
 void on_idle_timer(uv_timer_t* handle)
 {
-  ZoneScopedN("Idle timer");
   auto* state = (DeviceState*)handle->data;
   state->loopReason |= DeviceState::TIMER_EXPIRED;
 }
 
 void on_transition_requested_expired(uv_timer_t* handle)
 {
-  ZoneScopedN("Transition expired");
   auto* state = (DeviceState*)handle->data;
   state->loopReason |= DeviceState::TIMER_EXPIRED;
   LOGP(info, "Timer expired. Forcing transition to READY");
@@ -222,12 +220,10 @@ DataProcessingDevice::DataProcessingDevice(RunningDeviceRef running, ServiceRegi
 // one with the thread id. For the moment we simply use the first one.
 void run_callback(uv_work_t* handle)
 {
-  ZoneScopedN("run_callback");
   auto* task = (TaskStreamInfo*)handle->data;
   auto ref = ServiceRegistryRef{*task->registry, ServiceRegistry::globalStreamSalt(task->id.index + 1)};
   DataProcessingDevice::doPrepare(ref);
   DataProcessingDevice::doRun(ref);
-  //  FrameMark;
 }
 
 // Once the processing in a thread is done, this is executed on the main thread.
@@ -268,7 +264,6 @@ void run_completion(uv_work_t* handle, int status)
   quotaEvaluator.handleExpired(reportExpiredOffer);
   quotaEvaluator.dispose(task->id.index);
   task->running = false;
-  ZoneScopedN("run_completion");
 }
 
 // Context for polling
@@ -377,8 +372,6 @@ void DataProcessingDevice::Init()
   context.statefulProcess = nullptr;
   context.error = spec.algorithm.onError;
   context.initError = spec.algorithm.onInitError;
-  TracyAppInfo(spec.name.data(), spec.name.size());
-  ZoneScopedN("DataProcessingDevice::Init");
 
   auto configStore = DeviceConfigurationHelpers::getConfiguration(mServiceRegistry, spec.name.c_str(), spec.options);
   if (configStore == nullptr) {
@@ -412,7 +405,6 @@ void DataProcessingDevice::Init()
   if (context.initError) {
     context.initErrorHandling = [&errorCallback = context.initError,
                                  &serviceRegistry = mServiceRegistry](RuntimeErrorRef e) {
-      ZoneScopedN("Error handling");
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
       ServiceRegistryRef ref{serviceRegistry, ServiceRegistry::globalDeviceSalt()};
@@ -426,7 +418,6 @@ void DataProcessingDevice::Init()
     };
   } else {
     context.initErrorHandling = [&serviceRegistry = mServiceRegistry](RuntimeErrorRef e) {
-      ZoneScopedN("Error handling");
       auto& err = error_from_ref(e);
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
@@ -449,7 +440,6 @@ void DataProcessingDevice::Init()
       try {
         context.statefulProcess = context.init(initContext);
       } catch (o2::framework::RuntimeErrorRef e) {
-        ZoneScopedN("error handling");
         if (context.initErrorHandling) {
           (context.initErrorHandling)(e);
         }
@@ -458,14 +448,12 @@ void DataProcessingDevice::Init()
       try {
         context.statefulProcess = context.init(initContext);
       } catch (std::exception& ex) {
-        ZoneScopedN("error handling");
         /// Convert a standard exception to a RuntimeErrorRef
         /// Notice how this will lose the backtrace information
         /// and report the exception coming from here.
         auto e = runtime_error(ex.what());
         (context.initErrorHandling)(e);
       } catch (o2::framework::RuntimeErrorRef e) {
-        ZoneScopedN("error handling");
         (context.initErrorHandling)(e);
       }
     }
@@ -621,7 +609,6 @@ static auto toBeforwardedMessageSet = [](std::vector<ChannelIndex>& cachedForwar
 // FIXME: do it in a smarter way than O(N^2)
 static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot, std::vector<MessageSet>& currentSetOfInputs,
                                TimesliceIndex::OldestOutputInfo oldestTimeslice, bool copy, bool consume = true) {
-  ZoneScopedN("forward inputs");
   auto& proxy = registry.get<FairMQDeviceProxy>();
   // we collect all messages per forward in a map and send them together
   std::vector<fair::mq::Parts> forwardedParts;
@@ -1078,7 +1065,6 @@ void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceCont
   if (context.error != nullptr) {
     context.errorHandling = [&errorCallback = context.error,
                              &serviceRegistry = mServiceRegistry](RuntimeErrorRef e, InputRecord& record) {
-      ZoneScopedN("Error handling");
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
       ServiceRegistryRef ref{serviceRegistry, ServiceRegistry::globalDeviceSalt()};
@@ -1093,7 +1079,6 @@ void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceCont
   } else {
     context.errorHandling = [&errorPolicy = mProcessingPolicies.error,
                              &serviceRegistry = mServiceRegistry](RuntimeErrorRef e, InputRecord& record) {
-      ZoneScopedN("Error handling");
       auto& err = error_from_ref(e);
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
@@ -1242,8 +1227,6 @@ void DataProcessingDevice::Run()
     // so that devices which do not have a timer can still start an
     // enumeration.
     {
-      ZoneScopedN("uv idle");
-      TracyPlot("past activity", (int64_t)mWasActive);
       ServiceRegistryRef ref{mServiceRegistry};
       ref.get<DriverClient>().flushPending(mServiceRegistry);
       auto shouldNotWait = (mWasActive &&
@@ -1303,7 +1286,6 @@ void DataProcessingDevice::Run()
       if (state.transitionHandling == TransitionHandlingState::Requested && state.streaming == StreamingState::Idle) {
         state.transitionHandling = TransitionHandlingState::Expired;
       }
-      TracyPlot("shouldNotWait", (int)shouldNotWait);
       if (state.severityStack.empty() == false) {
         fair::Logger::SetConsoleSeverity((fair::Severity)state.severityStack.back());
         state.severityStack.pop_back();
@@ -1349,7 +1331,6 @@ void DataProcessingDevice::Run()
         fair::Logger::SetConsoleSeverity((fair::Severity)state.severityStack.back());
         state.severityStack.pop_back();
       }
-      TracyPlot("loopReason", (int64_t)(uint64_t)state.loopReason);
       LOGP(debug, "Loop reason mask {:b} & {:b} = {:b}",
            state.loopReason, state.tracingFlags,
            state.loopReason & state.tracingFlags);
@@ -1433,7 +1414,6 @@ void DataProcessingDevice::Run()
     } else {
       mWasActive = false;
     }
-    FrameMark;
   }
   auto& spec = ref.get<DeviceSpec const>();
   /// Cleanup messages which are still pending on exit.
@@ -1448,12 +1428,10 @@ void DataProcessingDevice::Run()
 /// non-data triggers like those which are time based.
 void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
 {
-  ZoneScopedN("DataProcessingDevice::doPrepare");
   auto& context = ref.get<DataProcessorContext>();
 
   *context.wasActive = false;
   {
-    ZoneScopedN("CallbackService::Id::ClockTick");
     ref.get<CallbackService>().call<CallbackService::Id::ClockTick>();
   }
   // Whether or not we had something to do.
@@ -1734,7 +1712,6 @@ struct WaitBackpressurePolicy {
 void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo& info)
 {
   auto& context = ref.get<DataProcessorContext>();
-  ZoneScopedN("DataProcessingDevice::handleData");
 
   enum struct InputType : int {
     Invalid = 0,
@@ -1763,7 +1740,6 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
     auto& parts = info.parts;
     stats.updateStats({(int)ProcessingStatsId::TOTAL_INPUTS, DataProcessingStats::Op::Set, (int64_t)parts.Size()});
 
-    TracyPlot("messages received", (int64_t)parts.Size());
     std::vector<InputInfo> results;
     // we can reserve the upper limit
     results.reserve(parts.Size() / 2);
@@ -2060,7 +2036,6 @@ void update_maximum(std::atomic<T>& maximum_value, T const& value) noexcept
 bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::vector<DataRelayer::RecordAction>& completed)
 {
   auto& context = ref.get<DataProcessorContext>();
-  ZoneScopedN("DataProcessingDevice::tryDispatchComputation");
   LOGP(debug, "DataProcessingDevice::tryDispatchComputation");
   // This is the actual hidden state for the outer loop. In case we decide we
   // want to support multithreaded dispatching of operations, I can simply
@@ -2135,7 +2110,6 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   auto prepareAllocatorForCurrentTimeSlice = [ref](TimesliceSlot i) -> void {
     auto& relayer = ref.get<DataRelayer>();
     auto& timingInfo = ref.get<TimingInfo>();
-    ZoneScopedN("DataProcessingDevice::prepareForCurrentTimeslice");
     auto timeslice = relayer.getTimesliceForSlot(i);
 
     timingInfo.timeslice = timeslice.value;
@@ -2422,21 +2396,18 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       try {
         runNoCatch(action);
       } catch (o2::framework::RuntimeErrorRef e) {
-        ZoneScopedN("error handling");
         (context.errorHandling)(e, record);
       }
     } else {
       try {
         runNoCatch(action);
       } catch (std::exception& ex) {
-        ZoneScopedN("error handling");
         /// Convert a standard exception to a RuntimeErrorRef
         /// Notice how this will lose the backtrace information
         /// and report the exception coming from here.
         auto e = runtime_error(ex.what());
         (context.errorHandling)(e, record);
       } catch (o2::framework::RuntimeErrorRef e) {
-        ZoneScopedN("error handling");
         (context.errorHandling)(e, record);
       }
     }

From 23d38302df6414abe92559ca3a6badafb109434c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:19:01 +0100
Subject: [PATCH 367/726] DPL: drop unused Tracy APIs

ZoneScoped*, TracyAppInfo, FrameMark, TracyAlloc, TracyPlot were all removed
and will be replaced by Signposts as needed.
---
 .../Foundation/include/Framework/Tracing.h    | 24 -------------------
 1 file changed, 24 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Tracing.h b/Framework/Foundation/include/Framework/Tracing.h
index 6c9c65f4423e5..91c3b99eb8dd0 100644
--- a/Framework/Foundation/include/Framework/Tracing.h
+++ b/Framework/Foundation/include/Framework/Tracing.h
@@ -15,30 +15,6 @@
 #define DPL_HAS_TRACING
 #include <tracy/Tracy.hpp>
 #else
-#define ZoneScoped \
-  while (false) {  \
-  }
-#define FrameMark \
-  while (false) { \
-  }
-#define TracyPlot(...) \
-  while (false) {      \
-  }
-#define ZoneScopedN(...) \
-  while (false) {        \
-  }
-#define ZoneScopedNS(...) \
-  while (false) {         \
-  }
-#define TracyAlloc(...) \
-  while (false) {       \
-  }
-#define TracyFree(...) \
-  while (false) {      \
-  }
-#define TracyAppInfo(...) \
-  while (false) {         \
-  }
 #define O2_LOCKABLE_NAMED(T, V, N) T V
 #define O2_LOCKABLE(T) T
 #endif

From 196f66c99375ea0bc7292ecaef191d7cfc503508 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:19:01 +0100
Subject: [PATCH 368/726] DPL: drop tracy support from the GUI

No one is using it and Signposts are probably a better alternative.
---
 .../GUISupport/src/FrameworkGUIDeviceInspector.cxx  | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
index eb62fcc09d338..3ef8b99ed26bb 100644
--- a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
@@ -261,9 +261,6 @@ void displayDeviceInspector(DeviceSpec const& spec,
     ImGui::Text("Pid: %d (exit status: %d)", info.pid, info.exitStatus);
   }
   ImGui::Text("Device state: %s", info.deviceState.data());
-#ifdef DPL_ENABLE_TRACING
-  ImGui::Text("Tracy Port: %d", info.tracyPort);
-#endif
   ImGui::Text("Rank: %zu/%zu%%%zu/%zu", spec.rank, spec.nSlots, spec.inputTimesliceId, spec.maxInputTimeslices);
 
   if (ImGui::Button(ICON_FA_BUG "Attach debugger")) {
@@ -324,16 +321,6 @@ void displayDeviceInspector(DeviceSpec const& spec,
   }
 #endif
 
-#if DPL_ENABLE_TRACING
-  ImGui::SameLine();
-  if (ImGui::Button("Tracy")) {
-    std::string tracyPort = std::to_string(info.tracyPort);
-    auto cmd = fmt::format("tracy-profiler -p {} -a 127.0.0.1 &", info.tracyPort);
-    LOG(debug) << cmd;
-    int retVal = system(cmd.c_str());
-    (void)retVal;
-  }
-#endif
   if (control.controller) {
     if (ImGui::Button("Offer SHM")) {
       control.controller->write("/shm-offer 1000", strlen("/shm-offer 1000"));

From 20fada9f64355bf67f0ad2caba88997641e623bd Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:19:01 +0100
Subject: [PATCH 369/726] DPL: drop tracy support for good

---
 Framework/Core/src/DataProcessingDevice.cxx      | 4 ----
 Framework/Foundation/include/Framework/Tracing.h | 7 ++++---
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 75ff3df50c3ea..4a9f5009b342e 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -8,10 +8,6 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
-#ifdef DPL_ENABLE_TRACING
-#define TRACY_ENABLE
-#include <tracy/TracyClient.cpp>
-#endif
 #include "Framework/AsyncQueue.h"
 #include "Framework/DataProcessingDevice.h"
 #include "Framework/ChannelMatching.h"
diff --git a/Framework/Foundation/include/Framework/Tracing.h b/Framework/Foundation/include/Framework/Tracing.h
index 91c3b99eb8dd0..72c052ca6c161 100644
--- a/Framework/Foundation/include/Framework/Tracing.h
+++ b/Framework/Foundation/include/Framework/Tracing.h
@@ -11,9 +11,10 @@
 #ifndef O2_FRAMEWORK_TRACING_H_
 #define O2_FRAMEWORK_TRACING_H_
 
-#if DPL_ENABLE_TRACING && __has_include(<tracy/Tracy.hpp>)
-#define DPL_HAS_TRACING
-#include <tracy/Tracy.hpp>
+#if DPL_ENABLE_TRACING
+// FIXME: not implemented yet in terms of Signposts
+#define O2_LOCKABLE_NAMED(T, V, N) T V
+#define O2_LOCKABLE(T) T
 #else
 #define O2_LOCKABLE_NAMED(T, V, N) T V
 #define O2_LOCKABLE(T) T

From 46d28a27e2472cf3e7215e7c467b2aa7d3cffbfb Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 22 Feb 2024 12:19:01 +0100
Subject: [PATCH 370/726] [FEAT] Drop support for Tracy

---
 Framework/CHANGELOG.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/Framework/CHANGELOG.md b/Framework/CHANGELOG.md
index e7278733de246..044274d9515c4 100644
--- a/Framework/CHANGELOG.md
+++ b/Framework/CHANGELOG.md
@@ -1,3 +1,12 @@
+# 2024-02-22: Drop Tracy support
+
+Tracy support never took off, so I am dropping it. This was mostly because people do not know about it and having a per process profile GUI was way unpractical. Moreover, needing an extra compile time flag meant one most likely did not have the support compiled in when needed.
+
+I have therefore decided to replace it with signposts, which hopefully will see better adoption thanks
+to the integration with Instruments on mac and the easy way they can be enabled dynamically.
+
+We could then reintroduce Tracy support as a hook on top of signposts, if really needed.
+
 # 2024-02-16: Improved Signposts.
 
 In particular:

From cf44ce925ac12722847fae3dd0d54d52c91e80c5 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 23 Feb 2024 06:56:31 +0100
Subject: [PATCH 371/726] [O2-4660] DPL: increase Dispatcher sending policy
 timeout

Now to 50 ms * 10 before switching to lossy.
---
 Framework/Core/src/SendingPolicy.cxx | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index 3dbbc451915b0..bf97d56b4a5b8 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -50,16 +50,16 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               auto &proxy = registry.get<FairMQDeviceProxy>();
               OutputChannelInfo const& info = proxy.getOutputChannelInfo(channelIndex);
               OutputChannelState& state = proxy.getOutputChannelState(channelIndex);
-              // Default timeout is 10ms.
+              // Default timeout is 50ms.
               // We count the number of consecutively dropped messages.
               // If we have more than 10, we switch to a completely
               // non-blocking approach.
-              int64_t timeout = 10;
+              int64_t timeout = 50;
               if (state.droppedMessages == 10 + 1) {
                 LOG(warning) << "Failed to send 10 messages with 10ms timeout in a row, switching to completely non-blocking mode.";
               }
               if (state.droppedMessages == 0) {
-                timeout = 10;
+                timeout = 50;
               }
               if (state.droppedMessages > 10) {
                 timeout = 0;

From 6700ec309da85f324d0738525910850a219ff180 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 23 Feb 2024 08:14:05 +0100
Subject: [PATCH 372/726] DPL: drop support for serialising / deserializing
 schema

---
 .../include/Framework/TMessageSerializer.h    |  8 ---
 Framework/Core/src/TMessageSerializer.cxx     | 64 -------------------
 2 files changed, 72 deletions(-)

diff --git a/Framework/Core/include/Framework/TMessageSerializer.h b/Framework/Core/include/Framework/TMessageSerializer.h
index b457f0ffc5f99..d9f0ca8edf12e 100644
--- a/Framework/Core/include/Framework/TMessageSerializer.h
+++ b/Framework/Core/include/Framework/TMessageSerializer.h
@@ -80,14 +80,6 @@ struct TMessageSerializer {
   template <typename T = TObject>
   static inline std::unique_ptr<T> deserialize(std::byte* buffer, size_t size);
 
-  // load the schema information from a message/buffer
-  static void loadSchema(const fair::mq::Message& msg);
-  static void loadSchema(gsl::span<std::byte> buffer);
-
-  // write the schema into an empty message/buffer
-  static void fillSchema(fair::mq::Message& msg, const StreamerList& streamers);
-  static void fillSchema(FairTMessage& msg, const StreamerList& streamers);
-
   // get the streamers
   static StreamerList getStreamers();
 
diff --git a/Framework/Core/src/TMessageSerializer.cxx b/Framework/Core/src/TMessageSerializer.cxx
index 6276cd74152cc..f3e8a37a2b8d2 100644
--- a/Framework/Core/src/TMessageSerializer.cxx
+++ b/Framework/Core/src/TMessageSerializer.cxx
@@ -17,70 +17,6 @@ using namespace o2::framework;
 TMessageSerializer::StreamerList TMessageSerializer::sStreamers{};
 std::mutex TMessageSerializer::sStreamersLock{};
 
-void TMessageSerializer::loadSchema(gsl::span<std::byte> buffer)
-{
-  std::unique_ptr<TObject> obj = deserialize(buffer);
-
-  TObjArray* pSchemas = dynamic_cast<TObjArray*>(obj.get());
-  if (!pSchemas) {
-    return;
-  }
-
-  // TODO: this is a bit of a problem in general: non-owning ROOT containers should become
-  // owners at deserialize, otherwise there is a leak. Switch to a better container.
-  pSchemas->SetOwner(kTRUE);
-
-  for (int i = 0; i < pSchemas->GetEntriesFast(); i++) {
-    TStreamerInfo* pSchema = dynamic_cast<TStreamerInfo*>(pSchemas->At(i));
-    if (!pSchema) {
-      continue;
-    }
-    int version = pSchema->GetClassVersion();
-    TClass* pClass = TClass::GetClass(pSchema->GetName());
-    if (!pClass) {
-      continue;
-    }
-    if (pClass->GetClassVersion() == version) {
-      continue;
-    }
-    TObjArray* pInfos = const_cast<TObjArray*>(pClass->GetStreamerInfos());
-    if (!pInfos) {
-      continue;
-    }
-    TVirtualStreamerInfo* pInfo = dynamic_cast<TVirtualStreamerInfo*>(pInfos->At(version));
-    if (pInfo) {
-      continue;
-    }
-    pSchema->SetClass(pClass);
-    pSchema->BuildOld();
-    pInfos->AddAtAndExpand(pSchema, version);
-    pSchemas->Remove(pSchema);
-  }
-}
-
-void TMessageSerializer::fillSchema(FairTMessage& msg, const StreamerList& streamers)
-{
-  // TODO: this is a bit of a problem in general: non-owning ROOT containers should become
-  // owners at deserialize, otherwise there is a leak. Switch to a better container.
-  TObjArray infoArray{};
-  for (const auto& info : streamers) {
-    infoArray.Add(info);
-  }
-  serialize(msg, &infoArray);
-}
-
-void TMessageSerializer::loadSchema(const fair::mq::Message& msg) { loadSchema(as_span(msg)); }
-void TMessageSerializer::fillSchema(fair::mq::Message& msg, const StreamerList& streamers)
-{
-  // TODO: this is a bit of a problem in general: non-owning ROOT containers should become
-  // owners at deserialize, otherwise there is a leak. Switch to a better container.
-  TObjArray infoArray{};
-  for (const auto& info : streamers) {
-    infoArray.Add(info);
-  }
-  Serialize(msg, &infoArray);
-}
-
 void TMessageSerializer::updateStreamers(const FairTMessage& message, StreamerList& streamers)
 {
   std::lock_guard<std::mutex> lock{TMessageSerializer::sStreamersLock};

From 9b7b506da0463f21a1d6db90fe62528b1c626e1e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 23 Feb 2024 08:19:26 +0100
Subject: [PATCH 373/726] [O2-4578] DPL: attempt at removing a race condition
 in the rate limiting

---
 Framework/Core/src/CommonDataProcessors.cxx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 48a3eb1da95b9..05950d9d1e855 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -580,11 +580,11 @@ DataProcessorSpec CommonDataProcessors::getDummySink(std::vector<InputSpec> cons
         auto oldestPossingTimeslice = timesliceIndex.getOldestPossibleOutput().timeslice.value;
         if (channel != device->GetChannels().end()) {
           fair::mq::MessagePtr payload(device->NewMessage());
-          size_t* consumed = (size_t*)malloc(sizeof(size_t));
+          auto* consumed = (size_t*)malloc(sizeof(size_t));
           *consumed = oldestPossingTimeslice;
           if (*consumed != lastTimeslice) {
             payload->Rebuild(consumed, sizeof(int64_t), nullptr, nullptr);
-            channel->second[0].Send(payload);
+            auto result = channel->second[0].Send(payload, 100);
             lastTimeslice = *consumed;
           }
         }

From 443c2d7b12d0ca2965fa6e04461f943724a57b1e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 23 Feb 2024 08:14:05 +0100
Subject: [PATCH 374/726] DPL: drop support for serialising / deserializing
 schema

---
 .../include/Framework/TMessageSerializer.h    | 55 ++----------
 Framework/Core/src/TMessageSerializer.cxx     | 89 -------------------
 2 files changed, 6 insertions(+), 138 deletions(-)

diff --git a/Framework/Core/include/Framework/TMessageSerializer.h b/Framework/Core/include/Framework/TMessageSerializer.h
index b457f0ffc5f99..52c4de984b221 100644
--- a/Framework/Core/include/Framework/TMessageSerializer.h
+++ b/Framework/Core/include/Framework/TMessageSerializer.h
@@ -26,9 +26,7 @@
 #include <mutex>
 #include <cstddef>
 
-namespace o2
-{
-namespace framework
+namespace o2::framework
 {
 class FairTMessage;
 
@@ -51,28 +49,23 @@ class FairTMessage : public TMessage
 struct TMessageSerializer {
   using StreamerList = std::vector<TVirtualStreamerInfo*>;
   using CompressionLevel = int;
-  enum class CacheStreamers { yes,
-                              no };
 
   static void Serialize(fair::mq::Message& msg, const TObject* input,
-                        CacheStreamers streamers = CacheStreamers::no,
                         CompressionLevel compressionLevel = -1);
 
   template <typename T>
   static void Serialize(fair::mq::Message& msg, const T* input, const TClass* cl, //
-                        CacheStreamers streamers = CacheStreamers::no,            //
                         CompressionLevel compressionLevel = -1);
 
   template <typename T = TObject>
   static void Deserialize(const fair::mq::Message& msg, std::unique_ptr<T>& output);
 
   static void serialize(FairTMessage& msg, const TObject* input,
-                        CacheStreamers streamers = CacheStreamers::no,
                         CompressionLevel compressionLevel = -1);
 
   template <typename T>
   static void serialize(FairTMessage& msg, const T* input,                               //
-                        const TClass* cl, CacheStreamers streamers = CacheStreamers::no, //
+                        const TClass* cl, 
                         CompressionLevel compressionLevel = -1);
 
   template <typename T = TObject>
@@ -80,45 +73,22 @@ struct TMessageSerializer {
   template <typename T = TObject>
   static inline std::unique_ptr<T> deserialize(std::byte* buffer, size_t size);
 
-  // load the schema information from a message/buffer
-  static void loadSchema(const fair::mq::Message& msg);
-  static void loadSchema(gsl::span<std::byte> buffer);
-
-  // write the schema into an empty message/buffer
-  static void fillSchema(fair::mq::Message& msg, const StreamerList& streamers);
-  static void fillSchema(FairTMessage& msg, const StreamerList& streamers);
-
   // get the streamers
   static StreamerList getStreamers();
 
-  // update the streamer list with infos appropriate for this type
-  static void updateStreamers(const TObject* object);
-
  private:
-  // update the cache of streamer infos for serialized classes
-  static void updateStreamers(const FairTMessage& message, StreamerList& streamers);
-
-  // for now this is a static, maybe it would be better to move the storage somewhere else?
-  static StreamerList sStreamers;
-  static std::mutex sStreamersLock;
 };
 
 inline void TMessageSerializer::serialize(FairTMessage& tm, const TObject* input,
-                                          CacheStreamers streamers,
                                           CompressionLevel compressionLevel)
 {
-  return serialize(tm, input, nullptr, streamers, compressionLevel);
+  return serialize(tm, input, nullptr, compressionLevel);
 }
 
 template <typename T>
 inline void TMessageSerializer::serialize(FairTMessage& tm, const T* input,           //
-                                          const TClass* cl, CacheStreamers streamers, //
-                                          CompressionLevel compressionLevel)
+                                          const TClass* cl, CompressionLevel compressionLevel)
 {
-  if (streamers == CacheStreamers::yes) {
-    tm.EnableSchemaEvolution(true);
-  }
-
   if (compressionLevel >= 0) {
     // if negative, skip to use ROOT default
     tm.SetCompressionLevel(compressionLevel);
@@ -130,10 +100,6 @@ inline void TMessageSerializer::serialize(FairTMessage& tm, const T* input,
   } else {
     tm.WriteObjectAny(input, cl);
   }
-
-  if (streamers == CacheStreamers::yes) {
-    updateStreamers(tm, sStreamers);
-  }
 }
 
 template <typename T>
@@ -172,12 +138,11 @@ inline void FairTMessage::free(void* /*data*/, void* hint)
 }
 
 inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const TObject* input,
-                                          TMessageSerializer::CacheStreamers streamers,
                                           TMessageSerializer::CompressionLevel compressionLevel)
 {
   std::unique_ptr<FairTMessage> tm = std::make_unique<FairTMessage>(kMESS_OBJECT);
 
-  serialize(*tm, input, input->Class(), streamers, compressionLevel);
+  serialize(*tm, input, input->Class(), compressionLevel);
 
   msg.Rebuild(tm->Buffer(), tm->BufferSize(), FairTMessage::free, tm.get());
   tm.release();
@@ -186,12 +151,11 @@ inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const TObject*
 template <typename T>
 inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const T* input,       //
                                           const TClass* cl,                             //
-                                          TMessageSerializer::CacheStreamers streamers, //
                                           TMessageSerializer::CompressionLevel compressionLevel)
 {
   std::unique_ptr<FairTMessage> tm = std::make_unique<FairTMessage>(kMESS_OBJECT);
 
-  serialize(*tm, input, cl, streamers, compressionLevel);
+  serialize(*tm, input, cl, compressionLevel);
 
   msg.Rebuild(tm->Buffer(), tm->BufferSize(), FairTMessage::free, tm.get());
   tm.release();
@@ -205,12 +169,6 @@ inline void TMessageSerializer::Deserialize(const fair::mq::Message& msg, std::u
   output = deserialize(as_span(msg));
 }
 
-inline TMessageSerializer::StreamerList TMessageSerializer::getStreamers()
-{
-  std::lock_guard<std::mutex> lock{TMessageSerializer::sStreamersLock};
-  return sStreamers;
-}
-
 // gsl::narrow is used to do a runtime narrowing check, this might be a bit paranoid,
 // we would probably be fine with e.g. gsl::narrow_cast (or just a static_cast)
 inline gsl::span<std::byte> as_span(const fair::mq::Message& msg)
@@ -224,6 +182,5 @@ inline gsl::span<std::byte> as_span(const FairTMessage& msg)
                              gsl::narrow<gsl::span<std::byte>::size_type>(msg.BufferSize())};
 }
 
-} // namespace framework
 } // namespace o2
 #endif // FRAMEWORK_TMESSAGESERIALIZER_H
diff --git a/Framework/Core/src/TMessageSerializer.cxx b/Framework/Core/src/TMessageSerializer.cxx
index 6276cd74152cc..5388a6d716cda 100644
--- a/Framework/Core/src/TMessageSerializer.cxx
+++ b/Framework/Core/src/TMessageSerializer.cxx
@@ -13,92 +13,3 @@
 #include <memory>
 
 using namespace o2::framework;
-
-TMessageSerializer::StreamerList TMessageSerializer::sStreamers{};
-std::mutex TMessageSerializer::sStreamersLock{};
-
-void TMessageSerializer::loadSchema(gsl::span<std::byte> buffer)
-{
-  std::unique_ptr<TObject> obj = deserialize(buffer);
-
-  TObjArray* pSchemas = dynamic_cast<TObjArray*>(obj.get());
-  if (!pSchemas) {
-    return;
-  }
-
-  // TODO: this is a bit of a problem in general: non-owning ROOT containers should become
-  // owners at deserialize, otherwise there is a leak. Switch to a better container.
-  pSchemas->SetOwner(kTRUE);
-
-  for (int i = 0; i < pSchemas->GetEntriesFast(); i++) {
-    TStreamerInfo* pSchema = dynamic_cast<TStreamerInfo*>(pSchemas->At(i));
-    if (!pSchema) {
-      continue;
-    }
-    int version = pSchema->GetClassVersion();
-    TClass* pClass = TClass::GetClass(pSchema->GetName());
-    if (!pClass) {
-      continue;
-    }
-    if (pClass->GetClassVersion() == version) {
-      continue;
-    }
-    TObjArray* pInfos = const_cast<TObjArray*>(pClass->GetStreamerInfos());
-    if (!pInfos) {
-      continue;
-    }
-    TVirtualStreamerInfo* pInfo = dynamic_cast<TVirtualStreamerInfo*>(pInfos->At(version));
-    if (pInfo) {
-      continue;
-    }
-    pSchema->SetClass(pClass);
-    pSchema->BuildOld();
-    pInfos->AddAtAndExpand(pSchema, version);
-    pSchemas->Remove(pSchema);
-  }
-}
-
-void TMessageSerializer::fillSchema(FairTMessage& msg, const StreamerList& streamers)
-{
-  // TODO: this is a bit of a problem in general: non-owning ROOT containers should become
-  // owners at deserialize, otherwise there is a leak. Switch to a better container.
-  TObjArray infoArray{};
-  for (const auto& info : streamers) {
-    infoArray.Add(info);
-  }
-  serialize(msg, &infoArray);
-}
-
-void TMessageSerializer::loadSchema(const fair::mq::Message& msg) { loadSchema(as_span(msg)); }
-void TMessageSerializer::fillSchema(fair::mq::Message& msg, const StreamerList& streamers)
-{
-  // TODO: this is a bit of a problem in general: non-owning ROOT containers should become
-  // owners at deserialize, otherwise there is a leak. Switch to a better container.
-  TObjArray infoArray{};
-  for (const auto& info : streamers) {
-    infoArray.Add(info);
-  }
-  Serialize(msg, &infoArray);
-}
-
-void TMessageSerializer::updateStreamers(const FairTMessage& message, StreamerList& streamers)
-{
-  std::lock_guard<std::mutex> lock{TMessageSerializer::sStreamersLock};
-
-  TIter nextStreamer(message.GetStreamerInfos()); // unfortunately ROOT uses TList* here
-  // this looks like we could use std::map here.
-  while (TVirtualStreamerInfo* in = static_cast<TVirtualStreamerInfo*>(nextStreamer())) {
-    auto found = std::find_if(streamers.begin(), streamers.end(), [&](const auto& old) {
-      return (old->GetName() == in->GetName() && old->GetClassVersion() == in->GetClassVersion());
-    });
-    if (found == streamers.end()) {
-      streamers.push_back(in);
-    }
-  }
-}
-
-void TMessageSerializer::updateStreamers(const TObject* object)
-{
-  FairTMessage msg(kMESS_OBJECT);
-  serialize(msg, object, CacheStreamers::yes, CompressionLevel{0});
-}

From fc7baac78d87738867856832de86f701454b97b9 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 23 Feb 2024 10:13:36 +0100
Subject: [PATCH 375/726] DPL: drop support for serialising / deserializing
 schema

---
 .../include/Framework/TMessageSerializer.h    | 72 +++------------
 Framework/Core/src/TMessageSerializer.cxx     | 89 -------------------
 2 files changed, 12 insertions(+), 149 deletions(-)

diff --git a/Framework/Core/include/Framework/TMessageSerializer.h b/Framework/Core/include/Framework/TMessageSerializer.h
index b457f0ffc5f99..1f08b456c0218 100644
--- a/Framework/Core/include/Framework/TMessageSerializer.h
+++ b/Framework/Core/include/Framework/TMessageSerializer.h
@@ -26,9 +26,7 @@
 #include <mutex>
 #include <cstddef>
 
-namespace o2
-{
-namespace framework
+namespace o2::framework
 {
 class FairTMessage;
 
@@ -51,74 +49,41 @@ class FairTMessage : public TMessage
 struct TMessageSerializer {
   using StreamerList = std::vector<TVirtualStreamerInfo*>;
   using CompressionLevel = int;
-  enum class CacheStreamers { yes,
-                              no };
 
   static void Serialize(fair::mq::Message& msg, const TObject* input,
-                        CacheStreamers streamers = CacheStreamers::no,
                         CompressionLevel compressionLevel = -1);
 
   template <typename T>
   static void Serialize(fair::mq::Message& msg, const T* input, const TClass* cl, //
-                        CacheStreamers streamers = CacheStreamers::no,            //
                         CompressionLevel compressionLevel = -1);
 
   template <typename T = TObject>
   static void Deserialize(const fair::mq::Message& msg, std::unique_ptr<T>& output);
 
   static void serialize(FairTMessage& msg, const TObject* input,
-                        CacheStreamers streamers = CacheStreamers::no,
                         CompressionLevel compressionLevel = -1);
 
   template <typename T>
-  static void serialize(FairTMessage& msg, const T* input,                               //
-                        const TClass* cl, CacheStreamers streamers = CacheStreamers::no, //
+  static void serialize(FairTMessage& msg, const T* input, //
+                        const TClass* cl,
                         CompressionLevel compressionLevel = -1);
 
   template <typename T = TObject>
   static std::unique_ptr<T> deserialize(gsl::span<std::byte> buffer);
   template <typename T = TObject>
   static inline std::unique_ptr<T> deserialize(std::byte* buffer, size_t size);
-
-  // load the schema information from a message/buffer
-  static void loadSchema(const fair::mq::Message& msg);
-  static void loadSchema(gsl::span<std::byte> buffer);
-
-  // write the schema into an empty message/buffer
-  static void fillSchema(fair::mq::Message& msg, const StreamerList& streamers);
-  static void fillSchema(FairTMessage& msg, const StreamerList& streamers);
-
-  // get the streamers
-  static StreamerList getStreamers();
-
-  // update the streamer list with infos appropriate for this type
-  static void updateStreamers(const TObject* object);
-
- private:
-  // update the cache of streamer infos for serialized classes
-  static void updateStreamers(const FairTMessage& message, StreamerList& streamers);
-
-  // for now this is a static, maybe it would be better to move the storage somewhere else?
-  static StreamerList sStreamers;
-  static std::mutex sStreamersLock;
 };
 
 inline void TMessageSerializer::serialize(FairTMessage& tm, const TObject* input,
-                                          CacheStreamers streamers,
                                           CompressionLevel compressionLevel)
 {
-  return serialize(tm, input, nullptr, streamers, compressionLevel);
+  return serialize(tm, input, nullptr, compressionLevel);
 }
 
 template <typename T>
-inline void TMessageSerializer::serialize(FairTMessage& tm, const T* input,           //
-                                          const TClass* cl, CacheStreamers streamers, //
-                                          CompressionLevel compressionLevel)
+inline void TMessageSerializer::serialize(FairTMessage& tm, const T* input, //
+                                          const TClass* cl, CompressionLevel compressionLevel)
 {
-  if (streamers == CacheStreamers::yes) {
-    tm.EnableSchemaEvolution(true);
-  }
-
   if (compressionLevel >= 0) {
     // if negative, skip to use ROOT default
     tm.SetCompressionLevel(compressionLevel);
@@ -130,10 +95,6 @@ inline void TMessageSerializer::serialize(FairTMessage& tm, const T* input,
   } else {
     tm.WriteObjectAny(input, cl);
   }
-
-  if (streamers == CacheStreamers::yes) {
-    updateStreamers(tm, sStreamers);
-  }
 }
 
 template <typename T>
@@ -172,26 +133,24 @@ inline void FairTMessage::free(void* /*data*/, void* hint)
 }
 
 inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const TObject* input,
-                                          TMessageSerializer::CacheStreamers streamers,
                                           TMessageSerializer::CompressionLevel compressionLevel)
 {
   std::unique_ptr<FairTMessage> tm = std::make_unique<FairTMessage>(kMESS_OBJECT);
 
-  serialize(*tm, input, input->Class(), streamers, compressionLevel);
+  serialize(*tm, input, input->Class(), compressionLevel);
 
   msg.Rebuild(tm->Buffer(), tm->BufferSize(), FairTMessage::free, tm.get());
   tm.release();
 }
 
 template <typename T>
-inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const T* input,       //
-                                          const TClass* cl,                             //
-                                          TMessageSerializer::CacheStreamers streamers, //
+inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const T* input, //
+                                          const TClass* cl,                       //
                                           TMessageSerializer::CompressionLevel compressionLevel)
 {
   std::unique_ptr<FairTMessage> tm = std::make_unique<FairTMessage>(kMESS_OBJECT);
 
-  serialize(*tm, input, cl, streamers, compressionLevel);
+  serialize(*tm, input, cl, compressionLevel);
 
   msg.Rebuild(tm->Buffer(), tm->BufferSize(), FairTMessage::free, tm.get());
   tm.release();
@@ -205,12 +164,6 @@ inline void TMessageSerializer::Deserialize(const fair::mq::Message& msg, std::u
   output = deserialize(as_span(msg));
 }
 
-inline TMessageSerializer::StreamerList TMessageSerializer::getStreamers()
-{
-  std::lock_guard<std::mutex> lock{TMessageSerializer::sStreamersLock};
-  return sStreamers;
-}
-
 // gsl::narrow is used to do a runtime narrowing check, this might be a bit paranoid,
 // we would probably be fine with e.g. gsl::narrow_cast (or just a static_cast)
 inline gsl::span<std::byte> as_span(const fair::mq::Message& msg)
@@ -221,9 +174,8 @@ inline gsl::span<std::byte> as_span(const fair::mq::Message& msg)
 inline gsl::span<std::byte> as_span(const FairTMessage& msg)
 {
   return gsl::span<std::byte>{reinterpret_cast<std::byte*>(msg.Buffer()),
-                             gsl::narrow<gsl::span<std::byte>::size_type>(msg.BufferSize())};
+                              gsl::narrow<gsl::span<std::byte>::size_type>(msg.BufferSize())};
 }
 
-} // namespace framework
-} // namespace o2
+} // namespace o2::framework
 #endif // FRAMEWORK_TMESSAGESERIALIZER_H
diff --git a/Framework/Core/src/TMessageSerializer.cxx b/Framework/Core/src/TMessageSerializer.cxx
index 6276cd74152cc..5388a6d716cda 100644
--- a/Framework/Core/src/TMessageSerializer.cxx
+++ b/Framework/Core/src/TMessageSerializer.cxx
@@ -13,92 +13,3 @@
 #include <memory>
 
 using namespace o2::framework;
-
-TMessageSerializer::StreamerList TMessageSerializer::sStreamers{};
-std::mutex TMessageSerializer::sStreamersLock{};
-
-void TMessageSerializer::loadSchema(gsl::span<std::byte> buffer)
-{
-  std::unique_ptr<TObject> obj = deserialize(buffer);
-
-  TObjArray* pSchemas = dynamic_cast<TObjArray*>(obj.get());
-  if (!pSchemas) {
-    return;
-  }
-
-  // TODO: this is a bit of a problem in general: non-owning ROOT containers should become
-  // owners at deserialize, otherwise there is a leak. Switch to a better container.
-  pSchemas->SetOwner(kTRUE);
-
-  for (int i = 0; i < pSchemas->GetEntriesFast(); i++) {
-    TStreamerInfo* pSchema = dynamic_cast<TStreamerInfo*>(pSchemas->At(i));
-    if (!pSchema) {
-      continue;
-    }
-    int version = pSchema->GetClassVersion();
-    TClass* pClass = TClass::GetClass(pSchema->GetName());
-    if (!pClass) {
-      continue;
-    }
-    if (pClass->GetClassVersion() == version) {
-      continue;
-    }
-    TObjArray* pInfos = const_cast<TObjArray*>(pClass->GetStreamerInfos());
-    if (!pInfos) {
-      continue;
-    }
-    TVirtualStreamerInfo* pInfo = dynamic_cast<TVirtualStreamerInfo*>(pInfos->At(version));
-    if (pInfo) {
-      continue;
-    }
-    pSchema->SetClass(pClass);
-    pSchema->BuildOld();
-    pInfos->AddAtAndExpand(pSchema, version);
-    pSchemas->Remove(pSchema);
-  }
-}
-
-void TMessageSerializer::fillSchema(FairTMessage& msg, const StreamerList& streamers)
-{
-  // TODO: this is a bit of a problem in general: non-owning ROOT containers should become
-  // owners at deserialize, otherwise there is a leak. Switch to a better container.
-  TObjArray infoArray{};
-  for (const auto& info : streamers) {
-    infoArray.Add(info);
-  }
-  serialize(msg, &infoArray);
-}
-
-void TMessageSerializer::loadSchema(const fair::mq::Message& msg) { loadSchema(as_span(msg)); }
-void TMessageSerializer::fillSchema(fair::mq::Message& msg, const StreamerList& streamers)
-{
-  // TODO: this is a bit of a problem in general: non-owning ROOT containers should become
-  // owners at deserialize, otherwise there is a leak. Switch to a better container.
-  TObjArray infoArray{};
-  for (const auto& info : streamers) {
-    infoArray.Add(info);
-  }
-  Serialize(msg, &infoArray);
-}
-
-void TMessageSerializer::updateStreamers(const FairTMessage& message, StreamerList& streamers)
-{
-  std::lock_guard<std::mutex> lock{TMessageSerializer::sStreamersLock};
-
-  TIter nextStreamer(message.GetStreamerInfos()); // unfortunately ROOT uses TList* here
-  // this looks like we could use std::map here.
-  while (TVirtualStreamerInfo* in = static_cast<TVirtualStreamerInfo*>(nextStreamer())) {
-    auto found = std::find_if(streamers.begin(), streamers.end(), [&](const auto& old) {
-      return (old->GetName() == in->GetName() && old->GetClassVersion() == in->GetClassVersion());
-    });
-    if (found == streamers.end()) {
-      streamers.push_back(in);
-    }
-  }
-}
-
-void TMessageSerializer::updateStreamers(const TObject* object)
-{
-  FairTMessage msg(kMESS_OBJECT);
-  serialize(msg, object, CacheStreamers::yes, CompressionLevel{0});
-}

From af2fb49149c828b2c28c6b67e36b12ebabe4c3d6 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 23 Feb 2024 20:36:58 +0100
Subject: [PATCH 376/726] DPL: avoid TMessage usage

TMessage does not allow for non owned buffers, so we end up having
an extra buffer in private memory for (de)serializing. Using TBufferFile
directly allows to avoid that, so this moves the whole ROOT serialization
support in DPL to use it.
---
 .../src/AODJAlienReaderHelpers.h              |   2 +
 .../Core/include/Framework/DataAllocator.h    |   1 +
 .../Core/include/Framework/DataRefUtils.h     |  13 +-
 .../include/Framework/RootMessageContext.h    |   3 +
 .../Framework/RootSerializationSupport.h      |   3 +-
 .../include/Framework/TMessageSerializer.h    | 130 ++++++++----------
 Framework/Core/src/CommonDataProcessors.cxx   |   4 +-
 Framework/Core/src/TMessageSerializer.cxx     |  31 +++++
 Framework/Core/test/test_DataRefUtils.cxx     |  28 +++-
 .../Core/test/test_TMessageSerializer.cxx     |  31 +++--
 Utilities/Mergers/src/ObjectStore.cxx         |  11 +-
 Utilities/Mergers/test/benchmark_Types.cxx    |  11 +-
 12 files changed, 167 insertions(+), 101 deletions(-)

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
index 655e4b6c0b439..4b9fd710aca14 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
@@ -16,7 +16,9 @@
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/Logger.h"
 #include <Monitoring/Monitoring.h>
+
 #include <uv.h>
+class TFile;
 
 namespace o2::framework::readers
 {
diff --git a/Framework/Core/include/Framework/DataAllocator.h b/Framework/Core/include/Framework/DataAllocator.h
index 8151d2f83c6c6..029e922aeb90b 100644
--- a/Framework/Core/include/Framework/DataAllocator.h
+++ b/Framework/Core/include/Framework/DataAllocator.h
@@ -359,6 +359,7 @@ class DataAllocator
     } else if constexpr (has_root_dictionary<T>::value == true || is_specialization_v<T, ROOTSerialized> == true) {
       // Serialize a snapshot of an object with root dictionary
       payloadMessage = proxy.createOutputMessage(routeIndex);
+      payloadMessage->Rebuild(4096, {64});
       if constexpr (is_specialization_v<T, ROOTSerialized> == true) {
         // Explicitely ROOT serialize a snapshot of object.
         // An object wrapped into type `ROOTSerialized` is explicitely marked to be ROOT serialized
diff --git a/Framework/Core/include/Framework/DataRefUtils.h b/Framework/Core/include/Framework/DataRefUtils.h
index defd10244bca5..e59f986f09250 100644
--- a/Framework/Core/include/Framework/DataRefUtils.h
+++ b/Framework/Core/include/Framework/DataRefUtils.h
@@ -71,12 +71,15 @@ struct DataRefUtils {
           throw runtime_error("Attempt to extract a TMessage from non-ROOT serialised message");
         }
 
-        typename RSS::FairTMessage ftm(const_cast<char*>(ref.payload), payloadSize);
-        auto* storedClass = ftm.GetClass();
+        typename RSS::FairInputTBuffer ftm(const_cast<char*>(ref.payload), payloadSize);
         auto* requestedClass = RSS::TClass::GetClass(typeid(T));
+        ftm.InitMap();
+        auto* storedClass = ftm.ReadClass();
         // should always have the class description if has_root_dictionary is true
         assert(requestedClass != nullptr);
 
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         auto* object = ftm.ReadObjectAny(storedClass);
         if (object == nullptr) {
           throw runtime_error_f("Failed to read object with name %s from message using ROOT serialization.",
@@ -146,7 +149,11 @@ struct DataRefUtils {
           throw runtime_error("ROOT serialization not supported, dictionary not found for data type");
         }
 
-        typename RSS::FairTMessage ftm(const_cast<char*>(ref.payload), payloadSize);
+        typename RSS::FairInputTBuffer ftm(const_cast<char*>(ref.payload), payloadSize);
+        ftm.InitMap();
+        auto *classInfo = ftm.ReadClass();
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         result.reset(static_cast<wrapped*>(ftm.ReadObjectAny(cl)));
         if (result.get() == nullptr) {
           throw runtime_error_f("Unable to extract class %s", cl == nullptr ? "<name not available>" : cl->GetName());
diff --git a/Framework/Core/include/Framework/RootMessageContext.h b/Framework/Core/include/Framework/RootMessageContext.h
index bef60ebbbf9f9..b1124880cf30f 100644
--- a/Framework/Core/include/Framework/RootMessageContext.h
+++ b/Framework/Core/include/Framework/RootMessageContext.h
@@ -72,6 +72,9 @@ class RootSerializedObject : public MessageContext::ContextObject
   fair::mq::Parts finalize() final
   {
     assert(mParts.Size() == 1);
+    if (mPayloadMsg->GetSize() < sizeof(char*)) {
+      mPayloadMsg->Rebuild(4096, {64});
+    }
     TMessageSerializer::Serialize(*mPayloadMsg, mObject.get(), nullptr);
     mParts.AddPart(std::move(mPayloadMsg));
     return ContextObject::finalize();
diff --git a/Framework/Core/include/Framework/RootSerializationSupport.h b/Framework/Core/include/Framework/RootSerializationSupport.h
index cbf7408b13c7d..a44093f9c02bf 100644
--- a/Framework/Core/include/Framework/RootSerializationSupport.h
+++ b/Framework/Core/include/Framework/RootSerializationSupport.h
@@ -21,7 +21,8 @@ namespace o2::framework
 /// compiler.
 struct RootSerializationSupport {
   using TClass = ::TClass;
-  using FairTMessage = o2::framework::FairTMessage;
+  using FairInputTBuffer = o2::framework::FairInputTBuffer;
+  using FairOutputBuffer = o2::framework::FairOutputTBuffer;
   using TObject = ::TObject;
 };
 
diff --git a/Framework/Core/include/Framework/TMessageSerializer.h b/Framework/Core/include/Framework/TMessageSerializer.h
index 1f08b456c0218..ca18eb21abfa1 100644
--- a/Framework/Core/include/Framework/TMessageSerializer.h
+++ b/Framework/Core/include/Framework/TMessageSerializer.h
@@ -16,9 +16,8 @@
 #include "Framework/RuntimeError.h"
 
 #include <TList.h>
-#include <TMessage.h>
+#include <TBufferFile.h>
 #include <TObjArray.h>
-#include <TStreamerInfo.h>
 #include <gsl/util>
 #include <gsl/span>
 #include <gsl/narrow>
@@ -28,67 +27,76 @@
 
 namespace o2::framework
 {
-class FairTMessage;
+class FairOutputTBuffer;
+class FairInputTBuffer;
 
 // utilities to produce a span over a byte buffer held by various message types
 // this is to avoid littering code with casts and conversions (span has a signed index type(!))
-gsl::span<std::byte> as_span(const FairTMessage& msg);
+gsl::span<std::byte> as_span(const FairInputTBuffer& msg);
+gsl::span<std::byte> as_span(const FairOutputTBuffer& msg);
 gsl::span<std::byte> as_span(const fair::mq::Message& msg);
 
-class FairTMessage : public TMessage
+// A TBufferFile which we can use to serialise data to a FairMQ message.
+class FairOutputTBuffer : public TBufferFile
 {
  public:
-  using TMessage::TMessage;
-  FairTMessage() : TMessage(kMESS_OBJECT) {}
-  FairTMessage(void* buf, Int_t len) : TMessage(buf, len) { ResetBit(kIsOwner); }
-  FairTMessage(gsl::span<std::byte> buf) : TMessage(buf.data(), buf.size()) { ResetBit(kIsOwner); }
+  // This is to serialise data to FairMQ. We embed the pointer to the message
+  // in the data itself, so that we can use it to reallocate the message if needed.
+  // The FairMQ message retains ownership of the data.
+  // When deserialising the root object, keep in mind one needs to skip the 8 bytes
+  // for the pointer.
+  FairOutputTBuffer(fair::mq::Message& msg)
+    : TBufferFile(TBuffer::kWrite, msg.GetSize() - sizeof(char*), embedInItself(msg), false, fairMQrealloc)
+  {
+  }
+  // Helper function to keep track of the FairMQ message that holds the data
+  // in the data itself. We can use this to make sure the message can be reallocated
+  // even if we simply have a pointer to the data. Hopefully ROOT will not play dirty
+  // with us.
+  void* embedInItself(fair::mq::Message& msg);
   // helper function to clean up the object holding the data after it is transported.
-  static void free(void* /*data*/, void* hint);
+  static char* fairMQrealloc(char* oldData, size_t newSize, size_t oldSize);
 };
 
-struct TMessageSerializer {
-  using StreamerList = std::vector<TVirtualStreamerInfo*>;
-  using CompressionLevel = int;
+class FairInputTBuffer : public TBufferFile
+{
+ public:
+  // This is to serialise data to FairMQ. The provided message is expeted to have 8 bytes
+  // of overhead, where the source embedded the pointer for the reallocation.
+  // Notice this will break if the sender and receiver are not using the same
+  // size for a pointer.
+  FairInputTBuffer(char * data, size_t size)
+    : TBufferFile(TBuffer::kRead, size-sizeof(char*), data + sizeof(char*), false, nullptr)
+  {
+  }
+};
 
-  static void Serialize(fair::mq::Message& msg, const TObject* input,
-                        CompressionLevel compressionLevel = -1);
+struct TMessageSerializer {
+  static void Serialize(fair::mq::Message& msg, const TObject* input);
 
   template <typename T>
-  static void Serialize(fair::mq::Message& msg, const T* input, const TClass* cl, //
-                        CompressionLevel compressionLevel = -1);
+  static void Serialize(fair::mq::Message& msg, const T* input, const TClass* cl);
 
   template <typename T = TObject>
   static void Deserialize(const fair::mq::Message& msg, std::unique_ptr<T>& output);
 
-  static void serialize(FairTMessage& msg, const TObject* input,
-                        CompressionLevel compressionLevel = -1);
+  static void serialize(o2::framework::FairOutputTBuffer& msg, const TObject* input);
 
   template <typename T>
-  static void serialize(FairTMessage& msg, const T* input, //
-                        const TClass* cl,
-                        CompressionLevel compressionLevel = -1);
+  static void serialize(o2::framework::FairOutputTBuffer& msg, const T* input, const TClass* cl);
 
   template <typename T = TObject>
-  static std::unique_ptr<T> deserialize(gsl::span<std::byte> buffer);
-  template <typename T = TObject>
-  static inline std::unique_ptr<T> deserialize(std::byte* buffer, size_t size);
+  static inline std::unique_ptr<T> deserialize(FairInputTBuffer & buffer);
 };
 
-inline void TMessageSerializer::serialize(FairTMessage& tm, const TObject* input,
-                                          CompressionLevel compressionLevel)
+inline void TMessageSerializer::serialize(FairOutputTBuffer& tm, const TObject* input)
 {
-  return serialize(tm, input, nullptr, compressionLevel);
+  return serialize(tm, input, nullptr);
 }
 
 template <typename T>
-inline void TMessageSerializer::serialize(FairTMessage& tm, const T* input, //
-                                          const TClass* cl, CompressionLevel compressionLevel)
+inline void TMessageSerializer::serialize(FairOutputTBuffer& tm, const T* input, const TClass* cl)
 {
-  if (compressionLevel >= 0) {
-    // if negative, skip to use ROOT default
-    tm.SetCompressionLevel(compressionLevel);
-  }
-
   // TODO: check what WriateObject and WriteObjectAny are doing
   if (cl == nullptr) {
     tm.WriteObject(input);
@@ -98,7 +106,7 @@ inline void TMessageSerializer::serialize(FairTMessage& tm, const T* input, //
 }
 
 template <typename T>
-inline std::unique_ptr<T> TMessageSerializer::deserialize(gsl::span<std::byte> buffer)
+inline std::unique_ptr<T> TMessageSerializer::deserialize(FairInputTBuffer & buffer)
 {
   TClass* tgtClass = TClass::GetClass(typeid(T));
   if (tgtClass == nullptr) {
@@ -107,53 +115,32 @@ inline std::unique_ptr<T> TMessageSerializer::deserialize(gsl::span<std::byte> b
   // FIXME: we need to add consistency check for buffer data to be serialized
   // at the moment, TMessage might simply crash if an invalid or inconsistent
   // buffer is provided
-  FairTMessage tm(buffer);
-  TClass* serializedClass = tm.GetClass();
+  buffer.InitMap();
+  TClass* serializedClass = buffer.ReadClass();
+  buffer.SetBufferOffset(0);
+  buffer.ResetMap();
   if (serializedClass == nullptr) {
     throw runtime_error_f("can not read class info from buffer");
   }
   if (tgtClass != serializedClass && serializedClass->GetBaseClass(tgtClass) == nullptr) {
     throw runtime_error_f("can not convert serialized class %s into target class %s",
-                          tm.GetClass()->GetName(),
+                          serializedClass->GetName(),
                           tgtClass->GetName());
   }
-  return std::unique_ptr<T>(reinterpret_cast<T*>(tm.ReadObjectAny(serializedClass)));
+  return std::unique_ptr<T>(reinterpret_cast<T*>(buffer.ReadObjectAny(serializedClass)));
 }
 
-template <typename T>
-inline std::unique_ptr<T> TMessageSerializer::deserialize(std::byte* buffer, size_t size)
+inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const TObject* input)
 {
-  return deserialize<T>(gsl::span<std::byte>(buffer, gsl::narrow<gsl::span<std::byte>::size_type>(size)));
-}
-
-inline void FairTMessage::free(void* /*data*/, void* hint)
-{
-  std::default_delete<FairTMessage> deleter;
-  deleter(static_cast<FairTMessage*>(hint));
-}
-
-inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const TObject* input,
-                                          TMessageSerializer::CompressionLevel compressionLevel)
-{
-  std::unique_ptr<FairTMessage> tm = std::make_unique<FairTMessage>(kMESS_OBJECT);
-
-  serialize(*tm, input, input->Class(), compressionLevel);
-
-  msg.Rebuild(tm->Buffer(), tm->BufferSize(), FairTMessage::free, tm.get());
-  tm.release();
+  FairOutputTBuffer output(msg);
+  serialize(output, input, input->Class());
 }
 
 template <typename T>
-inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const T* input, //
-                                          const TClass* cl,                       //
-                                          TMessageSerializer::CompressionLevel compressionLevel)
+inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const T* input, const TClass* cl)
 {
-  std::unique_ptr<FairTMessage> tm = std::make_unique<FairTMessage>(kMESS_OBJECT);
-
-  serialize(*tm, input, cl, compressionLevel);
-
-  msg.Rebuild(tm->Buffer(), tm->BufferSize(), FairTMessage::free, tm.get());
-  tm.release();
+  FairOutputTBuffer output(msg);
+  serialize(output, input, cl);
 }
 
 template <typename T>
@@ -161,7 +148,8 @@ inline void TMessageSerializer::Deserialize(const fair::mq::Message& msg, std::u
 {
   // we know the message will not be modified by this,
   // so const_cast should be OK here(IMHO).
-  output = deserialize(as_span(msg));
+  FairInputTBuffer input(static_cast<char*>(msg.GetData()), static_cast<int>(msg.GetSize()));
+  output = deserialize(input);
 }
 
 // gsl::narrow is used to do a runtime narrowing check, this might be a bit paranoid,
@@ -171,7 +159,7 @@ inline gsl::span<std::byte> as_span(const fair::mq::Message& msg)
   return gsl::span<std::byte>{static_cast<std::byte*>(msg.GetData()), gsl::narrow<gsl::span<std::byte>::size_type>(msg.GetSize())};
 }
 
-inline gsl::span<std::byte> as_span(const FairTMessage& msg)
+inline gsl::span<std::byte> as_span(const FairInputTBuffer& msg)
 {
   return gsl::span<std::byte>{reinterpret_cast<std::byte*>(msg.Buffer()),
                               gsl::narrow<gsl::span<std::byte>::size_type>(msg.BufferSize())};
diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 48a3eb1da95b9..02ef5c7bc5b3c 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -141,9 +141,9 @@ DataProcessorSpec CommonDataProcessors::getOutputObjHistSink(std::vector<OutputO
         return;
       }
 
-      FairTMessage tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
+      FairInputTBuffer tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
       InputObject obj;
-      obj.kind = tm.GetClass();
+      obj.kind = tm.ReadClass();
       if (obj.kind == nullptr) {
         LOG(error) << "Cannot read class info from buffer.";
         return;
diff --git a/Framework/Core/src/TMessageSerializer.cxx b/Framework/Core/src/TMessageSerializer.cxx
index 5388a6d716cda..9f09c3ade0089 100644
--- a/Framework/Core/src/TMessageSerializer.cxx
+++ b/Framework/Core/src/TMessageSerializer.cxx
@@ -9,7 +9,38 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 #include <Framework/TMessageSerializer.h>
+#include <FairMQTransportFactory.h>
 #include <algorithm>
 #include <memory>
 
 using namespace o2::framework;
+
+void* FairOutputTBuffer::embedInItself(fair::mq::Message& msg) {
+  // The first bytes of the message are used to store the pointer to the message itself
+  // so that we can reallocate it if needed.
+  if (sizeof(char*) > msg.GetSize()) {
+    throw std::runtime_error("Message size too small to embed pointer");
+  }
+  char* data = reinterpret_cast<char*>(msg.GetData());
+  char* ptr = reinterpret_cast<char*>(&msg);
+  std::memcpy(data, ptr, sizeof(char*));
+  return data + sizeof(char*);
+}
+
+// Reallocation function. Get the message pointer from the data and call Rebuild.
+char *FairOutputTBuffer::fairMQrealloc(char *oldData, size_t newSize, size_t oldSize) {
+  auto* msg = reinterpret_cast<fair::mq::Message*>(oldData - sizeof(char*));
+  if (newSize <= msg->GetSize()) {
+    // no need to reallocate, the message is already big enough
+    return oldData;
+  }
+  // Create a shallow copy of the message
+  fair::mq::MessagePtr oldMsg = msg->GetTransport()->CreateMessage();
+  oldMsg->Copy(*msg);
+  // Copy the old data while rebuilding. Reference counting should make
+  // sure the old message is not deleted until the new one is ready.
+  msg->Rebuild(newSize, fair::mq::Alignment{64});
+  memcpy(msg->GetData(), oldMsg->GetData(), oldSize);
+
+  return reinterpret_cast<char*>(msg->GetData()) + sizeof(char*);
+}
diff --git a/Framework/Core/test/test_DataRefUtils.cxx b/Framework/Core/test/test_DataRefUtils.cxx
index 37da7912bfe8b..081adc81ebf69 100644
--- a/Framework/Core/test/test_DataRefUtils.cxx
+++ b/Framework/Core/test/test_DataRefUtils.cxx
@@ -21,17 +21,37 @@
 
 using namespace o2::framework;
 
+TEST_CASE("PureRootTest") {
+  TBufferFile buffer(TBuffer::kWrite);
+  TObjString s("test");
+  buffer.WriteObject(&s);
+
+  TBufferFile buffer2(TBuffer::kRead, buffer.BufferSize(), buffer.Buffer(), false);
+  buffer2.SetReadMode();
+  buffer2.InitMap();
+  TClass *storedClass = buffer2.ReadClass();
+  // ReadClass advances the buffer, so we need to reset it.
+  buffer2.SetBufferOffset(0);
+  buffer2.ResetMap();
+  REQUIRE(storedClass != nullptr);
+  auto *outS = (TObjString*)buffer2.ReadObjectAny(storedClass);
+  REQUIRE(outS != nullptr);
+  REQUIRE(outS->GetString() == "test");
+}
+
 // Simple test to do root deserialization.
 TEST_CASE("TestRootSerialization")
 {
   DataRef ref;
-  TMessage* tm = new TMessage(kMESS_OBJECT);
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer tm(*msg);
   auto sOrig = std::make_unique<TObjString>("test");
-  tm->WriteObject(sOrig.get());
+  tm << sOrig.get();
   o2::header::DataHeader dh;
   dh.payloadSerializationMethod = o2::header::gSerializationMethodROOT;
-  ref.payload = tm->Buffer();
-  dh.payloadSize = tm->BufferSize();
+  ref.payload = (char*)msg->GetData();
+  dh.payloadSize = (size_t)msg->GetSize();
   ref.header = reinterpret_cast<char const*>(&dh);
 
   // Check by using the same type
diff --git a/Framework/Core/test/test_TMessageSerializer.cxx b/Framework/Core/test/test_TMessageSerializer.cxx
index bc5f817400a44..89de74edc8455 100644
--- a/Framework/Core/test/test_TMessageSerializer.cxx
+++ b/Framework/Core/test/test_TMessageSerializer.cxx
@@ -11,6 +11,7 @@
 
 #include "Framework/TMessageSerializer.h"
 #include "Framework/RuntimeError.h"
+#include <fairmq/TransportFactory.h>
 #include "TestClasses.h"
 #include <catch_amalgamated.hpp>
 #include <utility>
@@ -49,14 +50,14 @@ TEST_CASE("TestTMessageSerializer")
   array.SetOwner();
   array.Add(new TNamed(testname, testtitle));
 
-  FairTMessage msg;
-  TMessageSerializer::serialize(msg, &array);
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer buffer(*msg);
+  TMessageSerializer::serialize(buffer, &array);
 
-  auto buf = as_span(msg);
-  REQUIRE(buf.size() == msg.BufferSize());
-  REQUIRE(static_cast<void*>(buf.data()) == static_cast<void*>(msg.Buffer()));
+  FairInputTBuffer msg2((char*)msg->GetData(), msg->GetSize());
   // test deserialization with TObject as target class (default)
-  auto out = TMessageSerializer::deserialize(buf);
+  auto out = TMessageSerializer::deserialize(msg2);
 
   auto* outarr = dynamic_cast<TObjArray*>(out.get());
   REQUIRE(out.get() == outarr);
@@ -66,9 +67,9 @@ TEST_CASE("TestTMessageSerializer")
   REQUIRE(named->GetTitle() == std::string(testtitle));
 
   // test deserialization with a wrong target class and check the exception
-  REQUIRE_THROWS_AS(TMessageSerializer::deserialize<TNamed>(buf), o2::framework::RuntimeErrorRef);
+  REQUIRE_THROWS_AS(TMessageSerializer::deserialize<TNamed>(msg2), o2::framework::RuntimeErrorRef);
 
-  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<TNamed>(buf), o2::framework::RuntimeErrorRef,
+  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<TNamed>(msg2), o2::framework::RuntimeErrorRef,
                          ExceptionMatcher("can not convert serialized class TObjArray into target class TNamed"));
 }
 
@@ -87,18 +88,21 @@ TEST_CASE("TestTMessageSerializer_NonTObject")
   TClass* cl = TClass::GetClass("std::vector<o2::test::Polymorphic>");
   REQUIRE(cl != nullptr);
 
-  FairTMessage msg;
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer buffer(*msg);
   char* in = reinterpret_cast<char*>(&data);
-  TMessageSerializer::serialize(msg, in, cl);
+  TMessageSerializer::serialize(buffer, in, cl);
+  FairInputTBuffer msg2((char*)msg->GetData(), msg->GetSize());
 
-  auto out = TMessageSerializer::deserialize<std::vector<o2::test::Polymorphic>>(as_span(msg));
+  auto out = TMessageSerializer::deserialize<std::vector<o2::test::Polymorphic>>(msg2);
   REQUIRE(out);
   REQUIRE((*out.get()).size() == 2);
   REQUIRE((*out.get())[0] == o2::test::Polymorphic(0xaffe));
   REQUIRE((*out.get())[1] == o2::test::Polymorphic(0xd00f));
 
   // test deserialization with a wrong target class and check the exception
-  REQUIRE_THROWS_AS(TMessageSerializer::deserialize(as_span(msg)), RuntimeErrorRef);
+  REQUIRE_THROWS_AS(TMessageSerializer::deserialize(msg2), RuntimeErrorRef);
 }
 
 TEST_CASE("TestTMessageSerializer_InvalidBuffer")
@@ -119,5 +123,6 @@ TEST_CASE("TestTMessageSerializer_InvalidBuffer")
   struct Dummy {
   };
   auto matcher = ExceptionMatcher("class is not ROOT-serializable: ZL22CATCH2_INTERNAL_TEST_4vE5Dummy");
-  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<Dummy>((std::byte*)buffer, strlen(buffer)), o2::framework::RuntimeErrorRef, matcher);
+  FairInputTBuffer msg2((char*)buffer, strlen(buffer));
+  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<Dummy>(msg2), o2::framework::RuntimeErrorRef, matcher);
 }
diff --git a/Utilities/Mergers/src/ObjectStore.cxx b/Utilities/Mergers/src/ObjectStore.cxx
index e88358507c31e..3bb49f1dfc9d8 100644
--- a/Utilities/Mergers/src/ObjectStore.cxx
+++ b/Utilities/Mergers/src/ObjectStore.cxx
@@ -38,7 +38,7 @@ static std::string concat(Args&&... arguments)
   return std::move(ss.str());
 }
 
-void* readObject(const TClass* type, o2::framework::FairTMessage& ftm)
+void* readObject(const TClass* type, o2::framework::FairInputTBuffer& ftm)
 {
   using namespace std::string_view_literals;
   auto* object = ftm.ReadObjectAny(type);
@@ -60,7 +60,7 @@ MergeInterface* castToMergeInterface(bool inheritsFromTObject, void* object, TCl
   return objectAsMergeInterface;
 }
 
-std::optional<ObjectStore> extractVector(o2::framework::FairTMessage& ftm, const TClass* storedClass)
+std::optional<ObjectStore> extractVector(o2::framework::FairInputTBuffer& ftm, const TClass* storedClass)
 {
   if (!storedClass->InheritsFrom(TClass::GetClass(typeid(VectorOfRawTObjects)))) {
     return std::nullopt;
@@ -88,11 +88,14 @@ ObjectStore extractObjectFrom(const framework::DataRef& ref)
     throw std::runtime_error(concat(errorPrefix, "It is not ROOT-serialized"sv));
   }
 
-  o2::framework::FairTMessage ftm(const_cast<char*>(ref.payload), o2::framework::DataRefUtils::getPayloadSize(ref));
-  auto* storedClass = ftm.GetClass();
+  o2::framework::FairInputTBuffer ftm(const_cast<char*>(ref.payload), o2::framework::DataRefUtils::getPayloadSize(ref));
+  ftm.InitMap();
+  auto* storedClass = ftm.ReadClass();
   if (storedClass == nullptr) {
     throw std::runtime_error(concat(errorPrefix, "Unknown stored class"sv));
   }
+  ftm.SetBufferOffset(0);
+  ftm.ResetMap();
 
   if (const auto extractedVector = extractVector(ftm, storedClass)) {
     return extractedVector.value();
diff --git a/Utilities/Mergers/test/benchmark_Types.cxx b/Utilities/Mergers/test/benchmark_Types.cxx
index 790fd329185ea..736685c5746b8 100644
--- a/Utilities/Mergers/test/benchmark_Types.cxx
+++ b/Utilities/Mergers/test/benchmark_Types.cxx
@@ -165,11 +165,16 @@ auto measure = [](Measurement m, auto* o, auto* i) -> double {
         tm->WriteObject(o);
         start = std::chrono::high_resolution_clock::now();
 
-        o2::framework::FairTMessage ftm(const_cast<char*>(tm->Buffer()), tm->BufferSize());
-        auto* storedClass = ftm.GetClass();
+        // Needed to take into account that  FairInputTBuffer expects the first 8 bytes to be the
+        // allocator pointer, which is not present in the TMessage buffer.
+        o2::framework::FairInputTBuffer ftm(const_cast<char*>(tm->Buffer() - 8), tm->BufferSize() + 8);
+        ftm.InitMap();
+        auto* storedClass = ftm.ReadClass();
         if (storedClass == nullptr) {
           throw std::runtime_error("Unknown stored class");
         }
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         auto* tObjectClass = TClass::GetClass(typeid(TObject));
 
         if (!storedClass->InheritsFrom(tObjectClass)) {
@@ -738,4 +743,4 @@ int main(int argc, const char* argv[])
 
   file.close();
   return 0;
-}
\ No newline at end of file
+}

From eb215631f99b00576f45fb39889d76b1586ec989 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 23 Feb 2024 20:36:58 +0100
Subject: [PATCH 377/726] DPL: avoid TMessage usage

TMessage does not allow for non owned buffers, so we end up having
an extra buffer in private memory for (de)serializing. Using TBufferFile
directly allows to avoid that, so this moves the whole ROOT serialization
support in DPL to use it.
---
 .../src/AODJAlienReaderHelpers.h              |   2 +
 .../Core/include/Framework/DataAllocator.h    |   1 +
 .../Core/include/Framework/DataRefUtils.h     |  13 +-
 .../include/Framework/RootMessageContext.h    |   3 +
 .../Framework/RootSerializationSupport.h      |   3 +-
 .../include/Framework/TMessageSerializer.h    | 130 ++++++++----------
 Framework/Core/src/CommonDataProcessors.cxx   |   4 +-
 Framework/Core/src/TMessageSerializer.cxx     |  31 +++++
 Framework/Core/test/test_DataRefUtils.cxx     |  28 +++-
 .../Core/test/test_TMessageSerializer.cxx     |  31 +++--
 Framework/Utils/test/test_RootTreeWriter.cxx  |   1 +
 Utilities/Mergers/src/ObjectStore.cxx         |  11 +-
 Utilities/Mergers/test/benchmark_Types.cxx    |  11 +-
 13 files changed, 168 insertions(+), 101 deletions(-)

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
index 655e4b6c0b439..4b9fd710aca14 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
@@ -16,7 +16,9 @@
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/Logger.h"
 #include <Monitoring/Monitoring.h>
+
 #include <uv.h>
+class TFile;
 
 namespace o2::framework::readers
 {
diff --git a/Framework/Core/include/Framework/DataAllocator.h b/Framework/Core/include/Framework/DataAllocator.h
index 8151d2f83c6c6..029e922aeb90b 100644
--- a/Framework/Core/include/Framework/DataAllocator.h
+++ b/Framework/Core/include/Framework/DataAllocator.h
@@ -359,6 +359,7 @@ class DataAllocator
     } else if constexpr (has_root_dictionary<T>::value == true || is_specialization_v<T, ROOTSerialized> == true) {
       // Serialize a snapshot of an object with root dictionary
       payloadMessage = proxy.createOutputMessage(routeIndex);
+      payloadMessage->Rebuild(4096, {64});
       if constexpr (is_specialization_v<T, ROOTSerialized> == true) {
         // Explicitely ROOT serialize a snapshot of object.
         // An object wrapped into type `ROOTSerialized` is explicitely marked to be ROOT serialized
diff --git a/Framework/Core/include/Framework/DataRefUtils.h b/Framework/Core/include/Framework/DataRefUtils.h
index defd10244bca5..e59f986f09250 100644
--- a/Framework/Core/include/Framework/DataRefUtils.h
+++ b/Framework/Core/include/Framework/DataRefUtils.h
@@ -71,12 +71,15 @@ struct DataRefUtils {
           throw runtime_error("Attempt to extract a TMessage from non-ROOT serialised message");
         }
 
-        typename RSS::FairTMessage ftm(const_cast<char*>(ref.payload), payloadSize);
-        auto* storedClass = ftm.GetClass();
+        typename RSS::FairInputTBuffer ftm(const_cast<char*>(ref.payload), payloadSize);
         auto* requestedClass = RSS::TClass::GetClass(typeid(T));
+        ftm.InitMap();
+        auto* storedClass = ftm.ReadClass();
         // should always have the class description if has_root_dictionary is true
         assert(requestedClass != nullptr);
 
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         auto* object = ftm.ReadObjectAny(storedClass);
         if (object == nullptr) {
           throw runtime_error_f("Failed to read object with name %s from message using ROOT serialization.",
@@ -146,7 +149,11 @@ struct DataRefUtils {
           throw runtime_error("ROOT serialization not supported, dictionary not found for data type");
         }
 
-        typename RSS::FairTMessage ftm(const_cast<char*>(ref.payload), payloadSize);
+        typename RSS::FairInputTBuffer ftm(const_cast<char*>(ref.payload), payloadSize);
+        ftm.InitMap();
+        auto *classInfo = ftm.ReadClass();
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         result.reset(static_cast<wrapped*>(ftm.ReadObjectAny(cl)));
         if (result.get() == nullptr) {
           throw runtime_error_f("Unable to extract class %s", cl == nullptr ? "<name not available>" : cl->GetName());
diff --git a/Framework/Core/include/Framework/RootMessageContext.h b/Framework/Core/include/Framework/RootMessageContext.h
index bef60ebbbf9f9..b1124880cf30f 100644
--- a/Framework/Core/include/Framework/RootMessageContext.h
+++ b/Framework/Core/include/Framework/RootMessageContext.h
@@ -72,6 +72,9 @@ class RootSerializedObject : public MessageContext::ContextObject
   fair::mq::Parts finalize() final
   {
     assert(mParts.Size() == 1);
+    if (mPayloadMsg->GetSize() < sizeof(char*)) {
+      mPayloadMsg->Rebuild(4096, {64});
+    }
     TMessageSerializer::Serialize(*mPayloadMsg, mObject.get(), nullptr);
     mParts.AddPart(std::move(mPayloadMsg));
     return ContextObject::finalize();
diff --git a/Framework/Core/include/Framework/RootSerializationSupport.h b/Framework/Core/include/Framework/RootSerializationSupport.h
index cbf7408b13c7d..a44093f9c02bf 100644
--- a/Framework/Core/include/Framework/RootSerializationSupport.h
+++ b/Framework/Core/include/Framework/RootSerializationSupport.h
@@ -21,7 +21,8 @@ namespace o2::framework
 /// compiler.
 struct RootSerializationSupport {
   using TClass = ::TClass;
-  using FairTMessage = o2::framework::FairTMessage;
+  using FairInputTBuffer = o2::framework::FairInputTBuffer;
+  using FairOutputBuffer = o2::framework::FairOutputTBuffer;
   using TObject = ::TObject;
 };
 
diff --git a/Framework/Core/include/Framework/TMessageSerializer.h b/Framework/Core/include/Framework/TMessageSerializer.h
index 1f08b456c0218..ca18eb21abfa1 100644
--- a/Framework/Core/include/Framework/TMessageSerializer.h
+++ b/Framework/Core/include/Framework/TMessageSerializer.h
@@ -16,9 +16,8 @@
 #include "Framework/RuntimeError.h"
 
 #include <TList.h>
-#include <TMessage.h>
+#include <TBufferFile.h>
 #include <TObjArray.h>
-#include <TStreamerInfo.h>
 #include <gsl/util>
 #include <gsl/span>
 #include <gsl/narrow>
@@ -28,67 +27,76 @@
 
 namespace o2::framework
 {
-class FairTMessage;
+class FairOutputTBuffer;
+class FairInputTBuffer;
 
 // utilities to produce a span over a byte buffer held by various message types
 // this is to avoid littering code with casts and conversions (span has a signed index type(!))
-gsl::span<std::byte> as_span(const FairTMessage& msg);
+gsl::span<std::byte> as_span(const FairInputTBuffer& msg);
+gsl::span<std::byte> as_span(const FairOutputTBuffer& msg);
 gsl::span<std::byte> as_span(const fair::mq::Message& msg);
 
-class FairTMessage : public TMessage
+// A TBufferFile which we can use to serialise data to a FairMQ message.
+class FairOutputTBuffer : public TBufferFile
 {
  public:
-  using TMessage::TMessage;
-  FairTMessage() : TMessage(kMESS_OBJECT) {}
-  FairTMessage(void* buf, Int_t len) : TMessage(buf, len) { ResetBit(kIsOwner); }
-  FairTMessage(gsl::span<std::byte> buf) : TMessage(buf.data(), buf.size()) { ResetBit(kIsOwner); }
+  // This is to serialise data to FairMQ. We embed the pointer to the message
+  // in the data itself, so that we can use it to reallocate the message if needed.
+  // The FairMQ message retains ownership of the data.
+  // When deserialising the root object, keep in mind one needs to skip the 8 bytes
+  // for the pointer.
+  FairOutputTBuffer(fair::mq::Message& msg)
+    : TBufferFile(TBuffer::kWrite, msg.GetSize() - sizeof(char*), embedInItself(msg), false, fairMQrealloc)
+  {
+  }
+  // Helper function to keep track of the FairMQ message that holds the data
+  // in the data itself. We can use this to make sure the message can be reallocated
+  // even if we simply have a pointer to the data. Hopefully ROOT will not play dirty
+  // with us.
+  void* embedInItself(fair::mq::Message& msg);
   // helper function to clean up the object holding the data after it is transported.
-  static void free(void* /*data*/, void* hint);
+  static char* fairMQrealloc(char* oldData, size_t newSize, size_t oldSize);
 };
 
-struct TMessageSerializer {
-  using StreamerList = std::vector<TVirtualStreamerInfo*>;
-  using CompressionLevel = int;
+class FairInputTBuffer : public TBufferFile
+{
+ public:
+  // This is to serialise data to FairMQ. The provided message is expeted to have 8 bytes
+  // of overhead, where the source embedded the pointer for the reallocation.
+  // Notice this will break if the sender and receiver are not using the same
+  // size for a pointer.
+  FairInputTBuffer(char * data, size_t size)
+    : TBufferFile(TBuffer::kRead, size-sizeof(char*), data + sizeof(char*), false, nullptr)
+  {
+  }
+};
 
-  static void Serialize(fair::mq::Message& msg, const TObject* input,
-                        CompressionLevel compressionLevel = -1);
+struct TMessageSerializer {
+  static void Serialize(fair::mq::Message& msg, const TObject* input);
 
   template <typename T>
-  static void Serialize(fair::mq::Message& msg, const T* input, const TClass* cl, //
-                        CompressionLevel compressionLevel = -1);
+  static void Serialize(fair::mq::Message& msg, const T* input, const TClass* cl);
 
   template <typename T = TObject>
   static void Deserialize(const fair::mq::Message& msg, std::unique_ptr<T>& output);
 
-  static void serialize(FairTMessage& msg, const TObject* input,
-                        CompressionLevel compressionLevel = -1);
+  static void serialize(o2::framework::FairOutputTBuffer& msg, const TObject* input);
 
   template <typename T>
-  static void serialize(FairTMessage& msg, const T* input, //
-                        const TClass* cl,
-                        CompressionLevel compressionLevel = -1);
+  static void serialize(o2::framework::FairOutputTBuffer& msg, const T* input, const TClass* cl);
 
   template <typename T = TObject>
-  static std::unique_ptr<T> deserialize(gsl::span<std::byte> buffer);
-  template <typename T = TObject>
-  static inline std::unique_ptr<T> deserialize(std::byte* buffer, size_t size);
+  static inline std::unique_ptr<T> deserialize(FairInputTBuffer & buffer);
 };
 
-inline void TMessageSerializer::serialize(FairTMessage& tm, const TObject* input,
-                                          CompressionLevel compressionLevel)
+inline void TMessageSerializer::serialize(FairOutputTBuffer& tm, const TObject* input)
 {
-  return serialize(tm, input, nullptr, compressionLevel);
+  return serialize(tm, input, nullptr);
 }
 
 template <typename T>
-inline void TMessageSerializer::serialize(FairTMessage& tm, const T* input, //
-                                          const TClass* cl, CompressionLevel compressionLevel)
+inline void TMessageSerializer::serialize(FairOutputTBuffer& tm, const T* input, const TClass* cl)
 {
-  if (compressionLevel >= 0) {
-    // if negative, skip to use ROOT default
-    tm.SetCompressionLevel(compressionLevel);
-  }
-
   // TODO: check what WriateObject and WriteObjectAny are doing
   if (cl == nullptr) {
     tm.WriteObject(input);
@@ -98,7 +106,7 @@ inline void TMessageSerializer::serialize(FairTMessage& tm, const T* input, //
 }
 
 template <typename T>
-inline std::unique_ptr<T> TMessageSerializer::deserialize(gsl::span<std::byte> buffer)
+inline std::unique_ptr<T> TMessageSerializer::deserialize(FairInputTBuffer & buffer)
 {
   TClass* tgtClass = TClass::GetClass(typeid(T));
   if (tgtClass == nullptr) {
@@ -107,53 +115,32 @@ inline std::unique_ptr<T> TMessageSerializer::deserialize(gsl::span<std::byte> b
   // FIXME: we need to add consistency check for buffer data to be serialized
   // at the moment, TMessage might simply crash if an invalid or inconsistent
   // buffer is provided
-  FairTMessage tm(buffer);
-  TClass* serializedClass = tm.GetClass();
+  buffer.InitMap();
+  TClass* serializedClass = buffer.ReadClass();
+  buffer.SetBufferOffset(0);
+  buffer.ResetMap();
   if (serializedClass == nullptr) {
     throw runtime_error_f("can not read class info from buffer");
   }
   if (tgtClass != serializedClass && serializedClass->GetBaseClass(tgtClass) == nullptr) {
     throw runtime_error_f("can not convert serialized class %s into target class %s",
-                          tm.GetClass()->GetName(),
+                          serializedClass->GetName(),
                           tgtClass->GetName());
   }
-  return std::unique_ptr<T>(reinterpret_cast<T*>(tm.ReadObjectAny(serializedClass)));
+  return std::unique_ptr<T>(reinterpret_cast<T*>(buffer.ReadObjectAny(serializedClass)));
 }
 
-template <typename T>
-inline std::unique_ptr<T> TMessageSerializer::deserialize(std::byte* buffer, size_t size)
+inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const TObject* input)
 {
-  return deserialize<T>(gsl::span<std::byte>(buffer, gsl::narrow<gsl::span<std::byte>::size_type>(size)));
-}
-
-inline void FairTMessage::free(void* /*data*/, void* hint)
-{
-  std::default_delete<FairTMessage> deleter;
-  deleter(static_cast<FairTMessage*>(hint));
-}
-
-inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const TObject* input,
-                                          TMessageSerializer::CompressionLevel compressionLevel)
-{
-  std::unique_ptr<FairTMessage> tm = std::make_unique<FairTMessage>(kMESS_OBJECT);
-
-  serialize(*tm, input, input->Class(), compressionLevel);
-
-  msg.Rebuild(tm->Buffer(), tm->BufferSize(), FairTMessage::free, tm.get());
-  tm.release();
+  FairOutputTBuffer output(msg);
+  serialize(output, input, input->Class());
 }
 
 template <typename T>
-inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const T* input, //
-                                          const TClass* cl,                       //
-                                          TMessageSerializer::CompressionLevel compressionLevel)
+inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const T* input, const TClass* cl)
 {
-  std::unique_ptr<FairTMessage> tm = std::make_unique<FairTMessage>(kMESS_OBJECT);
-
-  serialize(*tm, input, cl, compressionLevel);
-
-  msg.Rebuild(tm->Buffer(), tm->BufferSize(), FairTMessage::free, tm.get());
-  tm.release();
+  FairOutputTBuffer output(msg);
+  serialize(output, input, cl);
 }
 
 template <typename T>
@@ -161,7 +148,8 @@ inline void TMessageSerializer::Deserialize(const fair::mq::Message& msg, std::u
 {
   // we know the message will not be modified by this,
   // so const_cast should be OK here(IMHO).
-  output = deserialize(as_span(msg));
+  FairInputTBuffer input(static_cast<char*>(msg.GetData()), static_cast<int>(msg.GetSize()));
+  output = deserialize(input);
 }
 
 // gsl::narrow is used to do a runtime narrowing check, this might be a bit paranoid,
@@ -171,7 +159,7 @@ inline gsl::span<std::byte> as_span(const fair::mq::Message& msg)
   return gsl::span<std::byte>{static_cast<std::byte*>(msg.GetData()), gsl::narrow<gsl::span<std::byte>::size_type>(msg.GetSize())};
 }
 
-inline gsl::span<std::byte> as_span(const FairTMessage& msg)
+inline gsl::span<std::byte> as_span(const FairInputTBuffer& msg)
 {
   return gsl::span<std::byte>{reinterpret_cast<std::byte*>(msg.Buffer()),
                               gsl::narrow<gsl::span<std::byte>::size_type>(msg.BufferSize())};
diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 48a3eb1da95b9..02ef5c7bc5b3c 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -141,9 +141,9 @@ DataProcessorSpec CommonDataProcessors::getOutputObjHistSink(std::vector<OutputO
         return;
       }
 
-      FairTMessage tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
+      FairInputTBuffer tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
       InputObject obj;
-      obj.kind = tm.GetClass();
+      obj.kind = tm.ReadClass();
       if (obj.kind == nullptr) {
         LOG(error) << "Cannot read class info from buffer.";
         return;
diff --git a/Framework/Core/src/TMessageSerializer.cxx b/Framework/Core/src/TMessageSerializer.cxx
index 5388a6d716cda..9f09c3ade0089 100644
--- a/Framework/Core/src/TMessageSerializer.cxx
+++ b/Framework/Core/src/TMessageSerializer.cxx
@@ -9,7 +9,38 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 #include <Framework/TMessageSerializer.h>
+#include <FairMQTransportFactory.h>
 #include <algorithm>
 #include <memory>
 
 using namespace o2::framework;
+
+void* FairOutputTBuffer::embedInItself(fair::mq::Message& msg) {
+  // The first bytes of the message are used to store the pointer to the message itself
+  // so that we can reallocate it if needed.
+  if (sizeof(char*) > msg.GetSize()) {
+    throw std::runtime_error("Message size too small to embed pointer");
+  }
+  char* data = reinterpret_cast<char*>(msg.GetData());
+  char* ptr = reinterpret_cast<char*>(&msg);
+  std::memcpy(data, ptr, sizeof(char*));
+  return data + sizeof(char*);
+}
+
+// Reallocation function. Get the message pointer from the data and call Rebuild.
+char *FairOutputTBuffer::fairMQrealloc(char *oldData, size_t newSize, size_t oldSize) {
+  auto* msg = reinterpret_cast<fair::mq::Message*>(oldData - sizeof(char*));
+  if (newSize <= msg->GetSize()) {
+    // no need to reallocate, the message is already big enough
+    return oldData;
+  }
+  // Create a shallow copy of the message
+  fair::mq::MessagePtr oldMsg = msg->GetTransport()->CreateMessage();
+  oldMsg->Copy(*msg);
+  // Copy the old data while rebuilding. Reference counting should make
+  // sure the old message is not deleted until the new one is ready.
+  msg->Rebuild(newSize, fair::mq::Alignment{64});
+  memcpy(msg->GetData(), oldMsg->GetData(), oldSize);
+
+  return reinterpret_cast<char*>(msg->GetData()) + sizeof(char*);
+}
diff --git a/Framework/Core/test/test_DataRefUtils.cxx b/Framework/Core/test/test_DataRefUtils.cxx
index 37da7912bfe8b..081adc81ebf69 100644
--- a/Framework/Core/test/test_DataRefUtils.cxx
+++ b/Framework/Core/test/test_DataRefUtils.cxx
@@ -21,17 +21,37 @@
 
 using namespace o2::framework;
 
+TEST_CASE("PureRootTest") {
+  TBufferFile buffer(TBuffer::kWrite);
+  TObjString s("test");
+  buffer.WriteObject(&s);
+
+  TBufferFile buffer2(TBuffer::kRead, buffer.BufferSize(), buffer.Buffer(), false);
+  buffer2.SetReadMode();
+  buffer2.InitMap();
+  TClass *storedClass = buffer2.ReadClass();
+  // ReadClass advances the buffer, so we need to reset it.
+  buffer2.SetBufferOffset(0);
+  buffer2.ResetMap();
+  REQUIRE(storedClass != nullptr);
+  auto *outS = (TObjString*)buffer2.ReadObjectAny(storedClass);
+  REQUIRE(outS != nullptr);
+  REQUIRE(outS->GetString() == "test");
+}
+
 // Simple test to do root deserialization.
 TEST_CASE("TestRootSerialization")
 {
   DataRef ref;
-  TMessage* tm = new TMessage(kMESS_OBJECT);
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer tm(*msg);
   auto sOrig = std::make_unique<TObjString>("test");
-  tm->WriteObject(sOrig.get());
+  tm << sOrig.get();
   o2::header::DataHeader dh;
   dh.payloadSerializationMethod = o2::header::gSerializationMethodROOT;
-  ref.payload = tm->Buffer();
-  dh.payloadSize = tm->BufferSize();
+  ref.payload = (char*)msg->GetData();
+  dh.payloadSize = (size_t)msg->GetSize();
   ref.header = reinterpret_cast<char const*>(&dh);
 
   // Check by using the same type
diff --git a/Framework/Core/test/test_TMessageSerializer.cxx b/Framework/Core/test/test_TMessageSerializer.cxx
index bc5f817400a44..89de74edc8455 100644
--- a/Framework/Core/test/test_TMessageSerializer.cxx
+++ b/Framework/Core/test/test_TMessageSerializer.cxx
@@ -11,6 +11,7 @@
 
 #include "Framework/TMessageSerializer.h"
 #include "Framework/RuntimeError.h"
+#include <fairmq/TransportFactory.h>
 #include "TestClasses.h"
 #include <catch_amalgamated.hpp>
 #include <utility>
@@ -49,14 +50,14 @@ TEST_CASE("TestTMessageSerializer")
   array.SetOwner();
   array.Add(new TNamed(testname, testtitle));
 
-  FairTMessage msg;
-  TMessageSerializer::serialize(msg, &array);
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer buffer(*msg);
+  TMessageSerializer::serialize(buffer, &array);
 
-  auto buf = as_span(msg);
-  REQUIRE(buf.size() == msg.BufferSize());
-  REQUIRE(static_cast<void*>(buf.data()) == static_cast<void*>(msg.Buffer()));
+  FairInputTBuffer msg2((char*)msg->GetData(), msg->GetSize());
   // test deserialization with TObject as target class (default)
-  auto out = TMessageSerializer::deserialize(buf);
+  auto out = TMessageSerializer::deserialize(msg2);
 
   auto* outarr = dynamic_cast<TObjArray*>(out.get());
   REQUIRE(out.get() == outarr);
@@ -66,9 +67,9 @@ TEST_CASE("TestTMessageSerializer")
   REQUIRE(named->GetTitle() == std::string(testtitle));
 
   // test deserialization with a wrong target class and check the exception
-  REQUIRE_THROWS_AS(TMessageSerializer::deserialize<TNamed>(buf), o2::framework::RuntimeErrorRef);
+  REQUIRE_THROWS_AS(TMessageSerializer::deserialize<TNamed>(msg2), o2::framework::RuntimeErrorRef);
 
-  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<TNamed>(buf), o2::framework::RuntimeErrorRef,
+  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<TNamed>(msg2), o2::framework::RuntimeErrorRef,
                          ExceptionMatcher("can not convert serialized class TObjArray into target class TNamed"));
 }
 
@@ -87,18 +88,21 @@ TEST_CASE("TestTMessageSerializer_NonTObject")
   TClass* cl = TClass::GetClass("std::vector<o2::test::Polymorphic>");
   REQUIRE(cl != nullptr);
 
-  FairTMessage msg;
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer buffer(*msg);
   char* in = reinterpret_cast<char*>(&data);
-  TMessageSerializer::serialize(msg, in, cl);
+  TMessageSerializer::serialize(buffer, in, cl);
+  FairInputTBuffer msg2((char*)msg->GetData(), msg->GetSize());
 
-  auto out = TMessageSerializer::deserialize<std::vector<o2::test::Polymorphic>>(as_span(msg));
+  auto out = TMessageSerializer::deserialize<std::vector<o2::test::Polymorphic>>(msg2);
   REQUIRE(out);
   REQUIRE((*out.get()).size() == 2);
   REQUIRE((*out.get())[0] == o2::test::Polymorphic(0xaffe));
   REQUIRE((*out.get())[1] == o2::test::Polymorphic(0xd00f));
 
   // test deserialization with a wrong target class and check the exception
-  REQUIRE_THROWS_AS(TMessageSerializer::deserialize(as_span(msg)), RuntimeErrorRef);
+  REQUIRE_THROWS_AS(TMessageSerializer::deserialize(msg2), RuntimeErrorRef);
 }
 
 TEST_CASE("TestTMessageSerializer_InvalidBuffer")
@@ -119,5 +123,6 @@ TEST_CASE("TestTMessageSerializer_InvalidBuffer")
   struct Dummy {
   };
   auto matcher = ExceptionMatcher("class is not ROOT-serializable: ZL22CATCH2_INTERNAL_TEST_4vE5Dummy");
-  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<Dummy>((std::byte*)buffer, strlen(buffer)), o2::framework::RuntimeErrorRef, matcher);
+  FairInputTBuffer msg2((char*)buffer, strlen(buffer));
+  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<Dummy>(msg2), o2::framework::RuntimeErrorRef, matcher);
 }
diff --git a/Framework/Utils/test/test_RootTreeWriter.cxx b/Framework/Utils/test/test_RootTreeWriter.cxx
index 3194508f3d775..62e1eb62cb4f1 100644
--- a/Framework/Utils/test/test_RootTreeWriter.cxx
+++ b/Framework/Utils/test/test_RootTreeWriter.cxx
@@ -179,6 +179,7 @@ TEST_CASE("test_RootTreeWriter")
 
   auto createSerializedMessage = [&transport, &store](DataHeader&& dh, auto& data) {
     fair::mq::MessagePtr payload = transport->CreateMessage();
+    payload->Rebuild(4096, {64});
     auto* cl = TClass::GetClass(typeid(decltype(data)));
     TMessageSerializer().Serialize(*payload, &data, cl);
     dh.payloadSize = payload->GetSize();
diff --git a/Utilities/Mergers/src/ObjectStore.cxx b/Utilities/Mergers/src/ObjectStore.cxx
index e88358507c31e..3bb49f1dfc9d8 100644
--- a/Utilities/Mergers/src/ObjectStore.cxx
+++ b/Utilities/Mergers/src/ObjectStore.cxx
@@ -38,7 +38,7 @@ static std::string concat(Args&&... arguments)
   return std::move(ss.str());
 }
 
-void* readObject(const TClass* type, o2::framework::FairTMessage& ftm)
+void* readObject(const TClass* type, o2::framework::FairInputTBuffer& ftm)
 {
   using namespace std::string_view_literals;
   auto* object = ftm.ReadObjectAny(type);
@@ -60,7 +60,7 @@ MergeInterface* castToMergeInterface(bool inheritsFromTObject, void* object, TCl
   return objectAsMergeInterface;
 }
 
-std::optional<ObjectStore> extractVector(o2::framework::FairTMessage& ftm, const TClass* storedClass)
+std::optional<ObjectStore> extractVector(o2::framework::FairInputTBuffer& ftm, const TClass* storedClass)
 {
   if (!storedClass->InheritsFrom(TClass::GetClass(typeid(VectorOfRawTObjects)))) {
     return std::nullopt;
@@ -88,11 +88,14 @@ ObjectStore extractObjectFrom(const framework::DataRef& ref)
     throw std::runtime_error(concat(errorPrefix, "It is not ROOT-serialized"sv));
   }
 
-  o2::framework::FairTMessage ftm(const_cast<char*>(ref.payload), o2::framework::DataRefUtils::getPayloadSize(ref));
-  auto* storedClass = ftm.GetClass();
+  o2::framework::FairInputTBuffer ftm(const_cast<char*>(ref.payload), o2::framework::DataRefUtils::getPayloadSize(ref));
+  ftm.InitMap();
+  auto* storedClass = ftm.ReadClass();
   if (storedClass == nullptr) {
     throw std::runtime_error(concat(errorPrefix, "Unknown stored class"sv));
   }
+  ftm.SetBufferOffset(0);
+  ftm.ResetMap();
 
   if (const auto extractedVector = extractVector(ftm, storedClass)) {
     return extractedVector.value();
diff --git a/Utilities/Mergers/test/benchmark_Types.cxx b/Utilities/Mergers/test/benchmark_Types.cxx
index 790fd329185ea..736685c5746b8 100644
--- a/Utilities/Mergers/test/benchmark_Types.cxx
+++ b/Utilities/Mergers/test/benchmark_Types.cxx
@@ -165,11 +165,16 @@ auto measure = [](Measurement m, auto* o, auto* i) -> double {
         tm->WriteObject(o);
         start = std::chrono::high_resolution_clock::now();
 
-        o2::framework::FairTMessage ftm(const_cast<char*>(tm->Buffer()), tm->BufferSize());
-        auto* storedClass = ftm.GetClass();
+        // Needed to take into account that  FairInputTBuffer expects the first 8 bytes to be the
+        // allocator pointer, which is not present in the TMessage buffer.
+        o2::framework::FairInputTBuffer ftm(const_cast<char*>(tm->Buffer() - 8), tm->BufferSize() + 8);
+        ftm.InitMap();
+        auto* storedClass = ftm.ReadClass();
         if (storedClass == nullptr) {
           throw std::runtime_error("Unknown stored class");
         }
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         auto* tObjectClass = TClass::GetClass(typeid(TObject));
 
         if (!storedClass->InheritsFrom(tObjectClass)) {
@@ -738,4 +743,4 @@ int main(int argc, const char* argv[])
 
   file.close();
   return 0;
-}
\ No newline at end of file
+}

From 045edb4da0ebf74d224fe0792f0cd61edbc04870 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 24 Feb 2024 00:21:18 +0100
Subject: [PATCH 378/726] DPL: avoid TMessage usage

TMessage does not allow for non owned buffers, so we end up having
an extra buffer in private memory for (de)serializing. Using TBufferFile
directly allows to avoid that, so this moves the whole ROOT serialization
support in DPL to use it.
---
 .../src/AODJAlienReaderHelpers.h              |   2 +
 .../Core/include/Framework/DataAllocator.h    |   1 +
 .../Core/include/Framework/DataRefUtils.h     |  13 +-
 .../include/Framework/RootMessageContext.h    |   3 +
 .../Framework/RootSerializationSupport.h      |   3 +-
 .../include/Framework/TMessageSerializer.h    | 130 ++++++++----------
 Framework/Core/src/CommonDataProcessors.cxx   |   4 +-
 Framework/Core/src/TMessageSerializer.cxx     |  31 +++++
 Framework/Core/test/test_DataRefUtils.cxx     |  28 +++-
 .../Core/test/test_TMessageSerializer.cxx     |  34 +++--
 Framework/Utils/test/test_RootTreeWriter.cxx  |   1 +
 Utilities/Mergers/src/ObjectStore.cxx         |  11 +-
 Utilities/Mergers/test/benchmark_Types.cxx    |  11 +-
 13 files changed, 171 insertions(+), 101 deletions(-)

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
index 655e4b6c0b439..4b9fd710aca14 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
@@ -16,7 +16,9 @@
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/Logger.h"
 #include <Monitoring/Monitoring.h>
+
 #include <uv.h>
+class TFile;
 
 namespace o2::framework::readers
 {
diff --git a/Framework/Core/include/Framework/DataAllocator.h b/Framework/Core/include/Framework/DataAllocator.h
index 8151d2f83c6c6..029e922aeb90b 100644
--- a/Framework/Core/include/Framework/DataAllocator.h
+++ b/Framework/Core/include/Framework/DataAllocator.h
@@ -359,6 +359,7 @@ class DataAllocator
     } else if constexpr (has_root_dictionary<T>::value == true || is_specialization_v<T, ROOTSerialized> == true) {
       // Serialize a snapshot of an object with root dictionary
       payloadMessage = proxy.createOutputMessage(routeIndex);
+      payloadMessage->Rebuild(4096, {64});
       if constexpr (is_specialization_v<T, ROOTSerialized> == true) {
         // Explicitely ROOT serialize a snapshot of object.
         // An object wrapped into type `ROOTSerialized` is explicitely marked to be ROOT serialized
diff --git a/Framework/Core/include/Framework/DataRefUtils.h b/Framework/Core/include/Framework/DataRefUtils.h
index defd10244bca5..e59f986f09250 100644
--- a/Framework/Core/include/Framework/DataRefUtils.h
+++ b/Framework/Core/include/Framework/DataRefUtils.h
@@ -71,12 +71,15 @@ struct DataRefUtils {
           throw runtime_error("Attempt to extract a TMessage from non-ROOT serialised message");
         }
 
-        typename RSS::FairTMessage ftm(const_cast<char*>(ref.payload), payloadSize);
-        auto* storedClass = ftm.GetClass();
+        typename RSS::FairInputTBuffer ftm(const_cast<char*>(ref.payload), payloadSize);
         auto* requestedClass = RSS::TClass::GetClass(typeid(T));
+        ftm.InitMap();
+        auto* storedClass = ftm.ReadClass();
         // should always have the class description if has_root_dictionary is true
         assert(requestedClass != nullptr);
 
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         auto* object = ftm.ReadObjectAny(storedClass);
         if (object == nullptr) {
           throw runtime_error_f("Failed to read object with name %s from message using ROOT serialization.",
@@ -146,7 +149,11 @@ struct DataRefUtils {
           throw runtime_error("ROOT serialization not supported, dictionary not found for data type");
         }
 
-        typename RSS::FairTMessage ftm(const_cast<char*>(ref.payload), payloadSize);
+        typename RSS::FairInputTBuffer ftm(const_cast<char*>(ref.payload), payloadSize);
+        ftm.InitMap();
+        auto *classInfo = ftm.ReadClass();
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         result.reset(static_cast<wrapped*>(ftm.ReadObjectAny(cl)));
         if (result.get() == nullptr) {
           throw runtime_error_f("Unable to extract class %s", cl == nullptr ? "<name not available>" : cl->GetName());
diff --git a/Framework/Core/include/Framework/RootMessageContext.h b/Framework/Core/include/Framework/RootMessageContext.h
index bef60ebbbf9f9..b1124880cf30f 100644
--- a/Framework/Core/include/Framework/RootMessageContext.h
+++ b/Framework/Core/include/Framework/RootMessageContext.h
@@ -72,6 +72,9 @@ class RootSerializedObject : public MessageContext::ContextObject
   fair::mq::Parts finalize() final
   {
     assert(mParts.Size() == 1);
+    if (mPayloadMsg->GetSize() < sizeof(char*)) {
+      mPayloadMsg->Rebuild(4096, {64});
+    }
     TMessageSerializer::Serialize(*mPayloadMsg, mObject.get(), nullptr);
     mParts.AddPart(std::move(mPayloadMsg));
     return ContextObject::finalize();
diff --git a/Framework/Core/include/Framework/RootSerializationSupport.h b/Framework/Core/include/Framework/RootSerializationSupport.h
index cbf7408b13c7d..a44093f9c02bf 100644
--- a/Framework/Core/include/Framework/RootSerializationSupport.h
+++ b/Framework/Core/include/Framework/RootSerializationSupport.h
@@ -21,7 +21,8 @@ namespace o2::framework
 /// compiler.
 struct RootSerializationSupport {
   using TClass = ::TClass;
-  using FairTMessage = o2::framework::FairTMessage;
+  using FairInputTBuffer = o2::framework::FairInputTBuffer;
+  using FairOutputBuffer = o2::framework::FairOutputTBuffer;
   using TObject = ::TObject;
 };
 
diff --git a/Framework/Core/include/Framework/TMessageSerializer.h b/Framework/Core/include/Framework/TMessageSerializer.h
index 1f08b456c0218..ca18eb21abfa1 100644
--- a/Framework/Core/include/Framework/TMessageSerializer.h
+++ b/Framework/Core/include/Framework/TMessageSerializer.h
@@ -16,9 +16,8 @@
 #include "Framework/RuntimeError.h"
 
 #include <TList.h>
-#include <TMessage.h>
+#include <TBufferFile.h>
 #include <TObjArray.h>
-#include <TStreamerInfo.h>
 #include <gsl/util>
 #include <gsl/span>
 #include <gsl/narrow>
@@ -28,67 +27,76 @@
 
 namespace o2::framework
 {
-class FairTMessage;
+class FairOutputTBuffer;
+class FairInputTBuffer;
 
 // utilities to produce a span over a byte buffer held by various message types
 // this is to avoid littering code with casts and conversions (span has a signed index type(!))
-gsl::span<std::byte> as_span(const FairTMessage& msg);
+gsl::span<std::byte> as_span(const FairInputTBuffer& msg);
+gsl::span<std::byte> as_span(const FairOutputTBuffer& msg);
 gsl::span<std::byte> as_span(const fair::mq::Message& msg);
 
-class FairTMessage : public TMessage
+// A TBufferFile which we can use to serialise data to a FairMQ message.
+class FairOutputTBuffer : public TBufferFile
 {
  public:
-  using TMessage::TMessage;
-  FairTMessage() : TMessage(kMESS_OBJECT) {}
-  FairTMessage(void* buf, Int_t len) : TMessage(buf, len) { ResetBit(kIsOwner); }
-  FairTMessage(gsl::span<std::byte> buf) : TMessage(buf.data(), buf.size()) { ResetBit(kIsOwner); }
+  // This is to serialise data to FairMQ. We embed the pointer to the message
+  // in the data itself, so that we can use it to reallocate the message if needed.
+  // The FairMQ message retains ownership of the data.
+  // When deserialising the root object, keep in mind one needs to skip the 8 bytes
+  // for the pointer.
+  FairOutputTBuffer(fair::mq::Message& msg)
+    : TBufferFile(TBuffer::kWrite, msg.GetSize() - sizeof(char*), embedInItself(msg), false, fairMQrealloc)
+  {
+  }
+  // Helper function to keep track of the FairMQ message that holds the data
+  // in the data itself. We can use this to make sure the message can be reallocated
+  // even if we simply have a pointer to the data. Hopefully ROOT will not play dirty
+  // with us.
+  void* embedInItself(fair::mq::Message& msg);
   // helper function to clean up the object holding the data after it is transported.
-  static void free(void* /*data*/, void* hint);
+  static char* fairMQrealloc(char* oldData, size_t newSize, size_t oldSize);
 };
 
-struct TMessageSerializer {
-  using StreamerList = std::vector<TVirtualStreamerInfo*>;
-  using CompressionLevel = int;
+class FairInputTBuffer : public TBufferFile
+{
+ public:
+  // This is to serialise data to FairMQ. The provided message is expeted to have 8 bytes
+  // of overhead, where the source embedded the pointer for the reallocation.
+  // Notice this will break if the sender and receiver are not using the same
+  // size for a pointer.
+  FairInputTBuffer(char * data, size_t size)
+    : TBufferFile(TBuffer::kRead, size-sizeof(char*), data + sizeof(char*), false, nullptr)
+  {
+  }
+};
 
-  static void Serialize(fair::mq::Message& msg, const TObject* input,
-                        CompressionLevel compressionLevel = -1);
+struct TMessageSerializer {
+  static void Serialize(fair::mq::Message& msg, const TObject* input);
 
   template <typename T>
-  static void Serialize(fair::mq::Message& msg, const T* input, const TClass* cl, //
-                        CompressionLevel compressionLevel = -1);
+  static void Serialize(fair::mq::Message& msg, const T* input, const TClass* cl);
 
   template <typename T = TObject>
   static void Deserialize(const fair::mq::Message& msg, std::unique_ptr<T>& output);
 
-  static void serialize(FairTMessage& msg, const TObject* input,
-                        CompressionLevel compressionLevel = -1);
+  static void serialize(o2::framework::FairOutputTBuffer& msg, const TObject* input);
 
   template <typename T>
-  static void serialize(FairTMessage& msg, const T* input, //
-                        const TClass* cl,
-                        CompressionLevel compressionLevel = -1);
+  static void serialize(o2::framework::FairOutputTBuffer& msg, const T* input, const TClass* cl);
 
   template <typename T = TObject>
-  static std::unique_ptr<T> deserialize(gsl::span<std::byte> buffer);
-  template <typename T = TObject>
-  static inline std::unique_ptr<T> deserialize(std::byte* buffer, size_t size);
+  static inline std::unique_ptr<T> deserialize(FairInputTBuffer & buffer);
 };
 
-inline void TMessageSerializer::serialize(FairTMessage& tm, const TObject* input,
-                                          CompressionLevel compressionLevel)
+inline void TMessageSerializer::serialize(FairOutputTBuffer& tm, const TObject* input)
 {
-  return serialize(tm, input, nullptr, compressionLevel);
+  return serialize(tm, input, nullptr);
 }
 
 template <typename T>
-inline void TMessageSerializer::serialize(FairTMessage& tm, const T* input, //
-                                          const TClass* cl, CompressionLevel compressionLevel)
+inline void TMessageSerializer::serialize(FairOutputTBuffer& tm, const T* input, const TClass* cl)
 {
-  if (compressionLevel >= 0) {
-    // if negative, skip to use ROOT default
-    tm.SetCompressionLevel(compressionLevel);
-  }
-
   // TODO: check what WriateObject and WriteObjectAny are doing
   if (cl == nullptr) {
     tm.WriteObject(input);
@@ -98,7 +106,7 @@ inline void TMessageSerializer::serialize(FairTMessage& tm, const T* input, //
 }
 
 template <typename T>
-inline std::unique_ptr<T> TMessageSerializer::deserialize(gsl::span<std::byte> buffer)
+inline std::unique_ptr<T> TMessageSerializer::deserialize(FairInputTBuffer & buffer)
 {
   TClass* tgtClass = TClass::GetClass(typeid(T));
   if (tgtClass == nullptr) {
@@ -107,53 +115,32 @@ inline std::unique_ptr<T> TMessageSerializer::deserialize(gsl::span<std::byte> b
   // FIXME: we need to add consistency check for buffer data to be serialized
   // at the moment, TMessage might simply crash if an invalid or inconsistent
   // buffer is provided
-  FairTMessage tm(buffer);
-  TClass* serializedClass = tm.GetClass();
+  buffer.InitMap();
+  TClass* serializedClass = buffer.ReadClass();
+  buffer.SetBufferOffset(0);
+  buffer.ResetMap();
   if (serializedClass == nullptr) {
     throw runtime_error_f("can not read class info from buffer");
   }
   if (tgtClass != serializedClass && serializedClass->GetBaseClass(tgtClass) == nullptr) {
     throw runtime_error_f("can not convert serialized class %s into target class %s",
-                          tm.GetClass()->GetName(),
+                          serializedClass->GetName(),
                           tgtClass->GetName());
   }
-  return std::unique_ptr<T>(reinterpret_cast<T*>(tm.ReadObjectAny(serializedClass)));
+  return std::unique_ptr<T>(reinterpret_cast<T*>(buffer.ReadObjectAny(serializedClass)));
 }
 
-template <typename T>
-inline std::unique_ptr<T> TMessageSerializer::deserialize(std::byte* buffer, size_t size)
+inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const TObject* input)
 {
-  return deserialize<T>(gsl::span<std::byte>(buffer, gsl::narrow<gsl::span<std::byte>::size_type>(size)));
-}
-
-inline void FairTMessage::free(void* /*data*/, void* hint)
-{
-  std::default_delete<FairTMessage> deleter;
-  deleter(static_cast<FairTMessage*>(hint));
-}
-
-inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const TObject* input,
-                                          TMessageSerializer::CompressionLevel compressionLevel)
-{
-  std::unique_ptr<FairTMessage> tm = std::make_unique<FairTMessage>(kMESS_OBJECT);
-
-  serialize(*tm, input, input->Class(), compressionLevel);
-
-  msg.Rebuild(tm->Buffer(), tm->BufferSize(), FairTMessage::free, tm.get());
-  tm.release();
+  FairOutputTBuffer output(msg);
+  serialize(output, input, input->Class());
 }
 
 template <typename T>
-inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const T* input, //
-                                          const TClass* cl,                       //
-                                          TMessageSerializer::CompressionLevel compressionLevel)
+inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const T* input, const TClass* cl)
 {
-  std::unique_ptr<FairTMessage> tm = std::make_unique<FairTMessage>(kMESS_OBJECT);
-
-  serialize(*tm, input, cl, compressionLevel);
-
-  msg.Rebuild(tm->Buffer(), tm->BufferSize(), FairTMessage::free, tm.get());
-  tm.release();
+  FairOutputTBuffer output(msg);
+  serialize(output, input, cl);
 }
 
 template <typename T>
@@ -161,7 +148,8 @@ inline void TMessageSerializer::Deserialize(const fair::mq::Message& msg, std::u
 {
   // we know the message will not be modified by this,
   // so const_cast should be OK here(IMHO).
-  output = deserialize(as_span(msg));
+  FairInputTBuffer input(static_cast<char*>(msg.GetData()), static_cast<int>(msg.GetSize()));
+  output = deserialize(input);
 }
 
 // gsl::narrow is used to do a runtime narrowing check, this might be a bit paranoid,
@@ -171,7 +159,7 @@ inline gsl::span<std::byte> as_span(const fair::mq::Message& msg)
   return gsl::span<std::byte>{static_cast<std::byte*>(msg.GetData()), gsl::narrow<gsl::span<std::byte>::size_type>(msg.GetSize())};
 }
 
-inline gsl::span<std::byte> as_span(const FairTMessage& msg)
+inline gsl::span<std::byte> as_span(const FairInputTBuffer& msg)
 {
   return gsl::span<std::byte>{reinterpret_cast<std::byte*>(msg.Buffer()),
                               gsl::narrow<gsl::span<std::byte>::size_type>(msg.BufferSize())};
diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 48a3eb1da95b9..02ef5c7bc5b3c 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -141,9 +141,9 @@ DataProcessorSpec CommonDataProcessors::getOutputObjHistSink(std::vector<OutputO
         return;
       }
 
-      FairTMessage tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
+      FairInputTBuffer tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
       InputObject obj;
-      obj.kind = tm.GetClass();
+      obj.kind = tm.ReadClass();
       if (obj.kind == nullptr) {
         LOG(error) << "Cannot read class info from buffer.";
         return;
diff --git a/Framework/Core/src/TMessageSerializer.cxx b/Framework/Core/src/TMessageSerializer.cxx
index 5388a6d716cda..9f09c3ade0089 100644
--- a/Framework/Core/src/TMessageSerializer.cxx
+++ b/Framework/Core/src/TMessageSerializer.cxx
@@ -9,7 +9,38 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 #include <Framework/TMessageSerializer.h>
+#include <FairMQTransportFactory.h>
 #include <algorithm>
 #include <memory>
 
 using namespace o2::framework;
+
+void* FairOutputTBuffer::embedInItself(fair::mq::Message& msg) {
+  // The first bytes of the message are used to store the pointer to the message itself
+  // so that we can reallocate it if needed.
+  if (sizeof(char*) > msg.GetSize()) {
+    throw std::runtime_error("Message size too small to embed pointer");
+  }
+  char* data = reinterpret_cast<char*>(msg.GetData());
+  char* ptr = reinterpret_cast<char*>(&msg);
+  std::memcpy(data, ptr, sizeof(char*));
+  return data + sizeof(char*);
+}
+
+// Reallocation function. Get the message pointer from the data and call Rebuild.
+char *FairOutputTBuffer::fairMQrealloc(char *oldData, size_t newSize, size_t oldSize) {
+  auto* msg = reinterpret_cast<fair::mq::Message*>(oldData - sizeof(char*));
+  if (newSize <= msg->GetSize()) {
+    // no need to reallocate, the message is already big enough
+    return oldData;
+  }
+  // Create a shallow copy of the message
+  fair::mq::MessagePtr oldMsg = msg->GetTransport()->CreateMessage();
+  oldMsg->Copy(*msg);
+  // Copy the old data while rebuilding. Reference counting should make
+  // sure the old message is not deleted until the new one is ready.
+  msg->Rebuild(newSize, fair::mq::Alignment{64});
+  memcpy(msg->GetData(), oldMsg->GetData(), oldSize);
+
+  return reinterpret_cast<char*>(msg->GetData()) + sizeof(char*);
+}
diff --git a/Framework/Core/test/test_DataRefUtils.cxx b/Framework/Core/test/test_DataRefUtils.cxx
index 37da7912bfe8b..081adc81ebf69 100644
--- a/Framework/Core/test/test_DataRefUtils.cxx
+++ b/Framework/Core/test/test_DataRefUtils.cxx
@@ -21,17 +21,37 @@
 
 using namespace o2::framework;
 
+TEST_CASE("PureRootTest") {
+  TBufferFile buffer(TBuffer::kWrite);
+  TObjString s("test");
+  buffer.WriteObject(&s);
+
+  TBufferFile buffer2(TBuffer::kRead, buffer.BufferSize(), buffer.Buffer(), false);
+  buffer2.SetReadMode();
+  buffer2.InitMap();
+  TClass *storedClass = buffer2.ReadClass();
+  // ReadClass advances the buffer, so we need to reset it.
+  buffer2.SetBufferOffset(0);
+  buffer2.ResetMap();
+  REQUIRE(storedClass != nullptr);
+  auto *outS = (TObjString*)buffer2.ReadObjectAny(storedClass);
+  REQUIRE(outS != nullptr);
+  REQUIRE(outS->GetString() == "test");
+}
+
 // Simple test to do root deserialization.
 TEST_CASE("TestRootSerialization")
 {
   DataRef ref;
-  TMessage* tm = new TMessage(kMESS_OBJECT);
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer tm(*msg);
   auto sOrig = std::make_unique<TObjString>("test");
-  tm->WriteObject(sOrig.get());
+  tm << sOrig.get();
   o2::header::DataHeader dh;
   dh.payloadSerializationMethod = o2::header::gSerializationMethodROOT;
-  ref.payload = tm->Buffer();
-  dh.payloadSize = tm->BufferSize();
+  ref.payload = (char*)msg->GetData();
+  dh.payloadSize = (size_t)msg->GetSize();
   ref.header = reinterpret_cast<char const*>(&dh);
 
   // Check by using the same type
diff --git a/Framework/Core/test/test_TMessageSerializer.cxx b/Framework/Core/test/test_TMessageSerializer.cxx
index bc5f817400a44..395b3779421a2 100644
--- a/Framework/Core/test/test_TMessageSerializer.cxx
+++ b/Framework/Core/test/test_TMessageSerializer.cxx
@@ -11,6 +11,7 @@
 
 #include "Framework/TMessageSerializer.h"
 #include "Framework/RuntimeError.h"
+#include <fairmq/TransportFactory.h>
 #include "TestClasses.h"
 #include <catch_amalgamated.hpp>
 #include <utility>
@@ -49,14 +50,14 @@ TEST_CASE("TestTMessageSerializer")
   array.SetOwner();
   array.Add(new TNamed(testname, testtitle));
 
-  FairTMessage msg;
-  TMessageSerializer::serialize(msg, &array);
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer buffer(*msg);
+  TMessageSerializer::serialize(buffer, &array);
 
-  auto buf = as_span(msg);
-  REQUIRE(buf.size() == msg.BufferSize());
-  REQUIRE(static_cast<void*>(buf.data()) == static_cast<void*>(msg.Buffer()));
+  FairInputTBuffer msg2((char*)msg->GetData(), msg->GetSize());
   // test deserialization with TObject as target class (default)
-  auto out = TMessageSerializer::deserialize(buf);
+  auto out = TMessageSerializer::deserialize(msg2);
 
   auto* outarr = dynamic_cast<TObjArray*>(out.get());
   REQUIRE(out.get() == outarr);
@@ -66,9 +67,9 @@ TEST_CASE("TestTMessageSerializer")
   REQUIRE(named->GetTitle() == std::string(testtitle));
 
   // test deserialization with a wrong target class and check the exception
-  REQUIRE_THROWS_AS(TMessageSerializer::deserialize<TNamed>(buf), o2::framework::RuntimeErrorRef);
+  REQUIRE_THROWS_AS(TMessageSerializer::deserialize<TNamed>(msg2), o2::framework::RuntimeErrorRef);
 
-  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<TNamed>(buf), o2::framework::RuntimeErrorRef,
+  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<TNamed>(msg2), o2::framework::RuntimeErrorRef,
                          ExceptionMatcher("can not convert serialized class TObjArray into target class TNamed"));
 }
 
@@ -87,23 +88,29 @@ TEST_CASE("TestTMessageSerializer_NonTObject")
   TClass* cl = TClass::GetClass("std::vector<o2::test::Polymorphic>");
   REQUIRE(cl != nullptr);
 
-  FairTMessage msg;
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer buffer(*msg);
   char* in = reinterpret_cast<char*>(&data);
-  TMessageSerializer::serialize(msg, in, cl);
+  TMessageSerializer::serialize(buffer, in, cl);
+  FairInputTBuffer msg2((char*)msg->GetData(), msg->GetSize());
 
-  auto out = TMessageSerializer::deserialize<std::vector<o2::test::Polymorphic>>(as_span(msg));
+  auto out = TMessageSerializer::deserialize<std::vector<o2::test::Polymorphic>>(msg2);
   REQUIRE(out);
   REQUIRE((*out.get()).size() == 2);
   REQUIRE((*out.get())[0] == o2::test::Polymorphic(0xaffe));
   REQUIRE((*out.get())[1] == o2::test::Polymorphic(0xd00f));
 
   // test deserialization with a wrong target class and check the exception
-  REQUIRE_THROWS_AS(TMessageSerializer::deserialize(as_span(msg)), RuntimeErrorRef);
+  REQUIRE_THROWS_AS(TMessageSerializer::deserialize(msg2), RuntimeErrorRef);
 }
 
 TEST_CASE("TestTMessageSerializer_InvalidBuffer")
 {
   const char* buffer = "this is for sure not a serialized ROOT object";
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(strlen(buffer) + 8);
+  memcpy((char*)msg->GetData() + 8, buffer, strlen(buffer));
   // test deserialization of invalid buffer and check the exception
   // FIXME: at the moment, TMessage fails directly with a segfault, which it shouldn't do
   /*
@@ -119,5 +126,6 @@ TEST_CASE("TestTMessageSerializer_InvalidBuffer")
   struct Dummy {
   };
   auto matcher = ExceptionMatcher("class is not ROOT-serializable: ZL22CATCH2_INTERNAL_TEST_4vE5Dummy");
-  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<Dummy>((std::byte*)buffer, strlen(buffer)), o2::framework::RuntimeErrorRef, matcher);
+  FairInputTBuffer msg2((char*)msg->GetData(), msg->GetSize());
+  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<Dummy>(msg2), o2::framework::RuntimeErrorRef, matcher);
 }
diff --git a/Framework/Utils/test/test_RootTreeWriter.cxx b/Framework/Utils/test/test_RootTreeWriter.cxx
index 3194508f3d775..62e1eb62cb4f1 100644
--- a/Framework/Utils/test/test_RootTreeWriter.cxx
+++ b/Framework/Utils/test/test_RootTreeWriter.cxx
@@ -179,6 +179,7 @@ TEST_CASE("test_RootTreeWriter")
 
   auto createSerializedMessage = [&transport, &store](DataHeader&& dh, auto& data) {
     fair::mq::MessagePtr payload = transport->CreateMessage();
+    payload->Rebuild(4096, {64});
     auto* cl = TClass::GetClass(typeid(decltype(data)));
     TMessageSerializer().Serialize(*payload, &data, cl);
     dh.payloadSize = payload->GetSize();
diff --git a/Utilities/Mergers/src/ObjectStore.cxx b/Utilities/Mergers/src/ObjectStore.cxx
index e88358507c31e..3bb49f1dfc9d8 100644
--- a/Utilities/Mergers/src/ObjectStore.cxx
+++ b/Utilities/Mergers/src/ObjectStore.cxx
@@ -38,7 +38,7 @@ static std::string concat(Args&&... arguments)
   return std::move(ss.str());
 }
 
-void* readObject(const TClass* type, o2::framework::FairTMessage& ftm)
+void* readObject(const TClass* type, o2::framework::FairInputTBuffer& ftm)
 {
   using namespace std::string_view_literals;
   auto* object = ftm.ReadObjectAny(type);
@@ -60,7 +60,7 @@ MergeInterface* castToMergeInterface(bool inheritsFromTObject, void* object, TCl
   return objectAsMergeInterface;
 }
 
-std::optional<ObjectStore> extractVector(o2::framework::FairTMessage& ftm, const TClass* storedClass)
+std::optional<ObjectStore> extractVector(o2::framework::FairInputTBuffer& ftm, const TClass* storedClass)
 {
   if (!storedClass->InheritsFrom(TClass::GetClass(typeid(VectorOfRawTObjects)))) {
     return std::nullopt;
@@ -88,11 +88,14 @@ ObjectStore extractObjectFrom(const framework::DataRef& ref)
     throw std::runtime_error(concat(errorPrefix, "It is not ROOT-serialized"sv));
   }
 
-  o2::framework::FairTMessage ftm(const_cast<char*>(ref.payload), o2::framework::DataRefUtils::getPayloadSize(ref));
-  auto* storedClass = ftm.GetClass();
+  o2::framework::FairInputTBuffer ftm(const_cast<char*>(ref.payload), o2::framework::DataRefUtils::getPayloadSize(ref));
+  ftm.InitMap();
+  auto* storedClass = ftm.ReadClass();
   if (storedClass == nullptr) {
     throw std::runtime_error(concat(errorPrefix, "Unknown stored class"sv));
   }
+  ftm.SetBufferOffset(0);
+  ftm.ResetMap();
 
   if (const auto extractedVector = extractVector(ftm, storedClass)) {
     return extractedVector.value();
diff --git a/Utilities/Mergers/test/benchmark_Types.cxx b/Utilities/Mergers/test/benchmark_Types.cxx
index 790fd329185ea..736685c5746b8 100644
--- a/Utilities/Mergers/test/benchmark_Types.cxx
+++ b/Utilities/Mergers/test/benchmark_Types.cxx
@@ -165,11 +165,16 @@ auto measure = [](Measurement m, auto* o, auto* i) -> double {
         tm->WriteObject(o);
         start = std::chrono::high_resolution_clock::now();
 
-        o2::framework::FairTMessage ftm(const_cast<char*>(tm->Buffer()), tm->BufferSize());
-        auto* storedClass = ftm.GetClass();
+        // Needed to take into account that  FairInputTBuffer expects the first 8 bytes to be the
+        // allocator pointer, which is not present in the TMessage buffer.
+        o2::framework::FairInputTBuffer ftm(const_cast<char*>(tm->Buffer() - 8), tm->BufferSize() + 8);
+        ftm.InitMap();
+        auto* storedClass = ftm.ReadClass();
         if (storedClass == nullptr) {
           throw std::runtime_error("Unknown stored class");
         }
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         auto* tObjectClass = TClass::GetClass(typeid(TObject));
 
         if (!storedClass->InheritsFrom(tObjectClass)) {
@@ -738,4 +743,4 @@ int main(int argc, const char* argv[])
 
   file.close();
   return 0;
-}
\ No newline at end of file
+}

From 8e513105561b7b1a03503d0cfa78b4ffae0b7e5a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 24 Feb 2024 00:21:18 +0100
Subject: [PATCH 379/726] DPL: avoid TMessage usage

TMessage does not allow for non owned buffers, so we end up having
an extra buffer in private memory for (de)serializing. Using TBufferFile
directly allows to avoid that, so this moves the whole ROOT serialization
support in DPL to use it.
---
 .../Base/include/DetectorsBase/Detector.h     |  15 +-
 Detectors/Base/src/Detector.cxx               |  34 ++---
 .../src/AODJAlienReaderHelpers.h              |   2 +
 .../Core/include/Framework/DataAllocator.h    |   1 +
 .../Core/include/Framework/DataRefUtils.h     |  13 +-
 .../include/Framework/RootMessageContext.h    |   3 +
 .../Framework/RootSerializationSupport.h      |   3 +-
 .../include/Framework/TMessageSerializer.h    | 130 ++++++++----------
 Framework/Core/src/CommonDataProcessors.cxx   |   4 +-
 Framework/Core/src/TMessageSerializer.cxx     |  31 +++++
 Framework/Core/test/test_DataRefUtils.cxx     |  28 +++-
 .../Core/test/test_TMessageSerializer.cxx     |  34 +++--
 Framework/Utils/test/test_RootTreeWriter.cxx  |   1 +
 Utilities/Mergers/src/ObjectStore.cxx         |  11 +-
 Utilities/Mergers/test/benchmark_Types.cxx    |  11 +-
 15 files changed, 191 insertions(+), 130 deletions(-)

diff --git a/Detectors/Base/include/DetectorsBase/Detector.h b/Detectors/Base/include/DetectorsBase/Detector.h
index 6acfa4f5cc46c..4dd0452f2c059 100644
--- a/Detectors/Base/include/DetectorsBase/Detector.h
+++ b/Detectors/Base/include/DetectorsBase/Detector.h
@@ -29,7 +29,6 @@
 #include <typeinfo>
 #include <type_traits>
 #include <string>
-#include <TMessage.h>
 #include "CommonUtils/ShmManager.h"
 #include "CommonUtils/ShmAllocator.h"
 #include <sys/shm.h>
@@ -42,9 +41,7 @@
 
 #include <fairmq/FwdDecls.h>
 
-namespace o2
-{
-namespace base
+namespace o2::base
 {
 
 /// This is the basic class for any AliceO2 detector module, whether it is
@@ -260,17 +257,12 @@ T decodeShmMessage(fair::mq::Parts& dataparts, int index, bool*& busy)
 }
 
 // this goes into the source
-void attachMessageBufferToParts(fair::mq::Parts& parts, fair::mq::Channel& channel,
-                                void* data, size_t size, void (*func_ptr)(void* data, void* hint), void* hint);
+void attachMessageBufferToParts(fair::mq::Parts& parts, fair::mq::Channel& channel, void* data, TClass* cl);
 
 template <typename Container>
 void attachTMessage(Container const& hits, fair::mq::Channel& channel, fair::mq::Parts& parts)
 {
-  TMessage* tmsg = new TMessage();
-  tmsg->WriteObjectAny((void*)&hits, TClass::GetClass(typeid(hits)));
-  attachMessageBufferToParts(
-    parts, channel, tmsg->Buffer(), tmsg->BufferSize(),
-    [](void* data, void* hint) { delete static_cast<TMessage*>(hint); }, tmsg);
+  attachMessageBufferToParts(parts, channel, (void*)&hits, TClass::GetClass(typeid(hits)));
 }
 
 void* decodeTMessageCore(fair::mq::Parts& dataparts, int index);
@@ -746,7 +738,6 @@ class DetImpl : public o2::base::Detector
 
   ClassDefOverride(DetImpl, 0);
 };
-} // namespace base
 } // namespace o2
 
 #endif
diff --git a/Detectors/Base/src/Detector.cxx b/Detectors/Base/src/Detector.cxx
index 3168e0e84e1f2..3dccf732517b6 100644
--- a/Detectors/Base/src/Detector.cxx
+++ b/Detectors/Base/src/Detector.cxx
@@ -17,6 +17,7 @@
 #include "DetectorsBase/MaterialManager.h"
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "Field/MagneticField.h"
+#include "Framework/TMessageSerializer.h"
 #include "TString.h" // for TString
 #include "TGeoManager.h"
 
@@ -196,16 +197,18 @@ int Detector::registerSensitiveVolumeAndGetVolID(std::string const& name)
 #include <fairmq/Message.h>
 #include <fairmq/Parts.h>
 #include <fairmq/Channel.h>
-namespace o2
-{
-namespace base
+namespace o2::base
 {
 // this goes into the source
-void attachMessageBufferToParts(fair::mq::Parts& parts, fair::mq::Channel& channel, void* data, size_t size,
-                                void (*free_func)(void* data, void* hint), void* hint)
-{
-  std::unique_ptr<fair::mq::Message> message(channel.NewMessage(data, size, free_func, hint));
-  parts.AddPart(std::move(message));
+void attachMessageBufferToParts(fair::mq::Parts& parts, fair::mq::Channel& channel, void* data, TClass* cl) {
+  auto msg = channel.Transport()->CreateMessage(4096, fair::mq::Alignment{64});
+  // This will serialize the data directly into the message buffer, without any further
+  // buffer or copying. Notice how the message will have 8 bytes of header and then
+  // the serialized data as TBufferFile. In principle one could construct a serialized TMessage payload
+  // however I did not manage to get it to work for every case.
+  o2::framework::FairOutputTBuffer buffer(*msg);
+  o2::framework::TMessageSerializer::serialize(buffer, data, cl);
+  parts.AddPart(std::move(msg));
 }
 void attachDetIDHeaderMessage(int id, fair::mq::Channel& channel, fair::mq::Parts& parts)
 {
@@ -246,17 +249,14 @@ void* decodeShmCore(fair::mq::Parts& dataparts, int index, bool*& busy)
 
 void* decodeTMessageCore(fair::mq::Parts& dataparts, int index)
 {
-  class TMessageWrapper : public TMessage
-  {
-   public:
-    TMessageWrapper(void* buf, Int_t len) : TMessage(buf, len) { ResetBit(kIsOwner); }
-    ~TMessageWrapper() override = default;
-  };
   auto rawmessage = std::move(dataparts.At(index));
-  auto message = std::make_unique<TMessageWrapper>(rawmessage->GetData(), rawmessage->GetSize());
-  return message.get()->ReadObjectAny(message.get()->GetClass());
+  o2::framework::FairInputTBuffer buffer((char*)rawmessage->GetData(), rawmessage->GetSize());
+  buffer.InitMap();
+  auto *cl = buffer.ReadClass();
+  buffer.SetBufferOffset(0);
+  buffer.ResetMap();
+  return buffer.ReadObjectAny(cl);
 }
 
-} // namespace base
 } // namespace o2
 ClassImp(o2::base::Detector);
diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
index 655e4b6c0b439..4b9fd710aca14 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
@@ -16,7 +16,9 @@
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/Logger.h"
 #include <Monitoring/Monitoring.h>
+
 #include <uv.h>
+class TFile;
 
 namespace o2::framework::readers
 {
diff --git a/Framework/Core/include/Framework/DataAllocator.h b/Framework/Core/include/Framework/DataAllocator.h
index 8151d2f83c6c6..029e922aeb90b 100644
--- a/Framework/Core/include/Framework/DataAllocator.h
+++ b/Framework/Core/include/Framework/DataAllocator.h
@@ -359,6 +359,7 @@ class DataAllocator
     } else if constexpr (has_root_dictionary<T>::value == true || is_specialization_v<T, ROOTSerialized> == true) {
       // Serialize a snapshot of an object with root dictionary
       payloadMessage = proxy.createOutputMessage(routeIndex);
+      payloadMessage->Rebuild(4096, {64});
       if constexpr (is_specialization_v<T, ROOTSerialized> == true) {
         // Explicitely ROOT serialize a snapshot of object.
         // An object wrapped into type `ROOTSerialized` is explicitely marked to be ROOT serialized
diff --git a/Framework/Core/include/Framework/DataRefUtils.h b/Framework/Core/include/Framework/DataRefUtils.h
index defd10244bca5..e59f986f09250 100644
--- a/Framework/Core/include/Framework/DataRefUtils.h
+++ b/Framework/Core/include/Framework/DataRefUtils.h
@@ -71,12 +71,15 @@ struct DataRefUtils {
           throw runtime_error("Attempt to extract a TMessage from non-ROOT serialised message");
         }
 
-        typename RSS::FairTMessage ftm(const_cast<char*>(ref.payload), payloadSize);
-        auto* storedClass = ftm.GetClass();
+        typename RSS::FairInputTBuffer ftm(const_cast<char*>(ref.payload), payloadSize);
         auto* requestedClass = RSS::TClass::GetClass(typeid(T));
+        ftm.InitMap();
+        auto* storedClass = ftm.ReadClass();
         // should always have the class description if has_root_dictionary is true
         assert(requestedClass != nullptr);
 
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         auto* object = ftm.ReadObjectAny(storedClass);
         if (object == nullptr) {
           throw runtime_error_f("Failed to read object with name %s from message using ROOT serialization.",
@@ -146,7 +149,11 @@ struct DataRefUtils {
           throw runtime_error("ROOT serialization not supported, dictionary not found for data type");
         }
 
-        typename RSS::FairTMessage ftm(const_cast<char*>(ref.payload), payloadSize);
+        typename RSS::FairInputTBuffer ftm(const_cast<char*>(ref.payload), payloadSize);
+        ftm.InitMap();
+        auto *classInfo = ftm.ReadClass();
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         result.reset(static_cast<wrapped*>(ftm.ReadObjectAny(cl)));
         if (result.get() == nullptr) {
           throw runtime_error_f("Unable to extract class %s", cl == nullptr ? "<name not available>" : cl->GetName());
diff --git a/Framework/Core/include/Framework/RootMessageContext.h b/Framework/Core/include/Framework/RootMessageContext.h
index bef60ebbbf9f9..b1124880cf30f 100644
--- a/Framework/Core/include/Framework/RootMessageContext.h
+++ b/Framework/Core/include/Framework/RootMessageContext.h
@@ -72,6 +72,9 @@ class RootSerializedObject : public MessageContext::ContextObject
   fair::mq::Parts finalize() final
   {
     assert(mParts.Size() == 1);
+    if (mPayloadMsg->GetSize() < sizeof(char*)) {
+      mPayloadMsg->Rebuild(4096, {64});
+    }
     TMessageSerializer::Serialize(*mPayloadMsg, mObject.get(), nullptr);
     mParts.AddPart(std::move(mPayloadMsg));
     return ContextObject::finalize();
diff --git a/Framework/Core/include/Framework/RootSerializationSupport.h b/Framework/Core/include/Framework/RootSerializationSupport.h
index cbf7408b13c7d..a44093f9c02bf 100644
--- a/Framework/Core/include/Framework/RootSerializationSupport.h
+++ b/Framework/Core/include/Framework/RootSerializationSupport.h
@@ -21,7 +21,8 @@ namespace o2::framework
 /// compiler.
 struct RootSerializationSupport {
   using TClass = ::TClass;
-  using FairTMessage = o2::framework::FairTMessage;
+  using FairInputTBuffer = o2::framework::FairInputTBuffer;
+  using FairOutputBuffer = o2::framework::FairOutputTBuffer;
   using TObject = ::TObject;
 };
 
diff --git a/Framework/Core/include/Framework/TMessageSerializer.h b/Framework/Core/include/Framework/TMessageSerializer.h
index 1f08b456c0218..ca18eb21abfa1 100644
--- a/Framework/Core/include/Framework/TMessageSerializer.h
+++ b/Framework/Core/include/Framework/TMessageSerializer.h
@@ -16,9 +16,8 @@
 #include "Framework/RuntimeError.h"
 
 #include <TList.h>
-#include <TMessage.h>
+#include <TBufferFile.h>
 #include <TObjArray.h>
-#include <TStreamerInfo.h>
 #include <gsl/util>
 #include <gsl/span>
 #include <gsl/narrow>
@@ -28,67 +27,76 @@
 
 namespace o2::framework
 {
-class FairTMessage;
+class FairOutputTBuffer;
+class FairInputTBuffer;
 
 // utilities to produce a span over a byte buffer held by various message types
 // this is to avoid littering code with casts and conversions (span has a signed index type(!))
-gsl::span<std::byte> as_span(const FairTMessage& msg);
+gsl::span<std::byte> as_span(const FairInputTBuffer& msg);
+gsl::span<std::byte> as_span(const FairOutputTBuffer& msg);
 gsl::span<std::byte> as_span(const fair::mq::Message& msg);
 
-class FairTMessage : public TMessage
+// A TBufferFile which we can use to serialise data to a FairMQ message.
+class FairOutputTBuffer : public TBufferFile
 {
  public:
-  using TMessage::TMessage;
-  FairTMessage() : TMessage(kMESS_OBJECT) {}
-  FairTMessage(void* buf, Int_t len) : TMessage(buf, len) { ResetBit(kIsOwner); }
-  FairTMessage(gsl::span<std::byte> buf) : TMessage(buf.data(), buf.size()) { ResetBit(kIsOwner); }
+  // This is to serialise data to FairMQ. We embed the pointer to the message
+  // in the data itself, so that we can use it to reallocate the message if needed.
+  // The FairMQ message retains ownership of the data.
+  // When deserialising the root object, keep in mind one needs to skip the 8 bytes
+  // for the pointer.
+  FairOutputTBuffer(fair::mq::Message& msg)
+    : TBufferFile(TBuffer::kWrite, msg.GetSize() - sizeof(char*), embedInItself(msg), false, fairMQrealloc)
+  {
+  }
+  // Helper function to keep track of the FairMQ message that holds the data
+  // in the data itself. We can use this to make sure the message can be reallocated
+  // even if we simply have a pointer to the data. Hopefully ROOT will not play dirty
+  // with us.
+  void* embedInItself(fair::mq::Message& msg);
   // helper function to clean up the object holding the data after it is transported.
-  static void free(void* /*data*/, void* hint);
+  static char* fairMQrealloc(char* oldData, size_t newSize, size_t oldSize);
 };
 
-struct TMessageSerializer {
-  using StreamerList = std::vector<TVirtualStreamerInfo*>;
-  using CompressionLevel = int;
+class FairInputTBuffer : public TBufferFile
+{
+ public:
+  // This is to serialise data to FairMQ. The provided message is expeted to have 8 bytes
+  // of overhead, where the source embedded the pointer for the reallocation.
+  // Notice this will break if the sender and receiver are not using the same
+  // size for a pointer.
+  FairInputTBuffer(char * data, size_t size)
+    : TBufferFile(TBuffer::kRead, size-sizeof(char*), data + sizeof(char*), false, nullptr)
+  {
+  }
+};
 
-  static void Serialize(fair::mq::Message& msg, const TObject* input,
-                        CompressionLevel compressionLevel = -1);
+struct TMessageSerializer {
+  static void Serialize(fair::mq::Message& msg, const TObject* input);
 
   template <typename T>
-  static void Serialize(fair::mq::Message& msg, const T* input, const TClass* cl, //
-                        CompressionLevel compressionLevel = -1);
+  static void Serialize(fair::mq::Message& msg, const T* input, const TClass* cl);
 
   template <typename T = TObject>
   static void Deserialize(const fair::mq::Message& msg, std::unique_ptr<T>& output);
 
-  static void serialize(FairTMessage& msg, const TObject* input,
-                        CompressionLevel compressionLevel = -1);
+  static void serialize(o2::framework::FairOutputTBuffer& msg, const TObject* input);
 
   template <typename T>
-  static void serialize(FairTMessage& msg, const T* input, //
-                        const TClass* cl,
-                        CompressionLevel compressionLevel = -1);
+  static void serialize(o2::framework::FairOutputTBuffer& msg, const T* input, const TClass* cl);
 
   template <typename T = TObject>
-  static std::unique_ptr<T> deserialize(gsl::span<std::byte> buffer);
-  template <typename T = TObject>
-  static inline std::unique_ptr<T> deserialize(std::byte* buffer, size_t size);
+  static inline std::unique_ptr<T> deserialize(FairInputTBuffer & buffer);
 };
 
-inline void TMessageSerializer::serialize(FairTMessage& tm, const TObject* input,
-                                          CompressionLevel compressionLevel)
+inline void TMessageSerializer::serialize(FairOutputTBuffer& tm, const TObject* input)
 {
-  return serialize(tm, input, nullptr, compressionLevel);
+  return serialize(tm, input, nullptr);
 }
 
 template <typename T>
-inline void TMessageSerializer::serialize(FairTMessage& tm, const T* input, //
-                                          const TClass* cl, CompressionLevel compressionLevel)
+inline void TMessageSerializer::serialize(FairOutputTBuffer& tm, const T* input, const TClass* cl)
 {
-  if (compressionLevel >= 0) {
-    // if negative, skip to use ROOT default
-    tm.SetCompressionLevel(compressionLevel);
-  }
-
   // TODO: check what WriateObject and WriteObjectAny are doing
   if (cl == nullptr) {
     tm.WriteObject(input);
@@ -98,7 +106,7 @@ inline void TMessageSerializer::serialize(FairTMessage& tm, const T* input, //
 }
 
 template <typename T>
-inline std::unique_ptr<T> TMessageSerializer::deserialize(gsl::span<std::byte> buffer)
+inline std::unique_ptr<T> TMessageSerializer::deserialize(FairInputTBuffer & buffer)
 {
   TClass* tgtClass = TClass::GetClass(typeid(T));
   if (tgtClass == nullptr) {
@@ -107,53 +115,32 @@ inline std::unique_ptr<T> TMessageSerializer::deserialize(gsl::span<std::byte> b
   // FIXME: we need to add consistency check for buffer data to be serialized
   // at the moment, TMessage might simply crash if an invalid or inconsistent
   // buffer is provided
-  FairTMessage tm(buffer);
-  TClass* serializedClass = tm.GetClass();
+  buffer.InitMap();
+  TClass* serializedClass = buffer.ReadClass();
+  buffer.SetBufferOffset(0);
+  buffer.ResetMap();
   if (serializedClass == nullptr) {
     throw runtime_error_f("can not read class info from buffer");
   }
   if (tgtClass != serializedClass && serializedClass->GetBaseClass(tgtClass) == nullptr) {
     throw runtime_error_f("can not convert serialized class %s into target class %s",
-                          tm.GetClass()->GetName(),
+                          serializedClass->GetName(),
                           tgtClass->GetName());
   }
-  return std::unique_ptr<T>(reinterpret_cast<T*>(tm.ReadObjectAny(serializedClass)));
+  return std::unique_ptr<T>(reinterpret_cast<T*>(buffer.ReadObjectAny(serializedClass)));
 }
 
-template <typename T>
-inline std::unique_ptr<T> TMessageSerializer::deserialize(std::byte* buffer, size_t size)
+inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const TObject* input)
 {
-  return deserialize<T>(gsl::span<std::byte>(buffer, gsl::narrow<gsl::span<std::byte>::size_type>(size)));
-}
-
-inline void FairTMessage::free(void* /*data*/, void* hint)
-{
-  std::default_delete<FairTMessage> deleter;
-  deleter(static_cast<FairTMessage*>(hint));
-}
-
-inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const TObject* input,
-                                          TMessageSerializer::CompressionLevel compressionLevel)
-{
-  std::unique_ptr<FairTMessage> tm = std::make_unique<FairTMessage>(kMESS_OBJECT);
-
-  serialize(*tm, input, input->Class(), compressionLevel);
-
-  msg.Rebuild(tm->Buffer(), tm->BufferSize(), FairTMessage::free, tm.get());
-  tm.release();
+  FairOutputTBuffer output(msg);
+  serialize(output, input, input->Class());
 }
 
 template <typename T>
-inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const T* input, //
-                                          const TClass* cl,                       //
-                                          TMessageSerializer::CompressionLevel compressionLevel)
+inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const T* input, const TClass* cl)
 {
-  std::unique_ptr<FairTMessage> tm = std::make_unique<FairTMessage>(kMESS_OBJECT);
-
-  serialize(*tm, input, cl, compressionLevel);
-
-  msg.Rebuild(tm->Buffer(), tm->BufferSize(), FairTMessage::free, tm.get());
-  tm.release();
+  FairOutputTBuffer output(msg);
+  serialize(output, input, cl);
 }
 
 template <typename T>
@@ -161,7 +148,8 @@ inline void TMessageSerializer::Deserialize(const fair::mq::Message& msg, std::u
 {
   // we know the message will not be modified by this,
   // so const_cast should be OK here(IMHO).
-  output = deserialize(as_span(msg));
+  FairInputTBuffer input(static_cast<char*>(msg.GetData()), static_cast<int>(msg.GetSize()));
+  output = deserialize(input);
 }
 
 // gsl::narrow is used to do a runtime narrowing check, this might be a bit paranoid,
@@ -171,7 +159,7 @@ inline gsl::span<std::byte> as_span(const fair::mq::Message& msg)
   return gsl::span<std::byte>{static_cast<std::byte*>(msg.GetData()), gsl::narrow<gsl::span<std::byte>::size_type>(msg.GetSize())};
 }
 
-inline gsl::span<std::byte> as_span(const FairTMessage& msg)
+inline gsl::span<std::byte> as_span(const FairInputTBuffer& msg)
 {
   return gsl::span<std::byte>{reinterpret_cast<std::byte*>(msg.Buffer()),
                               gsl::narrow<gsl::span<std::byte>::size_type>(msg.BufferSize())};
diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 48a3eb1da95b9..02ef5c7bc5b3c 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -141,9 +141,9 @@ DataProcessorSpec CommonDataProcessors::getOutputObjHistSink(std::vector<OutputO
         return;
       }
 
-      FairTMessage tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
+      FairInputTBuffer tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
       InputObject obj;
-      obj.kind = tm.GetClass();
+      obj.kind = tm.ReadClass();
       if (obj.kind == nullptr) {
         LOG(error) << "Cannot read class info from buffer.";
         return;
diff --git a/Framework/Core/src/TMessageSerializer.cxx b/Framework/Core/src/TMessageSerializer.cxx
index 5388a6d716cda..9f09c3ade0089 100644
--- a/Framework/Core/src/TMessageSerializer.cxx
+++ b/Framework/Core/src/TMessageSerializer.cxx
@@ -9,7 +9,38 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 #include <Framework/TMessageSerializer.h>
+#include <FairMQTransportFactory.h>
 #include <algorithm>
 #include <memory>
 
 using namespace o2::framework;
+
+void* FairOutputTBuffer::embedInItself(fair::mq::Message& msg) {
+  // The first bytes of the message are used to store the pointer to the message itself
+  // so that we can reallocate it if needed.
+  if (sizeof(char*) > msg.GetSize()) {
+    throw std::runtime_error("Message size too small to embed pointer");
+  }
+  char* data = reinterpret_cast<char*>(msg.GetData());
+  char* ptr = reinterpret_cast<char*>(&msg);
+  std::memcpy(data, ptr, sizeof(char*));
+  return data + sizeof(char*);
+}
+
+// Reallocation function. Get the message pointer from the data and call Rebuild.
+char *FairOutputTBuffer::fairMQrealloc(char *oldData, size_t newSize, size_t oldSize) {
+  auto* msg = reinterpret_cast<fair::mq::Message*>(oldData - sizeof(char*));
+  if (newSize <= msg->GetSize()) {
+    // no need to reallocate, the message is already big enough
+    return oldData;
+  }
+  // Create a shallow copy of the message
+  fair::mq::MessagePtr oldMsg = msg->GetTransport()->CreateMessage();
+  oldMsg->Copy(*msg);
+  // Copy the old data while rebuilding. Reference counting should make
+  // sure the old message is not deleted until the new one is ready.
+  msg->Rebuild(newSize, fair::mq::Alignment{64});
+  memcpy(msg->GetData(), oldMsg->GetData(), oldSize);
+
+  return reinterpret_cast<char*>(msg->GetData()) + sizeof(char*);
+}
diff --git a/Framework/Core/test/test_DataRefUtils.cxx b/Framework/Core/test/test_DataRefUtils.cxx
index 37da7912bfe8b..081adc81ebf69 100644
--- a/Framework/Core/test/test_DataRefUtils.cxx
+++ b/Framework/Core/test/test_DataRefUtils.cxx
@@ -21,17 +21,37 @@
 
 using namespace o2::framework;
 
+TEST_CASE("PureRootTest") {
+  TBufferFile buffer(TBuffer::kWrite);
+  TObjString s("test");
+  buffer.WriteObject(&s);
+
+  TBufferFile buffer2(TBuffer::kRead, buffer.BufferSize(), buffer.Buffer(), false);
+  buffer2.SetReadMode();
+  buffer2.InitMap();
+  TClass *storedClass = buffer2.ReadClass();
+  // ReadClass advances the buffer, so we need to reset it.
+  buffer2.SetBufferOffset(0);
+  buffer2.ResetMap();
+  REQUIRE(storedClass != nullptr);
+  auto *outS = (TObjString*)buffer2.ReadObjectAny(storedClass);
+  REQUIRE(outS != nullptr);
+  REQUIRE(outS->GetString() == "test");
+}
+
 // Simple test to do root deserialization.
 TEST_CASE("TestRootSerialization")
 {
   DataRef ref;
-  TMessage* tm = new TMessage(kMESS_OBJECT);
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer tm(*msg);
   auto sOrig = std::make_unique<TObjString>("test");
-  tm->WriteObject(sOrig.get());
+  tm << sOrig.get();
   o2::header::DataHeader dh;
   dh.payloadSerializationMethod = o2::header::gSerializationMethodROOT;
-  ref.payload = tm->Buffer();
-  dh.payloadSize = tm->BufferSize();
+  ref.payload = (char*)msg->GetData();
+  dh.payloadSize = (size_t)msg->GetSize();
   ref.header = reinterpret_cast<char const*>(&dh);
 
   // Check by using the same type
diff --git a/Framework/Core/test/test_TMessageSerializer.cxx b/Framework/Core/test/test_TMessageSerializer.cxx
index bc5f817400a44..395b3779421a2 100644
--- a/Framework/Core/test/test_TMessageSerializer.cxx
+++ b/Framework/Core/test/test_TMessageSerializer.cxx
@@ -11,6 +11,7 @@
 
 #include "Framework/TMessageSerializer.h"
 #include "Framework/RuntimeError.h"
+#include <fairmq/TransportFactory.h>
 #include "TestClasses.h"
 #include <catch_amalgamated.hpp>
 #include <utility>
@@ -49,14 +50,14 @@ TEST_CASE("TestTMessageSerializer")
   array.SetOwner();
   array.Add(new TNamed(testname, testtitle));
 
-  FairTMessage msg;
-  TMessageSerializer::serialize(msg, &array);
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer buffer(*msg);
+  TMessageSerializer::serialize(buffer, &array);
 
-  auto buf = as_span(msg);
-  REQUIRE(buf.size() == msg.BufferSize());
-  REQUIRE(static_cast<void*>(buf.data()) == static_cast<void*>(msg.Buffer()));
+  FairInputTBuffer msg2((char*)msg->GetData(), msg->GetSize());
   // test deserialization with TObject as target class (default)
-  auto out = TMessageSerializer::deserialize(buf);
+  auto out = TMessageSerializer::deserialize(msg2);
 
   auto* outarr = dynamic_cast<TObjArray*>(out.get());
   REQUIRE(out.get() == outarr);
@@ -66,9 +67,9 @@ TEST_CASE("TestTMessageSerializer")
   REQUIRE(named->GetTitle() == std::string(testtitle));
 
   // test deserialization with a wrong target class and check the exception
-  REQUIRE_THROWS_AS(TMessageSerializer::deserialize<TNamed>(buf), o2::framework::RuntimeErrorRef);
+  REQUIRE_THROWS_AS(TMessageSerializer::deserialize<TNamed>(msg2), o2::framework::RuntimeErrorRef);
 
-  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<TNamed>(buf), o2::framework::RuntimeErrorRef,
+  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<TNamed>(msg2), o2::framework::RuntimeErrorRef,
                          ExceptionMatcher("can not convert serialized class TObjArray into target class TNamed"));
 }
 
@@ -87,23 +88,29 @@ TEST_CASE("TestTMessageSerializer_NonTObject")
   TClass* cl = TClass::GetClass("std::vector<o2::test::Polymorphic>");
   REQUIRE(cl != nullptr);
 
-  FairTMessage msg;
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer buffer(*msg);
   char* in = reinterpret_cast<char*>(&data);
-  TMessageSerializer::serialize(msg, in, cl);
+  TMessageSerializer::serialize(buffer, in, cl);
+  FairInputTBuffer msg2((char*)msg->GetData(), msg->GetSize());
 
-  auto out = TMessageSerializer::deserialize<std::vector<o2::test::Polymorphic>>(as_span(msg));
+  auto out = TMessageSerializer::deserialize<std::vector<o2::test::Polymorphic>>(msg2);
   REQUIRE(out);
   REQUIRE((*out.get()).size() == 2);
   REQUIRE((*out.get())[0] == o2::test::Polymorphic(0xaffe));
   REQUIRE((*out.get())[1] == o2::test::Polymorphic(0xd00f));
 
   // test deserialization with a wrong target class and check the exception
-  REQUIRE_THROWS_AS(TMessageSerializer::deserialize(as_span(msg)), RuntimeErrorRef);
+  REQUIRE_THROWS_AS(TMessageSerializer::deserialize(msg2), RuntimeErrorRef);
 }
 
 TEST_CASE("TestTMessageSerializer_InvalidBuffer")
 {
   const char* buffer = "this is for sure not a serialized ROOT object";
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(strlen(buffer) + 8);
+  memcpy((char*)msg->GetData() + 8, buffer, strlen(buffer));
   // test deserialization of invalid buffer and check the exception
   // FIXME: at the moment, TMessage fails directly with a segfault, which it shouldn't do
   /*
@@ -119,5 +126,6 @@ TEST_CASE("TestTMessageSerializer_InvalidBuffer")
   struct Dummy {
   };
   auto matcher = ExceptionMatcher("class is not ROOT-serializable: ZL22CATCH2_INTERNAL_TEST_4vE5Dummy");
-  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<Dummy>((std::byte*)buffer, strlen(buffer)), o2::framework::RuntimeErrorRef, matcher);
+  FairInputTBuffer msg2((char*)msg->GetData(), msg->GetSize());
+  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<Dummy>(msg2), o2::framework::RuntimeErrorRef, matcher);
 }
diff --git a/Framework/Utils/test/test_RootTreeWriter.cxx b/Framework/Utils/test/test_RootTreeWriter.cxx
index 3194508f3d775..62e1eb62cb4f1 100644
--- a/Framework/Utils/test/test_RootTreeWriter.cxx
+++ b/Framework/Utils/test/test_RootTreeWriter.cxx
@@ -179,6 +179,7 @@ TEST_CASE("test_RootTreeWriter")
 
   auto createSerializedMessage = [&transport, &store](DataHeader&& dh, auto& data) {
     fair::mq::MessagePtr payload = transport->CreateMessage();
+    payload->Rebuild(4096, {64});
     auto* cl = TClass::GetClass(typeid(decltype(data)));
     TMessageSerializer().Serialize(*payload, &data, cl);
     dh.payloadSize = payload->GetSize();
diff --git a/Utilities/Mergers/src/ObjectStore.cxx b/Utilities/Mergers/src/ObjectStore.cxx
index e88358507c31e..3bb49f1dfc9d8 100644
--- a/Utilities/Mergers/src/ObjectStore.cxx
+++ b/Utilities/Mergers/src/ObjectStore.cxx
@@ -38,7 +38,7 @@ static std::string concat(Args&&... arguments)
   return std::move(ss.str());
 }
 
-void* readObject(const TClass* type, o2::framework::FairTMessage& ftm)
+void* readObject(const TClass* type, o2::framework::FairInputTBuffer& ftm)
 {
   using namespace std::string_view_literals;
   auto* object = ftm.ReadObjectAny(type);
@@ -60,7 +60,7 @@ MergeInterface* castToMergeInterface(bool inheritsFromTObject, void* object, TCl
   return objectAsMergeInterface;
 }
 
-std::optional<ObjectStore> extractVector(o2::framework::FairTMessage& ftm, const TClass* storedClass)
+std::optional<ObjectStore> extractVector(o2::framework::FairInputTBuffer& ftm, const TClass* storedClass)
 {
   if (!storedClass->InheritsFrom(TClass::GetClass(typeid(VectorOfRawTObjects)))) {
     return std::nullopt;
@@ -88,11 +88,14 @@ ObjectStore extractObjectFrom(const framework::DataRef& ref)
     throw std::runtime_error(concat(errorPrefix, "It is not ROOT-serialized"sv));
   }
 
-  o2::framework::FairTMessage ftm(const_cast<char*>(ref.payload), o2::framework::DataRefUtils::getPayloadSize(ref));
-  auto* storedClass = ftm.GetClass();
+  o2::framework::FairInputTBuffer ftm(const_cast<char*>(ref.payload), o2::framework::DataRefUtils::getPayloadSize(ref));
+  ftm.InitMap();
+  auto* storedClass = ftm.ReadClass();
   if (storedClass == nullptr) {
     throw std::runtime_error(concat(errorPrefix, "Unknown stored class"sv));
   }
+  ftm.SetBufferOffset(0);
+  ftm.ResetMap();
 
   if (const auto extractedVector = extractVector(ftm, storedClass)) {
     return extractedVector.value();
diff --git a/Utilities/Mergers/test/benchmark_Types.cxx b/Utilities/Mergers/test/benchmark_Types.cxx
index 790fd329185ea..736685c5746b8 100644
--- a/Utilities/Mergers/test/benchmark_Types.cxx
+++ b/Utilities/Mergers/test/benchmark_Types.cxx
@@ -165,11 +165,16 @@ auto measure = [](Measurement m, auto* o, auto* i) -> double {
         tm->WriteObject(o);
         start = std::chrono::high_resolution_clock::now();
 
-        o2::framework::FairTMessage ftm(const_cast<char*>(tm->Buffer()), tm->BufferSize());
-        auto* storedClass = ftm.GetClass();
+        // Needed to take into account that  FairInputTBuffer expects the first 8 bytes to be the
+        // allocator pointer, which is not present in the TMessage buffer.
+        o2::framework::FairInputTBuffer ftm(const_cast<char*>(tm->Buffer() - 8), tm->BufferSize() + 8);
+        ftm.InitMap();
+        auto* storedClass = ftm.ReadClass();
         if (storedClass == nullptr) {
           throw std::runtime_error("Unknown stored class");
         }
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         auto* tObjectClass = TClass::GetClass(typeid(TObject));
 
         if (!storedClass->InheritsFrom(tObjectClass)) {
@@ -738,4 +743,4 @@ int main(int argc, const char* argv[])
 
   file.close();
   return 0;
-}
\ No newline at end of file
+}

From 23c82aa37b9f1acfac022ec3c797d669c1c7b056 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 26 Feb 2024 14:40:12 +0100
Subject: [PATCH 380/726] DPL: move StringLiteral to FunctionalHelpers

---
 Framework/Core/include/Framework/AlgorithmSpec.h      | 11 -----------
 .../Foundation/include/Framework/FunctionalHelpers.h  | 10 ++++++++++
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/Framework/Core/include/Framework/AlgorithmSpec.h b/Framework/Core/include/Framework/AlgorithmSpec.h
index e98a3abefbce1..1f9bb750f0ef3 100644
--- a/Framework/Core/include/Framework/AlgorithmSpec.h
+++ b/Framework/Core/include/Framework/AlgorithmSpec.h
@@ -83,17 +83,6 @@ struct AlgorithmSpec {
 struct AlgorithmPlugin {
   virtual AlgorithmSpec create() = 0;
 };
-
-template <size_t N>
-struct StringLiteral {
-  constexpr StringLiteral(const char (&str)[N])
-  {
-    std::copy_n(str, N, value);
-  }
-
-  char value[N];
-};
-
 // Allow fetching inputs from the context using a string literal.
 template <StringLiteral lit, typename T>
 struct Input {
diff --git a/Framework/Foundation/include/Framework/FunctionalHelpers.h b/Framework/Foundation/include/Framework/FunctionalHelpers.h
index 7e060e8816e3b..12470a7eb0f1c 100644
--- a/Framework/Foundation/include/Framework/FunctionalHelpers.h
+++ b/Framework/Foundation/include/Framework/FunctionalHelpers.h
@@ -25,6 +25,16 @@ struct memfun_type {
 };
 } // namespace
 
+template <size_t N>
+struct StringLiteral {
+  constexpr StringLiteral(const char (&str)[N])
+  {
+    std::copy_n(str, N, value);
+  }
+
+  char value[N];
+};
+
 /// Type helper to hold metadata about a lambda or a class
 /// method.
 template <typename Ret, typename Class, typename... Args>

From 9368e259442fc5ec6521cb4ee9ff992b7ca7fd03 Mon Sep 17 00:00:00 2001
From: Ole Schmidt <ole.schmidt@cern.ch>
Date: Mon, 26 Feb 2024 17:07:18 +0100
Subject: [PATCH 381/726] Fix odc_expendable_task requirement in DDS XML

---
 Framework/Core/src/DDSConfigHelpers.cxx             | 4 ++--
 Framework/Core/test/test_FrameworkDataFlowToDDS.cxx | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/src/DDSConfigHelpers.cxx b/Framework/Core/src/DDSConfigHelpers.cxx
index e56afe34bc511..d17eeca394520 100644
--- a/Framework/Core/src/DDSConfigHelpers.cxx
+++ b/Framework/Core/src/DDSConfigHelpers.cxx
@@ -181,7 +181,7 @@ void DDSConfigHelpers::dumpDeviceSpec2DDS(std::ostream& out,
     out << fmt::format("cat ${{DDS_LOCATION}}/dpl_json{}.asset | o2-dpl-run --driver-mode embedded", workflowSuffix);
     out << R"(</exe>)"
         << "<requirements>\n"
-        << "  <requirement name=\"odc_expendable_task\" />\n"
+        << "  <name>odc_expendable_task</name>\n"
         << "</requirements>\n"
         << "\n";
     out << "</decltask>";
@@ -252,7 +252,7 @@ void DDSConfigHelpers::dumpDeviceSpec2DDS(std::ostream& out,
           return label.value == "expendable";
         }) != spec.labels.end()) {
       out << "       <requirements>\n";
-      out << "           <requirement name=\"odc_expendable_task\" />\n";
+      out << "           <name>odc_expendable_task</name>\n";
       out << "       </requirements>\n";
     }
     auto& rewriter = rewriters[di];
diff --git a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
index 49420e4bd99ad..e2082f13b40df 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
@@ -635,7 +635,7 @@ TEST_CASE("TestDDSExpendable")
        <assets><name>dpl_json</name></assets>
        <exe reachable="true">cat ${DDS_LOCATION}/dpl_json.asset | foo --id D_dds%TaskIndex%_%CollectionIndex% --shm-monitor false --log-color false --batch --color false --channel-config "name=from_B_to_D,type=pull,method=connect,address=ipc://@localhostworkflow-id_22002,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --channel-config "name=from_C_to_D,type=pull,method=connect,address=ipc://@localhostworkflow-id_22003,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --bad-alloc-attempt-interval 50 --bad-alloc-max-attempts 1 --early-forward-policy never --io-threads 1 --jobs 4 --severity info --shm-allocation rbtree_best_fit --shm-mlock-segment false --shm-mlock-segment-on-creation false --shm-no-cleanup false --shm-segment-id 0 --shm-throw-bad-alloc true --shm-zero-segment false --stacktrace-on-signal simple --timeframes-rate-limit 0 --a-param 1 --b-param "" --c-param "foo;bar" --session dpl_workflow-id --plugin odc</exe>
        <requirements>
-           <requirement name="odc_expendable_task" />
+           <name>odc_expendable_task</name>
        </requirements>
    </decltask>
    <declcollection name="DPL">

From 43b29ee7dd6d684bfd82f50368b610ad9f01dea8 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 27 Feb 2024 23:27:48 +0100
Subject: [PATCH 382/726] DPL: avoid TMessage usage

TMessage does not allow for non owned buffers, so we end up having
an extra buffer in private memory for (de)serializing. Using TBufferFile
directly allows to avoid that, so this moves the whole ROOT serialization
support in DPL to use it.
---
 .../src/AODJAlienReaderHelpers.h              |   2 +
 .../Core/include/Framework/DataAllocator.h    |   1 +
 .../Core/include/Framework/DataRefUtils.h     |  13 +-
 .../include/Framework/RootMessageContext.h    |   3 +
 .../Framework/RootSerializationSupport.h      |   3 +-
 .../include/Framework/TMessageSerializer.h    | 130 ++++++++----------
 Framework/Core/src/CommonDataProcessors.cxx   |   7 +-
 Framework/Core/src/TMessageSerializer.cxx     |  35 +++++
 Framework/Core/test/test_DataRefUtils.cxx     |  28 +++-
 .../Core/test/test_TMessageSerializer.cxx     |  51 +++++--
 Framework/Utils/test/test_RootTreeWriter.cxx  |   1 +
 Steer/DigitizerWorkflow/src/SimReaderSpec.cxx |   1 -
 Utilities/Mergers/src/ObjectStore.cxx         |  11 +-
 Utilities/Mergers/test/benchmark_Types.cxx    |  11 +-
 14 files changed, 195 insertions(+), 102 deletions(-)

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
index 655e4b6c0b439..4b9fd710aca14 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
@@ -16,7 +16,9 @@
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/Logger.h"
 #include <Monitoring/Monitoring.h>
+
 #include <uv.h>
+class TFile;
 
 namespace o2::framework::readers
 {
diff --git a/Framework/Core/include/Framework/DataAllocator.h b/Framework/Core/include/Framework/DataAllocator.h
index 8151d2f83c6c6..029e922aeb90b 100644
--- a/Framework/Core/include/Framework/DataAllocator.h
+++ b/Framework/Core/include/Framework/DataAllocator.h
@@ -359,6 +359,7 @@ class DataAllocator
     } else if constexpr (has_root_dictionary<T>::value == true || is_specialization_v<T, ROOTSerialized> == true) {
       // Serialize a snapshot of an object with root dictionary
       payloadMessage = proxy.createOutputMessage(routeIndex);
+      payloadMessage->Rebuild(4096, {64});
       if constexpr (is_specialization_v<T, ROOTSerialized> == true) {
         // Explicitely ROOT serialize a snapshot of object.
         // An object wrapped into type `ROOTSerialized` is explicitely marked to be ROOT serialized
diff --git a/Framework/Core/include/Framework/DataRefUtils.h b/Framework/Core/include/Framework/DataRefUtils.h
index defd10244bca5..e59f986f09250 100644
--- a/Framework/Core/include/Framework/DataRefUtils.h
+++ b/Framework/Core/include/Framework/DataRefUtils.h
@@ -71,12 +71,15 @@ struct DataRefUtils {
           throw runtime_error("Attempt to extract a TMessage from non-ROOT serialised message");
         }
 
-        typename RSS::FairTMessage ftm(const_cast<char*>(ref.payload), payloadSize);
-        auto* storedClass = ftm.GetClass();
+        typename RSS::FairInputTBuffer ftm(const_cast<char*>(ref.payload), payloadSize);
         auto* requestedClass = RSS::TClass::GetClass(typeid(T));
+        ftm.InitMap();
+        auto* storedClass = ftm.ReadClass();
         // should always have the class description if has_root_dictionary is true
         assert(requestedClass != nullptr);
 
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         auto* object = ftm.ReadObjectAny(storedClass);
         if (object == nullptr) {
           throw runtime_error_f("Failed to read object with name %s from message using ROOT serialization.",
@@ -146,7 +149,11 @@ struct DataRefUtils {
           throw runtime_error("ROOT serialization not supported, dictionary not found for data type");
         }
 
-        typename RSS::FairTMessage ftm(const_cast<char*>(ref.payload), payloadSize);
+        typename RSS::FairInputTBuffer ftm(const_cast<char*>(ref.payload), payloadSize);
+        ftm.InitMap();
+        auto *classInfo = ftm.ReadClass();
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         result.reset(static_cast<wrapped*>(ftm.ReadObjectAny(cl)));
         if (result.get() == nullptr) {
           throw runtime_error_f("Unable to extract class %s", cl == nullptr ? "<name not available>" : cl->GetName());
diff --git a/Framework/Core/include/Framework/RootMessageContext.h b/Framework/Core/include/Framework/RootMessageContext.h
index bef60ebbbf9f9..b1124880cf30f 100644
--- a/Framework/Core/include/Framework/RootMessageContext.h
+++ b/Framework/Core/include/Framework/RootMessageContext.h
@@ -72,6 +72,9 @@ class RootSerializedObject : public MessageContext::ContextObject
   fair::mq::Parts finalize() final
   {
     assert(mParts.Size() == 1);
+    if (mPayloadMsg->GetSize() < sizeof(char*)) {
+      mPayloadMsg->Rebuild(4096, {64});
+    }
     TMessageSerializer::Serialize(*mPayloadMsg, mObject.get(), nullptr);
     mParts.AddPart(std::move(mPayloadMsg));
     return ContextObject::finalize();
diff --git a/Framework/Core/include/Framework/RootSerializationSupport.h b/Framework/Core/include/Framework/RootSerializationSupport.h
index cbf7408b13c7d..a44093f9c02bf 100644
--- a/Framework/Core/include/Framework/RootSerializationSupport.h
+++ b/Framework/Core/include/Framework/RootSerializationSupport.h
@@ -21,7 +21,8 @@ namespace o2::framework
 /// compiler.
 struct RootSerializationSupport {
   using TClass = ::TClass;
-  using FairTMessage = o2::framework::FairTMessage;
+  using FairInputTBuffer = o2::framework::FairInputTBuffer;
+  using FairOutputBuffer = o2::framework::FairOutputTBuffer;
   using TObject = ::TObject;
 };
 
diff --git a/Framework/Core/include/Framework/TMessageSerializer.h b/Framework/Core/include/Framework/TMessageSerializer.h
index 1f08b456c0218..ca18eb21abfa1 100644
--- a/Framework/Core/include/Framework/TMessageSerializer.h
+++ b/Framework/Core/include/Framework/TMessageSerializer.h
@@ -16,9 +16,8 @@
 #include "Framework/RuntimeError.h"
 
 #include <TList.h>
-#include <TMessage.h>
+#include <TBufferFile.h>
 #include <TObjArray.h>
-#include <TStreamerInfo.h>
 #include <gsl/util>
 #include <gsl/span>
 #include <gsl/narrow>
@@ -28,67 +27,76 @@
 
 namespace o2::framework
 {
-class FairTMessage;
+class FairOutputTBuffer;
+class FairInputTBuffer;
 
 // utilities to produce a span over a byte buffer held by various message types
 // this is to avoid littering code with casts and conversions (span has a signed index type(!))
-gsl::span<std::byte> as_span(const FairTMessage& msg);
+gsl::span<std::byte> as_span(const FairInputTBuffer& msg);
+gsl::span<std::byte> as_span(const FairOutputTBuffer& msg);
 gsl::span<std::byte> as_span(const fair::mq::Message& msg);
 
-class FairTMessage : public TMessage
+// A TBufferFile which we can use to serialise data to a FairMQ message.
+class FairOutputTBuffer : public TBufferFile
 {
  public:
-  using TMessage::TMessage;
-  FairTMessage() : TMessage(kMESS_OBJECT) {}
-  FairTMessage(void* buf, Int_t len) : TMessage(buf, len) { ResetBit(kIsOwner); }
-  FairTMessage(gsl::span<std::byte> buf) : TMessage(buf.data(), buf.size()) { ResetBit(kIsOwner); }
+  // This is to serialise data to FairMQ. We embed the pointer to the message
+  // in the data itself, so that we can use it to reallocate the message if needed.
+  // The FairMQ message retains ownership of the data.
+  // When deserialising the root object, keep in mind one needs to skip the 8 bytes
+  // for the pointer.
+  FairOutputTBuffer(fair::mq::Message& msg)
+    : TBufferFile(TBuffer::kWrite, msg.GetSize() - sizeof(char*), embedInItself(msg), false, fairMQrealloc)
+  {
+  }
+  // Helper function to keep track of the FairMQ message that holds the data
+  // in the data itself. We can use this to make sure the message can be reallocated
+  // even if we simply have a pointer to the data. Hopefully ROOT will not play dirty
+  // with us.
+  void* embedInItself(fair::mq::Message& msg);
   // helper function to clean up the object holding the data after it is transported.
-  static void free(void* /*data*/, void* hint);
+  static char* fairMQrealloc(char* oldData, size_t newSize, size_t oldSize);
 };
 
-struct TMessageSerializer {
-  using StreamerList = std::vector<TVirtualStreamerInfo*>;
-  using CompressionLevel = int;
+class FairInputTBuffer : public TBufferFile
+{
+ public:
+  // This is to serialise data to FairMQ. The provided message is expeted to have 8 bytes
+  // of overhead, where the source embedded the pointer for the reallocation.
+  // Notice this will break if the sender and receiver are not using the same
+  // size for a pointer.
+  FairInputTBuffer(char * data, size_t size)
+    : TBufferFile(TBuffer::kRead, size-sizeof(char*), data + sizeof(char*), false, nullptr)
+  {
+  }
+};
 
-  static void Serialize(fair::mq::Message& msg, const TObject* input,
-                        CompressionLevel compressionLevel = -1);
+struct TMessageSerializer {
+  static void Serialize(fair::mq::Message& msg, const TObject* input);
 
   template <typename T>
-  static void Serialize(fair::mq::Message& msg, const T* input, const TClass* cl, //
-                        CompressionLevel compressionLevel = -1);
+  static void Serialize(fair::mq::Message& msg, const T* input, const TClass* cl);
 
   template <typename T = TObject>
   static void Deserialize(const fair::mq::Message& msg, std::unique_ptr<T>& output);
 
-  static void serialize(FairTMessage& msg, const TObject* input,
-                        CompressionLevel compressionLevel = -1);
+  static void serialize(o2::framework::FairOutputTBuffer& msg, const TObject* input);
 
   template <typename T>
-  static void serialize(FairTMessage& msg, const T* input, //
-                        const TClass* cl,
-                        CompressionLevel compressionLevel = -1);
+  static void serialize(o2::framework::FairOutputTBuffer& msg, const T* input, const TClass* cl);
 
   template <typename T = TObject>
-  static std::unique_ptr<T> deserialize(gsl::span<std::byte> buffer);
-  template <typename T = TObject>
-  static inline std::unique_ptr<T> deserialize(std::byte* buffer, size_t size);
+  static inline std::unique_ptr<T> deserialize(FairInputTBuffer & buffer);
 };
 
-inline void TMessageSerializer::serialize(FairTMessage& tm, const TObject* input,
-                                          CompressionLevel compressionLevel)
+inline void TMessageSerializer::serialize(FairOutputTBuffer& tm, const TObject* input)
 {
-  return serialize(tm, input, nullptr, compressionLevel);
+  return serialize(tm, input, nullptr);
 }
 
 template <typename T>
-inline void TMessageSerializer::serialize(FairTMessage& tm, const T* input, //
-                                          const TClass* cl, CompressionLevel compressionLevel)
+inline void TMessageSerializer::serialize(FairOutputTBuffer& tm, const T* input, const TClass* cl)
 {
-  if (compressionLevel >= 0) {
-    // if negative, skip to use ROOT default
-    tm.SetCompressionLevel(compressionLevel);
-  }
-
   // TODO: check what WriateObject and WriteObjectAny are doing
   if (cl == nullptr) {
     tm.WriteObject(input);
@@ -98,7 +106,7 @@ inline void TMessageSerializer::serialize(FairTMessage& tm, const T* input, //
 }
 
 template <typename T>
-inline std::unique_ptr<T> TMessageSerializer::deserialize(gsl::span<std::byte> buffer)
+inline std::unique_ptr<T> TMessageSerializer::deserialize(FairInputTBuffer & buffer)
 {
   TClass* tgtClass = TClass::GetClass(typeid(T));
   if (tgtClass == nullptr) {
@@ -107,53 +115,32 @@ inline std::unique_ptr<T> TMessageSerializer::deserialize(gsl::span<std::byte> b
   // FIXME: we need to add consistency check for buffer data to be serialized
   // at the moment, TMessage might simply crash if an invalid or inconsistent
   // buffer is provided
-  FairTMessage tm(buffer);
-  TClass* serializedClass = tm.GetClass();
+  buffer.InitMap();
+  TClass* serializedClass = buffer.ReadClass();
+  buffer.SetBufferOffset(0);
+  buffer.ResetMap();
   if (serializedClass == nullptr) {
     throw runtime_error_f("can not read class info from buffer");
   }
   if (tgtClass != serializedClass && serializedClass->GetBaseClass(tgtClass) == nullptr) {
     throw runtime_error_f("can not convert serialized class %s into target class %s",
-                          tm.GetClass()->GetName(),
+                          serializedClass->GetName(),
                           tgtClass->GetName());
   }
-  return std::unique_ptr<T>(reinterpret_cast<T*>(tm.ReadObjectAny(serializedClass)));
+  return std::unique_ptr<T>(reinterpret_cast<T*>(buffer.ReadObjectAny(serializedClass)));
 }
 
-template <typename T>
-inline std::unique_ptr<T> TMessageSerializer::deserialize(std::byte* buffer, size_t size)
+inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const TObject* input)
 {
-  return deserialize<T>(gsl::span<std::byte>(buffer, gsl::narrow<gsl::span<std::byte>::size_type>(size)));
-}
-
-inline void FairTMessage::free(void* /*data*/, void* hint)
-{
-  std::default_delete<FairTMessage> deleter;
-  deleter(static_cast<FairTMessage*>(hint));
-}
-
-inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const TObject* input,
-                                          TMessageSerializer::CompressionLevel compressionLevel)
-{
-  std::unique_ptr<FairTMessage> tm = std::make_unique<FairTMessage>(kMESS_OBJECT);
-
-  serialize(*tm, input, input->Class(), compressionLevel);
-
-  msg.Rebuild(tm->Buffer(), tm->BufferSize(), FairTMessage::free, tm.get());
-  tm.release();
+  FairOutputTBuffer output(msg);
+  serialize(output, input, input->Class());
 }
 
 template <typename T>
-inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const T* input, //
-                                          const TClass* cl,                       //
-                                          TMessageSerializer::CompressionLevel compressionLevel)
+inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const T* input, const TClass* cl)
 {
-  std::unique_ptr<FairTMessage> tm = std::make_unique<FairTMessage>(kMESS_OBJECT);
-
-  serialize(*tm, input, cl, compressionLevel);
-
-  msg.Rebuild(tm->Buffer(), tm->BufferSize(), FairTMessage::free, tm.get());
-  tm.release();
+  FairOutputTBuffer output(msg);
+  serialize(output, input, cl);
 }
 
 template <typename T>
@@ -161,7 +148,8 @@ inline void TMessageSerializer::Deserialize(const fair::mq::Message& msg, std::u
 {
   // we know the message will not be modified by this,
   // so const_cast should be OK here(IMHO).
-  output = deserialize(as_span(msg));
+  FairInputTBuffer input(static_cast<char*>(msg.GetData()), static_cast<int>(msg.GetSize()));
+  output = deserialize(input);
 }
 
 // gsl::narrow is used to do a runtime narrowing check, this might be a bit paranoid,
@@ -171,7 +159,7 @@ inline gsl::span<std::byte> as_span(const fair::mq::Message& msg)
   return gsl::span<std::byte>{static_cast<std::byte*>(msg.GetData()), gsl::narrow<gsl::span<std::byte>::size_type>(msg.GetSize())};
 }
 
-inline gsl::span<std::byte> as_span(const FairTMessage& msg)
+inline gsl::span<std::byte> as_span(const FairInputTBuffer& msg)
 {
   return gsl::span<std::byte>{reinterpret_cast<std::byte*>(msg.Buffer()),
                               gsl::narrow<gsl::span<std::byte>::size_type>(msg.BufferSize())};
diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 48a3eb1da95b9..0cf7224c25ac8 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -141,9 +141,12 @@ DataProcessorSpec CommonDataProcessors::getOutputObjHistSink(std::vector<OutputO
         return;
       }
 
-      FairTMessage tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
       InputObject obj;
-      obj.kind = tm.GetClass();
+      FairInputTBuffer tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
+      tm.InitMap();
+      obj.kind = tm.ReadClass();
+      tm.SetBufferOffset(0);
+      tm.ResetMap();
       if (obj.kind == nullptr) {
         LOG(error) << "Cannot read class info from buffer.";
         return;
diff --git a/Framework/Core/src/TMessageSerializer.cxx b/Framework/Core/src/TMessageSerializer.cxx
index 5388a6d716cda..43113b2572191 100644
--- a/Framework/Core/src/TMessageSerializer.cxx
+++ b/Framework/Core/src/TMessageSerializer.cxx
@@ -9,7 +9,42 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 #include <Framework/TMessageSerializer.h>
+#include <FairMQTransportFactory.h>
 #include <algorithm>
 #include <memory>
 
 using namespace o2::framework;
+
+void* FairOutputTBuffer::embedInItself(fair::mq::Message& msg) {
+  // The first bytes of the message are used to store the pointer to the message itself
+  // so that we can reallocate it if needed.
+  if (sizeof(char*) > msg.GetSize()) {
+    throw std::runtime_error("Message size too small to embed pointer");
+  }
+  char* data = reinterpret_cast<char*>(msg.GetData());
+  char* ptr = reinterpret_cast<char*>(&msg);
+  std::memcpy(data, &ptr, sizeof(char*));
+  return data + sizeof(char*);
+}
+
+// Reallocation function. Get the message pointer from the data and call Rebuild.
+char *FairOutputTBuffer::fairMQrealloc(char *oldData, size_t newSize, size_t oldSize) {
+  // Old data is the pointer at the beginning of the message, so the pointer
+  // to the message is **stored** in the 8 bytes before it.
+  auto* msg = *(fair::mq::Message**)(oldData - sizeof(char*));
+  if (newSize <= msg->GetSize()) {
+    // no need to reallocate, the message is already big enough
+    return oldData;
+  }
+  // Create a shallow copy of the message
+  fair::mq::MessagePtr oldMsg = msg->GetTransport()->CreateMessage();
+  oldMsg->Copy(*msg);
+  // Copy the old data while rebuilding. Reference counting should make
+  // sure the old message is not deleted until the new one is ready.
+  // We need 8 extra bytes for the pointer to the message itself (realloc does not know about it)
+  // and we need to copy 8 bytes more than the old size (again, the extra pointer).
+  msg->Rebuild(newSize+8, fair::mq::Alignment{64});
+  memcpy(msg->GetData(), oldMsg->GetData(), oldSize+8);
+
+  return reinterpret_cast<char*>(msg->GetData()) + sizeof(char*);
+}
diff --git a/Framework/Core/test/test_DataRefUtils.cxx b/Framework/Core/test/test_DataRefUtils.cxx
index 37da7912bfe8b..081adc81ebf69 100644
--- a/Framework/Core/test/test_DataRefUtils.cxx
+++ b/Framework/Core/test/test_DataRefUtils.cxx
@@ -21,17 +21,37 @@
 
 using namespace o2::framework;
 
+TEST_CASE("PureRootTest") {
+  TBufferFile buffer(TBuffer::kWrite);
+  TObjString s("test");
+  buffer.WriteObject(&s);
+
+  TBufferFile buffer2(TBuffer::kRead, buffer.BufferSize(), buffer.Buffer(), false);
+  buffer2.SetReadMode();
+  buffer2.InitMap();
+  TClass *storedClass = buffer2.ReadClass();
+  // ReadClass advances the buffer, so we need to reset it.
+  buffer2.SetBufferOffset(0);
+  buffer2.ResetMap();
+  REQUIRE(storedClass != nullptr);
+  auto *outS = (TObjString*)buffer2.ReadObjectAny(storedClass);
+  REQUIRE(outS != nullptr);
+  REQUIRE(outS->GetString() == "test");
+}
+
 // Simple test to do root deserialization.
 TEST_CASE("TestRootSerialization")
 {
   DataRef ref;
-  TMessage* tm = new TMessage(kMESS_OBJECT);
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer tm(*msg);
   auto sOrig = std::make_unique<TObjString>("test");
-  tm->WriteObject(sOrig.get());
+  tm << sOrig.get();
   o2::header::DataHeader dh;
   dh.payloadSerializationMethod = o2::header::gSerializationMethodROOT;
-  ref.payload = tm->Buffer();
-  dh.payloadSize = tm->BufferSize();
+  ref.payload = (char*)msg->GetData();
+  dh.payloadSize = (size_t)msg->GetSize();
   ref.header = reinterpret_cast<char const*>(&dh);
 
   // Check by using the same type
diff --git a/Framework/Core/test/test_TMessageSerializer.cxx b/Framework/Core/test/test_TMessageSerializer.cxx
index bc5f817400a44..365d31e95dd30 100644
--- a/Framework/Core/test/test_TMessageSerializer.cxx
+++ b/Framework/Core/test/test_TMessageSerializer.cxx
@@ -11,6 +11,7 @@
 
 #include "Framework/TMessageSerializer.h"
 #include "Framework/RuntimeError.h"
+#include <fairmq/TransportFactory.h>
 #include "TestClasses.h"
 #include <catch_amalgamated.hpp>
 #include <utility>
@@ -49,14 +50,14 @@ TEST_CASE("TestTMessageSerializer")
   array.SetOwner();
   array.Add(new TNamed(testname, testtitle));
 
-  FairTMessage msg;
-  TMessageSerializer::serialize(msg, &array);
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer buffer(*msg);
+  TMessageSerializer::serialize(buffer, &array);
 
-  auto buf = as_span(msg);
-  REQUIRE(buf.size() == msg.BufferSize());
-  REQUIRE(static_cast<void*>(buf.data()) == static_cast<void*>(msg.Buffer()));
+  FairInputTBuffer msg2((char*)msg->GetData(), msg->GetSize());
   // test deserialization with TObject as target class (default)
-  auto out = TMessageSerializer::deserialize(buf);
+  auto out = TMessageSerializer::deserialize(msg2);
 
   auto* outarr = dynamic_cast<TObjArray*>(out.get());
   REQUIRE(out.get() == outarr);
@@ -66,9 +67,9 @@ TEST_CASE("TestTMessageSerializer")
   REQUIRE(named->GetTitle() == std::string(testtitle));
 
   // test deserialization with a wrong target class and check the exception
-  REQUIRE_THROWS_AS(TMessageSerializer::deserialize<TNamed>(buf), o2::framework::RuntimeErrorRef);
+  REQUIRE_THROWS_AS(TMessageSerializer::deserialize<TNamed>(msg2), o2::framework::RuntimeErrorRef);
 
-  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<TNamed>(buf), o2::framework::RuntimeErrorRef,
+  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<TNamed>(msg2), o2::framework::RuntimeErrorRef,
                          ExceptionMatcher("can not convert serialized class TObjArray into target class TNamed"));
 }
 
@@ -87,23 +88,29 @@ TEST_CASE("TestTMessageSerializer_NonTObject")
   TClass* cl = TClass::GetClass("std::vector<o2::test::Polymorphic>");
   REQUIRE(cl != nullptr);
 
-  FairTMessage msg;
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer buffer(*msg);
   char* in = reinterpret_cast<char*>(&data);
-  TMessageSerializer::serialize(msg, in, cl);
+  TMessageSerializer::serialize(buffer, in, cl);
+  FairInputTBuffer msg2((char*)msg->GetData(), msg->GetSize());
 
-  auto out = TMessageSerializer::deserialize<std::vector<o2::test::Polymorphic>>(as_span(msg));
+  auto out = TMessageSerializer::deserialize<std::vector<o2::test::Polymorphic>>(msg2);
   REQUIRE(out);
   REQUIRE((*out.get()).size() == 2);
   REQUIRE((*out.get())[0] == o2::test::Polymorphic(0xaffe));
   REQUIRE((*out.get())[1] == o2::test::Polymorphic(0xd00f));
 
   // test deserialization with a wrong target class and check the exception
-  REQUIRE_THROWS_AS(TMessageSerializer::deserialize(as_span(msg)), RuntimeErrorRef);
+  REQUIRE_THROWS_AS(TMessageSerializer::deserialize(msg2), RuntimeErrorRef);
 }
 
 TEST_CASE("TestTMessageSerializer_InvalidBuffer")
 {
   const char* buffer = "this is for sure not a serialized ROOT object";
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(strlen(buffer) + 8);
+  memcpy((char*)msg->GetData() + 8, buffer, strlen(buffer));
   // test deserialization of invalid buffer and check the exception
   // FIXME: at the moment, TMessage fails directly with a segfault, which it shouldn't do
   /*
@@ -119,5 +126,23 @@ TEST_CASE("TestTMessageSerializer_InvalidBuffer")
   struct Dummy {
   };
   auto matcher = ExceptionMatcher("class is not ROOT-serializable: ZL22CATCH2_INTERNAL_TEST_4vE5Dummy");
-  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<Dummy>((std::byte*)buffer, strlen(buffer)), o2::framework::RuntimeErrorRef, matcher);
+  FairInputTBuffer msg2((char*)msg->GetData(), msg->GetSize());
+  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<Dummy>(msg2), o2::framework::RuntimeErrorRef, matcher);
+}
+
+TEST_CASE("TestTMessageSerializer_CheckExpansion")
+{
+  const char* buffer = "this is for sure not a serialized ROOT object";
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(strlen(buffer) + 8);
+  FairOutputTBuffer msg2(*msg);
+  // The buffer starts after 8 bytes.
+  REQUIRE(msg2.Buffer() == (char*)msg->GetData()+8);
+  // The first 8 bytes of the buffer store the pointer to the message itself.
+  REQUIRE(*(fair::mq::Message**)msg->GetData() == msg.get());
+  // Notice that TBuffer does the same trick with the reallocation function,
+  // so in the end the useful buffer size is the message size minus 16.
+  REQUIRE(msg2.BufferSize() == (msg->GetSize()-16));
+  // This will not fit the original buffer size, so the buffer will be expanded.
+  msg2.Expand(100);
 }
diff --git a/Framework/Utils/test/test_RootTreeWriter.cxx b/Framework/Utils/test/test_RootTreeWriter.cxx
index 3194508f3d775..62e1eb62cb4f1 100644
--- a/Framework/Utils/test/test_RootTreeWriter.cxx
+++ b/Framework/Utils/test/test_RootTreeWriter.cxx
@@ -179,6 +179,7 @@ TEST_CASE("test_RootTreeWriter")
 
   auto createSerializedMessage = [&transport, &store](DataHeader&& dh, auto& data) {
     fair::mq::MessagePtr payload = transport->CreateMessage();
+    payload->Rebuild(4096, {64});
     auto* cl = TClass::GetClass(typeid(decltype(data)));
     TMessageSerializer().Serialize(*payload, &data, cl);
     dh.payloadSize = payload->GetSize();
diff --git a/Steer/DigitizerWorkflow/src/SimReaderSpec.cxx b/Steer/DigitizerWorkflow/src/SimReaderSpec.cxx
index 03bfa2eb23ede..6f8502f74a85b 100644
--- a/Steer/DigitizerWorkflow/src/SimReaderSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/SimReaderSpec.cxx
@@ -25,7 +25,6 @@
 #include "DetectorsRaw/HBFUtils.h"
 #include <CCDB/BasicCCDBManager.h>
 #include <fairlogger/Logger.h>
-#include <TMessage.h> // object serialization
 #include <memory>     // std::unique_ptr
 #include <cstring>    // memcpy
 #include <string>     // std::string
diff --git a/Utilities/Mergers/src/ObjectStore.cxx b/Utilities/Mergers/src/ObjectStore.cxx
index e88358507c31e..3bb49f1dfc9d8 100644
--- a/Utilities/Mergers/src/ObjectStore.cxx
+++ b/Utilities/Mergers/src/ObjectStore.cxx
@@ -38,7 +38,7 @@ static std::string concat(Args&&... arguments)
   return std::move(ss.str());
 }
 
-void* readObject(const TClass* type, o2::framework::FairTMessage& ftm)
+void* readObject(const TClass* type, o2::framework::FairInputTBuffer& ftm)
 {
   using namespace std::string_view_literals;
   auto* object = ftm.ReadObjectAny(type);
@@ -60,7 +60,7 @@ MergeInterface* castToMergeInterface(bool inheritsFromTObject, void* object, TCl
   return objectAsMergeInterface;
 }
 
-std::optional<ObjectStore> extractVector(o2::framework::FairTMessage& ftm, const TClass* storedClass)
+std::optional<ObjectStore> extractVector(o2::framework::FairInputTBuffer& ftm, const TClass* storedClass)
 {
   if (!storedClass->InheritsFrom(TClass::GetClass(typeid(VectorOfRawTObjects)))) {
     return std::nullopt;
@@ -88,11 +88,14 @@ ObjectStore extractObjectFrom(const framework::DataRef& ref)
     throw std::runtime_error(concat(errorPrefix, "It is not ROOT-serialized"sv));
   }
 
-  o2::framework::FairTMessage ftm(const_cast<char*>(ref.payload), o2::framework::DataRefUtils::getPayloadSize(ref));
-  auto* storedClass = ftm.GetClass();
+  o2::framework::FairInputTBuffer ftm(const_cast<char*>(ref.payload), o2::framework::DataRefUtils::getPayloadSize(ref));
+  ftm.InitMap();
+  auto* storedClass = ftm.ReadClass();
   if (storedClass == nullptr) {
     throw std::runtime_error(concat(errorPrefix, "Unknown stored class"sv));
   }
+  ftm.SetBufferOffset(0);
+  ftm.ResetMap();
 
   if (const auto extractedVector = extractVector(ftm, storedClass)) {
     return extractedVector.value();
diff --git a/Utilities/Mergers/test/benchmark_Types.cxx b/Utilities/Mergers/test/benchmark_Types.cxx
index 790fd329185ea..736685c5746b8 100644
--- a/Utilities/Mergers/test/benchmark_Types.cxx
+++ b/Utilities/Mergers/test/benchmark_Types.cxx
@@ -165,11 +165,16 @@ auto measure = [](Measurement m, auto* o, auto* i) -> double {
         tm->WriteObject(o);
         start = std::chrono::high_resolution_clock::now();
 
-        o2::framework::FairTMessage ftm(const_cast<char*>(tm->Buffer()), tm->BufferSize());
-        auto* storedClass = ftm.GetClass();
+        // Needed to take into account that  FairInputTBuffer expects the first 8 bytes to be the
+        // allocator pointer, which is not present in the TMessage buffer.
+        o2::framework::FairInputTBuffer ftm(const_cast<char*>(tm->Buffer() - 8), tm->BufferSize() + 8);
+        ftm.InitMap();
+        auto* storedClass = ftm.ReadClass();
         if (storedClass == nullptr) {
           throw std::runtime_error("Unknown stored class");
         }
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         auto* tObjectClass = TClass::GetClass(typeid(TObject));
 
         if (!storedClass->InheritsFrom(tObjectClass)) {
@@ -738,4 +743,4 @@ int main(int argc, const char* argv[])
 
   file.close();
   return 0;
-}
\ No newline at end of file
+}

From 03228df7d530292972830710a1bcc5bbf53845c8 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 28 Feb 2024 14:08:03 +0100
Subject: [PATCH 383/726] DPL: avoid TMessage usage

TMessage does not allow for non owned buffers, so we end up having
an extra buffer in private memory for (de)serializing. Using TBufferFile
directly allows to avoid that, so this moves the whole ROOT serialization
support in DPL to use it.
---
 .../src/AODJAlienReaderHelpers.h              |   2 +
 .../Core/include/Framework/DataAllocator.h    |   1 +
 .../Core/include/Framework/DataRefUtils.h     |  13 +-
 .../include/Framework/RootMessageContext.h    |   3 +
 .../Framework/RootSerializationSupport.h      |   3 +-
 .../include/Framework/TMessageSerializer.h    | 130 ++++++++----------
 Framework/Core/src/CommonDataProcessors.cxx   |   7 +-
 Framework/Core/src/TMessageSerializer.cxx     |  37 +++++
 Framework/Core/test/test_DataRefUtils.cxx     |  29 +++-
 .../Core/test/test_TMessageSerializer.cxx     |  51 +++++--
 Framework/Utils/test/test_RootTreeWriter.cxx  |   1 +
 Steer/DigitizerWorkflow/src/SimReaderSpec.cxx |   1 -
 Utilities/Mergers/src/ObjectStore.cxx         |  11 +-
 Utilities/Mergers/test/benchmark_Types.cxx    |  11 +-
 14 files changed, 198 insertions(+), 102 deletions(-)

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
index 655e4b6c0b439..4b9fd710aca14 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
@@ -16,7 +16,9 @@
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/Logger.h"
 #include <Monitoring/Monitoring.h>
+
 #include <uv.h>
+class TFile;
 
 namespace o2::framework::readers
 {
diff --git a/Framework/Core/include/Framework/DataAllocator.h b/Framework/Core/include/Framework/DataAllocator.h
index 8151d2f83c6c6..029e922aeb90b 100644
--- a/Framework/Core/include/Framework/DataAllocator.h
+++ b/Framework/Core/include/Framework/DataAllocator.h
@@ -359,6 +359,7 @@ class DataAllocator
     } else if constexpr (has_root_dictionary<T>::value == true || is_specialization_v<T, ROOTSerialized> == true) {
       // Serialize a snapshot of an object with root dictionary
       payloadMessage = proxy.createOutputMessage(routeIndex);
+      payloadMessage->Rebuild(4096, {64});
       if constexpr (is_specialization_v<T, ROOTSerialized> == true) {
         // Explicitely ROOT serialize a snapshot of object.
         // An object wrapped into type `ROOTSerialized` is explicitely marked to be ROOT serialized
diff --git a/Framework/Core/include/Framework/DataRefUtils.h b/Framework/Core/include/Framework/DataRefUtils.h
index defd10244bca5..264533def326d 100644
--- a/Framework/Core/include/Framework/DataRefUtils.h
+++ b/Framework/Core/include/Framework/DataRefUtils.h
@@ -71,12 +71,15 @@ struct DataRefUtils {
           throw runtime_error("Attempt to extract a TMessage from non-ROOT serialised message");
         }
 
-        typename RSS::FairTMessage ftm(const_cast<char*>(ref.payload), payloadSize);
-        auto* storedClass = ftm.GetClass();
+        typename RSS::FairInputTBuffer ftm(const_cast<char*>(ref.payload), payloadSize);
         auto* requestedClass = RSS::TClass::GetClass(typeid(T));
+        ftm.InitMap();
+        auto* storedClass = ftm.ReadClass();
         // should always have the class description if has_root_dictionary is true
         assert(requestedClass != nullptr);
 
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         auto* object = ftm.ReadObjectAny(storedClass);
         if (object == nullptr) {
           throw runtime_error_f("Failed to read object with name %s from message using ROOT serialization.",
@@ -146,7 +149,11 @@ struct DataRefUtils {
           throw runtime_error("ROOT serialization not supported, dictionary not found for data type");
         }
 
-        typename RSS::FairTMessage ftm(const_cast<char*>(ref.payload), payloadSize);
+        typename RSS::FairInputTBuffer ftm(const_cast<char*>(ref.payload), payloadSize);
+        ftm.InitMap();
+        auto* classInfo = ftm.ReadClass();
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         result.reset(static_cast<wrapped*>(ftm.ReadObjectAny(cl)));
         if (result.get() == nullptr) {
           throw runtime_error_f("Unable to extract class %s", cl == nullptr ? "<name not available>" : cl->GetName());
diff --git a/Framework/Core/include/Framework/RootMessageContext.h b/Framework/Core/include/Framework/RootMessageContext.h
index bef60ebbbf9f9..b1124880cf30f 100644
--- a/Framework/Core/include/Framework/RootMessageContext.h
+++ b/Framework/Core/include/Framework/RootMessageContext.h
@@ -72,6 +72,9 @@ class RootSerializedObject : public MessageContext::ContextObject
   fair::mq::Parts finalize() final
   {
     assert(mParts.Size() == 1);
+    if (mPayloadMsg->GetSize() < sizeof(char*)) {
+      mPayloadMsg->Rebuild(4096, {64});
+    }
     TMessageSerializer::Serialize(*mPayloadMsg, mObject.get(), nullptr);
     mParts.AddPart(std::move(mPayloadMsg));
     return ContextObject::finalize();
diff --git a/Framework/Core/include/Framework/RootSerializationSupport.h b/Framework/Core/include/Framework/RootSerializationSupport.h
index cbf7408b13c7d..a44093f9c02bf 100644
--- a/Framework/Core/include/Framework/RootSerializationSupport.h
+++ b/Framework/Core/include/Framework/RootSerializationSupport.h
@@ -21,7 +21,8 @@ namespace o2::framework
 /// compiler.
 struct RootSerializationSupport {
   using TClass = ::TClass;
-  using FairTMessage = o2::framework::FairTMessage;
+  using FairInputTBuffer = o2::framework::FairInputTBuffer;
+  using FairOutputBuffer = o2::framework::FairOutputTBuffer;
   using TObject = ::TObject;
 };
 
diff --git a/Framework/Core/include/Framework/TMessageSerializer.h b/Framework/Core/include/Framework/TMessageSerializer.h
index 1f08b456c0218..34a5156074b81 100644
--- a/Framework/Core/include/Framework/TMessageSerializer.h
+++ b/Framework/Core/include/Framework/TMessageSerializer.h
@@ -16,9 +16,8 @@
 #include "Framework/RuntimeError.h"
 
 #include <TList.h>
-#include <TMessage.h>
+#include <TBufferFile.h>
 #include <TObjArray.h>
-#include <TStreamerInfo.h>
 #include <gsl/util>
 #include <gsl/span>
 #include <gsl/narrow>
@@ -28,67 +27,76 @@
 
 namespace o2::framework
 {
-class FairTMessage;
+class FairOutputTBuffer;
+class FairInputTBuffer;
 
 // utilities to produce a span over a byte buffer held by various message types
 // this is to avoid littering code with casts and conversions (span has a signed index type(!))
-gsl::span<std::byte> as_span(const FairTMessage& msg);
+gsl::span<std::byte> as_span(const FairInputTBuffer& msg);
+gsl::span<std::byte> as_span(const FairOutputTBuffer& msg);
 gsl::span<std::byte> as_span(const fair::mq::Message& msg);
 
-class FairTMessage : public TMessage
+// A TBufferFile which we can use to serialise data to a FairMQ message.
+class FairOutputTBuffer : public TBufferFile
 {
  public:
-  using TMessage::TMessage;
-  FairTMessage() : TMessage(kMESS_OBJECT) {}
-  FairTMessage(void* buf, Int_t len) : TMessage(buf, len) { ResetBit(kIsOwner); }
-  FairTMessage(gsl::span<std::byte> buf) : TMessage(buf.data(), buf.size()) { ResetBit(kIsOwner); }
+  // This is to serialise data to FairMQ. We embed the pointer to the message
+  // in the data itself, so that we can use it to reallocate the message if needed.
+  // The FairMQ message retains ownership of the data.
+  // When deserialising the root object, keep in mind one needs to skip the 8 bytes
+  // for the pointer.
+  FairOutputTBuffer(fair::mq::Message& msg)
+    : TBufferFile(TBuffer::kWrite, msg.GetSize() - sizeof(char*), embedInItself(msg), false, fairMQrealloc)
+  {
+  }
+  // Helper function to keep track of the FairMQ message that holds the data
+  // in the data itself. We can use this to make sure the message can be reallocated
+  // even if we simply have a pointer to the data. Hopefully ROOT will not play dirty
+  // with us.
+  void* embedInItself(fair::mq::Message& msg);
   // helper function to clean up the object holding the data after it is transported.
-  static void free(void* /*data*/, void* hint);
+  static char* fairMQrealloc(char* oldData, size_t newSize, size_t oldSize);
 };
 
-struct TMessageSerializer {
-  using StreamerList = std::vector<TVirtualStreamerInfo*>;
-  using CompressionLevel = int;
+class FairInputTBuffer : public TBufferFile
+{
+ public:
+  // This is to serialise data to FairMQ. The provided message is expeted to have 8 bytes
+  // of overhead, where the source embedded the pointer for the reallocation.
+  // Notice this will break if the sender and receiver are not using the same
+  // size for a pointer.
+  FairInputTBuffer(char* data, size_t size)
+    : TBufferFile(TBuffer::kRead, size - sizeof(char*), data + sizeof(char*), false, nullptr)
+  {
+  }
+};
 
-  static void Serialize(fair::mq::Message& msg, const TObject* input,
-                        CompressionLevel compressionLevel = -1);
+struct TMessageSerializer {
+  static void Serialize(fair::mq::Message& msg, const TObject* input);
 
   template <typename T>
-  static void Serialize(fair::mq::Message& msg, const T* input, const TClass* cl, //
-                        CompressionLevel compressionLevel = -1);
+  static void Serialize(fair::mq::Message& msg, const T* input, const TClass* cl);
 
   template <typename T = TObject>
   static void Deserialize(const fair::mq::Message& msg, std::unique_ptr<T>& output);
 
-  static void serialize(FairTMessage& msg, const TObject* input,
-                        CompressionLevel compressionLevel = -1);
+  static void serialize(o2::framework::FairOutputTBuffer& msg, const TObject* input);
 
   template <typename T>
-  static void serialize(FairTMessage& msg, const T* input, //
-                        const TClass* cl,
-                        CompressionLevel compressionLevel = -1);
+  static void serialize(o2::framework::FairOutputTBuffer& msg, const T* input, const TClass* cl);
 
   template <typename T = TObject>
-  static std::unique_ptr<T> deserialize(gsl::span<std::byte> buffer);
-  template <typename T = TObject>
-  static inline std::unique_ptr<T> deserialize(std::byte* buffer, size_t size);
+  static inline std::unique_ptr<T> deserialize(FairInputTBuffer& buffer);
 };
 
-inline void TMessageSerializer::serialize(FairTMessage& tm, const TObject* input,
-                                          CompressionLevel compressionLevel)
+inline void TMessageSerializer::serialize(FairOutputTBuffer& tm, const TObject* input)
 {
-  return serialize(tm, input, nullptr, compressionLevel);
+  return serialize(tm, input, nullptr);
 }
 
 template <typename T>
-inline void TMessageSerializer::serialize(FairTMessage& tm, const T* input, //
-                                          const TClass* cl, CompressionLevel compressionLevel)
+inline void TMessageSerializer::serialize(FairOutputTBuffer& tm, const T* input, const TClass* cl)
 {
-  if (compressionLevel >= 0) {
-    // if negative, skip to use ROOT default
-    tm.SetCompressionLevel(compressionLevel);
-  }
-
   // TODO: check what WriateObject and WriteObjectAny are doing
   if (cl == nullptr) {
     tm.WriteObject(input);
@@ -98,7 +106,7 @@ inline void TMessageSerializer::serialize(FairTMessage& tm, const T* input, //
 }
 
 template <typename T>
-inline std::unique_ptr<T> TMessageSerializer::deserialize(gsl::span<std::byte> buffer)
+inline std::unique_ptr<T> TMessageSerializer::deserialize(FairInputTBuffer& buffer)
 {
   TClass* tgtClass = TClass::GetClass(typeid(T));
   if (tgtClass == nullptr) {
@@ -107,53 +115,32 @@ inline std::unique_ptr<T> TMessageSerializer::deserialize(gsl::span<std::byte> b
   // FIXME: we need to add consistency check for buffer data to be serialized
   // at the moment, TMessage might simply crash if an invalid or inconsistent
   // buffer is provided
-  FairTMessage tm(buffer);
-  TClass* serializedClass = tm.GetClass();
+  buffer.InitMap();
+  TClass* serializedClass = buffer.ReadClass();
+  buffer.SetBufferOffset(0);
+  buffer.ResetMap();
   if (serializedClass == nullptr) {
     throw runtime_error_f("can not read class info from buffer");
   }
   if (tgtClass != serializedClass && serializedClass->GetBaseClass(tgtClass) == nullptr) {
     throw runtime_error_f("can not convert serialized class %s into target class %s",
-                          tm.GetClass()->GetName(),
+                          serializedClass->GetName(),
                           tgtClass->GetName());
   }
-  return std::unique_ptr<T>(reinterpret_cast<T*>(tm.ReadObjectAny(serializedClass)));
+  return std::unique_ptr<T>(reinterpret_cast<T*>(buffer.ReadObjectAny(serializedClass)));
 }
 
-template <typename T>
-inline std::unique_ptr<T> TMessageSerializer::deserialize(std::byte* buffer, size_t size)
+inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const TObject* input)
 {
-  return deserialize<T>(gsl::span<std::byte>(buffer, gsl::narrow<gsl::span<std::byte>::size_type>(size)));
-}
-
-inline void FairTMessage::free(void* /*data*/, void* hint)
-{
-  std::default_delete<FairTMessage> deleter;
-  deleter(static_cast<FairTMessage*>(hint));
-}
-
-inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const TObject* input,
-                                          TMessageSerializer::CompressionLevel compressionLevel)
-{
-  std::unique_ptr<FairTMessage> tm = std::make_unique<FairTMessage>(kMESS_OBJECT);
-
-  serialize(*tm, input, input->Class(), compressionLevel);
-
-  msg.Rebuild(tm->Buffer(), tm->BufferSize(), FairTMessage::free, tm.get());
-  tm.release();
+  FairOutputTBuffer output(msg);
+  serialize(output, input, input->Class());
 }
 
 template <typename T>
-inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const T* input, //
-                                          const TClass* cl,                       //
-                                          TMessageSerializer::CompressionLevel compressionLevel)
+inline void TMessageSerializer::Serialize(fair::mq::Message& msg, const T* input, const TClass* cl)
 {
-  std::unique_ptr<FairTMessage> tm = std::make_unique<FairTMessage>(kMESS_OBJECT);
-
-  serialize(*tm, input, cl, compressionLevel);
-
-  msg.Rebuild(tm->Buffer(), tm->BufferSize(), FairTMessage::free, tm.get());
-  tm.release();
+  FairOutputTBuffer output(msg);
+  serialize(output, input, cl);
 }
 
 template <typename T>
@@ -161,7 +148,8 @@ inline void TMessageSerializer::Deserialize(const fair::mq::Message& msg, std::u
 {
   // we know the message will not be modified by this,
   // so const_cast should be OK here(IMHO).
-  output = deserialize(as_span(msg));
+  FairInputTBuffer input(static_cast<char*>(msg.GetData()), static_cast<int>(msg.GetSize()));
+  output = deserialize(input);
 }
 
 // gsl::narrow is used to do a runtime narrowing check, this might be a bit paranoid,
@@ -171,7 +159,7 @@ inline gsl::span<std::byte> as_span(const fair::mq::Message& msg)
   return gsl::span<std::byte>{static_cast<std::byte*>(msg.GetData()), gsl::narrow<gsl::span<std::byte>::size_type>(msg.GetSize())};
 }
 
-inline gsl::span<std::byte> as_span(const FairTMessage& msg)
+inline gsl::span<std::byte> as_span(const FairInputTBuffer& msg)
 {
   return gsl::span<std::byte>{reinterpret_cast<std::byte*>(msg.Buffer()),
                               gsl::narrow<gsl::span<std::byte>::size_type>(msg.BufferSize())};
diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 48a3eb1da95b9..0cf7224c25ac8 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -141,9 +141,12 @@ DataProcessorSpec CommonDataProcessors::getOutputObjHistSink(std::vector<OutputO
         return;
       }
 
-      FairTMessage tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
       InputObject obj;
-      obj.kind = tm.GetClass();
+      FairInputTBuffer tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
+      tm.InitMap();
+      obj.kind = tm.ReadClass();
+      tm.SetBufferOffset(0);
+      tm.ResetMap();
       if (obj.kind == nullptr) {
         LOG(error) << "Cannot read class info from buffer.";
         return;
diff --git a/Framework/Core/src/TMessageSerializer.cxx b/Framework/Core/src/TMessageSerializer.cxx
index 5388a6d716cda..c5da4cc576242 100644
--- a/Framework/Core/src/TMessageSerializer.cxx
+++ b/Framework/Core/src/TMessageSerializer.cxx
@@ -9,7 +9,44 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 #include <Framework/TMessageSerializer.h>
+#include <FairMQTransportFactory.h>
 #include <algorithm>
 #include <memory>
 
 using namespace o2::framework;
+
+void* FairOutputTBuffer::embedInItself(fair::mq::Message& msg)
+{
+  // The first bytes of the message are used to store the pointer to the message itself
+  // so that we can reallocate it if needed.
+  if (sizeof(char*) > msg.GetSize()) {
+    throw std::runtime_error("Message size too small to embed pointer");
+  }
+  char* data = reinterpret_cast<char*>(msg.GetData());
+  char* ptr = reinterpret_cast<char*>(&msg);
+  std::memcpy(data, &ptr, sizeof(char*));
+  return data + sizeof(char*);
+}
+
+// Reallocation function. Get the message pointer from the data and call Rebuild.
+char* FairOutputTBuffer::fairMQrealloc(char* oldData, size_t newSize, size_t oldSize)
+{
+  // Old data is the pointer at the beginning of the message, so the pointer
+  // to the message is **stored** in the 8 bytes before it.
+  auto* msg = *(fair::mq::Message**)(oldData - sizeof(char*));
+  if (newSize <= msg->GetSize()) {
+    // no need to reallocate, the message is already big enough
+    return oldData;
+  }
+  // Create a shallow copy of the message
+  fair::mq::MessagePtr oldMsg = msg->GetTransport()->CreateMessage();
+  oldMsg->Copy(*msg);
+  // Copy the old data while rebuilding. Reference counting should make
+  // sure the old message is not deleted until the new one is ready.
+  // We need 8 extra bytes for the pointer to the message itself (realloc does not know about it)
+  // and we need to copy 8 bytes more than the old size (again, the extra pointer).
+  msg->Rebuild(newSize + 8, fair::mq::Alignment{64});
+  memcpy(msg->GetData(), oldMsg->GetData(), oldSize + 8);
+
+  return reinterpret_cast<char*>(msg->GetData()) + sizeof(char*);
+}
diff --git a/Framework/Core/test/test_DataRefUtils.cxx b/Framework/Core/test/test_DataRefUtils.cxx
index 37da7912bfe8b..d4accde0fecf0 100644
--- a/Framework/Core/test/test_DataRefUtils.cxx
+++ b/Framework/Core/test/test_DataRefUtils.cxx
@@ -21,17 +21,38 @@
 
 using namespace o2::framework;
 
+TEST_CASE("PureRootTest")
+{
+  TBufferFile buffer(TBuffer::kWrite);
+  TObjString s("test");
+  buffer.WriteObject(&s);
+
+  TBufferFile buffer2(TBuffer::kRead, buffer.BufferSize(), buffer.Buffer(), false);
+  buffer2.SetReadMode();
+  buffer2.InitMap();
+  TClass* storedClass = buffer2.ReadClass();
+  // ReadClass advances the buffer, so we need to reset it.
+  buffer2.SetBufferOffset(0);
+  buffer2.ResetMap();
+  REQUIRE(storedClass != nullptr);
+  auto* outS = (TObjString*)buffer2.ReadObjectAny(storedClass);
+  REQUIRE(outS != nullptr);
+  REQUIRE(outS->GetString() == "test");
+}
+
 // Simple test to do root deserialization.
 TEST_CASE("TestRootSerialization")
 {
   DataRef ref;
-  TMessage* tm = new TMessage(kMESS_OBJECT);
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer tm(*msg);
   auto sOrig = std::make_unique<TObjString>("test");
-  tm->WriteObject(sOrig.get());
+  tm << sOrig.get();
   o2::header::DataHeader dh;
   dh.payloadSerializationMethod = o2::header::gSerializationMethodROOT;
-  ref.payload = tm->Buffer();
-  dh.payloadSize = tm->BufferSize();
+  ref.payload = (char*)msg->GetData();
+  dh.payloadSize = (size_t)msg->GetSize();
   ref.header = reinterpret_cast<char const*>(&dh);
 
   // Check by using the same type
diff --git a/Framework/Core/test/test_TMessageSerializer.cxx b/Framework/Core/test/test_TMessageSerializer.cxx
index bc5f817400a44..2807351058c1d 100644
--- a/Framework/Core/test/test_TMessageSerializer.cxx
+++ b/Framework/Core/test/test_TMessageSerializer.cxx
@@ -11,6 +11,7 @@
 
 #include "Framework/TMessageSerializer.h"
 #include "Framework/RuntimeError.h"
+#include <fairmq/TransportFactory.h>
 #include "TestClasses.h"
 #include <catch_amalgamated.hpp>
 #include <utility>
@@ -49,14 +50,14 @@ TEST_CASE("TestTMessageSerializer")
   array.SetOwner();
   array.Add(new TNamed(testname, testtitle));
 
-  FairTMessage msg;
-  TMessageSerializer::serialize(msg, &array);
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer buffer(*msg);
+  TMessageSerializer::serialize(buffer, &array);
 
-  auto buf = as_span(msg);
-  REQUIRE(buf.size() == msg.BufferSize());
-  REQUIRE(static_cast<void*>(buf.data()) == static_cast<void*>(msg.Buffer()));
+  FairInputTBuffer msg2((char*)msg->GetData(), msg->GetSize());
   // test deserialization with TObject as target class (default)
-  auto out = TMessageSerializer::deserialize(buf);
+  auto out = TMessageSerializer::deserialize(msg2);
 
   auto* outarr = dynamic_cast<TObjArray*>(out.get());
   REQUIRE(out.get() == outarr);
@@ -66,9 +67,9 @@ TEST_CASE("TestTMessageSerializer")
   REQUIRE(named->GetTitle() == std::string(testtitle));
 
   // test deserialization with a wrong target class and check the exception
-  REQUIRE_THROWS_AS(TMessageSerializer::deserialize<TNamed>(buf), o2::framework::RuntimeErrorRef);
+  REQUIRE_THROWS_AS(TMessageSerializer::deserialize<TNamed>(msg2), o2::framework::RuntimeErrorRef);
 
-  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<TNamed>(buf), o2::framework::RuntimeErrorRef,
+  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<TNamed>(msg2), o2::framework::RuntimeErrorRef,
                          ExceptionMatcher("can not convert serialized class TObjArray into target class TNamed"));
 }
 
@@ -87,23 +88,29 @@ TEST_CASE("TestTMessageSerializer_NonTObject")
   TClass* cl = TClass::GetClass("std::vector<o2::test::Polymorphic>");
   REQUIRE(cl != nullptr);
 
-  FairTMessage msg;
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(4096);
+  FairOutputTBuffer buffer(*msg);
   char* in = reinterpret_cast<char*>(&data);
-  TMessageSerializer::serialize(msg, in, cl);
+  TMessageSerializer::serialize(buffer, in, cl);
+  FairInputTBuffer msg2((char*)msg->GetData(), msg->GetSize());
 
-  auto out = TMessageSerializer::deserialize<std::vector<o2::test::Polymorphic>>(as_span(msg));
+  auto out = TMessageSerializer::deserialize<std::vector<o2::test::Polymorphic>>(msg2);
   REQUIRE(out);
   REQUIRE((*out.get()).size() == 2);
   REQUIRE((*out.get())[0] == o2::test::Polymorphic(0xaffe));
   REQUIRE((*out.get())[1] == o2::test::Polymorphic(0xd00f));
 
   // test deserialization with a wrong target class and check the exception
-  REQUIRE_THROWS_AS(TMessageSerializer::deserialize(as_span(msg)), RuntimeErrorRef);
+  REQUIRE_THROWS_AS(TMessageSerializer::deserialize(msg2), RuntimeErrorRef);
 }
 
 TEST_CASE("TestTMessageSerializer_InvalidBuffer")
 {
   const char* buffer = "this is for sure not a serialized ROOT object";
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(strlen(buffer) + 8);
+  memcpy((char*)msg->GetData() + 8, buffer, strlen(buffer));
   // test deserialization of invalid buffer and check the exception
   // FIXME: at the moment, TMessage fails directly with a segfault, which it shouldn't do
   /*
@@ -119,5 +126,23 @@ TEST_CASE("TestTMessageSerializer_InvalidBuffer")
   struct Dummy {
   };
   auto matcher = ExceptionMatcher("class is not ROOT-serializable: ZL22CATCH2_INTERNAL_TEST_4vE5Dummy");
-  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<Dummy>((std::byte*)buffer, strlen(buffer)), o2::framework::RuntimeErrorRef, matcher);
+  FairInputTBuffer msg2((char*)msg->GetData(), msg->GetSize());
+  REQUIRE_THROWS_MATCHES(TMessageSerializer::deserialize<Dummy>(msg2), o2::framework::RuntimeErrorRef, matcher);
+}
+
+TEST_CASE("TestTMessageSerializer_CheckExpansion")
+{
+  const char* buffer = "this is for sure not a serialized ROOT object";
+  auto transport = fair::mq::TransportFactory::CreateTransportFactory("zeromq");
+  auto msg = transport->CreateMessage(strlen(buffer) + 8);
+  FairOutputTBuffer msg2(*msg);
+  // The buffer starts after 8 bytes.
+  REQUIRE(msg2.Buffer() == (char*)msg->GetData() + 8);
+  // The first 8 bytes of the buffer store the pointer to the message itself.
+  REQUIRE(*(fair::mq::Message**)msg->GetData() == msg.get());
+  // Notice that TBuffer does the same trick with the reallocation function,
+  // so in the end the useful buffer size is the message size minus 16.
+  REQUIRE(msg2.BufferSize() == (msg->GetSize() - 16));
+  // This will not fit the original buffer size, so the buffer will be expanded.
+  msg2.Expand(100);
 }
diff --git a/Framework/Utils/test/test_RootTreeWriter.cxx b/Framework/Utils/test/test_RootTreeWriter.cxx
index 3194508f3d775..62e1eb62cb4f1 100644
--- a/Framework/Utils/test/test_RootTreeWriter.cxx
+++ b/Framework/Utils/test/test_RootTreeWriter.cxx
@@ -179,6 +179,7 @@ TEST_CASE("test_RootTreeWriter")
 
   auto createSerializedMessage = [&transport, &store](DataHeader&& dh, auto& data) {
     fair::mq::MessagePtr payload = transport->CreateMessage();
+    payload->Rebuild(4096, {64});
     auto* cl = TClass::GetClass(typeid(decltype(data)));
     TMessageSerializer().Serialize(*payload, &data, cl);
     dh.payloadSize = payload->GetSize();
diff --git a/Steer/DigitizerWorkflow/src/SimReaderSpec.cxx b/Steer/DigitizerWorkflow/src/SimReaderSpec.cxx
index 03bfa2eb23ede..6f8502f74a85b 100644
--- a/Steer/DigitizerWorkflow/src/SimReaderSpec.cxx
+++ b/Steer/DigitizerWorkflow/src/SimReaderSpec.cxx
@@ -25,7 +25,6 @@
 #include "DetectorsRaw/HBFUtils.h"
 #include <CCDB/BasicCCDBManager.h>
 #include <fairlogger/Logger.h>
-#include <TMessage.h> // object serialization
 #include <memory>     // std::unique_ptr
 #include <cstring>    // memcpy
 #include <string>     // std::string
diff --git a/Utilities/Mergers/src/ObjectStore.cxx b/Utilities/Mergers/src/ObjectStore.cxx
index e88358507c31e..3bb49f1dfc9d8 100644
--- a/Utilities/Mergers/src/ObjectStore.cxx
+++ b/Utilities/Mergers/src/ObjectStore.cxx
@@ -38,7 +38,7 @@ static std::string concat(Args&&... arguments)
   return std::move(ss.str());
 }
 
-void* readObject(const TClass* type, o2::framework::FairTMessage& ftm)
+void* readObject(const TClass* type, o2::framework::FairInputTBuffer& ftm)
 {
   using namespace std::string_view_literals;
   auto* object = ftm.ReadObjectAny(type);
@@ -60,7 +60,7 @@ MergeInterface* castToMergeInterface(bool inheritsFromTObject, void* object, TCl
   return objectAsMergeInterface;
 }
 
-std::optional<ObjectStore> extractVector(o2::framework::FairTMessage& ftm, const TClass* storedClass)
+std::optional<ObjectStore> extractVector(o2::framework::FairInputTBuffer& ftm, const TClass* storedClass)
 {
   if (!storedClass->InheritsFrom(TClass::GetClass(typeid(VectorOfRawTObjects)))) {
     return std::nullopt;
@@ -88,11 +88,14 @@ ObjectStore extractObjectFrom(const framework::DataRef& ref)
     throw std::runtime_error(concat(errorPrefix, "It is not ROOT-serialized"sv));
   }
 
-  o2::framework::FairTMessage ftm(const_cast<char*>(ref.payload), o2::framework::DataRefUtils::getPayloadSize(ref));
-  auto* storedClass = ftm.GetClass();
+  o2::framework::FairInputTBuffer ftm(const_cast<char*>(ref.payload), o2::framework::DataRefUtils::getPayloadSize(ref));
+  ftm.InitMap();
+  auto* storedClass = ftm.ReadClass();
   if (storedClass == nullptr) {
     throw std::runtime_error(concat(errorPrefix, "Unknown stored class"sv));
   }
+  ftm.SetBufferOffset(0);
+  ftm.ResetMap();
 
   if (const auto extractedVector = extractVector(ftm, storedClass)) {
     return extractedVector.value();
diff --git a/Utilities/Mergers/test/benchmark_Types.cxx b/Utilities/Mergers/test/benchmark_Types.cxx
index 790fd329185ea..736685c5746b8 100644
--- a/Utilities/Mergers/test/benchmark_Types.cxx
+++ b/Utilities/Mergers/test/benchmark_Types.cxx
@@ -165,11 +165,16 @@ auto measure = [](Measurement m, auto* o, auto* i) -> double {
         tm->WriteObject(o);
         start = std::chrono::high_resolution_clock::now();
 
-        o2::framework::FairTMessage ftm(const_cast<char*>(tm->Buffer()), tm->BufferSize());
-        auto* storedClass = ftm.GetClass();
+        // Needed to take into account that  FairInputTBuffer expects the first 8 bytes to be the
+        // allocator pointer, which is not present in the TMessage buffer.
+        o2::framework::FairInputTBuffer ftm(const_cast<char*>(tm->Buffer() - 8), tm->BufferSize() + 8);
+        ftm.InitMap();
+        auto* storedClass = ftm.ReadClass();
         if (storedClass == nullptr) {
           throw std::runtime_error("Unknown stored class");
         }
+        ftm.SetBufferOffset(0);
+        ftm.ResetMap();
         auto* tObjectClass = TClass::GetClass(typeid(TObject));
 
         if (!storedClass->InheritsFrom(tObjectClass)) {
@@ -738,4 +743,4 @@ int main(int argc, const char* argv[])
 
   file.close();
   return 0;
-}
\ No newline at end of file
+}

From f7073e5acb3f9c5334d6b8018e7f88f5aef5f472 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 28 Feb 2024 14:55:55 +0100
Subject: [PATCH 384/726] Require C++20 for the whole AliceO2

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index bf1d83fbb6644..b754fc9ce9551 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -37,7 +37,7 @@ set_property(GLOBAL PROPERTY JOB_POOLS analysis=${ANALYSIS_COMPILE_POOL})
 
 include(O2BuildSanityChecks)
 o2_build_sanity_checks()
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
 
 find_package(ONNXRuntime::ONNXRuntime CONFIG)

From b9937bce006aefd1b41bc02878f3483156171600 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 28 Feb 2024 16:10:54 +0100
Subject: [PATCH 385/726] [O2-4578] DPL: attempt at removing a race condition
 in the rate limiting

---
 Framework/Core/src/CommonDataProcessors.cxx | 48 +++++++++++++++------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 0cf7224c25ac8..7f29b9bf42554 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -568,29 +568,49 @@ DataProcessorSpec CommonDataProcessors::getGlobalFairMQSink(std::vector<InputSpe
   return specifyFairMQDeviceOutputProxy("internal-dpl-injected-output-proxy", danglingOutputInputs, defaultChannelConfig.c_str());
 }
 
+void retryMetricCallback(uv_async_t* async)
+{
+  static size_t lastTimeslice = -1;
+  auto* services = (ServiceRegistryRef*)async->data;
+  auto& timesliceIndex = services->get<TimesliceIndex>();
+  auto* device = services->get<RawDeviceService>().device();
+  auto channel = device->GetChannels().find("metric-feedback");
+  auto oldestPossingTimeslice = timesliceIndex.getOldestPossibleOutput().timeslice.value;
+  if (channel == device->GetChannels().end()) {
+    return;
+  }
+  fair::mq::MessagePtr payload(device->NewMessage());
+  payload->Rebuild(&oldestPossingTimeslice, sizeof(int64_t), nullptr, nullptr);
+  auto* consumed = (size_t*)malloc(sizeof(size_t));
+  *consumed = oldestPossingTimeslice;
+
+  int64_t result = channel->second[0].Send(payload, 100);
+  // If the sending worked, we do not retry.
+  if (result != 0) {
+    // If the sending did not work, we keep trying until it actually works.
+    // This will schedule other tasks in the queue, so the processing of the
+    // data will still happen.
+    uv_async_send(async);
+  } else {
+    lastTimeslice = *consumed;
+  }
+}
+
 DataProcessorSpec CommonDataProcessors::getDummySink(std::vector<InputSpec> const& danglingOutputInputs, std::string rateLimitingChannelConfig)
 {
   return DataProcessorSpec{
     .name = "internal-dpl-injected-dummy-sink",
     .inputs = danglingOutputInputs,
-    .algorithm = AlgorithmSpec{adaptStateful([](CallbackService& callbacks) {
+    .algorithm = AlgorithmSpec{adaptStateful([](CallbackService& callbacks, DeviceState& deviceState, ServiceRegistry& services) {
+      static uv_async_t async;
+      // The callback will only have access to the
+      async.data = new ServiceRegistryRef{services};
+      uv_async_init(deviceState.loop, &async, retryMetricCallback);
       auto domainInfoUpdated = [](ServiceRegistryRef services, size_t timeslice, ChannelIndex channelIndex) {
         LOGP(debug, "Domain info updated with timeslice {}", timeslice);
-        static size_t lastTimeslice = -1;
+        retryMetricCallback(&async);
         auto& timesliceIndex = services.get<TimesliceIndex>();
-        auto device = services.get<RawDeviceService>().device();
-        auto channel = device->GetChannels().find("metric-feedback");
         auto oldestPossingTimeslice = timesliceIndex.getOldestPossibleOutput().timeslice.value;
-        if (channel != device->GetChannels().end()) {
-          fair::mq::MessagePtr payload(device->NewMessage());
-          size_t* consumed = (size_t*)malloc(sizeof(size_t));
-          *consumed = oldestPossingTimeslice;
-          if (*consumed != lastTimeslice) {
-            payload->Rebuild(consumed, sizeof(int64_t), nullptr, nullptr);
-            channel->second[0].Send(payload);
-            lastTimeslice = *consumed;
-          }
-        }
         auto& stats = services.get<DataProcessingStats>();
         stats.updateStats({(int)ProcessingStatsId::CONSUMED_TIMEFRAMES, DataProcessingStats::Op::Set, (int64_t)oldestPossingTimeslice});
       };

From f829c9c14961639d1d65b4df755c6d8533feb105 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 29 Feb 2024 08:56:51 +0100
Subject: [PATCH 386/726] DPL: allow a given buffer to be deserialised multiple
 times

This is only exercised in one of the tests, AFAICT, but it should not
harm.
---
 Framework/Core/include/Framework/TMessageSerializer.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Framework/Core/include/Framework/TMessageSerializer.h b/Framework/Core/include/Framework/TMessageSerializer.h
index 34a5156074b81..17f3f65ce2030 100644
--- a/Framework/Core/include/Framework/TMessageSerializer.h
+++ b/Framework/Core/include/Framework/TMessageSerializer.h
@@ -115,6 +115,7 @@ inline std::unique_ptr<T> TMessageSerializer::deserialize(FairInputTBuffer& buff
   // FIXME: we need to add consistency check for buffer data to be serialized
   // at the moment, TMessage might simply crash if an invalid or inconsistent
   // buffer is provided
+  buffer.SetBufferOffset(0);
   buffer.InitMap();
   TClass* serializedClass = buffer.ReadClass();
   buffer.SetBufferOffset(0);

From 0d941464737b8673f1f3ca7452b95501183e1aa9 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 1 Mar 2024 09:29:49 +0100
Subject: [PATCH 387/726] DPL: attempt to have some dropping logic which does
 not break test

I suspect what is happening in the test is the consumers start
too late (and the lossy policy drops messages on the sender side
when that happens).

This should avoid the problem by increasing the delay up to 1 second,
before switching to lossy, giving enough time to the consumers to start.
---
 Framework/Core/src/SendingPolicy.cxx | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index bf97d56b4a5b8..c575301064d8a 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -54,17 +54,17 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               // We count the number of consecutively dropped messages.
               // If we have more than 10, we switch to a completely
               // non-blocking approach.
-              int64_t timeout = 50;
+              int64_t timeout = 100;
               if (state.droppedMessages == 10 + 1) {
                 LOG(warning) << "Failed to send 10 messages with 10ms timeout in a row, switching to completely non-blocking mode.";
               }
               if (state.droppedMessages == 0) {
-                timeout = 50;
+                timeout = 100;
               }
               if (state.droppedMessages > 10) {
                 timeout = 0;
               }
-              int64_t result = info.channel.Send(parts, timeout);
+              int64_t result = info.channel.Send(parts, timeout*(state.droppedMessages+1));
               if (result >= 0) {
                 state.droppedMessages = 0;
               } else if (state.droppedMessages < std::numeric_limits<decltype(state.droppedMessages)>::max()) {

From d8b3ce7b76c7a02abedda16751bec5590d62a659 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 1 Mar 2024 09:32:07 +0100
Subject: [PATCH 388/726] DPL: attempt to have some dropping logic which does
 not break test

I suspect what is happening in the test is the consumers start
too late (and the lossy policy drops messages on the sender side
when that happens).

This should avoid the problem by increasing the delay up to 1 second,
before switching to lossy, giving enough time to the consumers to start.
---
 Framework/Core/src/SendingPolicy.cxx | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index bf97d56b4a5b8..fb0a590920b76 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -54,17 +54,17 @@ std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
               // We count the number of consecutively dropped messages.
               // If we have more than 10, we switch to a completely
               // non-blocking approach.
-              int64_t timeout = 50;
+              int64_t timeout = 100;
               if (state.droppedMessages == 10 + 1) {
                 LOG(warning) << "Failed to send 10 messages with 10ms timeout in a row, switching to completely non-blocking mode.";
               }
               if (state.droppedMessages == 0) {
-                timeout = 50;
+                timeout = 100;
               }
               if (state.droppedMessages > 10) {
                 timeout = 0;
               }
-              int64_t result = info.channel.Send(parts, timeout);
+              int64_t result = info.channel.Send(parts, timeout * (state.droppedMessages + 1));
               if (result >= 0) {
                 state.droppedMessages = 0;
               } else if (state.droppedMessages < std::numeric_limits<decltype(state.droppedMessages)>::max()) {

From a147ca6aebb5932e875e999fad94168c5f921911 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 1 Mar 2024 10:08:40 +0100
Subject: [PATCH 389/726] DPL: drop special policy for Dispatcher

---
 Framework/Core/src/SendingPolicy.cxx | 43 ----------------------------
 1 file changed, 43 deletions(-)

diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index fb0a590920b76..e2c00104a8951 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -28,49 +28,6 @@ namespace o2::framework
 std::vector<SendingPolicy> SendingPolicy::createDefaultPolicies()
 {
   return {SendingPolicy{
-            .name = "dispatcher",
-            .matcher = [](DataProcessorSpec const& source, DataProcessorSpec const& dest, ConfigContext const&) {
-                if (source.name == "Dispatcher") {
-                  return true;
-                }
-                // Check if any of the labels has "Dispatcher" as prefix
-                for (auto const& label : source.labels) {
-                  if (label.value.find("Dispatcher") == 0) {
-                    return true;
-                  }
-                }
-                // Check if any of the destination's labels is "expendable"
-                for (auto const& label : dest.labels) {
-                  if (label.value == "expendable") {
-                    return true;
-                  }
-                }
-                return false; },
-            .send = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
-              auto &proxy = registry.get<FairMQDeviceProxy>();
-              OutputChannelInfo const& info = proxy.getOutputChannelInfo(channelIndex);
-              OutputChannelState& state = proxy.getOutputChannelState(channelIndex);
-              // Default timeout is 50ms.
-              // We count the number of consecutively dropped messages.
-              // If we have more than 10, we switch to a completely
-              // non-blocking approach.
-              int64_t timeout = 100;
-              if (state.droppedMessages == 10 + 1) {
-                LOG(warning) << "Failed to send 10 messages with 10ms timeout in a row, switching to completely non-blocking mode.";
-              }
-              if (state.droppedMessages == 0) {
-                timeout = 100;
-              }
-              if (state.droppedMessages > 10) {
-                timeout = 0;
-              }
-              int64_t result = info.channel.Send(parts, timeout * (state.droppedMessages + 1));
-              if (result >= 0) {
-                state.droppedMessages = 0;
-              } else if (state.droppedMessages < std::numeric_limits<decltype(state.droppedMessages)>::max()) {
-                state.droppedMessages++;
-              } }},
-          SendingPolicy{
             .name = "profiling",
             .matcher = [](DataProcessorSpec const&, DataProcessorSpec const&, ConfigContext const&) { return getenv("DPL_DEBUG_MESSAGE_SIZE"); },
             .send = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {

From 1f1428e507090c2c9d11aa1237531e82ca481d29 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 4 Mar 2024 10:41:23 +0100
Subject: [PATCH 390/726] DPL CCDB: handle the Cache-Valid-Until header

This will not recheck the etag and consider the cache valid until
the timestamp in milliseconds in the Cache-Valid-Until is not passed
by the data timestamp.
---
 Framework/CCDBSupport/src/CCDBHelpers.cxx | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Framework/CCDBSupport/src/CCDBHelpers.cxx b/Framework/CCDBSupport/src/CCDBHelpers.cxx
index 837b20cf4262b..fb7a369bc1741 100644
--- a/Framework/CCDBSupport/src/CCDBHelpers.cxx
+++ b/Framework/CCDBSupport/src/CCDBHelpers.cxx
@@ -33,6 +33,7 @@ namespace o2::framework
 struct CCDBFetcherHelper {
   struct CCDBCacheInfo {
     std::string etag;
+    size_t cacheValidUntil = 0;
     size_t cacheMiss = 0;
     size_t cacheHit = 0;
     size_t minSize = -1ULL;
@@ -217,7 +218,9 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
     const auto url2uuid = helper->mapURL2UUID.find(path);
     if (url2uuid != helper->mapURL2UUID.end()) {
       etag = url2uuid->second.etag;
-      checkValidity = std::abs(int(timingInfo.tfCounter - url2uuid->second.lastCheckedTF)) >= chRate;
+      // We check validity every chRate timeslices or if the cache is expired
+      uint64_t validUntil = url2uuid->second.cacheValidUntil;
+      checkValidity = (std::abs(int(timingInfo.tfCounter - url2uuid->second.lastCheckedTF)) >= chRate) || (validUntil <= timestamp);
     } else {
       checkValidity = true; // never skip check if the cache is empty
     }
@@ -240,6 +243,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       helper->mapURL2UUID[path].lastCheckedTF = timingInfo.tfCounter;
       if (etag.empty()) {
         helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
+        helper->mapURL2UUID[path].cacheValidUntil = headers["Cache-Valid-Until"].empty() ? 0 : std::stoul(headers["Cache-Valid-Until"]);
         helper->mapURL2UUID[path].cacheMiss++;
         helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
@@ -251,6 +255,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       if (v.size()) { // but should be overridden by fresh object
         // somewhere here pruneFromCache should be called
         helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
+        helper->mapURL2UUID[path].cacheValidUntil = headers["Cache-Valid-Until"].empty() ? 0 : std::stoul(headers["Cache-Valid-Until"]);
         helper->mapURL2UUID[path].cacheMiss++;
         helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);

From 18c9baae745cfc0af5ef78665e1817949dd2ec8b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 4 Mar 2024 11:32:56 +0100
Subject: [PATCH 391/726] DPL CCDB: handle the Cache-Valid-Until header

This will not recheck the etag and consider the cache valid until
the timestamp in milliseconds in the Cache-Valid-Until is not passed
by the data timestamp.
---
 Framework/CCDBSupport/src/CCDBHelpers.cxx | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/Framework/CCDBSupport/src/CCDBHelpers.cxx b/Framework/CCDBSupport/src/CCDBHelpers.cxx
index 837b20cf4262b..c72c6b4d0ff7f 100644
--- a/Framework/CCDBSupport/src/CCDBHelpers.cxx
+++ b/Framework/CCDBSupport/src/CCDBHelpers.cxx
@@ -33,6 +33,7 @@ namespace o2::framework
 struct CCDBFetcherHelper {
   struct CCDBCacheInfo {
     std::string etag;
+    size_t cacheValidUntil = 0;
     size_t cacheMiss = 0;
     size_t cacheHit = 0;
     size_t minSize = -1ULL;
@@ -176,6 +177,11 @@ auto getOrbitResetTime(o2::pmr::vector<char> const& v) -> Long64_t
   return (*ctp)[0];
 };
 
+bool isOnlineRun(DataTakingContext const& dtc)
+{
+  return dtc.deploymentMode == DeploymentMode::OnlineAUX || dtc.deploymentMode == DeploymentMode::OnlineDDS || dtc.deploymentMode == DeploymentMode::OnlineECS;
+}
+
 auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
                        int64_t timestamp,
                        TimingInfo& timingInfo,
@@ -186,6 +192,8 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
   int objCnt = -1;
   // We use the timeslice, so that we hook into the same interval as the rest of the
   // callback.
+  static bool isOnline = isOnlineRun(dtc);
+
   auto sid = _o2_signpost_id_t{(int64_t)timingInfo.timeslice};
   O2_SIGNPOST_START(ccdb, sid, "populateCacheWith", "Starting to populate cache with CCDB objects");
   for (auto& route : helper->routes) {
@@ -217,7 +225,9 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
     const auto url2uuid = helper->mapURL2UUID.find(path);
     if (url2uuid != helper->mapURL2UUID.end()) {
       etag = url2uuid->second.etag;
-      checkValidity = std::abs(int(timingInfo.tfCounter - url2uuid->second.lastCheckedTF)) >= chRate;
+      // We check validity every chRate timeslices or if the cache is expired
+      uint64_t validUntil = url2uuid->second.cacheValidUntil;
+      checkValidity = (std::abs(int(timingInfo.tfCounter - url2uuid->second.lastCheckedTF)) >= chRate) && (isOnline || (validUntil <= timestamp));
     } else {
       checkValidity = true; // never skip check if the cache is empty
     }
@@ -240,6 +250,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       helper->mapURL2UUID[path].lastCheckedTF = timingInfo.tfCounter;
       if (etag.empty()) {
         helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
+        helper->mapURL2UUID[path].cacheValidUntil = headers["Cache-Valid-Until"].empty() ? 0 : std::stoul(headers["Cache-Valid-Until"]);
         helper->mapURL2UUID[path].cacheMiss++;
         helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
@@ -251,6 +262,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       if (v.size()) { // but should be overridden by fresh object
         // somewhere here pruneFromCache should be called
         helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
+        helper->mapURL2UUID[path].cacheValidUntil = headers["Cache-Valid-Until"].empty() ? 0 : std::stoul(headers["Cache-Valid-Until"]);
         helper->mapURL2UUID[path].cacheMiss++;
         helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);

From d7ed1a3bc07b57f22bb16fa167a60070da97e09f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 4 Mar 2024 15:44:42 +0100
Subject: [PATCH 392/726] DPL: Move DefaultsHelpers to Framework/Foundation

This allows non-DPL code to use the same helpers
---
 CCDB/src/CcdbApi.cxx                          |  1 +
 Framework/CCDBSupport/src/CCDBHelpers.cxx     |  1 +
 Framework/Core/CMakeLists.txt                 |  1 -
 .../include/Framework/DataTakingContext.h     | 13 +++--------
 Framework/Core/src/CallbacksPolicy.cxx        |  1 +
 Framework/Core/src/CommonServices.cxx         |  1 +
 Framework/Core/src/RateLimiter.cxx            |  1 +
 Framework/Core/src/WorkflowHelpers.cxx        |  1 +
 Framework/Core/src/runDataProcessing.cxx      |  1 +
 Framework/Foundation/CMakeLists.txt           |  1 +
 .../include/Framework/DefaultsHelpers.h       |  0
 .../include/Framework/DeploymentMode.h        | 22 +++++++++++++++++++
 .../src/DefaultsHelpers.cxx                   |  2 +-
 13 files changed, 34 insertions(+), 12 deletions(-)
 rename Framework/{Core => Foundation}/include/Framework/DefaultsHelpers.h (100%)
 create mode 100644 Framework/Foundation/include/Framework/DeploymentMode.h
 rename Framework/{Core => Foundation}/src/DefaultsHelpers.cxx (98%)

diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx
index 0661eff071e46..fd73305542c26 100644
--- a/CCDB/src/CcdbApi.cxx
+++ b/CCDB/src/CcdbApi.cxx
@@ -22,6 +22,7 @@
 #include "CommonUtils/MemFileHelper.h"
 #include "Framework/DefaultsHelpers.h"
 #include "Framework/DataTakingContext.h"
+#include "Framework/DeploymentMode.h"
 #include <chrono>
 #include <memory>
 #include <sstream>
diff --git a/Framework/CCDBSupport/src/CCDBHelpers.cxx b/Framework/CCDBSupport/src/CCDBHelpers.cxx
index 837b20cf4262b..415f7b4218261 100644
--- a/Framework/CCDBSupport/src/CCDBHelpers.cxx
+++ b/Framework/CCDBSupport/src/CCDBHelpers.cxx
@@ -17,6 +17,7 @@
 #include "Framework/DataTakingContext.h"
 #include "Framework/RawDeviceService.h"
 #include "Framework/DataSpecUtils.h"
+#include "Framework/DeploymentMode.h"
 #include "CCDB/CcdbApi.h"
 #include "CommonConstants/LHCConstants.h"
 #include "Framework/Signpost.h"
diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index 05bcb1bd53083..9953f13cf5e10 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -43,7 +43,6 @@ o2_add_library(Framework
                        src/DataSender.cxx
                        src/DataProcessingStats.cxx
                        src/DataProcessingStates.cxx
-                       src/DefaultsHelpers.cxx
                        src/DomainInfoHeader.cxx
                        src/ProcessingPoliciesHelpers.cxx
                        src/ConfigParamStore.cxx
diff --git a/Framework/Core/include/Framework/DataTakingContext.h b/Framework/Core/include/Framework/DataTakingContext.h
index 05bf8f34ad74d..2ae298de9f5a3 100644
--- a/Framework/Core/include/Framework/DataTakingContext.h
+++ b/Framework/Core/include/Framework/DataTakingContext.h
@@ -12,20 +12,13 @@
 #define O2_FRAMEWORK_DATATAKINGCONTEXT_H_
 
 #include "Framework/ServiceHandle.h"
+
 #include <string>
 #include <cstdint>
 
 namespace o2::framework
 {
-
-enum struct DeploymentMode {
-  Local,     // if nothing special is specified
-  OnlineECS, // Running online (sync processing) a P2 on FLP steered by ECS
-  OnlineDDS, // Running online (sync processing) a P2 on EPN steered by DDS/ODC
-  OnlineAUX, // Running online (sync processing) a P2 as auxiliary process
-  Grid,      // Running as GRID job with Alien job id
-  FST        // Running 8 GPU FST on EPNs (ALICE_O2_FST=1 set)
-};
+enum struct DeploymentMode;
 
 struct DataTakingContext {
   constexpr static ServiceKind service_kind = ServiceKind::Stream;
@@ -46,7 +39,7 @@ struct DataTakingContext {
   bool forcedRaw{false};
 
   /// Where we thing this is running
-  DeploymentMode deploymentMode{DeploymentMode::Local};
+  DeploymentMode deploymentMode{};
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/CallbacksPolicy.cxx b/Framework/Core/src/CallbacksPolicy.cxx
index 9be04cdef290d..98211f3c2d297 100644
--- a/Framework/Core/src/CallbacksPolicy.cxx
+++ b/Framework/Core/src/CallbacksPolicy.cxx
@@ -17,6 +17,7 @@
 #include "Framework/CommonServices.h"
 #include "Framework/DataTakingContext.h"
 #include "Framework/DefaultsHelpers.h"
+#include "Framework/DeploymentMode.h"
 #include <cstdlib>
 #include <uv.h>
 
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index a0f7fcdc88201..803c467457b9a 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -29,6 +29,7 @@
 #include "Framework/CommonMessageBackends.h"
 #include "Framework/DanglingContext.h"
 #include "Framework/DataProcessingHelpers.h"
+#include "Framework/DeploymentMode.h"
 #include "InputRouteHelpers.h"
 #include "Framework/EndOfStreamContext.h"
 #include "Framework/RawDeviceService.h"
diff --git a/Framework/Core/src/RateLimiter.cxx b/Framework/Core/src/RateLimiter.cxx
index 5f6ff24adfd7c..c242880b7e523 100644
--- a/Framework/Core/src/RateLimiter.cxx
+++ b/Framework/Core/src/RateLimiter.cxx
@@ -16,6 +16,7 @@
 #include "Framework/DataTakingContext.h"
 #include "Framework/DeviceState.h"
 #include "Framework/DeviceContext.h"
+#include "Framework/DeploymentMode.h"
 #include <fairmq/Device.h>
 #include <uv.h>
 #include <fairmq/shmem/Monitor.h>
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 6b4ad35f826be..52a98f5d9f810 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -24,6 +24,7 @@
 #include "Framework/Plugins.h"
 #include "Framework/DataTakingContext.h"
 #include "Framework/DefaultsHelpers.h"
+#include "Framework/DeploymentMode.h"
 
 #include "Headers/DataHeader.h"
 #include <algorithm>
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 64c034ebb8533..59ae3a2a167f9 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -10,6 +10,7 @@
 // or submit itself to any jurisdiction.
 #define BOOST_BIND_GLOBAL_PLACEHOLDERS
 #include <stdexcept>
+#include "Framework/DeploymentMode.h"
 #include "Framework/BoostOptionsRetriever.h"
 #include "Framework/CallbacksPolicy.h"
 #include "Framework/ChannelConfigurationPolicy.h"
diff --git a/Framework/Foundation/CMakeLists.txt b/Framework/Foundation/CMakeLists.txt
index bcde0f7a69f34..77e2fac4d2c10 100644
--- a/Framework/Foundation/CMakeLists.txt
+++ b/Framework/Foundation/CMakeLists.txt
@@ -16,6 +16,7 @@ install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/include/Framework
 o2_add_library(FrameworkFoundation
                SOURCES src/RuntimeError.cxx
                src/Signpost.cxx
+               src/DefaultsHelpers.cxx
                TARGETVARNAME targetName
                PUBLIC_LINK_LIBRARIES O2::FrameworkFoundation3rdparty
               )
diff --git a/Framework/Core/include/Framework/DefaultsHelpers.h b/Framework/Foundation/include/Framework/DefaultsHelpers.h
similarity index 100%
rename from Framework/Core/include/Framework/DefaultsHelpers.h
rename to Framework/Foundation/include/Framework/DefaultsHelpers.h
diff --git a/Framework/Foundation/include/Framework/DeploymentMode.h b/Framework/Foundation/include/Framework/DeploymentMode.h
new file mode 100644
index 0000000000000..194e0fa5a807b
--- /dev/null
+++ b/Framework/Foundation/include/Framework/DeploymentMode.h
@@ -0,0 +1,22 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+namespace o2::framework
+{
+enum struct DeploymentMode {
+  Local,     // if nothing special is specified
+  OnlineECS, // Running online (sync processing) a P2 on FLP steered by ECS
+  OnlineDDS, // Running online (sync processing) a P2 on EPN steered by DDS/ODC
+  OnlineAUX, // Running online (sync processing) a P2 as auxiliary process
+  Grid,      // Running as GRID job with Alien job id
+  FST        // Running 8 GPU FST on EPNs (ALICE_O2_FST=1 set)
+};
+}
diff --git a/Framework/Core/src/DefaultsHelpers.cxx b/Framework/Foundation/src/DefaultsHelpers.cxx
similarity index 98%
rename from Framework/Core/src/DefaultsHelpers.cxx
rename to Framework/Foundation/src/DefaultsHelpers.cxx
index 88956b93a855d..658e7a2500d25 100644
--- a/Framework/Core/src/DefaultsHelpers.cxx
+++ b/Framework/Foundation/src/DefaultsHelpers.cxx
@@ -10,7 +10,7 @@
 // or submit itself to any jurisdiction.
 
 #include "Framework/DefaultsHelpers.h"
-#include "Framework/DataTakingContext.h"
+#include "Framework/DeploymentMode.h"
 #include <cstdlib>
 #include <cstring>
 #include <stdexcept>

From f355f45f64aac4121d0a32f3971d652e8e0565ce Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 4 Mar 2024 11:32:56 +0100
Subject: [PATCH 393/726] DPL CCDB: handle the Cache-Valid-Until header

This will not recheck the etag and consider the cache valid until
the timestamp in milliseconds in the Cache-Valid-Until is not passed
by the data timestamp.
---
 Framework/CCDBSupport/src/CCDBHelpers.cxx | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/Framework/CCDBSupport/src/CCDBHelpers.cxx b/Framework/CCDBSupport/src/CCDBHelpers.cxx
index 415f7b4218261..515e503bd77c8 100644
--- a/Framework/CCDBSupport/src/CCDBHelpers.cxx
+++ b/Framework/CCDBSupport/src/CCDBHelpers.cxx
@@ -34,6 +34,7 @@ namespace o2::framework
 struct CCDBFetcherHelper {
   struct CCDBCacheInfo {
     std::string etag;
+    size_t cacheValidUntil = 0;
     size_t cacheMiss = 0;
     size_t cacheHit = 0;
     size_t minSize = -1ULL;
@@ -177,6 +178,11 @@ auto getOrbitResetTime(o2::pmr::vector<char> const& v) -> Long64_t
   return (*ctp)[0];
 };
 
+bool isOnlineRun(DataTakingContext const& dtc)
+{
+  return dtc.deploymentMode == DeploymentMode::OnlineAUX || dtc.deploymentMode == DeploymentMode::OnlineDDS || dtc.deploymentMode == DeploymentMode::OnlineECS;
+}
+
 auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
                        int64_t timestamp,
                        TimingInfo& timingInfo,
@@ -187,6 +193,8 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
   int objCnt = -1;
   // We use the timeslice, so that we hook into the same interval as the rest of the
   // callback.
+  static bool isOnline = isOnlineRun(dtc);
+
   auto sid = _o2_signpost_id_t{(int64_t)timingInfo.timeslice};
   O2_SIGNPOST_START(ccdb, sid, "populateCacheWith", "Starting to populate cache with CCDB objects");
   for (auto& route : helper->routes) {
@@ -218,7 +226,9 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
     const auto url2uuid = helper->mapURL2UUID.find(path);
     if (url2uuid != helper->mapURL2UUID.end()) {
       etag = url2uuid->second.etag;
-      checkValidity = std::abs(int(timingInfo.tfCounter - url2uuid->second.lastCheckedTF)) >= chRate;
+      // We check validity every chRate timeslices or if the cache is expired
+      uint64_t validUntil = url2uuid->second.cacheValidUntil;
+      checkValidity = (std::abs(int(timingInfo.tfCounter - url2uuid->second.lastCheckedTF)) >= chRate) && (isOnline || (validUntil <= timestamp));
     } else {
       checkValidity = true; // never skip check if the cache is empty
     }
@@ -241,6 +251,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       helper->mapURL2UUID[path].lastCheckedTF = timingInfo.tfCounter;
       if (etag.empty()) {
         helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
+        helper->mapURL2UUID[path].cacheValidUntil = headers["Cache-Valid-Until"].empty() ? 0 : std::stoul(headers["Cache-Valid-Until"]);
         helper->mapURL2UUID[path].cacheMiss++;
         helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
@@ -252,6 +263,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       if (v.size()) { // but should be overridden by fresh object
         // somewhere here pruneFromCache should be called
         helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
+        helper->mapURL2UUID[path].cacheValidUntil = headers["Cache-Valid-Until"].empty() ? 0 : std::stoul(headers["Cache-Valid-Until"]);
         helper->mapURL2UUID[path].cacheMiss++;
         helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);

From 956b26710723080d559b30835eb15cadaba0d503 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 4 Mar 2024 11:32:56 +0100
Subject: [PATCH 394/726] DPL CCDB: handle the Cache-Valid-Until header

This will not recheck the etag and consider the cache valid until
the timestamp in milliseconds in the Cache-Valid-Until is not passed
by the data timestamp.
---
 Framework/CCDBSupport/src/CCDBHelpers.cxx | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/Framework/CCDBSupport/src/CCDBHelpers.cxx b/Framework/CCDBSupport/src/CCDBHelpers.cxx
index 837b20cf4262b..c72c6b4d0ff7f 100644
--- a/Framework/CCDBSupport/src/CCDBHelpers.cxx
+++ b/Framework/CCDBSupport/src/CCDBHelpers.cxx
@@ -33,6 +33,7 @@ namespace o2::framework
 struct CCDBFetcherHelper {
   struct CCDBCacheInfo {
     std::string etag;
+    size_t cacheValidUntil = 0;
     size_t cacheMiss = 0;
     size_t cacheHit = 0;
     size_t minSize = -1ULL;
@@ -176,6 +177,11 @@ auto getOrbitResetTime(o2::pmr::vector<char> const& v) -> Long64_t
   return (*ctp)[0];
 };
 
+bool isOnlineRun(DataTakingContext const& dtc)
+{
+  return dtc.deploymentMode == DeploymentMode::OnlineAUX || dtc.deploymentMode == DeploymentMode::OnlineDDS || dtc.deploymentMode == DeploymentMode::OnlineECS;
+}
+
 auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
                        int64_t timestamp,
                        TimingInfo& timingInfo,
@@ -186,6 +192,8 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
   int objCnt = -1;
   // We use the timeslice, so that we hook into the same interval as the rest of the
   // callback.
+  static bool isOnline = isOnlineRun(dtc);
+
   auto sid = _o2_signpost_id_t{(int64_t)timingInfo.timeslice};
   O2_SIGNPOST_START(ccdb, sid, "populateCacheWith", "Starting to populate cache with CCDB objects");
   for (auto& route : helper->routes) {
@@ -217,7 +225,9 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
     const auto url2uuid = helper->mapURL2UUID.find(path);
     if (url2uuid != helper->mapURL2UUID.end()) {
       etag = url2uuid->second.etag;
-      checkValidity = std::abs(int(timingInfo.tfCounter - url2uuid->second.lastCheckedTF)) >= chRate;
+      // We check validity every chRate timeslices or if the cache is expired
+      uint64_t validUntil = url2uuid->second.cacheValidUntil;
+      checkValidity = (std::abs(int(timingInfo.tfCounter - url2uuid->second.lastCheckedTF)) >= chRate) && (isOnline || (validUntil <= timestamp));
     } else {
       checkValidity = true; // never skip check if the cache is empty
     }
@@ -240,6 +250,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       helper->mapURL2UUID[path].lastCheckedTF = timingInfo.tfCounter;
       if (etag.empty()) {
         helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
+        helper->mapURL2UUID[path].cacheValidUntil = headers["Cache-Valid-Until"].empty() ? 0 : std::stoul(headers["Cache-Valid-Until"]);
         helper->mapURL2UUID[path].cacheMiss++;
         helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
@@ -251,6 +262,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       if (v.size()) { // but should be overridden by fresh object
         // somewhere here pruneFromCache should be called
         helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
+        helper->mapURL2UUID[path].cacheValidUntil = headers["Cache-Valid-Until"].empty() ? 0 : std::stoul(headers["Cache-Valid-Until"]);
         helper->mapURL2UUID[path].cacheMiss++;
         helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);

From db18c53c2659aa4d755bc41a7aeb392f27fedf35 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 5 Mar 2024 09:27:26 +0100
Subject: [PATCH 395/726] DPL CCDB: handle the Cache-Valid-Until header

This will not recheck the etag and consider the cache valid until
the timestamp in milliseconds in the Cache-Valid-Until is not passed
by the data timestamp.
---
 Framework/CCDBSupport/src/CCDBHelpers.cxx | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/Framework/CCDBSupport/src/CCDBHelpers.cxx b/Framework/CCDBSupport/src/CCDBHelpers.cxx
index 837b20cf4262b..f1d948e184e13 100644
--- a/Framework/CCDBSupport/src/CCDBHelpers.cxx
+++ b/Framework/CCDBSupport/src/CCDBHelpers.cxx
@@ -33,6 +33,8 @@ namespace o2::framework
 struct CCDBFetcherHelper {
   struct CCDBCacheInfo {
     std::string etag;
+    size_t cacheValidUntil = 0;
+    size_t cachePopulatedAt = 0;
     size_t cacheMiss = 0;
     size_t cacheHit = 0;
     size_t minSize = -1ULL;
@@ -176,6 +178,11 @@ auto getOrbitResetTime(o2::pmr::vector<char> const& v) -> Long64_t
   return (*ctp)[0];
 };
 
+bool isOnlineRun(DataTakingContext const& dtc)
+{
+  return dtc.deploymentMode == DeploymentMode::OnlineAUX || dtc.deploymentMode == DeploymentMode::OnlineDDS || dtc.deploymentMode == DeploymentMode::OnlineECS;
+}
+
 auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
                        int64_t timestamp,
                        TimingInfo& timingInfo,
@@ -186,6 +193,8 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
   int objCnt = -1;
   // We use the timeslice, so that we hook into the same interval as the rest of the
   // callback.
+  static bool isOnline = isOnlineRun(dtc);
+
   auto sid = _o2_signpost_id_t{(int64_t)timingInfo.timeslice};
   O2_SIGNPOST_START(ccdb, sid, "populateCacheWith", "Starting to populate cache with CCDB objects");
   for (auto& route : helper->routes) {
@@ -217,7 +226,14 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
     const auto url2uuid = helper->mapURL2UUID.find(path);
     if (url2uuid != helper->mapURL2UUID.end()) {
       etag = url2uuid->second.etag;
-      checkValidity = std::abs(int(timingInfo.tfCounter - url2uuid->second.lastCheckedTF)) >= chRate;
+      // We check validity every chRate timeslices or if the cache is expired
+      uint64_t validUntil = url2uuid->second.cacheValidUntil;
+      // When the cache was populated. If the cache was populated after the timestamp, we need to check validity.
+      uint64_t cachePopulatedAt = url2uuid->second.cachePopulatedAt;
+      // If timestamp is before the time the element was cached or after the claimed validity, we need to check validity, again
+      // when online.
+      bool cacheExipred = (validUntil <= timestamp) && (timestamp <= cachePopulatedAt);
+      checkValidity = (std::abs(int(timingInfo.tfCounter - url2uuid->second.lastCheckedTF)) >= chRate) && (isOnline || cacheExipred);
     } else {
       checkValidity = true; // never skip check if the cache is empty
     }
@@ -240,6 +256,8 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       helper->mapURL2UUID[path].lastCheckedTF = timingInfo.tfCounter;
       if (etag.empty()) {
         helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
+        helper->mapURL2UUID[path].cacheValidUntil = headers["Cache-Valid-Until"].empty() ? 0 : std::stoul(headers["Cache-Valid-Until"]);
+        helper->mapURL2UUID[path].cachePopulatedAt = timestamp;
         helper->mapURL2UUID[path].cacheMiss++;
         helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
@@ -251,6 +269,8 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       if (v.size()) { // but should be overridden by fresh object
         // somewhere here pruneFromCache should be called
         helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
+        helper->mapURL2UUID[path].cacheValidUntil = headers["Cache-Valid-Until"].empty() ? 0 : std::stoul(headers["Cache-Valid-Until"]);
+        helper->mapURL2UUID[path].cachePopulatedAt = timestamp;
         helper->mapURL2UUID[path].cacheMiss++;
         helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);

From db34e4303c9d6c28a2875a3472a19e4e2c57cf4c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 5 Mar 2024 09:35:26 +0100
Subject: [PATCH 396/726] DPL CCDB: handle the Cache-Valid-Until header

This will not recheck the etag and consider the cache valid until
the timestamp in milliseconds in the Cache-Valid-Until is not passed
by the data timestamp.
---
 Framework/CCDBSupport/src/CCDBHelpers.cxx | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/Framework/CCDBSupport/src/CCDBHelpers.cxx b/Framework/CCDBSupport/src/CCDBHelpers.cxx
index 837b20cf4262b..04dd0b13a7434 100644
--- a/Framework/CCDBSupport/src/CCDBHelpers.cxx
+++ b/Framework/CCDBSupport/src/CCDBHelpers.cxx
@@ -33,6 +33,8 @@ namespace o2::framework
 struct CCDBFetcherHelper {
   struct CCDBCacheInfo {
     std::string etag;
+    size_t cacheValidUntil = 0;
+    size_t cachePopulatedAt = 0;
     size_t cacheMiss = 0;
     size_t cacheHit = 0;
     size_t minSize = -1ULL;
@@ -176,6 +178,11 @@ auto getOrbitResetTime(o2::pmr::vector<char> const& v) -> Long64_t
   return (*ctp)[0];
 };
 
+bool isOnlineRun(DataTakingContext const& dtc)
+{
+  return dtc.deploymentMode == DeploymentMode::OnlineAUX || dtc.deploymentMode == DeploymentMode::OnlineDDS || dtc.deploymentMode == DeploymentMode::OnlineECS;
+}
+
 auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
                        int64_t timestamp,
                        TimingInfo& timingInfo,
@@ -186,6 +193,8 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
   int objCnt = -1;
   // We use the timeslice, so that we hook into the same interval as the rest of the
   // callback.
+  static bool isOnline = isOnlineRun(dtc);
+
   auto sid = _o2_signpost_id_t{(int64_t)timingInfo.timeslice};
   O2_SIGNPOST_START(ccdb, sid, "populateCacheWith", "Starting to populate cache with CCDB objects");
   for (auto& route : helper->routes) {
@@ -217,7 +226,14 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
     const auto url2uuid = helper->mapURL2UUID.find(path);
     if (url2uuid != helper->mapURL2UUID.end()) {
       etag = url2uuid->second.etag;
-      checkValidity = std::abs(int(timingInfo.tfCounter - url2uuid->second.lastCheckedTF)) >= chRate;
+      // We check validity every chRate timeslices or if the cache is expired
+      uint64_t validUntil = url2uuid->second.cacheValidUntil;
+      // When the cache was populated. If the cache was populated after the timestamp, we need to check validity.
+      uint64_t cachePopulatedAt = url2uuid->second.cachePopulatedAt;
+      // If timestamp is before the time the element was cached or after the claimed validity, we need to check validity, again
+      // when online.
+      bool cacheExipred = (validUntil <= timestamp) && (timestamp <= cachePopulatedAt);
+      checkValidity = (std::abs(int(timingInfo.tfCounter - url2uuid->second.lastCheckedTF)) >= chRate) && (isOnline || cacheExipred);
     } else {
       checkValidity = true; // never skip check if the cache is empty
     }
@@ -240,6 +256,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       helper->mapURL2UUID[path].lastCheckedTF = timingInfo.tfCounter;
       if (etag.empty()) {
         helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
+        helper->mapURL2UUID[path].cachePopulatedAt = timestamp;
         helper->mapURL2UUID[path].cacheMiss++;
         helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
@@ -251,6 +268,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       if (v.size()) { // but should be overridden by fresh object
         // somewhere here pruneFromCache should be called
         helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
+        helper->mapURL2UUID[path].cachePopulatedAt = timestamp;
         helper->mapURL2UUID[path].cacheMiss++;
         helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
@@ -260,6 +278,9 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
         // one could modify the    adoptContainer to take optional old cacheID to clean:
         // mapURL2DPLCache[URL] = ctx.outputs().adoptContainer(output, std::move(outputBuffer), DataAllocator::CacheStrategy::Always, mapURL2DPLCache[URL]);
         continue;
+      } else {
+        // Only once the etag is actually used, we get the information on how long the object is valid
+        helper->mapURL2UUID[path].cacheValidUntil = headers["Cache-Valid-Until"].empty() ? 0 : std::stoul(headers["Cache-Valid-Until"]);
       }
     }
     // cached object is fine

From 8ca9c1783c09c3cf2424475f26fd1d6d7284be2f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 5 Mar 2024 09:35:26 +0100
Subject: [PATCH 397/726] DPL CCDB: handle the Cache-Valid-Until header

This will not recheck the etag and consider the cache valid until
the timestamp in milliseconds in the Cache-Valid-Until is not passed
by the data timestamp.
---
 Framework/CCDBSupport/src/CCDBHelpers.cxx | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/Framework/CCDBSupport/src/CCDBHelpers.cxx b/Framework/CCDBSupport/src/CCDBHelpers.cxx
index 837b20cf4262b..37c4affe2877e 100644
--- a/Framework/CCDBSupport/src/CCDBHelpers.cxx
+++ b/Framework/CCDBSupport/src/CCDBHelpers.cxx
@@ -33,6 +33,8 @@ namespace o2::framework
 struct CCDBFetcherHelper {
   struct CCDBCacheInfo {
     std::string etag;
+    size_t cacheValidUntil = 0;
+    size_t cachePopulatedAt = 0;
     size_t cacheMiss = 0;
     size_t cacheHit = 0;
     size_t minSize = -1ULL;
@@ -176,6 +178,11 @@ auto getOrbitResetTime(o2::pmr::vector<char> const& v) -> Long64_t
   return (*ctp)[0];
 };
 
+bool isOnlineRun(DataTakingContext const& dtc)
+{
+  return dtc.deploymentMode == DeploymentMode::OnlineAUX || dtc.deploymentMode == DeploymentMode::OnlineDDS || dtc.deploymentMode == DeploymentMode::OnlineECS;
+}
+
 auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
                        int64_t timestamp,
                        TimingInfo& timingInfo,
@@ -186,6 +193,8 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
   int objCnt = -1;
   // We use the timeslice, so that we hook into the same interval as the rest of the
   // callback.
+  static bool isOnline = isOnlineRun(dtc);
+
   auto sid = _o2_signpost_id_t{(int64_t)timingInfo.timeslice};
   O2_SIGNPOST_START(ccdb, sid, "populateCacheWith", "Starting to populate cache with CCDB objects");
   for (auto& route : helper->routes) {
@@ -217,7 +226,14 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
     const auto url2uuid = helper->mapURL2UUID.find(path);
     if (url2uuid != helper->mapURL2UUID.end()) {
       etag = url2uuid->second.etag;
-      checkValidity = std::abs(int(timingInfo.tfCounter - url2uuid->second.lastCheckedTF)) >= chRate;
+      // We check validity every chRate timeslices or if the cache is expired
+      uint64_t validUntil = url2uuid->second.cacheValidUntil;
+      // When the cache was populated. If the cache was populated after the timestamp, we need to check validity.
+      uint64_t cachePopulatedAt = url2uuid->second.cachePopulatedAt;
+      // If timestamp is before the time the element was cached or after the claimed validity, we need to check validity, again
+      // when online.
+      bool cacheExpired = (validUntil <= timestamp) && (timestamp <= cachePopulatedAt);
+      checkValidity = (std::abs(int(timingInfo.tfCounter - url2uuid->second.lastCheckedTF)) >= chRate) && (isOnline || cacheExpired);
     } else {
       checkValidity = true; // never skip check if the cache is empty
     }
@@ -240,6 +256,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       helper->mapURL2UUID[path].lastCheckedTF = timingInfo.tfCounter;
       if (etag.empty()) {
         helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
+        helper->mapURL2UUID[path].cachePopulatedAt = timestamp;
         helper->mapURL2UUID[path].cacheMiss++;
         helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
@@ -251,6 +268,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
       if (v.size()) { // but should be overridden by fresh object
         // somewhere here pruneFromCache should be called
         helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
+        helper->mapURL2UUID[path].cachePopulatedAt = timestamp;
         helper->mapURL2UUID[path].cacheMiss++;
         helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
         helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
@@ -260,6 +278,9 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
         // one could modify the    adoptContainer to take optional old cacheID to clean:
         // mapURL2DPLCache[URL] = ctx.outputs().adoptContainer(output, std::move(outputBuffer), DataAllocator::CacheStrategy::Always, mapURL2DPLCache[URL]);
         continue;
+      } else {
+        // Only once the etag is actually used, we get the information on how long the object is valid
+        helper->mapURL2UUID[path].cacheValidUntil = headers["Cache-Valid-Until"].empty() ? 0 : std::stoul(headers["Cache-Valid-Until"]);
       }
     }
     // cached object is fine

From b12b5bbfca6f4f1cf126b940415b69b9de096f5a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 5 Mar 2024 10:48:10 +0100
Subject: [PATCH 398/726] DPL: add helper to get InitContext when using
 adaptStateful

Needed to have callbacks which can be executed in the global context.
---
 Framework/Core/include/Framework/AlgorithmSpec.h | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Framework/Core/include/Framework/AlgorithmSpec.h b/Framework/Core/include/Framework/AlgorithmSpec.h
index e98a3abefbce1..39d0582a0f1ee 100644
--- a/Framework/Core/include/Framework/AlgorithmSpec.h
+++ b/Framework/Core/include/Framework/AlgorithmSpec.h
@@ -164,6 +164,14 @@ struct ContextElementTraits<ProcessingContext> {
   }
 };
 
+template <>
+struct ContextElementTraits<InitContext> {
+  static InitContext& get(InitContext& ctx)
+  {
+    return ctx;
+  }
+};
+
 template <StringLiteral L, typename T>
 struct ContextElementTraits<Input<L, T> const> {
   static Input<L, T> get(ProcessingContext& ctx)

From b00c9d0e19028bfb92625a2c0a88fefcbced5b3c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 5 Mar 2024 10:46:27 +0100
Subject: [PATCH 399/726] [O2-4578] DPL: attempt at removing a race condition
 in the rate limiting

---
 Framework/Core/src/CommonDataProcessors.cxx | 48 +++++++++++++++------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 0cf7224c25ac8..423627ec4f7a7 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -568,29 +568,49 @@ DataProcessorSpec CommonDataProcessors::getGlobalFairMQSink(std::vector<InputSpe
   return specifyFairMQDeviceOutputProxy("internal-dpl-injected-output-proxy", danglingOutputInputs, defaultChannelConfig.c_str());
 }
 
+void retryMetricCallback(uv_async_t* async)
+{
+  static size_t lastTimeslice = -1;
+  auto* services = (ServiceRegistryRef*)async->data;
+  auto& timesliceIndex = services->get<TimesliceIndex>();
+  auto* device = services->get<RawDeviceService>().device();
+  auto channel = device->GetChannels().find("metric-feedback");
+  auto oldestPossingTimeslice = timesliceIndex.getOldestPossibleOutput().timeslice.value;
+  if (channel == device->GetChannels().end()) {
+    return;
+  }
+  fair::mq::MessagePtr payload(device->NewMessage());
+  payload->Rebuild(&oldestPossingTimeslice, sizeof(int64_t), nullptr, nullptr);
+  auto* consumed = (size_t*)malloc(sizeof(size_t));
+  *consumed = oldestPossingTimeslice;
+
+  int64_t result = channel->second[0].Send(payload, 100);
+  // If the sending worked, we do not retry.
+  if (result != 0) {
+    // If the sending did not work, we keep trying until it actually works.
+    // This will schedule other tasks in the queue, so the processing of the
+    // data will still happen.
+    uv_async_send(async);
+  } else {
+    lastTimeslice = *consumed;
+  }
+}
+
 DataProcessorSpec CommonDataProcessors::getDummySink(std::vector<InputSpec> const& danglingOutputInputs, std::string rateLimitingChannelConfig)
 {
   return DataProcessorSpec{
     .name = "internal-dpl-injected-dummy-sink",
     .inputs = danglingOutputInputs,
-    .algorithm = AlgorithmSpec{adaptStateful([](CallbackService& callbacks) {
+    .algorithm = AlgorithmSpec{adaptStateful([](CallbackService& callbacks, DeviceState& deviceState, InitContext& ic) {
+      static uv_async_t async;
+      // The callback will only have access to the
+      async.data = new ServiceRegistryRef{ic.services()};
+      uv_async_init(deviceState.loop, &async, retryMetricCallback);
       auto domainInfoUpdated = [](ServiceRegistryRef services, size_t timeslice, ChannelIndex channelIndex) {
         LOGP(debug, "Domain info updated with timeslice {}", timeslice);
-        static size_t lastTimeslice = -1;
+        retryMetricCallback(&async);
         auto& timesliceIndex = services.get<TimesliceIndex>();
-        auto device = services.get<RawDeviceService>().device();
-        auto channel = device->GetChannels().find("metric-feedback");
         auto oldestPossingTimeslice = timesliceIndex.getOldestPossibleOutput().timeslice.value;
-        if (channel != device->GetChannels().end()) {
-          fair::mq::MessagePtr payload(device->NewMessage());
-          size_t* consumed = (size_t*)malloc(sizeof(size_t));
-          *consumed = oldestPossingTimeslice;
-          if (*consumed != lastTimeslice) {
-            payload->Rebuild(consumed, sizeof(int64_t), nullptr, nullptr);
-            channel->second[0].Send(payload);
-            lastTimeslice = *consumed;
-          }
-        }
         auto& stats = services.get<DataProcessingStats>();
         stats.updateStats({(int)ProcessingStatsId::CONSUMED_TIMEFRAMES, DataProcessingStats::Op::Set, (int64_t)oldestPossingTimeslice});
       };

From a622a4836fa9378c3d6d90a43c72874cc76d6dbb Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 5 Mar 2024 14:01:11 +0100
Subject: [PATCH 400/726] DPL: do not spin when stdout / stderr from child is
 not connected

---
 Framework/Core/src/runDataProcessing.cxx | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 64c034ebb8533..ce1b2647b30af 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -422,11 +422,20 @@ void log_callback(uv_poll_t* handle, int status, int events)
   if (status < 0) {
     info.active = false;
   }
+  // We are in writeable mode. This means that something connected and
+  // therefore we can read from the child. Switch to readable mode.
+  if (events & UV_WRITABLE) {
+    uv_poll_start(handle, UV_READABLE | UV_DISCONNECT | UV_PRIORITIZED, log_callback);
+    return;
+  }
   if (events & UV_READABLE) {
     getChildData(logContext->fd, info);
   }
   if (events & UV_DISCONNECT) {
     info.active = false;
+    // This makes sure we do not get any more events until it reconnects (which
+    // does not happen in the current implementation).
+    uv_poll_start(handle, UV_WRITABLE, log_callback);
   }
 }
 
@@ -606,7 +615,10 @@ void handleChildrenStdio(uv_loop_t* loop,
       auto handle = handles.back();
       handle->data = context;
       uv_poll_init(loop, handle, fd);
-      uv_poll_start(handle, UV_READABLE, log_callback);
+      // We need to start with the poller in writeable mode so that we
+      // do not unnecessarily block get notified when the child is not yet
+      // connected.
+      uv_poll_start(handle, UV_WRITABLE, log_callback);
     };
 
     addPoller(i, childstdout[0]);

From 771983da98e48ca2c3173a26526c41fa37ff5a23 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 5 Mar 2024 14:01:11 +0100
Subject: [PATCH 401/726] DPL: improve signposts when processing messages

---
 Framework/Core/src/runDataProcessing.cxx | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index ce1b2647b30af..8409809bd87bb 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -838,7 +838,7 @@ void processChildrenOutput(DriverInfo& driverInfo,
   std::match_results<std::string_view::const_iterator> match;
   ParsedMetricMatch metricMatch;
   ParsedConfigMatch configMatch;
-  const std::string delimiter("\n");
+  constexpr auto delimiter = "\n";
 
   for (size_t di = 0, de = infos.size(); di < de; ++di) {
     DeviceInfo& info = infos[di];
@@ -851,7 +851,7 @@ void processChildrenOutput(DriverInfo& driverInfo,
     }
 
     O2_SIGNPOST_ID_FROM_POINTER(sid, driver, &info);
-    O2_SIGNPOST_START(driver, sid, "bytes_processed", "bytes processed by %{xcode:pid}d", info.pid);
+    O2_SIGNPOST_START(driver, sid, "bytes_processed", "Staring processing of logs from pid %{public}s (%{xcode:pid}d)", spec.name.data(), info.pid);
 
     std::string_view s = info.unprinted;
     size_t pos = 0;
@@ -894,12 +894,12 @@ void processChildrenOutput(DriverInfo& driverInfo,
           info.firstSevereError = token;
         }
       }
-      s.remove_prefix(pos + delimiter.length());
+      s.remove_prefix(pos + strlen(delimiter));
     }
     size_t oldSize = info.unprinted.size();
     info.unprinted = std::string(s);
     int64_t bytesProcessed = oldSize - info.unprinted.size();
-    O2_SIGNPOST_END(driver, sid, "bytes_processed", "bytes processed by %{xcode:network-size-in-bytes}" PRIi64, bytesProcessed);
+    O2_SIGNPOST_END(driver, sid, "bytes_processed", "Done processing %{xcode:network-size-in-bytes}" PRIi64 " bytes by %{public}s", bytesProcessed, spec.name.data());
   }
 }
 

From 02d80f8bff296e54964fd499911b4a97b5679a76 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 6 Mar 2024 08:52:44 +0100
Subject: [PATCH 402/726] DPL: improve processing of logs from devices

- Log processing now happens in an asynchronous action which is performed only
when some logs are actually read. All logs are read in any case, because
libuv will coalesce those callbacks into one.
- Improved signposts to print out what the driver event loop is doing.
- Improved signposts to debug the GUI with a separate stream.
---
 .../include/Framework/GuiCallbackContext.h    |  1 +
 Framework/Core/src/DriverServerContext.h      |  1 +
 Framework/Core/src/runDataProcessing.cxx      | 92 +++++++++++++++----
 3 files changed, 74 insertions(+), 20 deletions(-)

diff --git a/Framework/Core/include/Framework/GuiCallbackContext.h b/Framework/Core/include/Framework/GuiCallbackContext.h
index 9c0f21f3aec9c..1dbb6ec30e849 100644
--- a/Framework/Core/include/Framework/GuiCallbackContext.h
+++ b/Framework/Core/include/Framework/GuiCallbackContext.h
@@ -44,6 +44,7 @@ struct GuiCallbackContext {
   void* window = nullptr;
   bool* guiQuitRequested = nullptr;
   bool* allChildrenGone = nullptr;
+  bool* guiTimerExpired = nullptr;
   std::function<void(void)> callback;
   std::set<GuiRenderer*> renderers;
 };
diff --git a/Framework/Core/src/DriverServerContext.h b/Framework/Core/src/DriverServerContext.h
index c49ab460e4c6a..4d25c47bd172b 100644
--- a/Framework/Core/src/DriverServerContext.h
+++ b/Framework/Core/src/DriverServerContext.h
@@ -52,6 +52,7 @@ struct DriverServerContext {
   /// The handle to the server component of the
   /// driver.
   uv_tcp_t serverHandle;
+  uv_async_t* asyncLogProcessing = nullptr;
 };
 } // namespace o2::framework
 
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 64c034ebb8533..6b424ee6a4261 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -176,6 +176,7 @@ std::vector<DeviceMetricsInfo> gDeviceMetricsInfos;
 bpo::options_description gHiddenDeviceOptions("Hidden child options");
 
 O2_DECLARE_DYNAMIC_LOG(driver);
+O2_DECLARE_DYNAMIC_LOG(gui);
 
 void doBoostException(boost::exception& e, const char*);
 void doDPLException(o2::framework::RuntimeErrorRef& ref, char const*);
@@ -409,14 +410,14 @@ void spawnRemoteDevice(uv_loop_t* loop,
 struct DeviceLogContext {
   int fd;
   int index;
-  uv_loop_t* loop;
-  std::vector<DeviceInfo>* infos;
+  DriverServerContext* serverContext;
 };
 
 void log_callback(uv_poll_t* handle, int status, int events)
 {
+  O2_SIGNPOST_ID_FROM_POINTER(sid, driver, handle->loop);
   auto* logContext = reinterpret_cast<DeviceLogContext*>(handle->data);
-  std::vector<DeviceInfo>* infos = logContext->infos;
+  std::vector<DeviceInfo>* infos = logContext->serverContext->infos;
   DeviceInfo& info = infos->at(logContext->index);
 
   if (status < 0) {
@@ -428,16 +429,26 @@ void log_callback(uv_poll_t* handle, int status, int events)
   if (events & UV_DISCONNECT) {
     info.active = false;
   }
+  O2_SIGNPOST_EVENT_EMIT(driver, sid, "loop", "log_callback invoked by poller for device %{xcode:pid}d which is %{public}s%{public}s",
+                         info.pid, info.active ? "active" : "inactive",
+                         info.active ? " and still has data to read." : ".");
+  if (info.active == false) {
+    uv_poll_stop(handle);
+  }
+  uv_async_send(logContext->serverContext->asyncLogProcessing);
 }
 
 void close_websocket(uv_handle_t* handle)
 {
-  LOG(debug) << "Handle is being closed";
+  O2_SIGNPOST_ID_FROM_POINTER(sid, driver, handle->loop);
+  O2_SIGNPOST_EVENT_EMIT(driver, sid, "mainloop", "close_websocket");
   delete (WSDPLHandler*)handle->data;
 }
 
 void websocket_callback(uv_stream_t* stream, ssize_t nread, const uv_buf_t* buf)
 {
+  O2_SIGNPOST_ID_FROM_POINTER(sid, driver, stream->loop);
+  O2_SIGNPOST_EVENT_EMIT(driver, sid, "mainloop", "websocket_callback");
   auto* handler = (WSDPLHandler*)stream->data;
   if (nread == 0) {
     return;
@@ -481,6 +492,8 @@ static void my_alloc_cb(uv_handle_t*, size_t suggested_size, uv_buf_t* buf)
 /// A callback for the rest engine
 void ws_connect_callback(uv_stream_t* server, int status)
 {
+  O2_SIGNPOST_ID_FROM_POINTER(sid, driver, server->loop);
+  O2_SIGNPOST_EVENT_EMIT(driver, sid, "mainloop", "websocket_callback");
   auto* serverContext = reinterpret_cast<DriverServerContext*>(server->data);
   if (status < 0) {
     LOGF(error, "New connection error %s\n", uv_strerror(status));
@@ -574,9 +587,8 @@ void handleSignals()
   }
 }
 
-void handleChildrenStdio(uv_loop_t* loop,
+void handleChildrenStdio(DriverServerContext* serverContext,
                          std::string const& forwardedStdin,
-                         std::vector<DeviceInfo>& deviceInfos,
                          std::vector<DeviceStdioContext>& childFds,
                          std::vector<uv_poll_t*>& handles)
 {
@@ -586,7 +598,7 @@ void handleChildrenStdio(uv_loop_t* loop,
 
     auto* req = (uv_work_t*)malloc(sizeof(uv_work_t));
     req->data = new StreamConfigContext{forwardedStdin, childstdin[1]};
-    uv_queue_work(loop, req, stream_config, nullptr);
+    uv_queue_work(serverContext->loop, req, stream_config, nullptr);
 
     // Setting them to non-blocking to avoid haing the driver hang when
     // reading from child.
@@ -596,16 +608,15 @@ void handleChildrenStdio(uv_loop_t* loop,
     }
 
     /// Add pollers for stdout and stderr
-    auto addPoller = [&handles, &deviceInfos, &loop](int index, int fd) {
+    auto addPoller = [&handles, &serverContext](int index, int fd) {
       auto* context = new DeviceLogContext{};
       context->index = index;
       context->fd = fd;
-      context->loop = loop;
-      context->infos = &deviceInfos;
+      context->serverContext = serverContext;
       handles.push_back((uv_poll_t*)malloc(sizeof(uv_poll_t)));
       auto handle = handles.back();
       handle->data = context;
-      uv_poll_init(loop, handle, fd);
+      uv_poll_init(serverContext->loop, handle, fd);
       uv_poll_start(handle, UV_READABLE, log_callback);
     };
 
@@ -810,7 +821,8 @@ void spawnDevice(uv_loop_t* loop,
   gDeviceMetricsInfos.emplace_back(DeviceMetricsInfo{});
 }
 
-void processChildrenOutput(DriverInfo& driverInfo,
+void processChildrenOutput(uv_loop_t* loop,
+                           DriverInfo& driverInfo,
                            DeviceInfos& infos,
                            DeviceSpecs const& specs,
                            DeviceControls& controls)
@@ -826,8 +838,8 @@ void processChildrenOutput(DriverInfo& driverInfo,
   std::match_results<std::string_view::const_iterator> match;
   ParsedMetricMatch metricMatch;
   ParsedConfigMatch configMatch;
-  const std::string delimiter("\n");
 
+  int processed = 0;
   for (size_t di = 0, de = infos.size(); di < de; ++di) {
     DeviceInfo& info = infos[di];
     DeviceControl& control = controls[di];
@@ -837,6 +849,7 @@ void processChildrenOutput(DriverInfo& driverInfo,
     if (info.unprinted.empty()) {
       continue;
     }
+    processed++;
 
     O2_SIGNPOST_ID_FROM_POINTER(sid, driver, &info);
     O2_SIGNPOST_START(driver, sid, "bytes_processed", "bytes processed by %{xcode:pid}d", info.pid);
@@ -846,7 +859,7 @@ void processChildrenOutput(DriverInfo& driverInfo,
     info.history.resize(info.historySize);
     info.historyLevel.resize(info.historySize);
 
-    while ((pos = s.find(delimiter)) != std::string::npos) {
+    while ((pos = s.find("\n")) != std::string::npos) {
       std::string_view token{s.substr(0, pos)};
       auto logLevel = LogParsingHelpers::parseTokenLevel(token);
 
@@ -882,13 +895,18 @@ void processChildrenOutput(DriverInfo& driverInfo,
           info.firstSevereError = token;
         }
       }
-      s.remove_prefix(pos + delimiter.length());
+      // +1 is to skip the \n
+      s.remove_prefix(pos + 1);
     }
     size_t oldSize = info.unprinted.size();
     info.unprinted = std::string(s);
     int64_t bytesProcessed = oldSize - info.unprinted.size();
     O2_SIGNPOST_END(driver, sid, "bytes_processed", "bytes processed by %{xcode:network-size-in-bytes}" PRIi64, bytesProcessed);
   }
+  if (processed == 0) {
+    O2_SIGNPOST_ID_FROM_POINTER(lid, driver, loop);
+    O2_SIGNPOST_EVENT_EMIT(driver, lid, "mainloop", "processChildrenOutput invoked for nothing!");
+  }
 }
 
 // Process all the sigchld which are pending
@@ -1106,14 +1124,29 @@ void gui_callback(uv_timer_s* ctx)
 {
   auto* gui = reinterpret_cast<GuiCallbackContext*>(ctx->data);
   if (gui->plugin == nullptr) {
+    // The gui is not there. Why are we here?
+    O2_SIGNPOST_ID_FROM_POINTER(sid, driver, ctx->loop);
+    O2_SIGNPOST_EVENT_EMIT_ERROR(driver, sid, "gui", "GUI timer callback invoked without a GUI plugin.");
+    uv_timer_stop(ctx);
     return;
   }
+  *gui->guiTimerExpired = true;
+  static int counter = 0;
+  if ((counter++ % 6000) == 0) {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, driver, ctx->loop);
+    O2_SIGNPOST_EVENT_EMIT(driver, sid, "gui", "The GUI callback got called %d times.", counter);
+    *gui->guiTimerExpired = false;
+  }
+  // One interval per GUI invocation, using the loop as anchor.
+  O2_SIGNPOST_ID_FROM_POINTER(sid, gui, ctx->loop);
+  O2_SIGNPOST_START(gui, sid, "gui", "gui_callback");
 
   // New version which allows deferred closure of windows
   if (gui->plugin->supportsDeferredClose()) {
     // For now, there is nothing for which we want to defer the close
     // so if the flag is set, we simply exit
     if (*(gui->guiQuitRequested)) {
+      O2_SIGNPOST_END(gui, sid, "gui", "Quit requested by the GUI.");
       return;
     }
     void* draw_data = nullptr;
@@ -1123,6 +1156,7 @@ void gui_callback(uv_timer_s* ctx)
     // if less than 15ms have passed reuse old frame
     if (frameLatency / 1000000 <= 15) {
       draw_data = gui->lastFrame;
+      O2_SIGNPOST_END(gui, sid, "gui", "Reusing old frame.");
       return;
     }
     // The result of the pollGUIPreRender is used to determine if we
@@ -1148,6 +1182,7 @@ void gui_callback(uv_timer_s* ctx)
     if (frameLatency / 1000000 > 15) {
       if (!gui->plugin->pollGUIPreRender(gui->window, (float)frameLatency / 1000000000.0f)) {
         *(gui->guiQuitRequested) = true;
+        O2_SIGNPOST_END(gui, sid, "gui", "Reusing old frame.");
         return;
       }
       draw_data = gui->plugin->pollGUIRender(gui->callback);
@@ -1163,6 +1198,7 @@ void gui_callback(uv_timer_s* ctx)
       gui->frameLast = frameStart;
     }
   }
+  O2_SIGNPOST_END(gui, sid, "gui", "Gui redrawn.");
 }
 
 /// Force single stepping of the children
@@ -1429,12 +1465,14 @@ int runStateMachine(DataProcessorSpecs const& workflow,
   ServiceRegistryRef ref{serviceRegistry};
   ref.registerService(ServiceRegistryHelpers::handleForService<DevicesManager>(devicesManager));
 
+  bool guiTimerExpired = false;
   GuiCallbackContext guiContext;
   guiContext.plugin = debugGUI;
   guiContext.frameLast = uv_hrtime();
   guiContext.frameLatency = &driverInfo.frameLatency;
   guiContext.frameCost = &driverInfo.frameCost;
   guiContext.guiQuitRequested = &guiQuitRequested;
+  guiContext.guiTimerExpired = &guiTimerExpired;
 
   // This is to make sure we can process metrics, commands, configuration
   // changes coming from websocket (or even via any standard uv_stream_t, I guess).
@@ -1467,6 +1505,16 @@ int runStateMachine(DataProcessorSpecs const& workflow,
   metricDumpTimer.data = &serverContext;
   bool allChildrenGone = false;
   guiContext.allChildrenGone = &allChildrenGone;
+  O2_SIGNPOST_ID_FROM_POINTER(sid, driver, loop);
+  O2_SIGNPOST_START(driver, sid, "driver", "Starting driver loop");
+
+  // Async callback to process the output of the children, if needed.
+  serverContext.asyncLogProcessing = (uv_async_t*)malloc(sizeof(uv_async_t));
+  serverContext.asyncLogProcessing->data = &serverContext;
+  uv_async_init(loop, serverContext.asyncLogProcessing, [](uv_async_t* handle) {
+    auto* context = (DriverServerContext*)handle->data;
+    processChildrenOutput(context->loop, *context->driver, *context->infos, *context->specs, *context->controls);
+  });
 
   while (true) {
     // If control forced some transition on us, we push it to the queue.
@@ -2046,7 +2094,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
           }
         }
         handleSignals();
-        handleChildrenStdio(loop, forwardedStdin.str(), infos, childFds, pollHandles);
+        handleChildrenStdio(&serverContext, forwardedStdin.str(), childFds, pollHandles);
         for (auto& callback : postScheduleCallbacks) {
           callback(serviceRegistry, {varmap});
         }
@@ -2067,6 +2115,12 @@ int runStateMachine(DataProcessorSpecs const& workflow,
         // any, so that we do not consume CPU time when the driver is
         // idle.
         devicesManager->flush();
+        // We print the event loop for the gui only once every
+        // 6000 iterations (i.e. ~2 minutes). To avoid spamming, while still
+        // being able to see the event loop in case of a deadlock / systematic failure.
+        if (guiTimerExpired == false) {
+          O2_SIGNPOST_EVENT_EMIT(driver, sid, "mainloop", "Entering event loop with %{public}s", once ? "UV_RUN_ONCE" : "UV_RUN_NOWAIT");
+        }
         uv_run(loop, once ? UV_RUN_ONCE : UV_RUN_NOWAIT);
         once = true;
         // Calculate what we should do next and eventually
@@ -2099,9 +2153,6 @@ int runStateMachine(DataProcessorSpecs const& workflow,
         } else {
           driverInfo.states.push_back(DriverState::RUNNING);
         }
-        {
-          processChildrenOutput(driverInfo, infos, runningWorkflow.devices, controls);
-        }
         break;
       case DriverState::QUIT_REQUESTED:
         LOG(info) << "QUIT_REQUESTED";
@@ -2135,7 +2186,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
         }
         sigchld_requested = false;
         driverInfo.sigchldRequested = false;
-        processChildrenOutput(driverInfo, infos, runningWorkflow.devices, controls);
+        processChildrenOutput(loop, driverInfo, infos, runningWorkflow.devices, controls);
         hasError = processSigChild(infos, runningWorkflow.devices);
         allChildrenGone = areAllChildrenGone(infos);
         bool canExit = checkIfCanExit(infos);
@@ -2213,6 +2264,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
         driverInfo.states.push_back(DriverState::QUIT_REQUESTED);
     }
   }
+  O2_SIGNPOST_END(driver, sid, "driver", "End driver loop");
 }
 
 // Print help

From 37e4427c92e9dbc4bd3f6b6acefb9cd0570821e1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 6 Mar 2024 09:22:12 +0100
Subject: [PATCH 403/726] DPL TestWorkflows: add example for two synchronized
 timers

This reproduces the issue reported in https://its.cern.ch/jira/browse/O2-4328.
---
 Framework/TestWorkflows/CMakeLists.txt      |  4 ++
 Framework/TestWorkflows/src/o2TwoTimers.cxx | 61 +++++++++++++++++++++
 2 files changed, 65 insertions(+)
 create mode 100644 Framework/TestWorkflows/src/o2TwoTimers.cxx

diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index 848337200a4d2..8aaf01ff55dcf 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -33,6 +33,10 @@ o2_add_dpl_workflow(diamond-workflow
                   SOURCES src/o2DiamondWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
 
+o2_add_dpl_workflow(two-timers
+                  SOURCES src/o2TwoTimers.cxx
+                  COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(dummy-calibration-workflow
                   SOURCES src/o2DummyCalibrationWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
diff --git a/Framework/TestWorkflows/src/o2TwoTimers.cxx b/Framework/TestWorkflows/src/o2TwoTimers.cxx
new file mode 100644
index 0000000000000..b0bdc87378a83
--- /dev/null
+++ b/Framework/TestWorkflows/src/o2TwoTimers.cxx
@@ -0,0 +1,61 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/DataTakingContext.h"
+#include "Framework/CompletionPolicyHelpers.h"
+#include "Framework/DeviceSpec.h"
+#include "Framework/RawDeviceService.h"
+#include "Framework/ControlService.h"
+#include "Framework/Configurable.h"
+#include "Framework/RunningWorkflowInfo.h"
+#include "Framework/CallbackService.h"
+#include "Framework/RateLimiter.h"
+#include "Framework/ConfigContext.h"
+#include <fairmq/Device.h>
+
+#include <iostream>
+#include <chrono>
+#include <thread>
+#include <vector>
+
+using namespace o2::framework;
+#include "Framework/runDataProcessing.h"
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& specs)
+{
+  DataProcessorSpec timer1{
+    .name = "timer1",
+    .inputs = {InputSpec{"x", "TIM", "A1", Lifetime::Timer}},
+    .outputs = {OutputSpec{{"output"}, "TST", "A1"}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, RawDeviceService& device, DataTakingContext& context, ProcessingContext& pcx) {
+        auto& aData = outputs.make<int>(OutputRef{"output"}, 1);
+        LOG(info) << "timer1: " << aData[0];
+      })},
+    .options = {
+      ConfigParamSpec{"some-device-param", VariantType::Int, 1, {"Some device parameter"}},
+    }};
+  DataProcessorSpec timer2{
+    .name = "timer2",
+    .inputs = {InputSpec{"x", "TIM", "A1", Lifetime::Timer}},
+    .outputs = {OutputSpec{{"output"}, "TST", "A2"}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, RawDeviceService& device, DataTakingContext& context, ProcessingContext& pcx) {
+        auto& aData = outputs.make<int>(OutputRef{"output"}, 1);
+        LOG(info) << "timer2: " << aData[0];
+      })},
+    .options = {
+      ConfigParamSpec{"some-device-param", VariantType::Int, 1, {"Some device parameter"}},
+    }};
+
+  return workflow::concat(WorkflowSpec{timer1, timer2});
+}

From ccbf4ca6acbe77db50d281c1c50b1730ded4ec82 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 6 Mar 2024 14:04:31 +0100
Subject: [PATCH 404/726] DPL: correctly propagate timers with the same
 description

---
 Framework/Core/src/DeviceSpecHelpers.cxx | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 43ba0f2d250e5..514e6c91da54c 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -705,6 +705,12 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
         .channel = channel.name,
         .policy = forwardPolicyPtr,
       };
+      // In case we have a timer, the data it creates should be
+      // forwarded as a timeframe to the next device, so that
+      // we have synchronization.
+      if (route.matcher.lifetime == Lifetime::Timer) {
+        route.matcher.lifetime = Lifetime::Timeframe;
+      }
       device.forwards.emplace_back(route);
     }
   };
@@ -949,6 +955,13 @@ void DeviceSpecHelpers::processInEdgeActions(std::vector<DeviceSpec>& devices,
       }
     }
 
+    // In case we add a new route to the device, we remap any
+    // Lifetime::Timer to Lifetime::Timeframe, so that we can
+    // synchronize the devices without creating a new timer.
+    if (edge.isForward && route.matcher.lifetime == Lifetime::Timer) {
+      route.matcher.lifetime = Lifetime::Timeframe;
+    }
+
     consumerDevice.inputs.push_back(route);
   };
 

From e213a5c1330ccfb304b7996ed692e7fe07fc3815 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 6 Mar 2024 14:21:33 +0100
Subject: [PATCH 405/726] DPL: correctly propagate timers with the same
 description

---
 Framework/Core/src/DeviceSpecHelpers.cxx | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 43ba0f2d250e5..eb89b51193b54 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -705,6 +705,12 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
         .channel = channel.name,
         .policy = forwardPolicyPtr,
       };
+      // In case we have a timer, the data it creates should be
+      // forwarded as a timeframe to the next device, so that
+      // we have synchronization.
+      if (route.matcher.lifetime == Lifetime::Timer) {
+        route.matcher.lifetime = Lifetime::Timeframe;
+      }
       device.forwards.emplace_back(route);
     }
   };
@@ -923,6 +929,7 @@ void DeviceSpecHelpers::processInEdgeActions(std::vector<DeviceSpec>& devices,
   auto appendInputRouteToDestDeviceChannel = [&devices, &logicalEdges, &workflow](size_t ei, size_t di, size_t ci) {
     auto const& edge = logicalEdges[ei];
     auto const& consumer = workflow[edge.consumer];
+    auto const& producer = workflow[edge.producer];
     auto& consumerDevice = devices[di];
 
     auto const& inputSpec = consumer.inputs[edge.consumerInputIndex];
@@ -949,6 +956,19 @@ void DeviceSpecHelpers::processInEdgeActions(std::vector<DeviceSpec>& devices,
       }
     }
 
+    // In case we add a new route to the device, we remap any
+    // Lifetime::Timer to Lifetime::Timeframe, so that we can
+    // synchronize the devices without creating a new timer.
+    if (edge.isForward && route.matcher.lifetime == Lifetime::Timer) {
+      LOGP(warn, "Warning: Forwarding timer {} from {} to a {} as both requested it."
+                 " If this is undesired, please make sure to use two different data matchers for their InputSpec.", 
+          DataSpecUtils::describe(route.matcher).c_str(),
+          producer.name.c_str(),
+          consumer.name.c_str()
+          );
+      route.matcher.lifetime = Lifetime::Timeframe;
+    }
+
     consumerDevice.inputs.push_back(route);
   };
 

From 492e5c2262967f32d09ec731f3397314a8fbb9fe Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 7 Mar 2024 06:58:14 +0100
Subject: [PATCH 406/726] DPL: correctly propagate timers with the same
 description

---
 Framework/Core/src/DeviceSpecHelpers.cxx | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 43ba0f2d250e5..565f85e895f31 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -705,6 +705,12 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
         .channel = channel.name,
         .policy = forwardPolicyPtr,
       };
+      // In case we have a timer, the data it creates should be
+      // forwarded as a timeframe to the next device, so that
+      // we have synchronization.
+      if (route.matcher.lifetime == Lifetime::Timer) {
+        route.matcher.lifetime = Lifetime::Timeframe;
+      }
       device.forwards.emplace_back(route);
     }
   };
@@ -923,6 +929,7 @@ void DeviceSpecHelpers::processInEdgeActions(std::vector<DeviceSpec>& devices,
   auto appendInputRouteToDestDeviceChannel = [&devices, &logicalEdges, &workflow](size_t ei, size_t di, size_t ci) {
     auto const& edge = logicalEdges[ei];
     auto const& consumer = workflow[edge.consumer];
+    auto const& producer = workflow[edge.producer];
     auto& consumerDevice = devices[di];
 
     auto const& inputSpec = consumer.inputs[edge.consumerInputIndex];
@@ -949,6 +956,19 @@ void DeviceSpecHelpers::processInEdgeActions(std::vector<DeviceSpec>& devices,
       }
     }
 
+    // In case we add a new route to the device, we remap any
+    // Lifetime::Timer to Lifetime::Timeframe, so that we can
+    // synchronize the devices without creating a new timer.
+    if (edge.isForward && route.matcher.lifetime == Lifetime::Timer) {
+      LOGP(warn,
+           "Warning: Forwarding timer {} from {} to a {} as both requested it."
+           " If this is undesired, please make sure to use two different data matchers for their InputSpec.",
+           DataSpecUtils::describe(route.matcher).c_str(),
+           producer.name.c_str(),
+           consumer.name.c_str());
+      route.matcher.lifetime = Lifetime::Timeframe;
+    }
+
     consumerDevice.inputs.push_back(route);
   };
 

From ac86486ac80fe3e066cc6791192508d8ae700c17 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Mar 2024 09:30:37 +0100
Subject: [PATCH 407/726] O2Sim move to new ROOT serialization

---
 .../Base/include/DetectorsBase/Detector.h     | 17 +++------
 Detectors/Base/src/Detector.cxx               | 35 ++++++++++---------
 2 files changed, 22 insertions(+), 30 deletions(-)

diff --git a/Detectors/Base/include/DetectorsBase/Detector.h b/Detectors/Base/include/DetectorsBase/Detector.h
index 6acfa4f5cc46c..1432d93c53821 100644
--- a/Detectors/Base/include/DetectorsBase/Detector.h
+++ b/Detectors/Base/include/DetectorsBase/Detector.h
@@ -29,7 +29,6 @@
 #include <typeinfo>
 #include <type_traits>
 #include <string>
-#include <TMessage.h>
 #include "CommonUtils/ShmManager.h"
 #include "CommonUtils/ShmAllocator.h"
 #include <sys/shm.h>
@@ -42,9 +41,7 @@
 
 #include <fairmq/FwdDecls.h>
 
-namespace o2
-{
-namespace base
+namespace o2::base
 {
 
 /// This is the basic class for any AliceO2 detector module, whether it is
@@ -260,17 +257,12 @@ T decodeShmMessage(fair::mq::Parts& dataparts, int index, bool*& busy)
 }
 
 // this goes into the source
-void attachMessageBufferToParts(fair::mq::Parts& parts, fair::mq::Channel& channel,
-                                void* data, size_t size, void (*func_ptr)(void* data, void* hint), void* hint);
+void attachMessageBufferToParts(fair::mq::Parts& parts, fair::mq::Channel& channel, void* data, TClass* cl);
 
 template <typename Container>
 void attachTMessage(Container const& hits, fair::mq::Channel& channel, fair::mq::Parts& parts)
 {
-  TMessage* tmsg = new TMessage();
-  tmsg->WriteObjectAny((void*)&hits, TClass::GetClass(typeid(hits)));
-  attachMessageBufferToParts(
-    parts, channel, tmsg->Buffer(), tmsg->BufferSize(),
-    [](void* data, void* hint) { delete static_cast<TMessage*>(hint); }, tmsg);
+  attachMessageBufferToParts(parts, channel, (void*)&hits, TClass::GetClass(typeid(hits)));
 }
 
 void* decodeTMessageCore(fair::mq::Parts& dataparts, int index);
@@ -746,7 +738,6 @@ class DetImpl : public o2::base::Detector
 
   ClassDefOverride(DetImpl, 0);
 };
-} // namespace base
-} // namespace o2
+} // namespace o2::base
 
 #endif
diff --git a/Detectors/Base/src/Detector.cxx b/Detectors/Base/src/Detector.cxx
index 3168e0e84e1f2..f2b790ffccd5b 100644
--- a/Detectors/Base/src/Detector.cxx
+++ b/Detectors/Base/src/Detector.cxx
@@ -17,6 +17,7 @@
 #include "DetectorsBase/MaterialManager.h"
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "Field/MagneticField.h"
+#include "Framework/TMessageSerializer.h"
 #include "TString.h" // for TString
 #include "TGeoManager.h"
 
@@ -196,16 +197,19 @@ int Detector::registerSensitiveVolumeAndGetVolID(std::string const& name)
 #include <fairmq/Message.h>
 #include <fairmq/Parts.h>
 #include <fairmq/Channel.h>
-namespace o2
-{
-namespace base
+namespace o2::base
 {
 // this goes into the source
-void attachMessageBufferToParts(fair::mq::Parts& parts, fair::mq::Channel& channel, void* data, size_t size,
-                                void (*free_func)(void* data, void* hint), void* hint)
+void attachMessageBufferToParts(fair::mq::Parts& parts, fair::mq::Channel& channel, void* data, TClass* cl)
 {
-  std::unique_ptr<fair::mq::Message> message(channel.NewMessage(data, size, free_func, hint));
-  parts.AddPart(std::move(message));
+  auto msg = channel.Transport()->CreateMessage(4096, fair::mq::Alignment{64});
+  // This will serialize the data directly into the message buffer, without any further
+  // buffer or copying. Notice how the message will have 8 bytes of header and then
+  // the serialized data as TBufferFile. In principle one could construct a serialized TMessage payload
+  // however I did not manage to get it to work for every case.
+  o2::framework::FairOutputTBuffer buffer(*msg);
+  o2::framework::TMessageSerializer::serialize(buffer, data, cl);
+  parts.AddPart(std::move(msg));
 }
 void attachDetIDHeaderMessage(int id, fair::mq::Channel& channel, fair::mq::Parts& parts)
 {
@@ -246,17 +250,14 @@ void* decodeShmCore(fair::mq::Parts& dataparts, int index, bool*& busy)
 
 void* decodeTMessageCore(fair::mq::Parts& dataparts, int index)
 {
-  class TMessageWrapper : public TMessage
-  {
-   public:
-    TMessageWrapper(void* buf, Int_t len) : TMessage(buf, len) { ResetBit(kIsOwner); }
-    ~TMessageWrapper() override = default;
-  };
   auto rawmessage = std::move(dataparts.At(index));
-  auto message = std::make_unique<TMessageWrapper>(rawmessage->GetData(), rawmessage->GetSize());
-  return message.get()->ReadObjectAny(message.get()->GetClass());
+  o2::framework::FairInputTBuffer buffer((char*)rawmessage->GetData(), rawmessage->GetSize());
+  buffer.InitMap();
+  auto* cl = buffer.ReadClass();
+  buffer.SetBufferOffset(0);
+  buffer.ResetMap();
+  return buffer.ReadObjectAny(cl);
 }
 
-} // namespace base
-} // namespace o2
+} // namespace o2::base
 ClassImp(o2::base::Detector);

From 61774fd3332b690d086fdcb03377b5b88910d66e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Mar 2024 09:35:09 +0100
Subject: [PATCH 408/726] DPL: make sure O2_SIGNPOST_ID_FROM_POINTER allows for
 const pointers

---
 Framework/Foundation/include/Framework/Signpost.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index 7750687088a1d..9ea8aea1192a9 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -202,8 +202,6 @@ struct _o2_log_t {
 
 bool _o2_lock_free_stack_push(_o2_lock_free_stack& stack, const int& value, bool spin = false);
 bool _o2_lock_free_stack_pop(_o2_lock_free_stack& stack, int& value, bool spin = false);
-//_o2_signpost_id_t _o2_signpost_id_generate_local(_o2_log_t* log);
-//_o2_signpost_id_t _o2_signpost_id_make_with_pointer(_o2_log_t* log, void* pointer);
 void* _o2_log_create(char const* name, int stacktrace);
 void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...);
 void _o2_signpost_interval_begin(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, ...);
@@ -224,7 +222,7 @@ inline _o2_signpost_id_t _o2_signpost_id_generate_local(_o2_log_t* log)
 
 // Generate a unique id for a signpost. Do not use this directly, use O2_SIGNPOST_ID_FROM_POINTER instead.
 // Notice that this will fail for pointers to bytes as it might overlap with the id above.
-inline _o2_signpost_id_t _o2_signpost_id_make_with_pointer(_o2_log_t* log, void* pointer)
+inline _o2_signpost_id_t _o2_signpost_id_make_with_pointer(_o2_log_t* log, void const* pointer)
 {
   assert(((int64_t)pointer & 1) != 1);
   _o2_signpost_id_t uniqueId{(int64_t)pointer};

From cf3c2402f1e6e85d3128d300262a13d2e4e447fc Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Mar 2024 09:35:09 +0100
Subject: [PATCH 409/726] DPL: replace Tracy allocation profiler with Signposts

---
 Framework/Core/src/DataAllocator.cxx        |  7 ++++
 Framework/Core/src/DataProcessingDevice.cxx | 42 +++++++++++++++------
 Framework/Core/src/LifetimeHelpers.cxx      |  8 +++-
 3 files changed, 44 insertions(+), 13 deletions(-)

diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index de500cd6d6c4e..fe38283d5e2de 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -35,6 +35,7 @@
 #include <utility>
 
 O2_DECLARE_DYNAMIC_LOG(stream_context);
+O2_DECLARE_DYNAMIC_LOG(parts);
 
 namespace o2::framework
 {
@@ -129,6 +130,8 @@ void DataAllocator::addPartToContext(RouteIndex routeIndex, fair::mq::MessagePtr
                                      o2::header::SerializationMethod serializationMethod)
 {
   auto headerMessage = headerMessageFromOutput(spec, routeIndex, serializationMethod, 0);
+  O2_SIGNPOST_ID_FROM_POINTER(pid, parts, headerMessage->GetData());
+  O2_SIGNPOST_START(parts, pid, "parts", "addPartToContext %p", headerMessage->GetData());
 
   // FIXME: this is kind of ugly, we know that we can change the content of the
   // header message because we have just created it, but the API declares it const
@@ -150,6 +153,8 @@ void DataAllocator::adopt(const Output& spec, std::string* ptr)
   // the correct payload size is set later when sending the
   // StringContext, see DataProcessor::doSend
   auto header = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, 0);
+  O2_SIGNPOST_ID_FROM_POINTER(pid, parts, header->GetData());
+  O2_SIGNPOST_START(parts, pid, "parts", "addPartToContext %p", header->GetData());
   mRegistry.get<StringContext>().addString(std::move(header), std::move(payload), routeIndex);
   assert(payload.get() == nullptr);
 }
@@ -206,6 +211,8 @@ void DataAllocator::adopt(const Output& spec, LifetimeHolder<TableBuilder>& tb)
   auto& timingInfo = mRegistry.get<TimingInfo>();
   RouteIndex routeIndex = matchDataHeader(spec, timingInfo.timeslice);
   auto header = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodArrow, 0);
+  O2_SIGNPOST_ID_FROM_POINTER(pid, parts, header->GetData());
+  O2_SIGNPOST_START(parts, pid, "parts", "adopt %p", header->GetData());
   auto& context = mRegistry.get<ArrowContext>();
 
   auto creator = [transport = context.proxy().getOutputTransport(routeIndex)](size_t s) -> std::unique_ptr<fair::mq::Message> {
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 3e944b30ed11f..ca4ca71f50c2c 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -92,7 +92,10 @@ struct formatter<o2::framework::CompletionPolicy::CompletionOp> : ostream_format
 };
 } // namespace fmt
 
+// A log to use for general device logging
 O2_DECLARE_DYNAMIC_LOG(device);
+// Special log to keep track of the lifetime of the parts
+O2_DECLARE_DYNAMIC_LOG(parts);
 
 using namespace o2::framework;
 using ConfigurationInterface = o2::configuration::ConfigurationInterface;
@@ -1804,9 +1807,14 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
         LOGP(error, "DataHeader payloadSize mismatch");
         continue;
       }
-      TracyPlot("payload size", (int64_t)dh->payloadSize);
       auto dph = o2::header::get<DataProcessingHeader*>(headerData);
-      TracyAlloc(parts.At(pi + 1)->GetData(), parts.At(pi + 1)->GetSize());
+      // We only deal with the tracking of parts if the log is enabled.
+      // This is because in principle we should track the size of each of
+      // the parts and sum it up. Not for now.
+      if (O2_LOG_ENABLED(parts) == true) {
+        O2_SIGNPOST_ID_FROM_POINTER(pid, parts, headerData);
+        O2_SIGNPOST_START(parts, pid, "parts", "Processing DataHeader with splitPayloadParts %d and splitPayloadIndex %d", dh->splitPayloadParts, dh->splitPayloadIndex);
+      }
       if (!dph) {
         insertInputInfo(pi, 2, InputType::Invalid);
         LOGP(error, "Header stack does not contain DataProcessingHeader");
@@ -2180,10 +2188,12 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   // Function to cleanup record. For the moment we
   // simply use it to keep track of input messages
   // which are not needed, to display them in the GUI.
-#ifdef TRACY_ENABLE
   auto cleanupRecord = [](InputRecord& record) {
-    for (size_t ii = 0, ie = record.size(); ii < ie; ++ii) {
-      DataRef input = record.getByPos(ii);
+    if (O2_LOG_ENABLED(parts) == false) {
+      return;
+    }
+    for (size_t pi = 0, pe = record.size(); pi < pe; ++pi) {
+      DataRef input = record.getByPos(pi);
       if (input.header == nullptr) {
         continue;
       }
@@ -2196,10 +2206,21 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       if (!dh) {
         continue;
       }
-      TracyFree(input.payload);
+      // We use the address of the first header of a split payload
+      // to identify the interval.
+      O2_SIGNPOST_ID_FROM_POINTER(pid, parts, dh);
+      O2_SIGNPOST_END(parts, pid, "parts", "Cleaning up parts associated to %p", dh);
+
+      // No split parts, we simply skip the payload
+      if (dh->splitPayloadParts > 0 && dh->splitPayloadParts == dh->splitPayloadIndex) {
+        // this is indicating a sequence of payloads following the header
+        // FIXME: we will probably also set the DataHeader version
+        pi += dh->splitPayloadParts - 1;
+      } else {
+        size_t pi = pi + (dh->splitPayloadParts > 0 ? dh->splitPayloadParts : 1) * 2;
+      }
     }
   };
-#endif
 
   auto switchState = [ref](StreamingState newState) {
     auto& control = ref.get<ControlService>();
@@ -2432,6 +2453,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
     // We forward inputs only when we consume them. If we simply Process them,
     // we keep them for next message arriving.
     if (action.op == CompletionPolicy::CompletionOp::Consume) {
+      cleanupRecord(record);
       context.postDispatchingCallbacks(processContext);
       ref.get<CallbackService>().call<CallbackService::Id::DataConsumed>(o2::framework::ServiceRegistryRef{ref});
     }
@@ -2441,11 +2463,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       forwardInputs(ref, action.slot, currentSetOfInputs, timesliceIndex.getOldestPossibleOutput(), false, action.op == CompletionPolicy::CompletionOp::Consume);
     }
     context.postForwardingCallbacks(processContext);
-    if (action.op == CompletionPolicy::CompletionOp::Consume) {
-#ifdef TRACY_ENABLE
-      cleanupRecord(record);
-#endif
-    } else if (action.op == CompletionPolicy::CompletionOp::Process) {
+    if (action.op == CompletionPolicy::CompletionOp::Process) {
       cleanTimers(action.slot, record);
     }
   }
diff --git a/Framework/Core/src/LifetimeHelpers.cxx b/Framework/Core/src/LifetimeHelpers.cxx
index 1aa53fa0493ca..8c8605d41849c 100644
--- a/Framework/Core/src/LifetimeHelpers.cxx
+++ b/Framework/Core/src/LifetimeHelpers.cxx
@@ -25,6 +25,7 @@
 #include "Framework/FairMQDeviceProxy.h"
 #include "Framework/Formatters.h"
 #include "Framework/DeviceState.h"
+#include "Framework/Signpost.h"
 
 #include "Headers/DataHeader.h"
 #include "Headers/DataHeaderHelpers.h"
@@ -44,6 +45,8 @@
 using namespace o2::header;
 using namespace fair;
 
+O2_DECLARE_DYNAMIC_LOG(parts);
+
 namespace o2::framework
 {
 
@@ -411,7 +414,6 @@ ExpirationHandler::Handler LifetimeHelpers::enumerate(ConcreteDataMatcher const&
     assert(!ref.payload);
 
     auto timestamp = VariableContextHelpers::getTimeslice(variables).value;
-    LOGP(debug, "Enumerating record");
     DataHeader dh;
     dh.dataOrigin = matcher.origin;
     dh.dataDescription = matcher.description;
@@ -432,6 +434,8 @@ ExpirationHandler::Handler LifetimeHelpers::enumerate(ConcreteDataMatcher const&
     auto&& transport = deviceProxy.getInputChannel(channelIndex)->Transport();
     auto channelAlloc = o2::pmr::getTransportAllocator(transport);
     auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, dh, dph});
+    O2_SIGNPOST_ID_FROM_POINTER(hid, parts, header->GetData());
+    O2_SIGNPOST_START(parts, hid, "parts", "Enumerating part %p with timestamp %zu", header->GetData(), timestamp);
     ref.header = std::move(header);
 
     auto payload = transport->CreateMessage(sizeof(counter_t));
@@ -486,6 +490,8 @@ ExpirationHandler::Handler LifetimeHelpers::dummy(ConcreteDataMatcher const& mat
     auto channelAlloc = o2::pmr::getTransportAllocator(transport);
     auto header = o2::pmr::getMessage(o2::header::Stack{channelAlloc, dh, dph});
     ref.header = std::move(header);
+    O2_SIGNPOST_ID_FROM_POINTER(hid, parts, header->GetData());
+    O2_SIGNPOST_START(parts, hid, "parts", "Enumerating part %p with timestamp %zu", header->GetData(), timestamp);
     auto payload = transport->CreateMessage(0);
     ref.payload = std::move(payload);
   };

From afb25c5bf77078b0e7900692455033aee4f23d27 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Mar 2024 09:35:09 +0100
Subject: [PATCH 410/726] DPL: use signposts for OOB callback

---
 Framework/Core/src/DataProcessingDevice.cxx | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index ca4ca71f50c2c..ae7c19022e556 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -318,7 +318,7 @@ void on_socket_polled(uv_poll_t* poller, int status, int events)
       O2_SIGNPOST_END(device, sid, "socket_state", "Socket disconnected in context %{public}s", context->name);
     } break;
     case UV_PRIORITIZED: {
-      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "Data pending on socket for context %{public}s", context->name);
+      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "Socket prioritized for context %{public}s", context->name);
     } break;
   }
   // We do nothing, all the logic for now stays in DataProcessingDevice::doRun()
@@ -326,6 +326,7 @@ void on_socket_polled(uv_poll_t* poller, int status, int events)
 
 void on_out_of_band_polled(uv_poll_t* poller, int status, int events)
 {
+  O2_SIGNPOST_ID_FROM_POINTER(sid, device, poller);
   auto* context = (PollerContext*)poller->data;
   context->state->loopReason |= DeviceState::OOB_ACTIVITY;
   if (status < 0) {
@@ -334,32 +335,27 @@ void on_out_of_band_polled(uv_poll_t* poller, int status, int events)
   }
   switch (events) {
     case UV_READABLE: {
-      ZoneScopedN("socket readable event");
+      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "Data pending on socket for channel %{public}s", context->name);
       context->state->loopReason |= DeviceState::DATA_INCOMING;
       assert(context->channelInfo);
-      LOGP(debug, "oob socket {} polled UV_READABLE.",
-           context->name,
-           context->channelInfo->hasPendingEvents);
       context->channelInfo->readPolled = true;
     } break;
     case UV_WRITABLE: {
-      ZoneScopedN("socket writeable");
+      O2_SIGNPOST_END(device, sid, "socket_state", "OOB socket connected for channel %{public}s", context->name);
       if (context->read) {
-        LOG(debug) << "socket polled UV_CONNECT" << context->name;
+        O2_SIGNPOST_START(device, sid, "socket_state", "OOB socket connected for read in context %{public}s", context->name);
         uv_poll_start(poller, UV_READABLE | UV_DISCONNECT | UV_PRIORITIZED, &on_out_of_band_polled);
       } else {
-        LOG(debug) << "socket polled UV_WRITABLE" << context->name;
+        O2_SIGNPOST_START(device, sid, "socket_state", "OOB socket connected for write for channel %{public}s", context->name);
         context->state->loopReason |= DeviceState::DATA_OUTGOING;
       }
     } break;
     case UV_DISCONNECT: {
-      ZoneScopedN("socket disconnect");
-      LOG(debug) << "socket polled UV_DISCONNECT";
+      O2_SIGNPOST_END(device, sid, "socket_state", "OOB socket disconnected in context %{public}s", context->name);
       uv_poll_start(poller, UV_WRITABLE, &on_out_of_band_polled);
     } break;
     case UV_PRIORITIZED: {
-      ZoneScopedN("socket prioritized");
-      LOG(debug) << "socket polled UV_PRIORITIZED";
+      O2_SIGNPOST_EVENT_EMIT(device, sid, "socket_state", "OOB socket prioritized for context %{public}s", context->name);
     } break;
   }
   // We do nothing, all the logic for now stays in DataProcessingDevice::doRun()

From 7175a2746927054ef2904398967e602111b9ffd0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Mar 2024 09:35:09 +0100
Subject: [PATCH 411/726] DPL: rename TracyLockable to O2_LOCKABLE

---
 .../Core/include/Framework/DataRelayer.h      |  2 +-
 Framework/Core/include/Framework/DataSender.h |  2 +-
 Framework/Core/src/DataRelayer.cxx            | 32 +++++++++----------
 Framework/Core/src/DataSender.cxx             |  2 +-
 Framework/Core/src/ServiceRegistry.cxx        |  6 ++--
 .../Foundation/include/Framework/Tracing.h    |  4 +--
 6 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/Framework/Core/include/Framework/DataRelayer.h b/Framework/Core/include/Framework/DataRelayer.h
index 8fdedbdd1119e..1ebcf549d6a29 100644
--- a/Framework/Core/include/Framework/DataRelayer.h
+++ b/Framework/Core/include/Framework/DataRelayer.h
@@ -194,7 +194,7 @@ class DataRelayer
   std::vector<PruneOp> mPruneOps;
   size_t mMaxLanes;
 
-  TracyLockableN(std::recursive_mutex, mMutex, "data relayer mutex");
+  O2_LOCKABLE_NAMED(std::recursive_mutex, mMutex, "data relayer mutex");
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/include/Framework/DataSender.h b/Framework/Core/include/Framework/DataSender.h
index 7cdc0ea537bea..2937bd80f78f8 100644
--- a/Framework/Core/include/Framework/DataSender.h
+++ b/Framework/Core/include/Framework/DataSender.h
@@ -61,7 +61,7 @@ class DataSender
   std::vector<bool> mPresent;
   std::vector<bool> mPresentDefaults;
 
-  TracyLockableN(std::recursive_mutex, mMutex, "data relayer mutex");
+  O2_LOCKABLE_NAMED(std::recursive_mutex, mMutex, "data relayer mutex");
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index 0b459dd155a21..1daf4bbd2a20b 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -70,7 +70,7 @@ DataRelayer::DataRelayer(const CompletionPolicy& policy,
     mInputMatchers{DataRelayerHelpers::createInputMatchers(routes)},
     mMaxLanes{InputRouteHelpers::maxLanes(routes)}
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   if (policy.configureRelayer == nullptr) {
     static int pipelineLength = DefaultsHelpers::pipelineLength();
@@ -100,7 +100,7 @@ DataRelayer::DataRelayer(const CompletionPolicy& policy,
 
 TimesliceId DataRelayer::getTimesliceForSlot(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   auto& variables = mTimesliceIndex.getVariablesForSlot(slot);
   return VariableContextHelpers::getTimeslice(variables);
 }
@@ -109,7 +109,7 @@ DataRelayer::ActivityStats DataRelayer::processDanglingInputs(std::vector<Expira
                                                               ServiceRegistryRef services, bool createNew)
 {
   LOGP(debug, "DataRelayer::processDanglingInputs");
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   auto& deviceProxy = services.get<FairMQDeviceProxy>();
 
   ActivityStats activity;
@@ -394,7 +394,7 @@ DataRelayer::RelayChoice
                      size_t nPayloads,
                      std::function<void(TimesliceSlot, std::vector<MessageSet>&, TimesliceIndex::OldestOutputInfo)> onDrop)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   DataProcessingHeader const* dph = o2::header::get<DataProcessingHeader*>(rawHeader);
   // IMPLEMENTATION DETAILS
   //
@@ -616,7 +616,7 @@ DataRelayer::RelayChoice
 void DataRelayer::getReadyToProcess(std::vector<DataRelayer::RecordAction>& completed)
 {
   LOGP(debug, "DataRelayer::getReadyToProcess");
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   // THE STATE
   const auto& cache = mCache;
@@ -749,7 +749,7 @@ void DataRelayer::getReadyToProcess(std::vector<DataRelayer::RecordAction>& comp
 
 void DataRelayer::updateCacheStatus(TimesliceSlot slot, CacheEntryStatus oldStatus, CacheEntryStatus newStatus)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   const auto numInputTypes = mDistinctRoutesIndex.size();
 
   auto markInputDone = [&cachedStateMetrics = mCachedStateMetrics,
@@ -767,7 +767,7 @@ void DataRelayer::updateCacheStatus(TimesliceSlot slot, CacheEntryStatus oldStat
 
 std::vector<o2::framework::MessageSet> DataRelayer::consumeAllInputsForTimeslice(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   const auto numInputTypes = mDistinctRoutesIndex.size();
   // State of the computation
@@ -821,7 +821,7 @@ std::vector<o2::framework::MessageSet> DataRelayer::consumeAllInputsForTimeslice
 
 std::vector<o2::framework::MessageSet> DataRelayer::consumeExistingInputsForTimeslice(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   const auto numInputTypes = mDistinctRoutesIndex.size();
   // State of the computation
@@ -864,7 +864,7 @@ std::vector<o2::framework::MessageSet> DataRelayer::consumeExistingInputsForTime
 
 void DataRelayer::clear()
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   for (auto& cache : mCache) {
     cache.clear();
@@ -886,7 +886,7 @@ size_t
 /// the time pipelining.
 void DataRelayer::setPipelineLength(size_t s)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   mTimesliceIndex.resize(s);
   mVariableContextes.resize(s);
@@ -895,7 +895,7 @@ void DataRelayer::setPipelineLength(size_t s)
 
 void DataRelayer::publishMetrics()
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   auto numInputTypes = mDistinctRoutesIndex.size();
   // FIXME: many of the DataRelayer function rely on allocated cache, so its
@@ -932,31 +932,31 @@ void DataRelayer::publishMetrics()
 
 uint32_t DataRelayer::getFirstTFOrbitForSlot(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   return VariableContextHelpers::getFirstTFOrbit(mTimesliceIndex.getVariablesForSlot(slot));
 }
 
 uint32_t DataRelayer::getFirstTFCounterForSlot(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   return VariableContextHelpers::getFirstTFCounter(mTimesliceIndex.getVariablesForSlot(slot));
 }
 
 uint32_t DataRelayer::getRunNumberForSlot(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   return VariableContextHelpers::getRunNumber(mTimesliceIndex.getVariablesForSlot(slot));
 }
 
 uint64_t DataRelayer::getCreationTimeForSlot(TimesliceSlot slot)
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   return VariableContextHelpers::getCreationTime(mTimesliceIndex.getVariablesForSlot(slot));
 }
 
 void DataRelayer::sendContextState()
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
   auto& states = mContext.get<DataProcessingStates>();
   for (size_t ci = 0; ci < mTimesliceIndex.size(); ++ci) {
     auto slot = TimesliceSlot{ci};
diff --git a/Framework/Core/src/DataSender.cxx b/Framework/Core/src/DataSender.cxx
index f0753102e2d40..3acce1c4b9704 100644
--- a/Framework/Core/src/DataSender.cxx
+++ b/Framework/Core/src/DataSender.cxx
@@ -51,7 +51,7 @@ DataSender::DataSender(ServiceRegistryRef registry)
     mSpec{registry.get<DeviceSpec const>()},
     mDistinctRoutesIndex{createDistinctOutputRouteIndex(mSpec.outputs)}
 {
-  std::scoped_lock<LockableBase(std::recursive_mutex)> lock(mMutex);
+  std::scoped_lock<O2_LOCKABLE(std::recursive_mutex)> lock(mMutex);
 
   auto numInputTypes = mDistinctRoutesIndex.size();
   auto& routes = mSpec.outputs;
diff --git a/Framework/Core/src/ServiceRegistry.cxx b/Framework/Core/src/ServiceRegistry.cxx
index 7b77c521fe6a7..09921f6b4a7d4 100644
--- a/Framework/Core/src/ServiceRegistry.cxx
+++ b/Framework/Core/src/ServiceRegistry.cxx
@@ -271,7 +271,7 @@ void ServiceRegistry::postRenderGUICallbacks(ServiceRegistryRef ref)
 
 void ServiceRegistry::bindService(ServiceRegistry::Salt salt, ServiceSpec const& spec, void* service) const
 {
-  static TracyLockableN(std::mutex, bindMutex, "bind mutex");
+  static O2_LOCKABLE_NAMED(std::mutex, bindMutex, "bind mutex");
   // Stream services need to store their callbacks in the stream context.
   // This is to make sure we invoke the correct callback only once per
   // stream, since they could bind multiple times.
@@ -280,12 +280,12 @@ void ServiceRegistry::bindService(ServiceRegistry::Salt salt, ServiceSpec const&
   if (spec.kind == ServiceKind::Stream) {
     ServiceRegistryRef ref{const_cast<ServiceRegistry&>(*this), salt};
     auto& streamContext = ref.get<StreamContext>();
-    std::scoped_lock<LockableBase(std::mutex)> lock(bindMutex);
+    std::scoped_lock<O2_LOCKABLE(std::mutex)> lock(bindMutex);
     auto& dataProcessorContext = ref.get<DataProcessorContext>();
     ContextHelpers::bindStreamService(dataProcessorContext, streamContext, spec, service);
   } else {
     ServiceRegistryRef ref{const_cast<ServiceRegistry&>(*this), salt};
-    std::scoped_lock<LockableBase(std::mutex)> lock(bindMutex);
+    std::scoped_lock<O2_LOCKABLE(std::mutex)> lock(bindMutex);
     if (ref.active<DataProcessorContext>()) {
       auto& dataProcessorContext = ref.get<DataProcessorContext>();
       ContextHelpers::bindProcessorService(dataProcessorContext, spec, service);
diff --git a/Framework/Foundation/include/Framework/Tracing.h b/Framework/Foundation/include/Framework/Tracing.h
index f5bcecd3889d7..6c9c65f4423e5 100644
--- a/Framework/Foundation/include/Framework/Tracing.h
+++ b/Framework/Foundation/include/Framework/Tracing.h
@@ -39,8 +39,8 @@
 #define TracyAppInfo(...) \
   while (false) {         \
   }
-#define TracyLockableN(T, V, N) T V
-#define LockableBase(T) T
+#define O2_LOCKABLE_NAMED(T, V, N) T V
+#define O2_LOCKABLE(T) T
 #endif
 
 #endif // O2_FRAMEWORK_TRACING_H_

From 178e632ca967462a43420e470d4631f65b4c538f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Mar 2024 09:35:09 +0100
Subject: [PATCH 412/726] DPL: do not use Tracy anymore

Tracy usage never took off, so I am removing all the explicit calls to it and
I will replace them with Signpost usage, where it still makes sense.
---
 .../Core/src/CommonMessageBackendsHelpers.h   |  1 -
 Framework/Core/src/CommonServices.cxx         |  1 -
 Framework/Core/src/DataProcessingDevice.cxx   | 29 -------------------
 3 files changed, 31 deletions(-)

diff --git a/Framework/Core/src/CommonMessageBackendsHelpers.h b/Framework/Core/src/CommonMessageBackendsHelpers.h
index 10d8a9c5d347a..a3684906ff838 100644
--- a/Framework/Core/src/CommonMessageBackendsHelpers.h
+++ b/Framework/Core/src/CommonMessageBackendsHelpers.h
@@ -38,7 +38,6 @@ struct CommonMessageBackendsHelpers {
   static ServiceProcessingCallback sendCallback()
   {
     return [](ProcessingContext& ctx, void* service) {
-      ZoneScopedN("send message callback");
       T* context = reinterpret_cast<T*>(service);
       DataProcessor::doSend(ctx.services().get<DataSender>(), *context, ctx.services());
     };
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index a0f7fcdc88201..0d0828729ef42 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -741,7 +741,6 @@ auto sendRelayerMetrics(ServiceRegistryRef registry, DataProcessingStats& stats)
     stats.updateStats({static_cast<unsigned short>(static_cast<int>(ProcessingStatsId::AVAILABLE_MANAGED_SHM_BASE) + (runningWorkflow.shmSegmentId % 512)), DataProcessingStats::Op::SetIfPositive, freeMemory});
   }
 
-  ZoneScopedN("send metrics");
   auto device = registry.get<RawDeviceService>().device();
 
   int64_t totalBytesIn = 0;
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index ae7c19022e556..649aba2d6b502 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -115,14 +115,12 @@ struct ServiceKindExtractor<ConfigurationInterface> {
 /// Watching stdin for commands probably a better approach.
 void on_idle_timer(uv_timer_t* handle)
 {
-  ZoneScopedN("Idle timer");
   auto* state = (DeviceState*)handle->data;
   state->loopReason |= DeviceState::TIMER_EXPIRED;
 }
 
 void on_transition_requested_expired(uv_timer_t* handle)
 {
-  ZoneScopedN("Transition expired");
   auto* state = (DeviceState*)handle->data;
   state->loopReason |= DeviceState::TIMER_EXPIRED;
   LOGP(info, "Timer expired. Forcing transition to READY");
@@ -222,12 +220,10 @@ DataProcessingDevice::DataProcessingDevice(RunningDeviceRef running, ServiceRegi
 // one with the thread id. For the moment we simply use the first one.
 void run_callback(uv_work_t* handle)
 {
-  ZoneScopedN("run_callback");
   auto* task = (TaskStreamInfo*)handle->data;
   auto ref = ServiceRegistryRef{*task->registry, ServiceRegistry::globalStreamSalt(task->id.index + 1)};
   DataProcessingDevice::doPrepare(ref);
   DataProcessingDevice::doRun(ref);
-  //  FrameMark;
 }
 
 // Once the processing in a thread is done, this is executed on the main thread.
@@ -268,7 +264,6 @@ void run_completion(uv_work_t* handle, int status)
   quotaEvaluator.handleExpired(reportExpiredOffer);
   quotaEvaluator.dispose(task->id.index);
   task->running = false;
-  ZoneScopedN("run_completion");
 }
 
 // Context for polling
@@ -377,8 +372,6 @@ void DataProcessingDevice::Init()
   context.statefulProcess = nullptr;
   context.error = spec.algorithm.onError;
   context.initError = spec.algorithm.onInitError;
-  TracyAppInfo(spec.name.data(), spec.name.size());
-  ZoneScopedN("DataProcessingDevice::Init");
 
   auto configStore = DeviceConfigurationHelpers::getConfiguration(mServiceRegistry, spec.name.c_str(), spec.options);
   if (configStore == nullptr) {
@@ -412,7 +405,6 @@ void DataProcessingDevice::Init()
   if (context.initError) {
     context.initErrorHandling = [&errorCallback = context.initError,
                                  &serviceRegistry = mServiceRegistry](RuntimeErrorRef e) {
-      ZoneScopedN("Error handling");
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
       ServiceRegistryRef ref{serviceRegistry, ServiceRegistry::globalDeviceSalt()};
@@ -426,7 +418,6 @@ void DataProcessingDevice::Init()
     };
   } else {
     context.initErrorHandling = [&serviceRegistry = mServiceRegistry](RuntimeErrorRef e) {
-      ZoneScopedN("Error handling");
       auto& err = error_from_ref(e);
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
@@ -449,7 +440,6 @@ void DataProcessingDevice::Init()
       try {
         context.statefulProcess = context.init(initContext);
       } catch (o2::framework::RuntimeErrorRef e) {
-        ZoneScopedN("error handling");
         if (context.initErrorHandling) {
           (context.initErrorHandling)(e);
         }
@@ -458,14 +448,12 @@ void DataProcessingDevice::Init()
       try {
         context.statefulProcess = context.init(initContext);
       } catch (std::exception& ex) {
-        ZoneScopedN("error handling");
         /// Convert a standard exception to a RuntimeErrorRef
         /// Notice how this will lose the backtrace information
         /// and report the exception coming from here.
         auto e = runtime_error(ex.what());
         (context.initErrorHandling)(e);
       } catch (o2::framework::RuntimeErrorRef e) {
-        ZoneScopedN("error handling");
         (context.initErrorHandling)(e);
       }
     }
@@ -621,7 +609,6 @@ static auto toBeforwardedMessageSet = [](std::vector<ChannelIndex>& cachedForwar
 // FIXME: do it in a smarter way than O(N^2)
 static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot, std::vector<MessageSet>& currentSetOfInputs,
                                TimesliceIndex::OldestOutputInfo oldestTimeslice, bool copy, bool consume = true) {
-  ZoneScopedN("forward inputs");
   auto& proxy = registry.get<FairMQDeviceProxy>();
   // we collect all messages per forward in a map and send them together
   std::vector<fair::mq::Parts> forwardedParts;
@@ -1078,7 +1065,6 @@ void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceCont
   if (context.error != nullptr) {
     context.errorHandling = [&errorCallback = context.error,
                              &serviceRegistry = mServiceRegistry](RuntimeErrorRef e, InputRecord& record) {
-      ZoneScopedN("Error handling");
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
       ServiceRegistryRef ref{serviceRegistry, ServiceRegistry::globalDeviceSalt()};
@@ -1093,7 +1079,6 @@ void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceCont
   } else {
     context.errorHandling = [&errorPolicy = mProcessingPolicies.error,
                              &serviceRegistry = mServiceRegistry](RuntimeErrorRef e, InputRecord& record) {
-      ZoneScopedN("Error handling");
       auto& err = error_from_ref(e);
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
@@ -1242,8 +1227,6 @@ void DataProcessingDevice::Run()
     // so that devices which do not have a timer can still start an
     // enumeration.
     {
-      ZoneScopedN("uv idle");
-      TracyPlot("past activity", (int64_t)mWasActive);
       ServiceRegistryRef ref{mServiceRegistry};
       ref.get<DriverClient>().flushPending(mServiceRegistry);
       auto shouldNotWait = (mWasActive &&
@@ -1303,7 +1286,6 @@ void DataProcessingDevice::Run()
       if (state.transitionHandling == TransitionHandlingState::Requested && state.streaming == StreamingState::Idle) {
         state.transitionHandling = TransitionHandlingState::Expired;
       }
-      TracyPlot("shouldNotWait", (int)shouldNotWait);
       if (state.severityStack.empty() == false) {
         fair::Logger::SetConsoleSeverity((fair::Severity)state.severityStack.back());
         state.severityStack.pop_back();
@@ -1349,7 +1331,6 @@ void DataProcessingDevice::Run()
         fair::Logger::SetConsoleSeverity((fair::Severity)state.severityStack.back());
         state.severityStack.pop_back();
       }
-      TracyPlot("loopReason", (int64_t)(uint64_t)state.loopReason);
       LOGP(debug, "Loop reason mask {:b} & {:b} = {:b}",
            state.loopReason, state.tracingFlags,
            state.loopReason & state.tracingFlags);
@@ -1433,7 +1414,6 @@ void DataProcessingDevice::Run()
     } else {
       mWasActive = false;
     }
-    FrameMark;
   }
   auto& spec = ref.get<DeviceSpec const>();
   /// Cleanup messages which are still pending on exit.
@@ -1448,12 +1428,10 @@ void DataProcessingDevice::Run()
 /// non-data triggers like those which are time based.
 void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
 {
-  ZoneScopedN("DataProcessingDevice::doPrepare");
   auto& context = ref.get<DataProcessorContext>();
 
   *context.wasActive = false;
   {
-    ZoneScopedN("CallbackService::Id::ClockTick");
     ref.get<CallbackService>().call<CallbackService::Id::ClockTick>();
   }
   // Whether or not we had something to do.
@@ -1734,7 +1712,6 @@ struct WaitBackpressurePolicy {
 void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo& info)
 {
   auto& context = ref.get<DataProcessorContext>();
-  ZoneScopedN("DataProcessingDevice::handleData");
 
   enum struct InputType : int {
     Invalid = 0,
@@ -1763,7 +1740,6 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
     auto& parts = info.parts;
     stats.updateStats({(int)ProcessingStatsId::TOTAL_INPUTS, DataProcessingStats::Op::Set, (int64_t)parts.Size()});
 
-    TracyPlot("messages received", (int64_t)parts.Size());
     std::vector<InputInfo> results;
     // we can reserve the upper limit
     results.reserve(parts.Size() / 2);
@@ -2060,7 +2036,6 @@ void update_maximum(std::atomic<T>& maximum_value, T const& value) noexcept
 bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::vector<DataRelayer::RecordAction>& completed)
 {
   auto& context = ref.get<DataProcessorContext>();
-  ZoneScopedN("DataProcessingDevice::tryDispatchComputation");
   LOGP(debug, "DataProcessingDevice::tryDispatchComputation");
   // This is the actual hidden state for the outer loop. In case we decide we
   // want to support multithreaded dispatching of operations, I can simply
@@ -2135,7 +2110,6 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   auto prepareAllocatorForCurrentTimeSlice = [ref](TimesliceSlot i) -> void {
     auto& relayer = ref.get<DataRelayer>();
     auto& timingInfo = ref.get<TimingInfo>();
-    ZoneScopedN("DataProcessingDevice::prepareForCurrentTimeslice");
     auto timeslice = relayer.getTimesliceForSlot(i);
 
     timingInfo.timeslice = timeslice.value;
@@ -2422,21 +2396,18 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
       try {
         runNoCatch(action);
       } catch (o2::framework::RuntimeErrorRef e) {
-        ZoneScopedN("error handling");
         (context.errorHandling)(e, record);
       }
     } else {
       try {
         runNoCatch(action);
       } catch (std::exception& ex) {
-        ZoneScopedN("error handling");
         /// Convert a standard exception to a RuntimeErrorRef
         /// Notice how this will lose the backtrace information
         /// and report the exception coming from here.
         auto e = runtime_error(ex.what());
         (context.errorHandling)(e, record);
       } catch (o2::framework::RuntimeErrorRef e) {
-        ZoneScopedN("error handling");
         (context.errorHandling)(e, record);
       }
     }

From 7c3408d03132c468097d0357cafafa11d984e390 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Mar 2024 09:35:09 +0100
Subject: [PATCH 413/726] DPL: drop unused Tracy APIs

ZoneScoped*, TracyAppInfo, FrameMark, TracyAlloc, TracyPlot were all removed
and will be replaced by Signposts as needed.
---
 .../Foundation/include/Framework/Tracing.h    | 24 -------------------
 1 file changed, 24 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Tracing.h b/Framework/Foundation/include/Framework/Tracing.h
index 6c9c65f4423e5..91c3b99eb8dd0 100644
--- a/Framework/Foundation/include/Framework/Tracing.h
+++ b/Framework/Foundation/include/Framework/Tracing.h
@@ -15,30 +15,6 @@
 #define DPL_HAS_TRACING
 #include <tracy/Tracy.hpp>
 #else
-#define ZoneScoped \
-  while (false) {  \
-  }
-#define FrameMark \
-  while (false) { \
-  }
-#define TracyPlot(...) \
-  while (false) {      \
-  }
-#define ZoneScopedN(...) \
-  while (false) {        \
-  }
-#define ZoneScopedNS(...) \
-  while (false) {         \
-  }
-#define TracyAlloc(...) \
-  while (false) {       \
-  }
-#define TracyFree(...) \
-  while (false) {      \
-  }
-#define TracyAppInfo(...) \
-  while (false) {         \
-  }
 #define O2_LOCKABLE_NAMED(T, V, N) T V
 #define O2_LOCKABLE(T) T
 #endif

From 2a57f1a6a3e0c680225f2bf07f44f9c4b1beae1e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Mar 2024 09:35:09 +0100
Subject: [PATCH 414/726] DPL: drop tracy support from the GUI

No one is using it and Signposts are probably a better alternative.
---
 .../GUISupport/src/FrameworkGUIDeviceInspector.cxx  | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
index eb62fcc09d338..3ef8b99ed26bb 100644
--- a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
@@ -261,9 +261,6 @@ void displayDeviceInspector(DeviceSpec const& spec,
     ImGui::Text("Pid: %d (exit status: %d)", info.pid, info.exitStatus);
   }
   ImGui::Text("Device state: %s", info.deviceState.data());
-#ifdef DPL_ENABLE_TRACING
-  ImGui::Text("Tracy Port: %d", info.tracyPort);
-#endif
   ImGui::Text("Rank: %zu/%zu%%%zu/%zu", spec.rank, spec.nSlots, spec.inputTimesliceId, spec.maxInputTimeslices);
 
   if (ImGui::Button(ICON_FA_BUG "Attach debugger")) {
@@ -324,16 +321,6 @@ void displayDeviceInspector(DeviceSpec const& spec,
   }
 #endif
 
-#if DPL_ENABLE_TRACING
-  ImGui::SameLine();
-  if (ImGui::Button("Tracy")) {
-    std::string tracyPort = std::to_string(info.tracyPort);
-    auto cmd = fmt::format("tracy-profiler -p {} -a 127.0.0.1 &", info.tracyPort);
-    LOG(debug) << cmd;
-    int retVal = system(cmd.c_str());
-    (void)retVal;
-  }
-#endif
   if (control.controller) {
     if (ImGui::Button("Offer SHM")) {
       control.controller->write("/shm-offer 1000", strlen("/shm-offer 1000"));

From 101fc92ec1d9d6227b7744dd80939052cbfe5e92 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Mar 2024 09:35:09 +0100
Subject: [PATCH 415/726] DPL: drop tracy support for good

---
 Framework/Core/src/DataProcessingDevice.cxx      | 4 ----
 Framework/Foundation/include/Framework/Tracing.h | 7 ++++---
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 649aba2d6b502..58ea6524f0b7d 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -8,10 +8,6 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
-#ifdef DPL_ENABLE_TRACING
-#define TRACY_ENABLE
-#include <tracy/TracyClient.cpp>
-#endif
 #include "Framework/AsyncQueue.h"
 #include "Framework/DataProcessingDevice.h"
 #include "Framework/ChannelMatching.h"
diff --git a/Framework/Foundation/include/Framework/Tracing.h b/Framework/Foundation/include/Framework/Tracing.h
index 91c3b99eb8dd0..72c052ca6c161 100644
--- a/Framework/Foundation/include/Framework/Tracing.h
+++ b/Framework/Foundation/include/Framework/Tracing.h
@@ -11,9 +11,10 @@
 #ifndef O2_FRAMEWORK_TRACING_H_
 #define O2_FRAMEWORK_TRACING_H_
 
-#if DPL_ENABLE_TRACING && __has_include(<tracy/Tracy.hpp>)
-#define DPL_HAS_TRACING
-#include <tracy/Tracy.hpp>
+#if DPL_ENABLE_TRACING
+// FIXME: not implemented yet in terms of Signposts
+#define O2_LOCKABLE_NAMED(T, V, N) T V
+#define O2_LOCKABLE(T) T
 #else
 #define O2_LOCKABLE_NAMED(T, V, N) T V
 #define O2_LOCKABLE(T) T

From 2fbedffecc63d304c86d56e8b6d64f1fb82194ef Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Mar 2024 09:35:09 +0100
Subject: [PATCH 416/726] [FEAT] Drop support for Tracy

---
 Framework/CHANGELOG.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/Framework/CHANGELOG.md b/Framework/CHANGELOG.md
index e7278733de246..044274d9515c4 100644
--- a/Framework/CHANGELOG.md
+++ b/Framework/CHANGELOG.md
@@ -1,3 +1,12 @@
+# 2024-02-22: Drop Tracy support
+
+Tracy support never took off, so I am dropping it. This was mostly because people do not know about it and having a per process profile GUI was way unpractical. Moreover, needing an extra compile time flag meant one most likely did not have the support compiled in when needed.
+
+I have therefore decided to replace it with signposts, which hopefully will see better adoption thanks
+to the integration with Instruments on mac and the easy way they can be enabled dynamically.
+
+We could then reintroduce Tracy support as a hook on top of signposts, if really needed.
+
 # 2024-02-16: Improved Signposts.
 
 In particular:

From 473b9587320736412b3e576f6bbe7141d5c0069d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Mar 2024 10:21:15 +0100
Subject: [PATCH 417/726] DPL: drop some more code from the main header

---
 Framework/Core/include/Framework/runDataProcessing.h | 10 ++--------
 Framework/Core/src/runDataProcessing.cxx             |  9 +++++++++
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/Framework/Core/include/Framework/runDataProcessing.h b/Framework/Core/include/Framework/runDataProcessing.h
index 186d76499c1d6..75e2a964ccba1 100644
--- a/Framework/Core/include/Framework/runDataProcessing.h
+++ b/Framework/Core/include/Framework/runDataProcessing.h
@@ -31,7 +31,6 @@
 #include "Framework/StructToTuple.h"
 
 #include <vector>
-#include <cstring>
 
 namespace o2::framework
 {
@@ -211,6 +210,7 @@ int mainNoCatch(int argc, char** argv)
 }
 
 int callMain(int argc, char** argv, int (*)(int, char**));
+char *getIdString(int argc, char** argv);
 
 int main(int argc, char** argv)
 {
@@ -219,13 +219,7 @@ int main(int argc, char** argv)
 
   int result = callMain(argc, argv, mainNoCatch);
 
-  char* idstring = nullptr;
-  for (int argi = 0; argi < argc; argi++) {
-    if (strcmp(argv[argi], "--id") == 0 && argi + 1 < argc) {
-      idstring = argv[argi + 1];
-      break;
-    }
-  }
+  char* idstring = getIdString(argc, argv);
   o2::framework::OnWorkflowTerminationHook onWorkflowTerminationHook;
   UserCustomizationsHelper::userDefinedCustomization(onWorkflowTerminationHook, 0);
   onWorkflowTerminationHook(idstring);
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 6b424ee6a4261..f233eb5b72667 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -182,6 +182,15 @@ void doBoostException(boost::exception& e, const char*);
 void doDPLException(o2::framework::RuntimeErrorRef& ref, char const*);
 void doUnknownException(std::string const& s, char const*);
 
+char *getIdString(int argc, char** argv) {
+  for (int argi = 0; argi < argc; argi++) {
+    if (strcmp(argv[argi], "--id") == 0 && argi + 1 < argc) {
+      return argv[argi + 1];
+    }
+  }
+  return nullptr;
+}
+
 int callMain(int argc, char** argv, int (*mainNoCatch)(int, char**))
 {
   static bool noCatch = getenv("O2_NO_CATCHALL_EXCEPTIONS") && strcmp(getenv("O2_NO_CATCHALL_EXCEPTIONS"), "0");

From b98dafccee418dd10a249e9646c0a6d592fddc88 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Mar 2024 10:21:15 +0100
Subject: [PATCH 418/726] DPL: forward declaration of fill functions

---
 Framework/Core/include/Framework/HistogramRegistry.h | 4 ++++
 Framework/Core/src/HistogramRegistry.cxx             | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/Framework/Core/include/Framework/HistogramRegistry.h b/Framework/Core/include/Framework/HistogramRegistry.h
index 4b96ce75bbc98..e93972b27dcf9 100644
--- a/Framework/Core/include/Framework/HistogramRegistry.h
+++ b/Framework/Core/include/Framework/HistogramRegistry.h
@@ -417,6 +417,10 @@ void HistogramRegistry::fill(const HistName& histName, Ts&&... positionAndWeight
   std::visit([&positionAndWeight...](auto&& hist) { HistFiller::fillHistAny(hist, std::forward<Ts>(positionAndWeight)...); }, mRegistryValue[getHistIndex(histName)]);
 }
 
+extern template void HistogramRegistry::fill(const HistName& histName, double);
+extern template void HistogramRegistry::fill(const HistName& histName, float);
+extern template void HistogramRegistry::fill(const HistName& histName, int);
+
 template <typename... Cs, typename T>
 void HistogramRegistry::fill(const HistName& histName, const T& table, const o2::framework::expressions::Filter& filter)
 {
diff --git a/Framework/Core/src/HistogramRegistry.cxx b/Framework/Core/src/HistogramRegistry.cxx
index 86b2e128ebb2c..4b4bb2cf9be09 100644
--- a/Framework/Core/src/HistogramRegistry.cxx
+++ b/Framework/Core/src/HistogramRegistry.cxx
@@ -17,6 +17,10 @@
 namespace o2::framework
 {
 
+template void HistogramRegistry::fill(const HistName& histName, const double&);
+template void HistogramRegistry::fill(const HistName& histName, const float&);
+template void HistogramRegistry::fill(const HistName& histName, const int&);
+
 constexpr HistogramRegistry::HistName::HistName(char const* const name)
   : str(name),
     hash(runtime_hash(name)),

From 6eb36efa49d024c28349e20e1eee267071408afe Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Mar 2024 10:33:48 +0100
Subject: [PATCH 419/726] DPL Analysis: use voids to select packs

---
 Framework/Foundation/include/Framework/Pack.h | 29 ++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/Framework/Foundation/include/Framework/Pack.h b/Framework/Foundation/include/Framework/Pack.h
index f03476f4fdc0e..2d54ecb8ea5c8 100644
--- a/Framework/Foundation/include/Framework/Pack.h
+++ b/Framework/Foundation/include/Framework/Pack.h
@@ -99,6 +99,33 @@ constexpr auto interleave_pack(pack<Args1...>, pack<Args2...>)
 template <typename P1, typename P2>
 using interleaved_pack_t = decltype(interleave_pack(P1{}, P2{}));
 
+/// Marks as void the types that do not satisfy the condition
+template <template <typename...> typename Condition, typename... Ts>
+using with_condition_pack = pack<std::conditional_t<Condition<Ts>::value, Ts, void>...>;
+
+template <typename... Ts>
+constexpr auto count_non_void_pack(pack<Ts...> const&)
+{
+  return ((std::is_void_v<Ts> ? 0 : 1) + ...);
+}
+
+template <typename Result>
+constexpr auto prune_voids_pack(Result result, pack<>)
+{
+  return result;
+}
+
+template <typename... Rs, typename T, typename... Ts>
+constexpr auto prune_voids_pack(pack<Rs...> result, pack<T, Ts...>)
+{
+  constexpr auto rest = pack<Ts...>{};
+  if constexpr (std::is_void_v<T> == false) {
+    return prune_voids_pack(pack<Rs..., T>{}, rest);
+  } else {
+    return prune_voids_pack(pack<Rs...>{}, rest);
+  }
+}
+
 /// Selects from the pack types that satisfy the Condition
 /// Multicondition takes the type to check as first template parameter
 /// and any helper types as the following parameters
@@ -119,7 +146,7 @@ constexpr auto select_pack(Result result, pack<T, Ts...>, pack<Cs...> condPack)
 }
 
 template <template <typename...> typename Condition, typename... Types>
-using selected_pack = std::decay_t<decltype(select_pack<Condition>(pack<>{}, pack<Types...>{}, pack<>{}))>;
+using selected_pack = std::decay_t<decltype(prune_voids_pack(pack<>{}, with_condition_pack<Condition, Types...>{}))>;
 template <template <typename...> typename Condition, typename CondPack, typename Pack>
 using selected_pack_multicondition = std::decay_t<decltype(select_pack<Condition>(pack<>{}, Pack{}, CondPack{}))>;
 

From 23a9eead945353613bfbd618d2c6d67a8bf4c37e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Mar 2024 10:33:48 +0100
Subject: [PATCH 420/726] DPL Analysis: drop unneeded helper function for
 doCopyExternalIndices

---
 Framework/Core/include/Framework/ASoA.h | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h
index 5c877a0e8f476..0e1031f9ef4c3 100644
--- a/Framework/Core/include/Framework/ASoA.h
+++ b/Framework/Core/include/Framework/ASoA.h
@@ -1696,16 +1696,10 @@ class Table
     mBegin.bindExternalIndicesRaw(std::forward<std::vector<o2::soa::Binding>>(ptrs));
   }
 
-  template <typename T, typename... Cs>
-  void doCopyIndexBindings(framework::pack<Cs...>, T& dest) const
-  {
-    dest.bindExternalIndicesRaw(mBegin.getIndexBindings());
-  }
-
   template <typename T>
   void copyIndexBindings(T& dest) const
   {
-    doCopyIndexBindings(external_index_columns_t{}, dest);
+    dest.bindExternalIndicesRaw(mBegin.getIndexBindings());
   }
 
   auto select(framework::expressions::Filter const& f) const
@@ -2996,16 +2990,10 @@ class FilteredBase : public T
     mFilteredBegin.bindInternalIndices(ptr);
   }
 
-  template <typename T1, typename... Cs>
-  void doCopyIndexBindings(framework::pack<Cs...>, T1& dest) const
-  {
-    dest.bindExternalIndicesRaw(mFilteredBegin.getIndexBindings());
-  }
-
   template <typename T1>
   void copyIndexBindings(T1& dest) const
   {
-    doCopyIndexBindings(external_index_columns_t{}, dest);
+    dest.bindExternalIndicesRaw(mFilteredBegin.getIndexBindings());
   }
 
   template <typename T1>

From ccf49c3403ad4fd8d138aef937d5c2f9640f7e5c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Mar 2024 10:37:12 +0100
Subject: [PATCH 421/726] DPL: Make has_type_at non-recursive

---
 Framework/Foundation/include/Framework/Pack.h | 22 ++++++++-----------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Pack.h b/Framework/Foundation/include/Framework/Pack.h
index 2d54ecb8ea5c8..eb105480ef9f0 100644
--- a/Framework/Foundation/include/Framework/Pack.h
+++ b/Framework/Foundation/include/Framework/Pack.h
@@ -197,21 +197,17 @@ struct has_type_conditional<Condition, T, pack<Us...>> : std::disjunction<Condit
 template <template <typename, typename> typename Condition, typename T, typename... Us>
 inline constexpr bool has_type_conditional_v = has_type_conditional<Condition, T, Us...>::value;
 
-template <typename T>
-constexpr size_t has_type_at(pack<> const&)
-{
-  return static_cast<size_t>(-1);
-}
-
-template <typename T, typename T1, typename... Ts>
-constexpr size_t has_type_at(pack<T1, Ts...> const&)
+template <typename T, typename... Ts>
+consteval std::size_t has_type_at(pack<Ts...> const&)
 {
-  if constexpr (std::is_same_v<T, T1>) {
-    return 0;
-  } else if constexpr (has_type_v<T, pack<Ts...>>) {
-    return 1 + has_type_at<T>(pack<Ts...>{});
+  constexpr std::size_t count = []<std::size_t... Is>(std::index_sequence<Is...>) {
+    return ((std::is_same_v<T, Ts> ? Is + 1 : 0) + ...);
+  }(std::make_index_sequence<sizeof...(Ts)>());
+  if constexpr (count == 0) {
+    return sizeof...(Ts) + 1;
+  } else {
+    return count - 1;
   }
-  return sizeof...(Ts) + 2;
 }
 
 template <template <typename, typename> typename Condition, typename T>

From 10cafb71b35b5a217b776278da15009a78d94bef Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Mar 2024 10:43:58 +0100
Subject: [PATCH 422/726] [CLEANUP] DPL Analysis: Clean up helper functions for
 packs handling

- `select_pack` now first sets to void non-matching elements, and then prunes voids.
- `has_type_at` is now non-recursive
- `doCopyIndexBindings` no longer needed.
---
 Framework/CHANGELOG.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Framework/CHANGELOG.md b/Framework/CHANGELOG.md
index e7278733de246..d6e9060355fd0 100644
--- a/Framework/CHANGELOG.md
+++ b/Framework/CHANGELOG.md
@@ -1,3 +1,9 @@
+# 2024-03-08: Clean up helper functions for packs handling
+
+- `select_pack` now first sets to void non-matching elements, and then prunes voids.
+- `has_type_at` is now non-recursive
+- `doCopyIndexBindings` no longer needed.
+
 # 2024-02-16: Improved Signposts.
 
 In particular:

From 8aa9a30e941a9d7ff4d3644071ce26f83f0f7e39 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Mar 2024 14:02:29 +0100
Subject: [PATCH 423/726] DPL: drop some more code from the main header

---
 Framework/Core/include/Framework/runDataProcessing.h | 10 ++--------
 Framework/Core/src/runDataProcessing.cxx             | 10 ++++++++++
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/Framework/Core/include/Framework/runDataProcessing.h b/Framework/Core/include/Framework/runDataProcessing.h
index 186d76499c1d6..77a4c9f5ba8d6 100644
--- a/Framework/Core/include/Framework/runDataProcessing.h
+++ b/Framework/Core/include/Framework/runDataProcessing.h
@@ -31,7 +31,6 @@
 #include "Framework/StructToTuple.h"
 
 #include <vector>
-#include <cstring>
 
 namespace o2::framework
 {
@@ -211,6 +210,7 @@ int mainNoCatch(int argc, char** argv)
 }
 
 int callMain(int argc, char** argv, int (*)(int, char**));
+char* getIdString(int argc, char** argv);
 
 int main(int argc, char** argv)
 {
@@ -219,13 +219,7 @@ int main(int argc, char** argv)
 
   int result = callMain(argc, argv, mainNoCatch);
 
-  char* idstring = nullptr;
-  for (int argi = 0; argi < argc; argi++) {
-    if (strcmp(argv[argi], "--id") == 0 && argi + 1 < argc) {
-      idstring = argv[argi + 1];
-      break;
-    }
-  }
+  char* idstring = getIdString(argc, argv);
   o2::framework::OnWorkflowTerminationHook onWorkflowTerminationHook;
   UserCustomizationsHelper::userDefinedCustomization(onWorkflowTerminationHook, 0);
   onWorkflowTerminationHook(idstring);
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 6b424ee6a4261..c72b4da73a45a 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -182,6 +182,16 @@ void doBoostException(boost::exception& e, const char*);
 void doDPLException(o2::framework::RuntimeErrorRef& ref, char const*);
 void doUnknownException(std::string const& s, char const*);
 
+char* getIdString(int argc, char** argv)
+{
+  for (int argi = 0; argi < argc; argi++) {
+    if (strcmp(argv[argi], "--id") == 0 && argi + 1 < argc) {
+      return argv[argi + 1];
+    }
+  }
+  return nullptr;
+}
+
 int callMain(int argc, char** argv, int (*mainNoCatch)(int, char**))
 {
   static bool noCatch = getenv("O2_NO_CATCHALL_EXCEPTIONS") && strcmp(getenv("O2_NO_CATCHALL_EXCEPTIONS"), "0");

From 1a3cfca46612bed9114dc73e25f9026342b3ebf4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sun, 10 Mar 2024 00:47:05 +0100
Subject: [PATCH 424/726] DPL: fix leak in rate limiting

---
 Framework/Core/src/CommonDataProcessors.cxx | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 79a31af04cdcd..593cb8d1ac1f8 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -581,8 +581,7 @@ void retryMetricCallback(uv_async_t* async)
   }
   fair::mq::MessagePtr payload(device->NewMessage());
   payload->Rebuild(&oldestPossingTimeslice, sizeof(int64_t), nullptr, nullptr);
-  auto* consumed = (size_t*)malloc(sizeof(size_t));
-  *consumed = oldestPossingTimeslice;
+  auto consumed = oldestPossingTimeslice;
 
   int64_t result = channel->second[0].Send(payload, 100);
   // If the sending worked, we do not retry.
@@ -592,7 +591,7 @@ void retryMetricCallback(uv_async_t* async)
     // data will still happen.
     uv_async_send(async);
   } else {
-    lastTimeslice = *consumed;
+    lastTimeslice = consumed;
   }
 }
 

From 6595db0360a9eaff74ad7b0edaa49dcffb1ed94e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 12 Mar 2024 12:17:33 +0100
Subject: [PATCH 425/726] DPL: cleanup HistogramRegistry::fill function

* make sure that the fill method fills arithmetic types.
* instanciate templates for common case
---
 .../Core/include/Framework/HistogramRegistry.h | 18 +++++++++++++-----
 Framework/Core/src/HistogramRegistry.cxx       |  4 ++++
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/Framework/Core/include/Framework/HistogramRegistry.h b/Framework/Core/include/Framework/HistogramRegistry.h
index 4b96ce75bbc98..e26ad7bbe53c9 100644
--- a/Framework/Core/include/Framework/HistogramRegistry.h
+++ b/Framework/Core/include/Framework/HistogramRegistry.h
@@ -42,7 +42,8 @@ namespace o2::framework
 struct HistFiller {
   // fill any type of histogram (if weight was requested it must be the last argument)
   template <typename T, typename... Ts>
-  static void fillHistAny(std::shared_ptr<T> hist, const Ts&... positionAndWeight);
+  static void fillHistAny(std::shared_ptr<T> hist, Ts... positionAndWeight)
+    requires(std::is_arithmetic_v<Ts> && ...);
 
   // fill any type of histogram with columns (Cs) of a filtered table (if weight is requested it must reside the last specified column)
   template <typename... Cs, typename R, typename T>
@@ -127,7 +128,8 @@ class HistogramRegistry
 
   // fill hist with values
   template <typename... Ts>
-  void fill(const HistName& histName, Ts&&... positionAndWeight);
+  void fill(const HistName& histName, Ts... positionAndWeight)
+    requires(std::is_arithmetic_v<Ts> && ...);
 
   // fill hist with content of (filtered) table columns
   template <typename... Cs, typename T>
@@ -197,7 +199,8 @@ class HistogramRegistry
 //--------------------------------------------------------------------------------------------------
 
 template <typename T, typename... Ts>
-void HistFiller::fillHistAny(std::shared_ptr<T> hist, const Ts&... positionAndWeight)
+void HistFiller::fillHistAny(std::shared_ptr<T> hist, Ts... positionAndWeight)
+  requires(std::is_arithmetic_v<Ts> && ...)
 {
   constexpr int nArgs = sizeof...(Ts);
 
@@ -412,11 +415,16 @@ uint32_t HistogramRegistry::getHistIndex(const T& histName)
 }
 
 template <typename... Ts>
-void HistogramRegistry::fill(const HistName& histName, Ts&&... positionAndWeight)
+void HistogramRegistry::fill(const HistName& histName, Ts... positionAndWeight)
+  requires(std::is_arithmetic_v<Ts> && ...)
 {
-  std::visit([&positionAndWeight...](auto&& hist) { HistFiller::fillHistAny(hist, std::forward<Ts>(positionAndWeight)...); }, mRegistryValue[getHistIndex(histName)]);
+  std::visit([positionAndWeight...](auto&& hist) { HistFiller::fillHistAny(hist, positionAndWeight...); }, mRegistryValue[getHistIndex(histName)]);
 }
 
+extern template void HistogramRegistry::fill(const HistName& histName, double);
+extern template void HistogramRegistry::fill(const HistName& histName, float);
+extern template void HistogramRegistry::fill(const HistName& histName, int);
+
 template <typename... Cs, typename T>
 void HistogramRegistry::fill(const HistName& histName, const T& table, const o2::framework::expressions::Filter& filter)
 {
diff --git a/Framework/Core/src/HistogramRegistry.cxx b/Framework/Core/src/HistogramRegistry.cxx
index 86b2e128ebb2c..806aae93be95e 100644
--- a/Framework/Core/src/HistogramRegistry.cxx
+++ b/Framework/Core/src/HistogramRegistry.cxx
@@ -17,6 +17,10 @@
 namespace o2::framework
 {
 
+template void HistogramRegistry::fill(const HistName& histName, double);
+template void HistogramRegistry::fill(const HistName& histName, float);
+template void HistogramRegistry::fill(const HistName& histName, int);
+
 constexpr HistogramRegistry::HistName::HistName(char const* const name)
   : str(name),
     hash(runtime_hash(name)),

From 206c68b6e61ab40aed0d5f838bbf985a5202b7d6 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 13 Mar 2024 10:46:05 +0100
Subject: [PATCH 426/726] DPL: cleanup HistogramRegistry::fill function

* make sure that the fill method fills arithmetic types.
* instanciate templates for common case
---
 .../include/Framework/HistogramRegistry.h     | 21 ++++++++++++++-----
 Framework/Core/src/HistogramRegistry.cxx      |  4 ++++
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/Framework/Core/include/Framework/HistogramRegistry.h b/Framework/Core/include/Framework/HistogramRegistry.h
index 4b96ce75bbc98..1bd3bf9ece00c 100644
--- a/Framework/Core/include/Framework/HistogramRegistry.h
+++ b/Framework/Core/include/Framework/HistogramRegistry.h
@@ -39,10 +39,14 @@ namespace o2::framework
  * Static helper class to fill root histograms of any type. Contains functionality to fill once per call or a whole (filtered) table at once.
  */
 //**************************************************************************************************
+template <typename T>
+concept FillValue = std::is_integral_v<T> || std::is_floating_point_v<T> || std::is_enum_v<T>;
+
 struct HistFiller {
   // fill any type of histogram (if weight was requested it must be the last argument)
   template <typename T, typename... Ts>
-  static void fillHistAny(std::shared_ptr<T> hist, const Ts&... positionAndWeight);
+  static void fillHistAny(std::shared_ptr<T> hist, Ts... positionAndWeight)
+    requires(FillValue<Ts> && ...);
 
   // fill any type of histogram with columns (Cs) of a filtered table (if weight is requested it must reside the last specified column)
   template <typename... Cs, typename R, typename T>
@@ -127,7 +131,8 @@ class HistogramRegistry
 
   // fill hist with values
   template <typename... Ts>
-  void fill(const HistName& histName, Ts&&... positionAndWeight);
+  void fill(const HistName& histName, Ts... positionAndWeight)
+    requires(FillValue<Ts> && ...);
 
   // fill hist with content of (filtered) table columns
   template <typename... Cs, typename T>
@@ -197,7 +202,8 @@ class HistogramRegistry
 //--------------------------------------------------------------------------------------------------
 
 template <typename T, typename... Ts>
-void HistFiller::fillHistAny(std::shared_ptr<T> hist, const Ts&... positionAndWeight)
+void HistFiller::fillHistAny(std::shared_ptr<T> hist, Ts... positionAndWeight)
+  requires(FillValue<Ts> && ...)
 {
   constexpr int nArgs = sizeof...(Ts);
 
@@ -412,11 +418,16 @@ uint32_t HistogramRegistry::getHistIndex(const T& histName)
 }
 
 template <typename... Ts>
-void HistogramRegistry::fill(const HistName& histName, Ts&&... positionAndWeight)
+void HistogramRegistry::fill(const HistName& histName, Ts... positionAndWeight)
+  requires(FillValue<Ts> && ...)
 {
-  std::visit([&positionAndWeight...](auto&& hist) { HistFiller::fillHistAny(hist, std::forward<Ts>(positionAndWeight)...); }, mRegistryValue[getHistIndex(histName)]);
+  std::visit([positionAndWeight...](auto&& hist) { HistFiller::fillHistAny(hist, positionAndWeight...); }, mRegistryValue[getHistIndex(histName)]);
 }
 
+extern template void HistogramRegistry::fill(const HistName& histName, double);
+extern template void HistogramRegistry::fill(const HistName& histName, float);
+extern template void HistogramRegistry::fill(const HistName& histName, int);
+
 template <typename... Cs, typename T>
 void HistogramRegistry::fill(const HistName& histName, const T& table, const o2::framework::expressions::Filter& filter)
 {
diff --git a/Framework/Core/src/HistogramRegistry.cxx b/Framework/Core/src/HistogramRegistry.cxx
index 86b2e128ebb2c..806aae93be95e 100644
--- a/Framework/Core/src/HistogramRegistry.cxx
+++ b/Framework/Core/src/HistogramRegistry.cxx
@@ -17,6 +17,10 @@
 namespace o2::framework
 {
 
+template void HistogramRegistry::fill(const HistName& histName, double);
+template void HistogramRegistry::fill(const HistName& histName, float);
+template void HistogramRegistry::fill(const HistName& histName, int);
+
 constexpr HistogramRegistry::HistName::HistName(char const* const name)
   : str(name),
     hash(runtime_hash(name)),

From fed0d4a59c232d1d603cfd10c3d17aa88f046e8d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 13 Mar 2024 10:46:13 +0100
Subject: [PATCH 427/726] DPL: make helper for typeId consteval

---
 Framework/Foundation/include/Framework/TypeIdHelpers.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Foundation/include/Framework/TypeIdHelpers.h b/Framework/Foundation/include/Framework/TypeIdHelpers.h
index d4f72b8f3f7ff..2336e5dd7b8e0 100644
--- a/Framework/Foundation/include/Framework/TypeIdHelpers.h
+++ b/Framework/Foundation/include/Framework/TypeIdHelpers.h
@@ -72,7 +72,7 @@ constexpr static std::string_view type_name()
 
 struct TypeIdHelpers {
   template <typename T>
-  constexpr static uint32_t uniqueId()
+  consteval static uint32_t uniqueId()
   {
 #ifdef __CLING__
     constexpr uint32_t r = crc32(unique_type_id_v<T>.data(), unique_type_id_v<T>.size());

From bfe66668914726d2de262e7a9f86db988306fdaf Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 13 Mar 2024 13:16:33 +0100
Subject: [PATCH 428/726] Do not expose Devices to ROOT

---
 Detectors/PHOS/calib/CMakeLists.txt | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/Detectors/PHOS/calib/CMakeLists.txt b/Detectors/PHOS/calib/CMakeLists.txt
index c54001e08eee9..eca804037d53b 100644
--- a/Detectors/PHOS/calib/CMakeLists.txt
+++ b/Detectors/PHOS/calib/CMakeLists.txt
@@ -30,21 +30,14 @@ o2_add_library(PHOSCalibWorkflow
 o2_add_header_only_library(PHOSCalib)
 
 o2_target_root_dictionary(PHOSCalibWorkflow
-                          HEADERS include/PHOSCalibWorkflow/PHOSPedestalCalibDevice.h
-                                  include/PHOSCalibWorkflow/PHOSHGLGRatioCalibDevice.h
-                                  include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
+                          HEADERS include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
                                   include/PHOSCalibWorkflow/RingBuffer.h
                                   include/PHOSCalibWorkflow/ETCalibHistos.h
                                   include/PHOSCalibWorkflow/PHOSEnergyCalibrator.h
-                                  include/PHOSCalibWorkflow/PHOSEnergyCalibDevice.h
                                   include/PHOSCalibWorkflow/TurnOnHistos.h
                                   include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
-                                  include/PHOSCalibWorkflow/PHOSTurnonCalibDevice.h
                                   include/PHOSCalibWorkflow/PHOSRunbyrunCalibrator.h
-                                  include/PHOSCalibWorkflow/PHOSRunbyrunCalibDevice.h
                                   include/PHOSCalibWorkflow/PHOSL1phaseCalibrator.h
-                                  include/PHOSCalibWorkflow/PHOSL1phaseCalibDevice.h
-                                  include/PHOSCalibWorkflow/PHOSBadMapCalibDevice.h
                           LINKDEF src/PHOSCalibWorkflowLinkDef.h)
 
 o2_add_executable(calib-workflow

From c766640ef8a8e8657dcb0042103a087bb01418b3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 13 Mar 2024 10:46:13 +0100
Subject: [PATCH 429/726] DPL: make helper for typeId consteval

---
 Framework/Foundation/include/Framework/TypeIdHelpers.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Foundation/include/Framework/TypeIdHelpers.h b/Framework/Foundation/include/Framework/TypeIdHelpers.h
index d4f72b8f3f7ff..2336e5dd7b8e0 100644
--- a/Framework/Foundation/include/Framework/TypeIdHelpers.h
+++ b/Framework/Foundation/include/Framework/TypeIdHelpers.h
@@ -72,7 +72,7 @@ constexpr static std::string_view type_name()
 
 struct TypeIdHelpers {
   template <typename T>
-  constexpr static uint32_t uniqueId()
+  consteval static uint32_t uniqueId()
   {
 #ifdef __CLING__
     constexpr uint32_t r = crc32(unique_type_id_v<T>.data(), unique_type_id_v<T>.size());

From 7d28debc5bb69350f7cd04b6981aa9d74a1e6685 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 10:36:50 +0100
Subject: [PATCH 430/726] DPL: make helper for typeId consteval

---
 Framework/Core/include/Framework/ServiceRegistry.h     | 2 +-
 Framework/Foundation/include/Framework/TypeIdHelpers.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/include/Framework/ServiceRegistry.h b/Framework/Core/include/Framework/ServiceRegistry.h
index e3fa23294ee78..4d4ee1588aa36 100644
--- a/Framework/Core/include/Framework/ServiceRegistry.h
+++ b/Framework/Core/include/Framework/ServiceRegistry.h
@@ -197,7 +197,7 @@ struct ServiceRegistry {
   void throwError(const char* name, int64_t hash, int64_t streamId, int64_t dataprocessorId) const;
 
  public:
-  using hash_type = decltype(TypeIdHelpers::uniqueId<void>());
+  using hash_type = decltype(TypeIdHelpers::uniqueId<std::void_t<void>>());
   ServiceRegistry();
 
   ServiceRegistry(ServiceRegistry const& other);
diff --git a/Framework/Foundation/include/Framework/TypeIdHelpers.h b/Framework/Foundation/include/Framework/TypeIdHelpers.h
index d4f72b8f3f7ff..2336e5dd7b8e0 100644
--- a/Framework/Foundation/include/Framework/TypeIdHelpers.h
+++ b/Framework/Foundation/include/Framework/TypeIdHelpers.h
@@ -72,7 +72,7 @@ constexpr static std::string_view type_name()
 
 struct TypeIdHelpers {
   template <typename T>
-  constexpr static uint32_t uniqueId()
+  consteval static uint32_t uniqueId()
   {
 #ifdef __CLING__
     constexpr uint32_t r = crc32(unique_type_id_v<T>.data(), unique_type_id_v<T>.size());

From 21bfd3be8058e7dbb3d9ef545b3dab01d73b7e13 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 10:36:50 +0100
Subject: [PATCH 431/726] Do not expose Devices to ROOT

---
 Detectors/PHOS/calib/CMakeLists.txt | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/Detectors/PHOS/calib/CMakeLists.txt b/Detectors/PHOS/calib/CMakeLists.txt
index c54001e08eee9..eca804037d53b 100644
--- a/Detectors/PHOS/calib/CMakeLists.txt
+++ b/Detectors/PHOS/calib/CMakeLists.txt
@@ -30,21 +30,14 @@ o2_add_library(PHOSCalibWorkflow
 o2_add_header_only_library(PHOSCalib)
 
 o2_target_root_dictionary(PHOSCalibWorkflow
-                          HEADERS include/PHOSCalibWorkflow/PHOSPedestalCalibDevice.h
-                                  include/PHOSCalibWorkflow/PHOSHGLGRatioCalibDevice.h
-                                  include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
+                          HEADERS include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
                                   include/PHOSCalibWorkflow/RingBuffer.h
                                   include/PHOSCalibWorkflow/ETCalibHistos.h
                                   include/PHOSCalibWorkflow/PHOSEnergyCalibrator.h
-                                  include/PHOSCalibWorkflow/PHOSEnergyCalibDevice.h
                                   include/PHOSCalibWorkflow/TurnOnHistos.h
                                   include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
-                                  include/PHOSCalibWorkflow/PHOSTurnonCalibDevice.h
                                   include/PHOSCalibWorkflow/PHOSRunbyrunCalibrator.h
-                                  include/PHOSCalibWorkflow/PHOSRunbyrunCalibDevice.h
                                   include/PHOSCalibWorkflow/PHOSL1phaseCalibrator.h
-                                  include/PHOSCalibWorkflow/PHOSL1phaseCalibDevice.h
-                                  include/PHOSCalibWorkflow/PHOSBadMapCalibDevice.h
                           LINKDEF src/PHOSCalibWorkflowLinkDef.h)
 
 o2_add_executable(calib-workflow

From aa88f6f5c5050278888d525becc534178f367a8c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 12:07:31 +0100
Subject: [PATCH 432/726] DPL: make helper for typeId consteval

---
 Framework/Core/include/Framework/ServiceRegistry.h     | 2 +-
 Framework/Foundation/include/Framework/TypeIdHelpers.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/include/Framework/ServiceRegistry.h b/Framework/Core/include/Framework/ServiceRegistry.h
index e3fa23294ee78..f7314325efcd0 100644
--- a/Framework/Core/include/Framework/ServiceRegistry.h
+++ b/Framework/Core/include/Framework/ServiceRegistry.h
@@ -197,7 +197,7 @@ struct ServiceRegistry {
   void throwError(const char* name, int64_t hash, int64_t streamId, int64_t dataprocessorId) const;
 
  public:
-  using hash_type = decltype(TypeIdHelpers::uniqueId<void>());
+  using hash_type = decltype(TypeIdHelpers::uniqueId<int>());
   ServiceRegistry();
 
   ServiceRegistry(ServiceRegistry const& other);
diff --git a/Framework/Foundation/include/Framework/TypeIdHelpers.h b/Framework/Foundation/include/Framework/TypeIdHelpers.h
index d4f72b8f3f7ff..2336e5dd7b8e0 100644
--- a/Framework/Foundation/include/Framework/TypeIdHelpers.h
+++ b/Framework/Foundation/include/Framework/TypeIdHelpers.h
@@ -72,7 +72,7 @@ constexpr static std::string_view type_name()
 
 struct TypeIdHelpers {
   template <typename T>
-  constexpr static uint32_t uniqueId()
+  consteval static uint32_t uniqueId()
   {
 #ifdef __CLING__
     constexpr uint32_t r = crc32(unique_type_id_v<T>.data(), unique_type_id_v<T>.size());

From a6d3268ee6f985e6a874e409e2e0a6b3cfeb71b4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 14:53:02 +0100
Subject: [PATCH 433/726] DPL: make helper for typeId consteval

---
 Framework/Core/include/Framework/ServiceRegistry.h     | 2 +-
 Framework/Foundation/include/Framework/TypeIdHelpers.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/include/Framework/ServiceRegistry.h b/Framework/Core/include/Framework/ServiceRegistry.h
index e3fa23294ee78..360dc693d5798 100644
--- a/Framework/Core/include/Framework/ServiceRegistry.h
+++ b/Framework/Core/include/Framework/ServiceRegistry.h
@@ -197,7 +197,7 @@ struct ServiceRegistry {
   void throwError(const char* name, int64_t hash, int64_t streamId, int64_t dataprocessorId) const;
 
  public:
-  using hash_type = decltype(TypeIdHelpers::uniqueId<void>());
+  using hash_type = decltype(TypeIdHelpers::uniqueId<Key>());
   ServiceRegistry();
 
   ServiceRegistry(ServiceRegistry const& other);
diff --git a/Framework/Foundation/include/Framework/TypeIdHelpers.h b/Framework/Foundation/include/Framework/TypeIdHelpers.h
index d4f72b8f3f7ff..2336e5dd7b8e0 100644
--- a/Framework/Foundation/include/Framework/TypeIdHelpers.h
+++ b/Framework/Foundation/include/Framework/TypeIdHelpers.h
@@ -72,7 +72,7 @@ constexpr static std::string_view type_name()
 
 struct TypeIdHelpers {
   template <typename T>
-  constexpr static uint32_t uniqueId()
+  consteval static uint32_t uniqueId()
   {
 #ifdef __CLING__
     constexpr uint32_t r = crc32(unique_type_id_v<T>.data(), unique_type_id_v<T>.size());

From 1e023d2b6cd06fb0cdb1db0d4c1ec7d918fa628a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 15:35:49 +0100
Subject: [PATCH 434/726] DPL: use signposts in place of LOG

---
 Framework/Core/src/DataProcessingDevice.cxx | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 58ea6524f0b7d..917b60d4c4393 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -1584,9 +1584,13 @@ void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
 void DataProcessingDevice::doRun(ServiceRegistryRef ref)
 {
   auto& context = ref.get<DataProcessorContext>();
+  O2_SIGNPOST_ID_FROM_POINTER(dpid, device, &context);
   auto switchState = [ref](StreamingState newState) {
     auto& state = ref.get<DeviceState>();
-    LOG(detail) << "New state " << (int)newState << " old state " << (int)state.streaming;
+    auto& context = ref.get<DataProcessorContext>();
+    O2_SIGNPOST_ID_FROM_POINTER(dpid, device, &context);
+    O2_SIGNPOST_END(device, dpid, "state", "End of processing state %d", (int)state.streaming);
+    O2_SIGNPOST_START(device, dpid, "state", "Starting processing state %d", (int)newState);
     state.streaming = newState;
     ref.get<ControlService>().notifyStreamingState(state.streaming);
   };
@@ -1625,7 +1629,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
   }
 
   if (state.streaming == StreamingState::EndOfStreaming) {
-    LOGP(detail, "We are in EndOfStreaming. Flushing queues.");
+    O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "We are in EndOfStreaming. Flushing queues.");
     // We keep processing data until we are Idle.
     // FIXME: not sure this is the correct way to drain the queues, but
     // I guess we will see.
@@ -1646,7 +1650,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
     context.postEOSCallbacks(eosContext);
 
     for (auto& channel : spec.outputChannels) {
-      LOGP(detail, "Sending end of stream to {}", channel.name);
+      O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "Sending end of stream to %{public}s.", channel.name.c_str());
       DataProcessingHelpers::sendEndOfStream(ref, channel);
     }
     // This is needed because the transport is deleted before the device.
@@ -1658,7 +1662,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
       *context.wasActive = true;
     }
     // On end of stream we shut down all output pollers.
-    LOGP(detail, "Shutting down output pollers");
+    O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "Shutting down output pollers.");
     for (auto& poller : state.activeOutputPollers) {
       uv_poll_stop(poller);
     }
@@ -1667,7 +1671,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
 
   if (state.streaming == StreamingState::Idle) {
     // On end of stream we shut down all output pollers.
-    LOGP(detail, "We are in Idle. Shutting down output pollers.");
+    O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "Shutting down output pollers.");
     for (auto& poller : state.activeOutputPollers) {
       uv_poll_stop(poller);
     }

From 565847e2be64f3faed68a3159d53859392b0c8de Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 16:49:57 +0100
Subject: [PATCH 435/726] [FEAT] DPL: use Signposts everywhere in
 DataProcessingDevice

---
 Framework/CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Framework/CHANGELOG.md b/Framework/CHANGELOG.md
index 044274d9515c4..684c9d3c3f1bd 100644
--- a/Framework/CHANGELOG.md
+++ b/Framework/CHANGELOG.md
@@ -1,3 +1,8 @@
+# 2024-02-22: Move DataProcessingDevice to use Signposts
+
+All the messages from DataProcessingDevice have been migrated to use Signpost.
+This will hopefully simplify debugging.
+
 # 2024-02-22: Drop Tracy support
 
 Tracy support never took off, so I am dropping it. This was mostly because people do not know about it and having a per process profile GUI was way unpractical. Moreover, needing an extra compile time flag meant one most likely did not have the support compiled in when needed.

From 8a0de64ba4df40d6cf8cd4446268f3a8e1c17041 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 16:49:57 +0100
Subject: [PATCH 436/726] Use signposts in DataProcessingDevice

---
 Framework/Core/src/DataProcessingDevice.cxx | 49 ++++++++++++++-------
 1 file changed, 32 insertions(+), 17 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 917b60d4c4393..8d79567d56ed5 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -218,8 +218,13 @@ void run_callback(uv_work_t* handle)
 {
   auto* task = (TaskStreamInfo*)handle->data;
   auto ref = ServiceRegistryRef{*task->registry, ServiceRegistry::globalStreamSalt(task->id.index + 1)};
+  // We create a new signpost interval for this specific data processor. Same id, same data processor.
+  auto& dataProcessorContext = ref.get<DataProcessorContext>();
+  O2_SIGNPOST_ID_FROM_POINTER(sid, device, &dataProcessorContext);
+  O2_SIGNPOST_START(device, sid, "run_callback", "Starting run callback on stream %d", task->id.index);
   DataProcessingDevice::doPrepare(ref);
   DataProcessingDevice::doRun(ref);
+  O2_SIGNPOST_END(device, sid, "run_callback", "Done processing data for stream %d", task->id.index);
 }
 
 // Once the processing in a thread is done, this is executed on the main thread.
@@ -1207,6 +1212,8 @@ void DataProcessingDevice::Run()
   auto& state = ref.get<DeviceState>();
   state.loopReason = DeviceState::LoopReason::FIRST_LOOP;
   bool firstLoop = true;
+  O2_SIGNPOST_ID_FROM_POINTER(lid, device, state.loop);
+  O2_SIGNPOST_START(device, lid, "device_state", "First iteration of the device loop");
   while (state.transitionHandling != TransitionHandlingState::Expired) {
     if (state.nextFairMQState.empty() == false) {
       (void)this->ChangeState(state.nextFairMQState.back());
@@ -1236,6 +1243,7 @@ void DataProcessingDevice::Run()
         state.loopReason |= DeviceState::LoopReason::PREVIOUSLY_ACTIVE;
       }
       if (NewStatePending()) {
+        O2_SIGNPOST_EVENT_EMIT(device, lid, "run_loop", "New state pending. Waiting for it to be handled.");
         shouldNotWait = true;
         state.loopReason |= DeviceState::LoopReason::NEW_STATE_PENDING;
       }
@@ -1261,25 +1269,26 @@ void DataProcessingDevice::Run()
           uv_update_time(state.loop);
           uv_timer_start(deviceContext.gracePeriodTimer, on_transition_requested_expired, timeout * 1000, 0);
           if (mProcessingPolicies.termination == TerminationPolicy::QUIT) {
-            LOGP(info, "New state requested. Waiting for {} seconds before quitting.", timeout);
+            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state requested. Waiting for %d seconds before quitting.", (int)deviceContext.exitTransitionTimeout);
           } else {
-            LOGP(info, "New state requested. Waiting for {} seconds before switching to READY state.", timeout);
+            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state requested. Waiting for %d seconds before switching to READY state.", (int)deviceContext.exitTransitionTimeout);
           }
         } else {
           state.transitionHandling = TransitionHandlingState::Expired;
-          if (timeout == 0 && mProcessingPolicies.termination == TerminationPolicy::QUIT) {
-            LOGP(info, "New state requested. No timeout set, quitting immediately as per --completion-policy");
-          } else if (timeout == 0 && mProcessingPolicies.termination != TerminationPolicy::QUIT) {
-            LOGP(info, "New state requested. No timeout set, switching to READY state immediately");
+          if (deviceContext.exitTransitionTimeout == 0 && mProcessingPolicies.termination == TerminationPolicy::QUIT) {
+            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state requested. No timeout set, quitting immediately as per --completion-policy");
+          } else if (deviceContext.exitTransitionTimeout == 0 && mProcessingPolicies.termination != TerminationPolicy::QUIT) {
+            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state requested. No timeout set, switching to READY state immediately");
           } else if (mProcessingPolicies.termination == TerminationPolicy::QUIT) {
-            LOGP(info, "New state pending and we are already idle, quitting immediately as per --completion-policy");
+            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state pending and we are already idle, quitting immediately as per --completion-policy");
           } else {
-            LOGP(info, "New state pending and we are already idle, switching to READY immediately.");
+            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "runb_loop", "New state pending and we are already idle, switching to READY immediately.");
           }
         }
       }
       // If we are Idle, we can then consider the transition to be expired.
       if (state.transitionHandling == TransitionHandlingState::Requested && state.streaming == StreamingState::Idle) {
+        O2_SIGNPOST_EVENT_EMIT(device, lid, "run_loop", "State transition requested and we are now in Idle. We can consider it to be completed.");
         state.transitionHandling = TransitionHandlingState::Expired;
       }
       if (state.severityStack.empty() == false) {
@@ -1299,8 +1308,8 @@ void DataProcessingDevice::Run()
       // - we can trigger further events from the queue
       // - we can guarantee this is the last thing we do in the loop (
       //   assuming no one else is adding to the queue before this point).
-      auto onDrop = [&registry = mServiceRegistry](TimesliceSlot slot, std::vector<MessageSet>& dropped, TimesliceIndex::OldestOutputInfo oldestOutputInfo) {
-        LOGP(debug, "Dropping message from slot {}. Forwarding as needed.", slot.index);
+      auto onDrop = [&registry = mServiceRegistry, lid](TimesliceSlot slot, std::vector<MessageSet>& dropped, TimesliceIndex::OldestOutputInfo oldestOutputInfo) {
+        O2_SIGNPOST_START(device, lid, "run_loop", "Dropping message from slot %" PRIu64 ". Forwarding as needed.", (uint64_t)slot.index);
         ServiceRegistryRef ref{registry};
         ref.get<AsyncQueue>();
         ref.get<DecongestionService>();
@@ -1319,7 +1328,9 @@ void DataProcessingDevice::Run()
         auto& dpContext = ref.get<DataProcessorContext>();
         dpContext.preLoopCallbacks(ref);
       }
+      O2_SIGNPOST_END(device, lid, "run_loop", "Run loop completed. %{}s", shouldNotWait ? "Will immediately schedule a new one" : "Waiting for next event.");
       uv_run(state.loop, shouldNotWait ? UV_RUN_NOWAIT : UV_RUN_ONCE);
+      O2_SIGNPOST_START(device, lid, "run_loop", "Run loop started. Loop reason %d.", state.loopReason);
       if ((state.loopReason & state.tracingFlags) != 0) {
         state.severityStack.push_back((int)fair::Logger::GetConsoleSeverity());
         fair::Logger::SetConsoleSeverity(fair::Severity::trace);
@@ -1327,16 +1338,15 @@ void DataProcessingDevice::Run()
         fair::Logger::SetConsoleSeverity((fair::Severity)state.severityStack.back());
         state.severityStack.pop_back();
       }
-      LOGP(debug, "Loop reason mask {:b} & {:b} = {:b}",
-           state.loopReason, state.tracingFlags,
-           state.loopReason & state.tracingFlags);
+      O2_SIGNPOST_EVENT_EMIT(device, lid, "run_loop", "Loop reason mask %x & %x = %x", state.loopReason, state.tracingFlags, state.loopReason & state.tracingFlags);
 
       if ((state.loopReason & DeviceState::LoopReason::OOB_ACTIVITY) != 0) {
-        LOGP(debug, "We were awakened by a OOB event. Rescanning everything.");
+        O2_SIGNPOST_EVENT_EMIT(device, lid, "run_loop", "Out of band activity detected. Rescanning everything.");
         relayer.rescan();
       }
 
       if (!state.pendingOffers.empty()) {
+        O2_SIGNPOST_EVENT_EMIT(device, lid, "run_loop", "Pending %" PRIu64 " offers. updating the ComputingQuotaEvaluator.", (uint64_t)state.pendingOffers.size());
         ref.get<ComputingQuotaEvaluator>().updateOffers(state.pendingOffers, uv_now(state.loop));
       }
     }
@@ -1411,6 +1421,8 @@ void DataProcessingDevice::Run()
       mWasActive = false;
     }
   }
+
+  O2_SIGNPOST_END(device, lid, "run_loop", "Run loop completed. Transition handling state %d.", state.transitionHandling);
   auto& spec = ref.get<DeviceSpec const>();
   /// Cleanup messages which are still pending on exit.
   for (size_t ci = 0; ci < spec.inputChannels.size(); ++ci) {
@@ -1425,6 +1437,8 @@ void DataProcessingDevice::Run()
 void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
 {
   auto& context = ref.get<DataProcessorContext>();
+  O2_SIGNPOST_ID_FROM_POINTER(dpid, device, &context);
+  O2_SIGNPOST_START(device, dpid, "do_prepare", "Starting DataProcessorContext::doPrepare.");
 
   *context.wasActive = false;
   {
@@ -1451,7 +1465,7 @@ void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
   });
 
   // Whether or not all the channels are completed
-  LOGP(debug, "Processing {} input channels.", spec.inputChannels.size());
+  O2_SIGNPOST_EVENT_EMIT(device, dpid, "do_prepare", "Processing %zu input channels.", spec.inputChannels.size());
   /// Sort channels by oldest possible timeframe and
   /// process them in such order.
   static std::vector<int> pollOrder;
@@ -1463,13 +1477,14 @@ void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
 
   // Nothing to poll...
   if (pollOrder.empty()) {
+    O2_SIGNPOST_END(device, dpid, "do_prepare", "Nothing to poll. Waiting for next iteration.");
     return;
   }
   auto currentOldest = state.inputChannelInfos[pollOrder.front()].oldestForChannel;
   auto currentNewest = state.inputChannelInfos[pollOrder.back()].oldestForChannel;
   auto delta = currentNewest.value - currentOldest.value;
-  LOGP(debug, "oldest possible timeframe range {}, {} => {} delta", currentOldest.value, currentNewest.value,
-       delta);
+  O2_SIGNPOST_EVENT_EMIT(device, dpid, "do_prepare", "Oldest possible timeframe range %" PRIu64 " => %" PRIu64 " delta %" PRIu64,
+                         (int64_t)currentOldest.value, (int64_t)currentNewest.value, (int64_t)delta);
   auto& infos = state.inputChannelInfos;
 
   if (context.balancingInputs) {

From d8d64b7b6a4d3f1f281534b276fcf1edb0d3c0f7 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 16:49:57 +0100
Subject: [PATCH 437/726] DPL: introduce flag to signal that data should not be
 dropped at EoS

---
 DataFormats/Headers/include/Headers/DataHeader.h       |  5 +++--
 .../Core/include/Framework/DataProcessingHeader.h      |  3 +++
 Framework/Core/include/Framework/DeviceContext.h       | 10 ++++++++++
 Framework/Core/include/Framework/DeviceStateEnums.h    |  2 ++
 Framework/Core/include/Framework/TimingInfo.h          |  4 +++-
 Framework/Core/src/DataAllocator.cxx                   |  1 +
 Framework/Core/src/DataProcessingDevice.cxx            |  8 +++++++-
 Framework/Core/src/DeviceSpecHelpers.cxx               |  4 +++-
 Framework/Core/src/O2ControlHelpers.cxx                |  2 ++
 Framework/Core/src/runDataProcessing.cxx               |  2 ++
 10 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/DataFormats/Headers/include/Headers/DataHeader.h b/DataFormats/Headers/include/Headers/DataHeader.h
index c37eff9b34f20..e4ddaded20aba 100644
--- a/DataFormats/Headers/include/Headers/DataHeader.h
+++ b/DataFormats/Headers/include/Headers/DataHeader.h
@@ -372,8 +372,9 @@ struct BaseHeader {
   union {
     uint32_t flags;
     struct {
-      uint32_t flagsNextHeader : 1, // do we have a next header after this one?
-        flagsUnused : 31;           // currently unused
+      uint32_t flagsNextHeader : 1,        // do we have a next header after this one?
+               flagsReserved : 15,         // reserved for future use
+               flagsDerivedHeader : 16;    // reserved for usage by the derived header
     };
   };
 
diff --git a/Framework/Core/include/Framework/DataProcessingHeader.h b/Framework/Core/include/Framework/DataProcessingHeader.h
index 5c068b4e4179a..484dbb9d51a8e 100644
--- a/Framework/Core/include/Framework/DataProcessingHeader.h
+++ b/Framework/Core/include/Framework/DataProcessingHeader.h
@@ -42,6 +42,9 @@ namespace o2::framework
 /// @ingroup aliceo2_dataformats_dataheader
 struct DataProcessingHeader : public header::BaseHeader {
   static constexpr uint64_t DUMMY_CREATION_TIME_OFFSET = 0x8000000000000000;
+  // The following flags are used to indicate the behavior of the data processing
+  static constexpr int32_t KEEP_AT_EOS_FLAG = 1;
+
   /// We return some number of milliseconds, offsetting int by 0x8000000000000000
   /// to make sure we can understand when the dummy constructor of DataProcessingHeader was
   /// used without overriding it with an actual real time from epoch.
diff --git a/Framework/Core/include/Framework/DeviceContext.h b/Framework/Core/include/Framework/DeviceContext.h
index 3777e7f608b75..04cf663d5e276 100644
--- a/Framework/Core/include/Framework/DeviceContext.h
+++ b/Framework/Core/include/Framework/DeviceContext.h
@@ -28,8 +28,18 @@ struct ComputingQuotaStats;
 struct DeviceContext {
   ComputingQuotaStats* quotaStats = nullptr;
   uv_timer_t* gracePeriodTimer = nullptr;
+  uv_timer_t* dataProcessingGracePeriodTimer = nullptr;
   uv_signal_t* sigusr1Handle = nullptr;
   int expectedRegionCallbacks = 0;
+  // The timeout for the data processing to stop on this device.
+  // After this is reached, incoming data not marked to be kept will
+  // be dropped and the data processing will be stopped. However the 
+  // calibrations will still be done and objects resulting from calibrations
+  // will be marked to be kept.
+  int dataProcessingTimeout = 0;
+  // The timeout for the whole processing to stop on this device.
+  // This includes the grace period for processing and the time 
+  // for the calibrations to be done.
   int exitTransitionTimeout = 0;
 };
 
diff --git a/Framework/Core/include/Framework/DeviceStateEnums.h b/Framework/Core/include/Framework/DeviceStateEnums.h
index 291faac0ac982..a4c02c70c2bf6 100644
--- a/Framework/Core/include/Framework/DeviceStateEnums.h
+++ b/Framework/Core/include/Framework/DeviceStateEnums.h
@@ -30,6 +30,8 @@ enum struct TransitionHandlingState {
   NoTransition,
   /// A transition was notified to be requested
   Requested,
+  /// Only calibrations can be done
+  DataProcessingExpired,
   /// A transition needs to be fullfilled ASAP
   Expired
 };
diff --git a/Framework/Core/include/Framework/TimingInfo.h b/Framework/Core/include/Framework/TimingInfo.h
index 84f3971ad3a4a..1734da8dd3941 100644
--- a/Framework/Core/include/Framework/TimingInfo.h
+++ b/Framework/Core/include/Framework/TimingInfo.h
@@ -36,9 +36,11 @@ struct TimingInfo {
   /// from a new run, as being processed by the current stream.
   /// FIXME: for now this is the same as the above.
   bool streamRunNumberChanged = false;
+  /// Wether this kind of data should be flushed during end of stream.
+  bool keepAtEndOfStream = false;
 
   static bool timesliceIsTimer(size_t timeslice) { return timeslice > 1652945069870351; }
-  bool isTimer() const { return timesliceIsTimer(timeslice); };
+  [[nodiscard]] bool isTimer() const { return timesliceIsTimer(timeslice); };
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index fe38283d5e2de..b197f31c4f485 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -119,6 +119,7 @@ fair::mq::MessagePtr DataAllocator::headerMessageFromOutput(Output const& spec,
   dh.runNumber = timingInfo.runNumber;
 
   DataProcessingHeader dph{timingInfo.timeslice, 1, timingInfo.creation};
+  static_cast<o2::header::BaseHeader&>(dph).flagsDerivedHeader |= timingInfo.keepAtEndOfStream ? DataProcessingHeader::KEEP_AT_EOS_FLAG : 0;
   auto& proxy = mRegistry.get<FairMQDeviceProxy>();
   auto* transport = proxy.getOutputTransport(routeIndex);
 
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 70654bc43e84b..e0454a05094d5 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -957,6 +957,7 @@ void DataProcessingDevice::InitTask()
   }
 
   deviceContext.expectedRegionCallbacks = std::stoi(fConfig->GetValue<std::string>("expected-region-callbacks"));
+  deviceContext.dataProcessingTimeout = std::stoi(fConfig->GetValue<std::string>("data-processing-timeout"));
   deviceContext.exitTransitionTimeout = std::stoi(fConfig->GetValue<std::string>("exit-transition-timeout"));
 
   for (auto& channel : GetChannels()) {
@@ -1655,7 +1656,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
   }
 
   if (state.streaming == StreamingState::EndOfStreaming) {
-    O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "We are in EndOfStreaming. Flushing queues.");
+    O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "Flushing queues.");
     // We keep processing data until we are Idle.
     // FIXME: not sure this is the correct way to drain the queues, but
     // I guess we will see.
@@ -1666,6 +1667,11 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
     while (DataProcessingDevice::tryDispatchComputation(ref, context.completed) && hasOnlyGenerated == false) {
       relayer.processDanglingInputs(context.expirationHandlers, *context.registry, false);
     }
+
+    auto& timingInfo = ref.get<TimingInfo>();
+    // We should keep the data generated at end of stream only for those
+    // which are not sources.
+    timingInfo.keepAtEndOfStream = (hasOnlyGenerated == false);
     EndOfStreamContext eosContext{*context.registry, ref.get<DataAllocator>()};
 
     context.preEOSCallbacks(eosContext);
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 565f85e895f31..354c30b4781b2 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1495,6 +1495,7 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
         realOdesc.add_options()("child-driver", bpo::value<std::string>());
         realOdesc.add_options()("rate", bpo::value<std::string>());
         realOdesc.add_options()("exit-transition-timeout", bpo::value<std::string>());
+        realOdesc.add_options()("data-processing-timeout", bpo::value<std::string>());
         realOdesc.add_options()("expected-region-callbacks", bpo::value<std::string>());
         realOdesc.add_options()("timeframes-rate-limit", bpo::value<std::string>());
         realOdesc.add_options()("environment", bpo::value<std::string>());
@@ -1679,6 +1680,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
     ("control-port", bpo::value<std::string>(), "Utility port to be used by O2 Control")                                                                             //
     ("rate", bpo::value<std::string>(), "rate for a data source device (Hz)")                                                                                        //
     ("exit-transition-timeout", bpo::value<std::string>(), "timeout before switching to READY state")                                                                //
+    ("data-processing-timeout", bpo::value<std::string>(), "timeout before switching to calibration processing mode")                                                //
     ("expected-region-callbacks", bpo::value<std::string>(), "region callbacks to expect before starting")                                                           //
     ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframes can be in fly")                                                    //
     ("shm-monitor", bpo::value<std::string>(), "whether to use the shared memory monitor")                                                                           //
@@ -1709,7 +1711,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
     ("infologger-mode", bpo::value<std::string>(), "O2_INFOLOGGER_MODE override")                                                                                    //
     ("infologger-severity", bpo::value<std::string>(), "minimun FairLogger severity which goes to info logger")                                                      //
     ("dpl-tracing-flags", bpo::value<std::string>(), "pipe separated list of events to trace")                                                                       //
-    ("signposts", bpo::value<std::string>(),                                                                                                                         //
+    ("signposts", bpo::value<std::string>()->default_value(defaultSignposts),                                                                                        //
      "comma separated list of signposts to enable (any of `completion`, `data_processor_context`, `stream_context`, `device`, `monitoring_service`)")                //
     ("child-driver", bpo::value<std::string>(), "external driver to start childs with (e.g. valgrind)");                                                             //
 
diff --git a/Framework/Core/src/O2ControlHelpers.cxx b/Framework/Core/src/O2ControlHelpers.cxx
index 88132572a6210..e19e4a995d505 100644
--- a/Framework/Core/src/O2ControlHelpers.cxx
+++ b/Framework/Core/src/O2ControlHelpers.cxx
@@ -261,6 +261,8 @@ void dumpCommand(std::ostream& dumpOut, const DeviceExecution& execution, std::s
   dumpOut << indLevel << indScheme << "- \"-b\"\n";
   dumpOut << indLevel << indScheme << "- \"--exit-transition-timeout\"\n";
   dumpOut << indLevel << indScheme << "- \"'{{ exit_transition_timeout }}'\"\n";
+  dumpOut << indLevel << indScheme << "- \"--data-processing-timeout\"\n";
+  dumpOut << indLevel << indScheme << "- \"'{{ data_processing_timeout }}'\"\n";
   dumpOut << indLevel << indScheme << "- \"--monitoring-backend\"\n";
   dumpOut << indLevel << indScheme << "- \"'{{ monitoring_dpl_url }}'\"\n";
   dumpOut << indLevel << indScheme << "- \"--session\"\n";
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index c72b4da73a45a..6be625743c6d6 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1034,6 +1034,7 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
   // declared in the workflow definition are allowed.
   runner.AddHook<fair::mq::hooks::SetCustomCmdLineOptions>([&spec, driverConfig, defaultDriverClient](fair::mq::DeviceRunner& r) {
     std::string defaultExitTransitionTimeout = "0";
+    std::string defaultDataProcessingTimeout = "0";
     std::string defaultInfologgerMode = "";
     o2::framework::DeploymentMode deploymentMode = o2::framework::DefaultsHelpers::deploymentMode();
     if (deploymentMode == o2::framework::DeploymentMode::OnlineDDS) {
@@ -1052,6 +1053,7 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
       ("signposts", bpo::value<std::string>()->default_value(defaultSignposts ? defaultSignposts : ""), "comma separated list of signposts to enable")                                     //
       ("expected-region-callbacks", bpo::value<std::string>()->default_value("0"), "how many region callbacks we are expecting")                                                           //
       ("exit-transition-timeout", bpo::value<std::string>()->default_value(defaultExitTransitionTimeout), "how many second to wait before switching from RUN to READY")                    //
+      ("data-processing-timeout", bpo::value<std::string>()->default_value(defaultDataProcessingTimeout), "how many second to wait before switching from RUN to CALIBRATION")              //
       ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframe can be in fly at the same moment (0 disables)")                                         //
       ("configuration,cfg", bpo::value<std::string>()->default_value("command-line"), "configuration backend")                                                                             //
       ("infologger-mode", bpo::value<std::string>()->default_value(defaultInfologgerMode), "O2_INFOLOGGER_MODE override");

From b31dd6e6a5169838b65e8c4e9a98424f77bd337c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 16:49:57 +0100
Subject: [PATCH 438/726] DPL: move DataProcessingDevice to use signposts
 entirely

---
 Framework/Core/src/DataProcessingDevice.cxx | 55 ++++++++++++++-------
 1 file changed, 36 insertions(+), 19 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 8d79567d56ed5..70654bc43e84b 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -119,7 +119,8 @@ void on_transition_requested_expired(uv_timer_t* handle)
 {
   auto* state = (DeviceState*)handle->data;
   state->loopReason |= DeviceState::TIMER_EXPIRED;
-  LOGP(info, "Timer expired. Forcing transition to READY");
+  O2_SIGNPOST_ID_FROM_POINTER(cid, device, handle);
+  O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "callback", "Grace period for calibration expired. Exiting.");
   state->transitionHandling = TransitionHandlingState::Expired;
 }
 
@@ -1455,16 +1456,18 @@ void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
   // real data input channels, they have to signal EndOfStream themselves.
   auto& state = ref.get<DeviceState>();
   auto& spec = ref.get<DeviceSpec const>();
-  context.allDone = std::any_of(state.inputChannelInfos.begin(), state.inputChannelInfos.end(), [](const auto& info) {
+  O2_SIGNPOST_ID_FROM_POINTER(cid, device, state.inputChannelInfos.data());
+  O2_SIGNPOST_START(device, cid, "do_prepare", "Reported channel states.");
+  context.allDone = std::any_of(state.inputChannelInfos.begin(), state.inputChannelInfos.end(), [cid](const auto& info) {
     if (info.channel) {
-      LOGP(debug, "Input channel {}{} has {} parts left and is in state {}.", info.channel->GetName(), (info.id.value == ChannelIndex::INVALID ? " (non DPL)" : ""), info.parts.fParts.size(), (int)info.state);
+      O2_SIGNPOST_EVENT_EMIT(device, cid, "do_prepare", "Input channel %{public}s%{public}s has %zu parts left and is in state %d.",
+                             info.channel->GetName().c_str(), (info.id.value == ChannelIndex::INVALID ? " (non DPL)" : ""), info.parts.fParts.size(), (int)info.state);
     } else {
-      LOGP(debug, "External channel {} is in state {}.", info.id.value, (int)info.state);
+      O2_SIGNPOST_EVENT_EMIT(device, cid, "do_prepare", "External channel %d is in state %d.", info.id.value, (int)info.state);
     }
     return (info.parts.fParts.empty() == true && info.state != InputChannelState::Pull);
   });
-
-  // Whether or not all the channels are completed
+  O2_SIGNPOST_END(device, cid, "do_prepare", "End report.");
   O2_SIGNPOST_EVENT_EMIT(device, dpid, "do_prepare", "Processing %zu input channels.", spec.inputChannels.size());
   /// Sort channels by oldest possible timeframe and
   /// process them in such order.
@@ -1510,12 +1513,13 @@ void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
     }
     pollOrder.erase(newEnd, pollOrder.end());
   }
-  LOGP(debug, "processing {} channels", pollOrder.size());
+  O2_SIGNPOST_END(device, dpid, "do_prepare", "%zu channels pass the channel inbalance balance check.", pollOrder.size());
 
   for (auto sci : pollOrder) {
     auto& info = state.inputChannelInfos[sci];
     auto& channelSpec = spec.inputChannels[sci];
-    LOGP(debug, "Processing channel {}", channelSpec.name);
+    O2_SIGNPOST_ID_FROM_POINTER(cid, device, &info);
+    O2_SIGNPOST_START(device, cid, "channels", "Processing channel %s", channelSpec.name.c_str());
 
     if (info.state != InputChannelState::Completed && info.state != InputChannelState::Pull) {
       context.allDone = false;
@@ -1526,14 +1530,19 @@ void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
       if (info.parts.Size()) {
         DataProcessingDevice::handleData(ref, info);
       }
-      LOGP(debug, "Flushing channel {} which is in state {} and has {} parts still pending.", channelSpec.name, (int)info.state, info.parts.Size());
+      O2_SIGNPOST_END(device, cid, "channels", "Flushing channel %s which is in state %d and has %zu parts still pending.",
+                      channelSpec.name.c_str(), (int)info.state, info.parts.Size());
       continue;
     }
     if (info.channel == nullptr) {
+      O2_SIGNPOST_END(device, cid, "channels", "Channel %s which is in state %d is nullptr and has %zu parts still pending.",
+                      channelSpec.name.c_str(), (int)info.state, info.parts.Size());
       continue;
     }
     // Only poll DPL channels for now.
     if (info.channelType != ChannelAccountingType::DPL) {
+      O2_SIGNPOST_END(device, cid, "channels", "Channel %s which is in state %d is not a DPL channel and has %zu parts still pending.",
+                      channelSpec.name.c_str(), (int)info.state, info.parts.Size());
       continue;
     }
     auto& socket = info.channel->GetSocket();
@@ -1545,7 +1554,7 @@ void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
       socket.Events(&info.hasPendingEvents);
       // If we do not read, we can continue.
       if ((info.hasPendingEvents & 1) == 0 && (info.parts.Size() == 0)) {
-        LOGP(debug, "No pending events and no remaining parts to process for channel {}", channelSpec.name);
+        O2_SIGNPOST_END(device, cid, "channels", "No pending events and no remaining parts to process for channel %{public}s", channelSpec.name.c_str());
         continue;
       }
     }
@@ -1562,13 +1571,13 @@ void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
     // to process.
     bool newMessages = false;
     while (true) {
-      LOGP(debug, "Receiving loop called for channel {} ({}) with oldest possible timeslice {}",
-           info.channel->GetName(), info.id.value, info.oldestForChannel.value);
+      O2_SIGNPOST_EVENT_EMIT(device, cid, "channels", "Receiving loop called for channel %{public}s (%d) with oldest possible timeslice %zu",
+                             channelSpec.name.c_str(), info.id.value, info.oldestForChannel.value);
       if (info.parts.Size() < 64) {
         fair::mq::Parts parts;
         info.channel->Receive(parts, 0);
         if (parts.Size()) {
-          LOGP(debug, "Receiving some parts {}", parts.Size());
+          O2_SIGNPOST_EVENT_EMIT(device, cid, "channels", "Received %zu parts from channel %{public}s (%d).", parts.Size(), channelSpec.name.c_str(), info.id.value);
         }
         for (auto&& part : parts) {
           info.parts.fParts.emplace_back(std::move(part));
@@ -1593,6 +1602,8 @@ void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
       info.readPolled = false;
       *context.wasActive |= newMessages;
     }
+    O2_SIGNPOST_END(device, cid, "channels", "Done processing channel %{public}s (%d).",
+                    channelSpec.name.c_str(), info.id.value);
   }
 }
 
@@ -1727,6 +1738,10 @@ struct WaitBackpressurePolicy {
 void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo& info)
 {
   auto& context = ref.get<DataProcessorContext>();
+  // This is the same id as the upper level function, so we get the events
+  // associated with the same interval. We will simply use "handle_data" as
+  // the category.
+  O2_SIGNPOST_ID_FROM_POINTER(cid, device, &info);
 
   enum struct InputType : int {
     Invalid = 0,
@@ -1750,6 +1765,7 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
   // than an input, because we do not want the outer loop actually be exposed
   // to the implementation details of the messaging layer.
   auto getInputTypes = [&info, &context]() -> std::optional<std::vector<InputInfo>> {
+    O2_SIGNPOST_ID_FROM_POINTER(cid, device, &info);
     auto ref = ServiceRegistryRef{*context.registry};
     auto& stats = ref.get<DataProcessingStats>();
     auto& parts = info.parts;
@@ -1771,7 +1787,7 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
       auto* headerData = parts.At(pi)->GetData();
       auto sih = o2::header::get<SourceInfoHeader*>(headerData);
       if (sih) {
-        LOGP(debug, "Got SourceInfoHeader with state {}", (int)sih->state);
+        O2_SIGNPOST_EVENT_EMIT(device, cid, "handle_data", "Got SourceInfoHeader with state %d", (int)sih->state);
         info.state = sih->state;
         insertInputInfo(pi, 2, InputType::SourceInfo);
         *context.wasActive = true;
@@ -1786,12 +1802,12 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
       auto dh = o2::header::get<DataHeader*>(headerData);
       if (!dh) {
         insertInputInfo(pi, 0, InputType::Invalid);
-        LOGP(error, "Header is not a DataHeader?");
+        O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "Header is not a DataHeader?");
         continue;
       }
       if (dh->payloadSize > parts.At(pi + 1)->GetSize()) {
         insertInputInfo(pi, 0, InputType::Invalid);
-        LOGP(error, "DataHeader payloadSize mismatch");
+        O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "DataHeader payloadSize mismatch");
         continue;
       }
       auto dph = o2::header::get<DataProcessingHeader*>(headerData);
@@ -1804,7 +1820,7 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
       }
       if (!dph) {
         insertInputInfo(pi, 2, InputType::Invalid);
-        LOGP(error, "Header stack does not contain DataProcessingHeader");
+        O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "Header stack does not contain DataProcessingHeader");
         continue;
       }
       if (dh->splitPayloadParts > 0 && dh->splitPayloadParts == dh->splitPayloadIndex) {
@@ -1819,7 +1835,7 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
         // pair.
         size_t finalSplitPayloadIndex = pi + (dh->splitPayloadParts > 0 ? dh->splitPayloadParts : 1) * 2;
         if (finalSplitPayloadIndex > parts.Size()) {
-          LOGP(error, "DataHeader::splitPayloadParts invalid");
+          O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "DataHeader::splitPayloadParts invalid");
           insertInputInfo(pi, 0, InputType::Invalid);
           continue;
         }
@@ -1830,7 +1846,7 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
       }
     }
     if (results.size() + nTotalPayloads != parts.Size()) {
-      LOG(error) << "inconsistent number of inputs extracted";
+      O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "inconsistent number of inputs extracted. %zu vs parts (%zu)", results.size() + nTotalPayloads, parts.Size());
       return std::nullopt;
     }
     return results;
@@ -2448,6 +2464,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
     if (action.op == CompletionPolicy::CompletionOp::Process) {
       cleanTimers(action.slot, record);
     }
+    O2_SIGNPOST_END(device, aid, "device", "Done processing action on slot %lu for action %{public}s", action.slot.index, fmt::format("{}", action.op).c_str());
   }
   O2_SIGNPOST_END(device, sid, "device", "Start processing ready actions");
 

From eae4a88a4d834b22804f38834e989da634b8779c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 16:49:57 +0100
Subject: [PATCH 439/726] DPL: attempt to enable calibration mode

---
 Framework/Core/src/DataProcessingDevice.cxx | 25 ++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index e0454a05094d5..436b48f80a8be 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -124,6 +124,15 @@ void on_transition_requested_expired(uv_timer_t* handle)
   state->transitionHandling = TransitionHandlingState::Expired;
 }
 
+void on_data_processing_grace_expired(uv_timer_t* handle)
+{
+  auto* state = (DeviceState*)handle->data;
+  state->loopReason |= DeviceState::TIMER_EXPIRED;
+  O2_SIGNPOST_ID_FROM_POINTER(cid, device, handle);
+  O2_SIGNPOST_EVENT_EMIT_INFO(device, cid, "callback", "Grace period for Data Processing expired. Waiting for calibration");
+  state->transitionHandling = TransitionHandlingState::DataProcessingExpired;
+}
+
 void on_communication_requested(uv_async_t* s)
 {
   auto* state = (DeviceState*)s->data;
@@ -1252,7 +1261,6 @@ void DataProcessingDevice::Run()
       if (state.transitionHandling == TransitionHandlingState::NoTransition && NewStatePending()) {
         state.transitionHandling = TransitionHandlingState::Requested;
         auto& deviceContext = ref.get<DeviceContext>();
-        auto timeout = deviceContext.exitTransitionTimeout;
         // Check if we only have timers
         bool onlyTimers = true;
         auto& spec = ref.get<DeviceSpec const>();
@@ -1265,11 +1273,18 @@ void DataProcessingDevice::Run()
         if (onlyTimers) {
           state.streaming = StreamingState::EndOfStreaming;
         }
-        if (timeout != 0 && state.streaming != StreamingState::Idle) {
+
+        if (deviceContext.exitTransitionTimeout != 0 && state.streaming != StreamingState::Idle) {
           state.transitionHandling = TransitionHandlingState::Requested;
           ref.get<CallbackService>().call<CallbackService::Id::ExitRequested>(ServiceRegistryRef{ref});
           uv_update_time(state.loop);
-          uv_timer_start(deviceContext.gracePeriodTimer, on_transition_requested_expired, timeout * 1000, 0);
+          uv_timer_start(deviceContext.gracePeriodTimer, on_transition_requested_expired, deviceContext.exitTransitionTimeout * 1000, 0);
+          // In case we have a calibration grace period it will always be longer than the data processing timeout
+          if (deviceContext.dataProcessingTimeout != deviceContext.exitTransitionTimeout) {
+            uv_timer_start(deviceContext.dataProcessingGracePeriodTimer, on_data_processing_grace_expired, deviceContext.dataProcessingTimeout * 1000, 0);
+          } else {
+            deviceContext.dataProcessingGracePeriodTimer = nullptr;
+          }
           if (mProcessingPolicies.termination == TerminationPolicy::QUIT) {
             O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state requested. Waiting for %d seconds before quitting.", (int)deviceContext.exitTransitionTimeout);
           } else {
@@ -1288,8 +1303,8 @@ void DataProcessingDevice::Run()
           }
         }
       }
-      // If we are Idle, we can then consider the transition to be expired.
-      if (state.transitionHandling == TransitionHandlingState::Requested && state.streaming == StreamingState::Idle) {
+      // If we are Idle, we can then consider the transition to be expired when it was requested or when the data processing timeout expired.
+      if ((state.transitionHandling == TransitionHandlingState::Requested || state.transitionHandling == TransitionHandlingState::DataProcessingExpired) && state.streaming == StreamingState::Idle) {
         O2_SIGNPOST_EVENT_EMIT(device, lid, "run_loop", "State transition requested and we are now in Idle. We can consider it to be completed.");
         state.transitionHandling = TransitionHandlingState::Expired;
       }

From 205e6eb66d9c5827c088a5b0c306f5c5e656fa67 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 16:49:57 +0100
Subject: [PATCH 440/726] Fix documentation

---
 Framework/Core/COOKBOOK.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Framework/Core/COOKBOOK.md b/Framework/Core/COOKBOOK.md
index d92006e63d650..c327651ae53ca 100644
--- a/Framework/Core/COOKBOOK.md
+++ b/Framework/Core/COOKBOOK.md
@@ -524,8 +524,7 @@ Sometimes (e.g. when running a child inside valgrind) it might be useful to disa
 some-workflow --monitoring-backend=no-op://
 ```
 
-notice that the 
-will not function properly if you do so.
+notice that the will not function properly if you do so.
 
 ## Profiling
 

From 929fd63f3a46b88eb8ec5c86934f90857511231f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 16:50:06 +0100
Subject: [PATCH 441/726] [DRAFT] DPL: Introduce calibration mode

---
 Framework/CHANGELOG.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Framework/CHANGELOG.md b/Framework/CHANGELOG.md
index 684c9d3c3f1bd..818e7fc35e938 100644
--- a/Framework/CHANGELOG.md
+++ b/Framework/CHANGELOG.md
@@ -1,4 +1,10 @@
 # 2024-02-22: Move DataProcessingDevice to use Signposts
+# 2024-02-24: Introduce calibration mode
+
+This introduces a new --data-processing-timeout `<timeout>` option which can be used to specify
+that data processing should finish after `<timeout>` seconds. After it, only the messages which are produced on EndOfStream will be send.
+
+# 2024-02-23: Move DataProcessingDevice to use Signposts
 
 All the messages from DataProcessingDevice have been migrated to use Signpost.
 This will hopefully simplify debugging.

From 57106c4a9b940a70c76ab5aec3bb88f3236a5a7e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 16:56:58 +0100
Subject: [PATCH 442/726] [FEAT] DPL: use Signposts everywhere in
 DataProcessingDevice

---
 Framework/CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Framework/CHANGELOG.md b/Framework/CHANGELOG.md
index 044274d9515c4..331805f8538fd 100644
--- a/Framework/CHANGELOG.md
+++ b/Framework/CHANGELOG.md
@@ -1,3 +1,8 @@
+# 2024-03-14: Move DataProcessingDevice to use Signposts
+
+All the messages from DataProcessingDevice have been migrated to use Signpost.
+This will hopefully simplify debugging.
+
 # 2024-02-22: Drop Tracy support
 
 Tracy support never took off, so I am dropping it. This was mostly because people do not know about it and having a per process profile GUI was way unpractical. Moreover, needing an extra compile time flag meant one most likely did not have the support compiled in when needed.

From 842b81563782e49da6ec278051307d211b2abf75 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 16:57:06 +0100
Subject: [PATCH 443/726] DPL: introduce flag to signal that data should not be
 dropped at EoS

---
 DataFormats/Headers/include/Headers/DataHeader.h       |  5 +++--
 .../Core/include/Framework/DataProcessingHeader.h      |  3 +++
 Framework/Core/include/Framework/DeviceContext.h       | 10 ++++++++++
 Framework/Core/include/Framework/DeviceStateEnums.h    |  2 ++
 Framework/Core/include/Framework/TimingInfo.h          |  4 +++-
 Framework/Core/src/DataAllocator.cxx                   |  1 +
 Framework/Core/src/DataProcessingDevice.cxx            |  8 +++++++-
 Framework/Core/src/DeviceSpecHelpers.cxx               |  4 +++-
 Framework/Core/src/O2ControlHelpers.cxx                |  2 ++
 Framework/Core/src/runDataProcessing.cxx               |  2 ++
 10 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/DataFormats/Headers/include/Headers/DataHeader.h b/DataFormats/Headers/include/Headers/DataHeader.h
index c37eff9b34f20..e4ddaded20aba 100644
--- a/DataFormats/Headers/include/Headers/DataHeader.h
+++ b/DataFormats/Headers/include/Headers/DataHeader.h
@@ -372,8 +372,9 @@ struct BaseHeader {
   union {
     uint32_t flags;
     struct {
-      uint32_t flagsNextHeader : 1, // do we have a next header after this one?
-        flagsUnused : 31;           // currently unused
+      uint32_t flagsNextHeader : 1,        // do we have a next header after this one?
+               flagsReserved : 15,         // reserved for future use
+               flagsDerivedHeader : 16;    // reserved for usage by the derived header
     };
   };
 
diff --git a/Framework/Core/include/Framework/DataProcessingHeader.h b/Framework/Core/include/Framework/DataProcessingHeader.h
index 5c068b4e4179a..484dbb9d51a8e 100644
--- a/Framework/Core/include/Framework/DataProcessingHeader.h
+++ b/Framework/Core/include/Framework/DataProcessingHeader.h
@@ -42,6 +42,9 @@ namespace o2::framework
 /// @ingroup aliceo2_dataformats_dataheader
 struct DataProcessingHeader : public header::BaseHeader {
   static constexpr uint64_t DUMMY_CREATION_TIME_OFFSET = 0x8000000000000000;
+  // The following flags are used to indicate the behavior of the data processing
+  static constexpr int32_t KEEP_AT_EOS_FLAG = 1;
+
   /// We return some number of milliseconds, offsetting int by 0x8000000000000000
   /// to make sure we can understand when the dummy constructor of DataProcessingHeader was
   /// used without overriding it with an actual real time from epoch.
diff --git a/Framework/Core/include/Framework/DeviceContext.h b/Framework/Core/include/Framework/DeviceContext.h
index 3777e7f608b75..04cf663d5e276 100644
--- a/Framework/Core/include/Framework/DeviceContext.h
+++ b/Framework/Core/include/Framework/DeviceContext.h
@@ -28,8 +28,18 @@ struct ComputingQuotaStats;
 struct DeviceContext {
   ComputingQuotaStats* quotaStats = nullptr;
   uv_timer_t* gracePeriodTimer = nullptr;
+  uv_timer_t* dataProcessingGracePeriodTimer = nullptr;
   uv_signal_t* sigusr1Handle = nullptr;
   int expectedRegionCallbacks = 0;
+  // The timeout for the data processing to stop on this device.
+  // After this is reached, incoming data not marked to be kept will
+  // be dropped and the data processing will be stopped. However the 
+  // calibrations will still be done and objects resulting from calibrations
+  // will be marked to be kept.
+  int dataProcessingTimeout = 0;
+  // The timeout for the whole processing to stop on this device.
+  // This includes the grace period for processing and the time 
+  // for the calibrations to be done.
   int exitTransitionTimeout = 0;
 };
 
diff --git a/Framework/Core/include/Framework/DeviceStateEnums.h b/Framework/Core/include/Framework/DeviceStateEnums.h
index 291faac0ac982..a4c02c70c2bf6 100644
--- a/Framework/Core/include/Framework/DeviceStateEnums.h
+++ b/Framework/Core/include/Framework/DeviceStateEnums.h
@@ -30,6 +30,8 @@ enum struct TransitionHandlingState {
   NoTransition,
   /// A transition was notified to be requested
   Requested,
+  /// Only calibrations can be done
+  DataProcessingExpired,
   /// A transition needs to be fullfilled ASAP
   Expired
 };
diff --git a/Framework/Core/include/Framework/TimingInfo.h b/Framework/Core/include/Framework/TimingInfo.h
index 84f3971ad3a4a..1734da8dd3941 100644
--- a/Framework/Core/include/Framework/TimingInfo.h
+++ b/Framework/Core/include/Framework/TimingInfo.h
@@ -36,9 +36,11 @@ struct TimingInfo {
   /// from a new run, as being processed by the current stream.
   /// FIXME: for now this is the same as the above.
   bool streamRunNumberChanged = false;
+  /// Wether this kind of data should be flushed during end of stream.
+  bool keepAtEndOfStream = false;
 
   static bool timesliceIsTimer(size_t timeslice) { return timeslice > 1652945069870351; }
-  bool isTimer() const { return timesliceIsTimer(timeslice); };
+  [[nodiscard]] bool isTimer() const { return timesliceIsTimer(timeslice); };
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index fe38283d5e2de..b197f31c4f485 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -119,6 +119,7 @@ fair::mq::MessagePtr DataAllocator::headerMessageFromOutput(Output const& spec,
   dh.runNumber = timingInfo.runNumber;
 
   DataProcessingHeader dph{timingInfo.timeslice, 1, timingInfo.creation};
+  static_cast<o2::header::BaseHeader&>(dph).flagsDerivedHeader |= timingInfo.keepAtEndOfStream ? DataProcessingHeader::KEEP_AT_EOS_FLAG : 0;
   auto& proxy = mRegistry.get<FairMQDeviceProxy>();
   auto* transport = proxy.getOutputTransport(routeIndex);
 
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 70654bc43e84b..e0454a05094d5 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -957,6 +957,7 @@ void DataProcessingDevice::InitTask()
   }
 
   deviceContext.expectedRegionCallbacks = std::stoi(fConfig->GetValue<std::string>("expected-region-callbacks"));
+  deviceContext.dataProcessingTimeout = std::stoi(fConfig->GetValue<std::string>("data-processing-timeout"));
   deviceContext.exitTransitionTimeout = std::stoi(fConfig->GetValue<std::string>("exit-transition-timeout"));
 
   for (auto& channel : GetChannels()) {
@@ -1655,7 +1656,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
   }
 
   if (state.streaming == StreamingState::EndOfStreaming) {
-    O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "We are in EndOfStreaming. Flushing queues.");
+    O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "Flushing queues.");
     // We keep processing data until we are Idle.
     // FIXME: not sure this is the correct way to drain the queues, but
     // I guess we will see.
@@ -1666,6 +1667,11 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
     while (DataProcessingDevice::tryDispatchComputation(ref, context.completed) && hasOnlyGenerated == false) {
       relayer.processDanglingInputs(context.expirationHandlers, *context.registry, false);
     }
+
+    auto& timingInfo = ref.get<TimingInfo>();
+    // We should keep the data generated at end of stream only for those
+    // which are not sources.
+    timingInfo.keepAtEndOfStream = (hasOnlyGenerated == false);
     EndOfStreamContext eosContext{*context.registry, ref.get<DataAllocator>()};
 
     context.preEOSCallbacks(eosContext);
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 565f85e895f31..354c30b4781b2 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1495,6 +1495,7 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
         realOdesc.add_options()("child-driver", bpo::value<std::string>());
         realOdesc.add_options()("rate", bpo::value<std::string>());
         realOdesc.add_options()("exit-transition-timeout", bpo::value<std::string>());
+        realOdesc.add_options()("data-processing-timeout", bpo::value<std::string>());
         realOdesc.add_options()("expected-region-callbacks", bpo::value<std::string>());
         realOdesc.add_options()("timeframes-rate-limit", bpo::value<std::string>());
         realOdesc.add_options()("environment", bpo::value<std::string>());
@@ -1679,6 +1680,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
     ("control-port", bpo::value<std::string>(), "Utility port to be used by O2 Control")                                                                             //
     ("rate", bpo::value<std::string>(), "rate for a data source device (Hz)")                                                                                        //
     ("exit-transition-timeout", bpo::value<std::string>(), "timeout before switching to READY state")                                                                //
+    ("data-processing-timeout", bpo::value<std::string>(), "timeout before switching to calibration processing mode")                                                //
     ("expected-region-callbacks", bpo::value<std::string>(), "region callbacks to expect before starting")                                                           //
     ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframes can be in fly")                                                    //
     ("shm-monitor", bpo::value<std::string>(), "whether to use the shared memory monitor")                                                                           //
@@ -1709,7 +1711,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
     ("infologger-mode", bpo::value<std::string>(), "O2_INFOLOGGER_MODE override")                                                                                    //
     ("infologger-severity", bpo::value<std::string>(), "minimun FairLogger severity which goes to info logger")                                                      //
     ("dpl-tracing-flags", bpo::value<std::string>(), "pipe separated list of events to trace")                                                                       //
-    ("signposts", bpo::value<std::string>(),                                                                                                                         //
+    ("signposts", bpo::value<std::string>()->default_value(defaultSignposts),                                                                                        //
      "comma separated list of signposts to enable (any of `completion`, `data_processor_context`, `stream_context`, `device`, `monitoring_service`)")                //
     ("child-driver", bpo::value<std::string>(), "external driver to start childs with (e.g. valgrind)");                                                             //
 
diff --git a/Framework/Core/src/O2ControlHelpers.cxx b/Framework/Core/src/O2ControlHelpers.cxx
index 88132572a6210..e19e4a995d505 100644
--- a/Framework/Core/src/O2ControlHelpers.cxx
+++ b/Framework/Core/src/O2ControlHelpers.cxx
@@ -261,6 +261,8 @@ void dumpCommand(std::ostream& dumpOut, const DeviceExecution& execution, std::s
   dumpOut << indLevel << indScheme << "- \"-b\"\n";
   dumpOut << indLevel << indScheme << "- \"--exit-transition-timeout\"\n";
   dumpOut << indLevel << indScheme << "- \"'{{ exit_transition_timeout }}'\"\n";
+  dumpOut << indLevel << indScheme << "- \"--data-processing-timeout\"\n";
+  dumpOut << indLevel << indScheme << "- \"'{{ data_processing_timeout }}'\"\n";
   dumpOut << indLevel << indScheme << "- \"--monitoring-backend\"\n";
   dumpOut << indLevel << indScheme << "- \"'{{ monitoring_dpl_url }}'\"\n";
   dumpOut << indLevel << indScheme << "- \"--session\"\n";
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index c72b4da73a45a..6be625743c6d6 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1034,6 +1034,7 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
   // declared in the workflow definition are allowed.
   runner.AddHook<fair::mq::hooks::SetCustomCmdLineOptions>([&spec, driverConfig, defaultDriverClient](fair::mq::DeviceRunner& r) {
     std::string defaultExitTransitionTimeout = "0";
+    std::string defaultDataProcessingTimeout = "0";
     std::string defaultInfologgerMode = "";
     o2::framework::DeploymentMode deploymentMode = o2::framework::DefaultsHelpers::deploymentMode();
     if (deploymentMode == o2::framework::DeploymentMode::OnlineDDS) {
@@ -1052,6 +1053,7 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
       ("signposts", bpo::value<std::string>()->default_value(defaultSignposts ? defaultSignposts : ""), "comma separated list of signposts to enable")                                     //
       ("expected-region-callbacks", bpo::value<std::string>()->default_value("0"), "how many region callbacks we are expecting")                                                           //
       ("exit-transition-timeout", bpo::value<std::string>()->default_value(defaultExitTransitionTimeout), "how many second to wait before switching from RUN to READY")                    //
+      ("data-processing-timeout", bpo::value<std::string>()->default_value(defaultDataProcessingTimeout), "how many second to wait before switching from RUN to CALIBRATION")              //
       ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframe can be in fly at the same moment (0 disables)")                                         //
       ("configuration,cfg", bpo::value<std::string>()->default_value("command-line"), "configuration backend")                                                                             //
       ("infologger-mode", bpo::value<std::string>()->default_value(defaultInfologgerMode), "O2_INFOLOGGER_MODE override");

From 2d49d41a88be10dd88fc932c4d038fc634254679 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 16:57:06 +0100
Subject: [PATCH 444/726] DPL: attempt to enable calibration mode

---
 Framework/Core/src/DataProcessingDevice.cxx | 25 ++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index e0454a05094d5..436b48f80a8be 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -124,6 +124,15 @@ void on_transition_requested_expired(uv_timer_t* handle)
   state->transitionHandling = TransitionHandlingState::Expired;
 }
 
+void on_data_processing_grace_expired(uv_timer_t* handle)
+{
+  auto* state = (DeviceState*)handle->data;
+  state->loopReason |= DeviceState::TIMER_EXPIRED;
+  O2_SIGNPOST_ID_FROM_POINTER(cid, device, handle);
+  O2_SIGNPOST_EVENT_EMIT_INFO(device, cid, "callback", "Grace period for Data Processing expired. Waiting for calibration");
+  state->transitionHandling = TransitionHandlingState::DataProcessingExpired;
+}
+
 void on_communication_requested(uv_async_t* s)
 {
   auto* state = (DeviceState*)s->data;
@@ -1252,7 +1261,6 @@ void DataProcessingDevice::Run()
       if (state.transitionHandling == TransitionHandlingState::NoTransition && NewStatePending()) {
         state.transitionHandling = TransitionHandlingState::Requested;
         auto& deviceContext = ref.get<DeviceContext>();
-        auto timeout = deviceContext.exitTransitionTimeout;
         // Check if we only have timers
         bool onlyTimers = true;
         auto& spec = ref.get<DeviceSpec const>();
@@ -1265,11 +1273,18 @@ void DataProcessingDevice::Run()
         if (onlyTimers) {
           state.streaming = StreamingState::EndOfStreaming;
         }
-        if (timeout != 0 && state.streaming != StreamingState::Idle) {
+
+        if (deviceContext.exitTransitionTimeout != 0 && state.streaming != StreamingState::Idle) {
           state.transitionHandling = TransitionHandlingState::Requested;
           ref.get<CallbackService>().call<CallbackService::Id::ExitRequested>(ServiceRegistryRef{ref});
           uv_update_time(state.loop);
-          uv_timer_start(deviceContext.gracePeriodTimer, on_transition_requested_expired, timeout * 1000, 0);
+          uv_timer_start(deviceContext.gracePeriodTimer, on_transition_requested_expired, deviceContext.exitTransitionTimeout * 1000, 0);
+          // In case we have a calibration grace period it will always be longer than the data processing timeout
+          if (deviceContext.dataProcessingTimeout != deviceContext.exitTransitionTimeout) {
+            uv_timer_start(deviceContext.dataProcessingGracePeriodTimer, on_data_processing_grace_expired, deviceContext.dataProcessingTimeout * 1000, 0);
+          } else {
+            deviceContext.dataProcessingGracePeriodTimer = nullptr;
+          }
           if (mProcessingPolicies.termination == TerminationPolicy::QUIT) {
             O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state requested. Waiting for %d seconds before quitting.", (int)deviceContext.exitTransitionTimeout);
           } else {
@@ -1288,8 +1303,8 @@ void DataProcessingDevice::Run()
           }
         }
       }
-      // If we are Idle, we can then consider the transition to be expired.
-      if (state.transitionHandling == TransitionHandlingState::Requested && state.streaming == StreamingState::Idle) {
+      // If we are Idle, we can then consider the transition to be expired when it was requested or when the data processing timeout expired.
+      if ((state.transitionHandling == TransitionHandlingState::Requested || state.transitionHandling == TransitionHandlingState::DataProcessingExpired) && state.streaming == StreamingState::Idle) {
         O2_SIGNPOST_EVENT_EMIT(device, lid, "run_loop", "State transition requested and we are now in Idle. We can consider it to be completed.");
         state.transitionHandling = TransitionHandlingState::Expired;
       }

From 52c52fb1fe0848a9e8172d7e2f536391c91f6333 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 16:57:06 +0100
Subject: [PATCH 445/726] [DRAFT] DPL: Introduce calibration mode

---
 Framework/CHANGELOG.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Framework/CHANGELOG.md b/Framework/CHANGELOG.md
index 331805f8538fd..c74155d62c6e2 100644
--- a/Framework/CHANGELOG.md
+++ b/Framework/CHANGELOG.md
@@ -1,4 +1,10 @@
 # 2024-03-14: Move DataProcessingDevice to use Signposts
+# 2024-03-14: Introduce calibration mode
+
+This introduces a new --data-processing-timeout `<timeout>` option which can be used to specify
+that data processing should finish after `<timeout>` seconds. After it, only the messages which are produced on EndOfStream will be send.
+
+# 2024-02-23: Move DataProcessingDevice to use Signposts
 
 All the messages from DataProcessingDevice have been migrated to use Signpost.
 This will hopefully simplify debugging.

From 3a4af0418a406f31824d535794a8e5eb9f8a2cf7 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 16:57:06 +0100
Subject: [PATCH 446/726] [DRAFT] DPL: Introduce calibration mode

---
 Framework/CHANGELOG.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Framework/CHANGELOG.md b/Framework/CHANGELOG.md
index 331805f8538fd..a1c0accde0886 100644
--- a/Framework/CHANGELOG.md
+++ b/Framework/CHANGELOG.md
@@ -1,4 +1,9 @@
-# 2024-03-14: Move DataProcessingDevice to use Signposts
+# 2024-03-14: Introduce calibration mode
+
+This introduces a new --data-processing-timeout `<timeout>` option which can be used to specify
+that data processing should finish after `<timeout>` seconds. After it, only the messages which are produced on EndOfStream will be send.
+
+# 2024-02-23: Move DataProcessingDevice to use Signposts
 
 All the messages from DataProcessingDevice have been migrated to use Signpost.
 This will hopefully simplify debugging.

From da605f36d5fae1c2851f9bdb1fbed42a3d512cfc Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 10:36:50 +0100
Subject: [PATCH 447/726] Do not expose Devices to ROOT

---
 Detectors/PHOS/calib/CMakeLists.txt                   |  9 +--------
 .../include/PHOSCalibWorkflow/PHOSEnergyCalibrator.h  |  7 +------
 .../include/PHOSCalibWorkflow/PHOSL1phaseCalibrator.h |  7 +------
 .../PHOSCalibWorkflow/PHOSRunbyrunCalibrator.h        | 11 +++++------
 .../include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h  |  8 +-------
 Detectors/PHOS/calib/src/PHOSCalibWorkflowLinkDef.h   |  6 ------
 6 files changed, 9 insertions(+), 39 deletions(-)

diff --git a/Detectors/PHOS/calib/CMakeLists.txt b/Detectors/PHOS/calib/CMakeLists.txt
index c54001e08eee9..eca804037d53b 100644
--- a/Detectors/PHOS/calib/CMakeLists.txt
+++ b/Detectors/PHOS/calib/CMakeLists.txt
@@ -30,21 +30,14 @@ o2_add_library(PHOSCalibWorkflow
 o2_add_header_only_library(PHOSCalib)
 
 o2_target_root_dictionary(PHOSCalibWorkflow
-                          HEADERS include/PHOSCalibWorkflow/PHOSPedestalCalibDevice.h
-                                  include/PHOSCalibWorkflow/PHOSHGLGRatioCalibDevice.h
-                                  include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
+                          HEADERS include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
                                   include/PHOSCalibWorkflow/RingBuffer.h
                                   include/PHOSCalibWorkflow/ETCalibHistos.h
                                   include/PHOSCalibWorkflow/PHOSEnergyCalibrator.h
-                                  include/PHOSCalibWorkflow/PHOSEnergyCalibDevice.h
                                   include/PHOSCalibWorkflow/TurnOnHistos.h
                                   include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
-                                  include/PHOSCalibWorkflow/PHOSTurnonCalibDevice.h
                                   include/PHOSCalibWorkflow/PHOSRunbyrunCalibrator.h
-                                  include/PHOSCalibWorkflow/PHOSRunbyrunCalibDevice.h
                                   include/PHOSCalibWorkflow/PHOSL1phaseCalibrator.h
-                                  include/PHOSCalibWorkflow/PHOSL1phaseCalibDevice.h
-                                  include/PHOSCalibWorkflow/PHOSBadMapCalibDevice.h
                           LINKDEF src/PHOSCalibWorkflowLinkDef.h)
 
 o2_add_executable(calib-workflow
diff --git a/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSEnergyCalibrator.h b/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSEnergyCalibrator.h
index 65fb9978bd767..fcdd0bd7e1d2b 100644
--- a/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSEnergyCalibrator.h
+++ b/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSEnergyCalibrator.h
@@ -15,8 +15,6 @@
 /// @file   PHOSEnergyCalibtor.h
 /// @brief  Device to collect energy and time PHOS energy and time calibration.
 
-#include "Framework/Task.h"
-#include "Framework/WorkflowSpec.h"
 #include "DetectorsCalibration/TimeSlotCalibration.h"
 #include "DataFormatsPHOS/TriggerRecord.h"
 #include "DataFormatsPHOS/Cluster.h"
@@ -32,9 +30,7 @@
 
 using namespace o2::framework;
 
-namespace o2
-{
-namespace phos
+namespace o2::phos
 {
 
 // structure used to store digit info for re-calibration
@@ -156,7 +152,6 @@ class PHOSEnergyCalibrator final : public o2::calibration::TimeSlotCalibration<o
   std::unique_ptr<ETCalibHistos> mHistos; /// final histograms
 };
 
-} // namespace phos
 } // namespace o2
 
 #endif
diff --git a/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSL1phaseCalibrator.h b/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSL1phaseCalibrator.h
index 7e9f1a665dc87..e26fb9c5c88a5 100644
--- a/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSL1phaseCalibrator.h
+++ b/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSL1phaseCalibrator.h
@@ -15,8 +15,6 @@
 /// @file   PHOSRunbyrunCalibDevice.h
 /// @brief  Device to calculate PHOS time shift (L1phase)
 
-#include "Framework/Task.h"
-#include "Framework/ProcessingContext.h"
 #include "DataFormatsPHOS/BadChannelsMap.h"
 #include "DataFormatsPHOS/CalibParams.h"
 #include "DataFormatsPHOS/TriggerRecord.h"
@@ -25,9 +23,7 @@
 
 using namespace o2::framework;
 
-namespace o2
-{
-namespace phos
+namespace o2::phos
 {
 
 class PHOSL1phaseSlot
@@ -95,7 +91,6 @@ class PHOSL1phaseCalibrator final : public o2::calibration::TimeSlotCalibration<
   ClassDefOverride(PHOSL1phaseCalibrator, 3);
 };
 
-} // namespace phos
 } // namespace o2
 
 #endif
diff --git a/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSRunbyrunCalibrator.h b/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSRunbyrunCalibrator.h
index 5bd2d239c8786..2a71027ed4aa1 100644
--- a/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSRunbyrunCalibrator.h
+++ b/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSRunbyrunCalibrator.h
@@ -15,8 +15,6 @@
 /// @file   PHOSRunbyrunCalibDevice.h
 /// @brief  Device to calculate PHOS energy run by run corrections
 
-#include "Framework/Task.h"
-#include "Framework/ProcessingContext.h"
 #include "DataFormatsPHOS/BadChannelsMap.h"
 #include "DataFormatsPHOS/TriggerRecord.h"
 #include "DetectorsCalibration/TimeSlotCalibration.h"
@@ -28,9 +26,11 @@
 
 using namespace o2::framework;
 
-namespace o2
-{
-namespace phos
+namespace o2::framework {
+  struct ProcessingContext;
+}
+
+namespace o2::phos
 {
 
 class PHOSRunbyrunSlot
@@ -105,7 +105,6 @@ class PHOSRunbyrunCalibrator final : public o2::calibration::TimeSlotCalibration
   ClassDefOverride(PHOSRunbyrunCalibrator, 1);
 };
 
-} // namespace phos
 } // namespace o2
 
 #endif
diff --git a/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h b/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
index b0e8e165254b7..5a97c4db97cc0 100644
--- a/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
+++ b/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
@@ -15,8 +15,6 @@
 /// @file   PHOSTurnonCalibrator.h
 /// @brief  Device to calculate PHOS turn-on curve and bad map
 
-#include "Framework/Task.h"
-#include "Framework/WorkflowSpec.h"
 #include "DetectorsCalibration/TimeSlotCalibration.h"
 #include "DataFormatsPHOS/Cluster.h"
 #include "DataFormatsPHOS/Cell.h"
@@ -26,9 +24,7 @@
 
 using namespace o2::framework;
 
-namespace o2
-{
-namespace phos
+namespace o2::phos
 {
 
 class PHOSTurnonSlot
@@ -96,8 +92,6 @@ class PHOSTurnonCalibrator final : public o2::calibration::TimeSlotCalibration<o
   ClassDefOverride(PHOSTurnonCalibrator, 1);
 };
 
-o2::framework::DataProcessorSpec getPHOSTunronCalibDeviceSpec(bool useCCDB);
-} // namespace phos
 } // namespace o2
 
 #endif
diff --git a/Detectors/PHOS/calib/src/PHOSCalibWorkflowLinkDef.h b/Detectors/PHOS/calib/src/PHOSCalibWorkflowLinkDef.h
index 88d90d49d41c4..d439d21bf62e4 100644
--- a/Detectors/PHOS/calib/src/PHOSCalibWorkflowLinkDef.h
+++ b/Detectors/PHOS/calib/src/PHOSCalibWorkflowLinkDef.h
@@ -15,26 +15,20 @@
 #pragma link off all classes;
 #pragma link off all functions;
 
-#pragma link C++ class o2::phos::PHOSPedestalCalibDevice + ;
-#pragma link C++ class o2::phos::PHOSHGLGRatioCalibDevice + ;
-#pragma link C++ class o2::phos::PHOSBadMapCalibDevice + ;
 #pragma link C++ class o2::phos::ETCalibHistos + ;
 #pragma link C++ class o2::calibration::TimeSlot < o2::phos::PHOSEnergySlot> + ;
 #pragma link C++ class o2::calibration::TimeSlotCalibration < o2::phos::PHOSEnergySlot> + ;
 #pragma link C++ class o2::phos::TurnOnHistos + ;
 #pragma link C++ class o2::phos::PHOSTurnonSlot + ;
 #pragma link C++ class o2::phos::PHOSTurnonCalibrator + ;
-#pragma link C++ class o2::phos::PHOSTurnonCalibDevice + ;
 #pragma link C++ class o2::calibration::TimeSlot < o2::phos::PHOSTurnonSlot> + ;
 #pragma link C++ class o2::calibration::TimeSlotCalibration < o2::phos::PHOSTurnonSlot> + ;
 #pragma link C++ class o2::phos::PHOSRunbyrunSlot + ;
 #pragma link C++ class o2::phos::PHOSRunbyrunCalibrator + ;
-#pragma link C++ class o2::phos::PHOSRunbyrunCalibDevice + ;
 #pragma link C++ class o2::calibration::TimeSlot < o2::phos::PHOSRunbyrunSlot> + ;
 #pragma link C++ class o2::calibration::TimeSlotCalibration < o2::phos::PHOSRunbyrunSlot> + ;
 #pragma link C++ class o2::phos::PHOSL1phaseSlot + ;
 #pragma link C++ class o2::phos::PHOSL1phaseCalibrator + ;
-#pragma link C++ class o2::phos::PHOSL1phaseCalibDevice + ;
 #pragma link C++ class o2::calibration::TimeSlot < o2::phos::PHOSL1phaseSlot> + ;
 #pragma link C++ class o2::calibration::TimeSlotCalibration < o2::phos::PHOSL1phaseSlot> + ;
 #pragma link C++ class vector < int> + ;

From b8d90b91c3f80a3d4fb23c54dd4a456ee9437811 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 22:39:11 +0100
Subject: [PATCH 448/726] DPL: make helper for typeId consteval

---
 Framework/Foundation/include/Framework/TypeIdHelpers.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Foundation/include/Framework/TypeIdHelpers.h b/Framework/Foundation/include/Framework/TypeIdHelpers.h
index d4f72b8f3f7ff..2336e5dd7b8e0 100644
--- a/Framework/Foundation/include/Framework/TypeIdHelpers.h
+++ b/Framework/Foundation/include/Framework/TypeIdHelpers.h
@@ -72,7 +72,7 @@ constexpr static std::string_view type_name()
 
 struct TypeIdHelpers {
   template <typename T>
-  constexpr static uint32_t uniqueId()
+  consteval static uint32_t uniqueId()
   {
 #ifdef __CLING__
     constexpr uint32_t r = crc32(unique_type_id_v<T>.data(), unique_type_id_v<T>.size());

From 2a21b4576f61150500e39f07e55614451079b2b8 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 19:58:56 +0100
Subject: [PATCH 449/726] Do not expose Devices to ROOT

---
 Detectors/MUON/MCH/Align/CMakeLists.txt               |  3 +--
 Detectors/PHOS/calib/CMakeLists.txt                   |  9 +--------
 .../include/PHOSCalibWorkflow/PHOSEnergyCalibrator.h  |  7 +------
 .../include/PHOSCalibWorkflow/PHOSL1phaseCalibrator.h |  7 +------
 .../PHOSCalibWorkflow/PHOSRunbyrunCalibrator.h        | 11 +++++------
 .../include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h  |  8 +-------
 Detectors/PHOS/calib/src/PHOSCalibWorkflowLinkDef.h   |  6 ------
 7 files changed, 10 insertions(+), 41 deletions(-)

diff --git a/Detectors/MUON/MCH/Align/CMakeLists.txt b/Detectors/MUON/MCH/Align/CMakeLists.txt
index 4f9c3cd7f35b8..2ef8fd3495658 100644
--- a/Detectors/MUON/MCH/Align/CMakeLists.txt
+++ b/Detectors/MUON/MCH/Align/CMakeLists.txt
@@ -31,8 +31,7 @@ o2_add_library(MCHAlign
 
 o2_target_root_dictionary(MCHAlign
   HEADERS
-    include/MCHAlign/Aligner.h
-    include/MCHAlign/AlignmentSpec.h)
+    include/MCHAlign/Aligner.h)
 
 o2_add_executable(
         alignment-workflow
diff --git a/Detectors/PHOS/calib/CMakeLists.txt b/Detectors/PHOS/calib/CMakeLists.txt
index c54001e08eee9..eca804037d53b 100644
--- a/Detectors/PHOS/calib/CMakeLists.txt
+++ b/Detectors/PHOS/calib/CMakeLists.txt
@@ -30,21 +30,14 @@ o2_add_library(PHOSCalibWorkflow
 o2_add_header_only_library(PHOSCalib)
 
 o2_target_root_dictionary(PHOSCalibWorkflow
-                          HEADERS include/PHOSCalibWorkflow/PHOSPedestalCalibDevice.h
-                                  include/PHOSCalibWorkflow/PHOSHGLGRatioCalibDevice.h
-                                  include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
+                          HEADERS include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
                                   include/PHOSCalibWorkflow/RingBuffer.h
                                   include/PHOSCalibWorkflow/ETCalibHistos.h
                                   include/PHOSCalibWorkflow/PHOSEnergyCalibrator.h
-                                  include/PHOSCalibWorkflow/PHOSEnergyCalibDevice.h
                                   include/PHOSCalibWorkflow/TurnOnHistos.h
                                   include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
-                                  include/PHOSCalibWorkflow/PHOSTurnonCalibDevice.h
                                   include/PHOSCalibWorkflow/PHOSRunbyrunCalibrator.h
-                                  include/PHOSCalibWorkflow/PHOSRunbyrunCalibDevice.h
                                   include/PHOSCalibWorkflow/PHOSL1phaseCalibrator.h
-                                  include/PHOSCalibWorkflow/PHOSL1phaseCalibDevice.h
-                                  include/PHOSCalibWorkflow/PHOSBadMapCalibDevice.h
                           LINKDEF src/PHOSCalibWorkflowLinkDef.h)
 
 o2_add_executable(calib-workflow
diff --git a/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSEnergyCalibrator.h b/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSEnergyCalibrator.h
index 65fb9978bd767..fcdd0bd7e1d2b 100644
--- a/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSEnergyCalibrator.h
+++ b/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSEnergyCalibrator.h
@@ -15,8 +15,6 @@
 /// @file   PHOSEnergyCalibtor.h
 /// @brief  Device to collect energy and time PHOS energy and time calibration.
 
-#include "Framework/Task.h"
-#include "Framework/WorkflowSpec.h"
 #include "DetectorsCalibration/TimeSlotCalibration.h"
 #include "DataFormatsPHOS/TriggerRecord.h"
 #include "DataFormatsPHOS/Cluster.h"
@@ -32,9 +30,7 @@
 
 using namespace o2::framework;
 
-namespace o2
-{
-namespace phos
+namespace o2::phos
 {
 
 // structure used to store digit info for re-calibration
@@ -156,7 +152,6 @@ class PHOSEnergyCalibrator final : public o2::calibration::TimeSlotCalibration<o
   std::unique_ptr<ETCalibHistos> mHistos; /// final histograms
 };
 
-} // namespace phos
 } // namespace o2
 
 #endif
diff --git a/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSL1phaseCalibrator.h b/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSL1phaseCalibrator.h
index 7e9f1a665dc87..e26fb9c5c88a5 100644
--- a/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSL1phaseCalibrator.h
+++ b/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSL1phaseCalibrator.h
@@ -15,8 +15,6 @@
 /// @file   PHOSRunbyrunCalibDevice.h
 /// @brief  Device to calculate PHOS time shift (L1phase)
 
-#include "Framework/Task.h"
-#include "Framework/ProcessingContext.h"
 #include "DataFormatsPHOS/BadChannelsMap.h"
 #include "DataFormatsPHOS/CalibParams.h"
 #include "DataFormatsPHOS/TriggerRecord.h"
@@ -25,9 +23,7 @@
 
 using namespace o2::framework;
 
-namespace o2
-{
-namespace phos
+namespace o2::phos
 {
 
 class PHOSL1phaseSlot
@@ -95,7 +91,6 @@ class PHOSL1phaseCalibrator final : public o2::calibration::TimeSlotCalibration<
   ClassDefOverride(PHOSL1phaseCalibrator, 3);
 };
 
-} // namespace phos
 } // namespace o2
 
 #endif
diff --git a/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSRunbyrunCalibrator.h b/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSRunbyrunCalibrator.h
index 5bd2d239c8786..2a71027ed4aa1 100644
--- a/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSRunbyrunCalibrator.h
+++ b/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSRunbyrunCalibrator.h
@@ -15,8 +15,6 @@
 /// @file   PHOSRunbyrunCalibDevice.h
 /// @brief  Device to calculate PHOS energy run by run corrections
 
-#include "Framework/Task.h"
-#include "Framework/ProcessingContext.h"
 #include "DataFormatsPHOS/BadChannelsMap.h"
 #include "DataFormatsPHOS/TriggerRecord.h"
 #include "DetectorsCalibration/TimeSlotCalibration.h"
@@ -28,9 +26,11 @@
 
 using namespace o2::framework;
 
-namespace o2
-{
-namespace phos
+namespace o2::framework {
+  struct ProcessingContext;
+}
+
+namespace o2::phos
 {
 
 class PHOSRunbyrunSlot
@@ -105,7 +105,6 @@ class PHOSRunbyrunCalibrator final : public o2::calibration::TimeSlotCalibration
   ClassDefOverride(PHOSRunbyrunCalibrator, 1);
 };
 
-} // namespace phos
 } // namespace o2
 
 #endif
diff --git a/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h b/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
index b0e8e165254b7..5a97c4db97cc0 100644
--- a/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
+++ b/Detectors/PHOS/calib/include/PHOSCalibWorkflow/PHOSTurnonCalibrator.h
@@ -15,8 +15,6 @@
 /// @file   PHOSTurnonCalibrator.h
 /// @brief  Device to calculate PHOS turn-on curve and bad map
 
-#include "Framework/Task.h"
-#include "Framework/WorkflowSpec.h"
 #include "DetectorsCalibration/TimeSlotCalibration.h"
 #include "DataFormatsPHOS/Cluster.h"
 #include "DataFormatsPHOS/Cell.h"
@@ -26,9 +24,7 @@
 
 using namespace o2::framework;
 
-namespace o2
-{
-namespace phos
+namespace o2::phos
 {
 
 class PHOSTurnonSlot
@@ -96,8 +92,6 @@ class PHOSTurnonCalibrator final : public o2::calibration::TimeSlotCalibration<o
   ClassDefOverride(PHOSTurnonCalibrator, 1);
 };
 
-o2::framework::DataProcessorSpec getPHOSTunronCalibDeviceSpec(bool useCCDB);
-} // namespace phos
 } // namespace o2
 
 #endif
diff --git a/Detectors/PHOS/calib/src/PHOSCalibWorkflowLinkDef.h b/Detectors/PHOS/calib/src/PHOSCalibWorkflowLinkDef.h
index 88d90d49d41c4..d439d21bf62e4 100644
--- a/Detectors/PHOS/calib/src/PHOSCalibWorkflowLinkDef.h
+++ b/Detectors/PHOS/calib/src/PHOSCalibWorkflowLinkDef.h
@@ -15,26 +15,20 @@
 #pragma link off all classes;
 #pragma link off all functions;
 
-#pragma link C++ class o2::phos::PHOSPedestalCalibDevice + ;
-#pragma link C++ class o2::phos::PHOSHGLGRatioCalibDevice + ;
-#pragma link C++ class o2::phos::PHOSBadMapCalibDevice + ;
 #pragma link C++ class o2::phos::ETCalibHistos + ;
 #pragma link C++ class o2::calibration::TimeSlot < o2::phos::PHOSEnergySlot> + ;
 #pragma link C++ class o2::calibration::TimeSlotCalibration < o2::phos::PHOSEnergySlot> + ;
 #pragma link C++ class o2::phos::TurnOnHistos + ;
 #pragma link C++ class o2::phos::PHOSTurnonSlot + ;
 #pragma link C++ class o2::phos::PHOSTurnonCalibrator + ;
-#pragma link C++ class o2::phos::PHOSTurnonCalibDevice + ;
 #pragma link C++ class o2::calibration::TimeSlot < o2::phos::PHOSTurnonSlot> + ;
 #pragma link C++ class o2::calibration::TimeSlotCalibration < o2::phos::PHOSTurnonSlot> + ;
 #pragma link C++ class o2::phos::PHOSRunbyrunSlot + ;
 #pragma link C++ class o2::phos::PHOSRunbyrunCalibrator + ;
-#pragma link C++ class o2::phos::PHOSRunbyrunCalibDevice + ;
 #pragma link C++ class o2::calibration::TimeSlot < o2::phos::PHOSRunbyrunSlot> + ;
 #pragma link C++ class o2::calibration::TimeSlotCalibration < o2::phos::PHOSRunbyrunSlot> + ;
 #pragma link C++ class o2::phos::PHOSL1phaseSlot + ;
 #pragma link C++ class o2::phos::PHOSL1phaseCalibrator + ;
-#pragma link C++ class o2::phos::PHOSL1phaseCalibDevice + ;
 #pragma link C++ class o2::calibration::TimeSlot < o2::phos::PHOSL1phaseSlot> + ;
 #pragma link C++ class o2::calibration::TimeSlotCalibration < o2::phos::PHOSL1phaseSlot> + ;
 #pragma link C++ class vector < int> + ;

From dc4bf487e9dfe5bcaf7be2e73aa0fa0d3f6c0e20 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Mar 2024 22:39:16 +0100
Subject: [PATCH 450/726] DPL: make helper for typeId consteval

---
 Framework/Foundation/include/Framework/TypeIdHelpers.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Foundation/include/Framework/TypeIdHelpers.h b/Framework/Foundation/include/Framework/TypeIdHelpers.h
index d4f72b8f3f7ff..2336e5dd7b8e0 100644
--- a/Framework/Foundation/include/Framework/TypeIdHelpers.h
+++ b/Framework/Foundation/include/Framework/TypeIdHelpers.h
@@ -72,7 +72,7 @@ constexpr static std::string_view type_name()
 
 struct TypeIdHelpers {
   template <typename T>
-  constexpr static uint32_t uniqueId()
+  consteval static uint32_t uniqueId()
   {
 #ifdef __CLING__
     constexpr uint32_t r = crc32(unique_type_id_v<T>.data(), unique_type_id_v<T>.size());

From 5a15abefccd83711cdb2f3e9ad323781bf35e225 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 18 Mar 2024 21:20:33 +0100
Subject: [PATCH 451/726] DPL: drop need for ms_gsl

---
 .../include/Framework/TMessageSerializer.h    | 22 -------------------
 1 file changed, 22 deletions(-)

diff --git a/Framework/Core/include/Framework/TMessageSerializer.h b/Framework/Core/include/Framework/TMessageSerializer.h
index 17f3f65ce2030..769d23a7a3427 100644
--- a/Framework/Core/include/Framework/TMessageSerializer.h
+++ b/Framework/Core/include/Framework/TMessageSerializer.h
@@ -18,9 +18,6 @@
 #include <TList.h>
 #include <TBufferFile.h>
 #include <TObjArray.h>
-#include <gsl/util>
-#include <gsl/span>
-#include <gsl/narrow>
 #include <memory>
 #include <mutex>
 #include <cstddef>
@@ -30,12 +27,6 @@ namespace o2::framework
 class FairOutputTBuffer;
 class FairInputTBuffer;
 
-// utilities to produce a span over a byte buffer held by various message types
-// this is to avoid littering code with casts and conversions (span has a signed index type(!))
-gsl::span<std::byte> as_span(const FairInputTBuffer& msg);
-gsl::span<std::byte> as_span(const FairOutputTBuffer& msg);
-gsl::span<std::byte> as_span(const fair::mq::Message& msg);
-
 // A TBufferFile which we can use to serialise data to a FairMQ message.
 class FairOutputTBuffer : public TBufferFile
 {
@@ -153,18 +144,5 @@ inline void TMessageSerializer::Deserialize(const fair::mq::Message& msg, std::u
   output = deserialize(input);
 }
 
-// gsl::narrow is used to do a runtime narrowing check, this might be a bit paranoid,
-// we would probably be fine with e.g. gsl::narrow_cast (or just a static_cast)
-inline gsl::span<std::byte> as_span(const fair::mq::Message& msg)
-{
-  return gsl::span<std::byte>{static_cast<std::byte*>(msg.GetData()), gsl::narrow<gsl::span<std::byte>::size_type>(msg.GetSize())};
-}
-
-inline gsl::span<std::byte> as_span(const FairInputTBuffer& msg)
-{
-  return gsl::span<std::byte>{reinterpret_cast<std::byte*>(msg.Buffer()),
-                              gsl::narrow<gsl::span<std::byte>::size_type>(msg.BufferSize())};
-}
-
 } // namespace o2::framework
 #endif // FRAMEWORK_TMESSAGESERIALIZER_H

From 0e8f915ab19c0de0c805f1fb6130fbbf7721c785 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 19 Mar 2024 09:11:55 +0100
Subject: [PATCH 452/726] Fix case sensitiveness

---
 GPU/GPUTracking/CMakeLists.txt    | 2 +-
 dependencies/O2CompileFlags.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/GPU/GPUTracking/CMakeLists.txt b/GPU/GPUTracking/CMakeLists.txt
index 0a8e269e70dcb..396d6d20731d3 100644
--- a/GPU/GPUTracking/CMakeLists.txt
+++ b/GPU/GPUTracking/CMakeLists.txt
@@ -13,7 +13,7 @@ set(MODULE GPUTracking)
 
 # set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -O0") # to uncomment if needed, tired of typing this...
 
-if(NOT ${GPUCA_NO_FAST_MATH} AND NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+if(NOT ${GPUCA_NO_FAST_MATH} AND NOT $<CONFIG:Debug>)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffast-math")
 endif()
 
diff --git a/dependencies/O2CompileFlags.cmake b/dependencies/O2CompileFlags.cmake
index 1157a530a2ca8..b250209e0fd11 100644
--- a/dependencies/O2CompileFlags.cmake
+++ b/dependencies/O2CompileFlags.cmake
@@ -104,7 +104,7 @@ IF(ENABLE_CASSERT) #For the CI, we want to have <cassert> assertions enabled
 ELSE()
     set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
     set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG")
-    if (CMAKE_BUILD_TYPE STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE STREQUAL "RELWITHDEBINFO")
+    if ($<CONFIG:Release> OR $<CONFIG:RelWithDebInfo>)
       set(FAIR_MIN_SEVERITY "detail")
     endif()
 ENDIF()

From 857cab3849a03b1bbbe771d53bd1ec0dda910b3f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 19 Mar 2024 09:24:24 +0100
Subject: [PATCH 453/726] Fix case sensitiveness for CMAKE_BUILD_TYPE

---
 GPU/GPUTracking/CMakeLists.txt            | 2 +-
 GPU/GPUTracking/Standalone/CMakeLists.txt | 4 ++--
 dependencies/FindO2GPU.cmake              | 4 ++--
 dependencies/O2CompileFlags.cmake         | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/GPU/GPUTracking/CMakeLists.txt b/GPU/GPUTracking/CMakeLists.txt
index 0a8e269e70dcb..396d6d20731d3 100644
--- a/GPU/GPUTracking/CMakeLists.txt
+++ b/GPU/GPUTracking/CMakeLists.txt
@@ -13,7 +13,7 @@ set(MODULE GPUTracking)
 
 # set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -O0") # to uncomment if needed, tired of typing this...
 
-if(NOT ${GPUCA_NO_FAST_MATH} AND NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+if(NOT ${GPUCA_NO_FAST_MATH} AND NOT $<CONFIG:Debug>)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffast-math")
 endif()
 
diff --git a/GPU/GPUTracking/Standalone/CMakeLists.txt b/GPU/GPUTracking/Standalone/CMakeLists.txt
index 643b09a332b2c..4e456c2270c30 100644
--- a/GPU/GPUTracking/Standalone/CMakeLists.txt
+++ b/GPU/GPUTracking/Standalone/CMakeLists.txt
@@ -50,7 +50,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 if(BUILD_DEBUG)
   set(CMAKE_CXX_FLAGS "-O0 -ggdb") # -fsanitize=address,undefined -fno-sanitize=vptr
-  set(CMAKE_BUILD_TYPE DEBUG)
+  set(CMAKE_BUILD_TYPE Debug)
 else()
   set(CMAKE_CXX_FLAGS "-O3 -march=native -ggdb -minline-all-stringops -funroll-loops -fno-stack-protector")
   if(DEFINED GPUCA_NO_FAST_MATH AND ${GPUCA_NO_FAST_MATH})
@@ -61,7 +61,7 @@ else()
   if (NOT CMAKE_CXX_COMPILER STREQUAL "clang++")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ftracer -fprefetch-loop-arrays")
   endif()
-  set(CMAKE_BUILD_TYPE RELEASE)
+  set(CMAKE_BUILD_TYPE Release)
   add_definitions(-DNDEBUG)
 endif()
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error -Wall -Wextra -Wshadow -Wno-unused-function -Wno-unused-parameter -Wno-unused-local-typedefs -Wno-unknown-pragmas -Wno-write-strings")
diff --git a/dependencies/FindO2GPU.cmake b/dependencies/FindO2GPU.cmake
index da0dd7b85628f..a8d657acaebfd 100644
--- a/dependencies/FindO2GPU.cmake
+++ b/dependencies/FindO2GPU.cmake
@@ -113,13 +113,13 @@ if(ENABLE_CUDA)
       string(APPEND CMAKE_CUDA_FLAGS " -Xcudafe --diag_suppress=20257")
     endif()
     set(CMAKE_CUDA_FLAGS_DEBUG "${CMAKE_CUDA_FLAGS_DEBUG} -lineinfo -Xcompiler \"${CMAKE_CXX_FLAGS_DEBUG}\" -Xptxas -O0 -Xcompiler -O0")
-    if(NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+    if(NOT $<CONFIG:Debug>)
       set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -Xcompiler \"${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}}\" -Xptxas -O4 -Xcompiler -O4")
     endif()
     set(GPUCA_CUDA_NO_FAST_MATH_FLAGS "--ftz=false --prec-div=true --prec-sqrt=true --fmad false")
     if(DEFINED GPUCA_NO_FAST_MATH AND "${GPUCA_NO_FAST_MATH}")
       set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} ${GPUCA_CUDA_NO_FAST_MATH_FLAGS}")
-    elseif(NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+    elseif(NOT $<CONFIG:Debug>)
       set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -use_fast_math --ftz=true")#
     endif()
     if(CMAKE_CXX_FLAGS MATCHES "(^| )-Werror( |$)")
diff --git a/dependencies/O2CompileFlags.cmake b/dependencies/O2CompileFlags.cmake
index 1157a530a2ca8..b250209e0fd11 100644
--- a/dependencies/O2CompileFlags.cmake
+++ b/dependencies/O2CompileFlags.cmake
@@ -104,7 +104,7 @@ IF(ENABLE_CASSERT) #For the CI, we want to have <cassert> assertions enabled
 ELSE()
     set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
     set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG")
-    if (CMAKE_BUILD_TYPE STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE STREQUAL "RELWITHDEBINFO")
+    if ($<CONFIG:Release> OR $<CONFIG:RelWithDebInfo>)
       set(FAIR_MIN_SEVERITY "detail")
     endif()
 ENDIF()

From d2c07f53a6f46cfc10a655fe84350659ce728162 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 19 Mar 2024 10:34:25 +0100
Subject: [PATCH 454/726] Fix case sensitiveness for CMAKE_BUILD_TYPE

---
 GPU/GPUTracking/Base/cuda/CMakeLists.txt  |  2 +-
 GPU/GPUTracking/Base/hip/CMakeLists.txt   |  2 +-
 GPU/GPUTracking/CMakeLists.txt            |  4 ++--
 GPU/GPUTracking/Standalone/CMakeLists.txt |  4 ++--
 dependencies/FindO2GPU.cmake              | 12 ++++++------
 dependencies/O2CompileFlags.cmake         | 18 +++++++++---------
 6 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/GPU/GPUTracking/Base/cuda/CMakeLists.txt b/GPU/GPUTracking/Base/cuda/CMakeLists.txt
index 652cedeba557c..44bebc4e561a1 100644
--- a/GPU/GPUTracking/Base/cuda/CMakeLists.txt
+++ b/GPU/GPUTracking/Base/cuda/CMakeLists.txt
@@ -37,7 +37,7 @@ if(NOT ALIGPU_BUILD_TYPE STREQUAL "ALIROOT")
   endif()
   #set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -keep")
 
-  set(CUDARTC_FLAGS "${CMAKE_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -std=c++${CMAKE_CUDA_STANDARD}")
+  set(CUDARTC_FLAGS "${CMAKE_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS_$<UPPER_CASE:{CMAKE_BUILD_TYPE}>} -std=c++${CMAKE_CUDA_STANDARD}")
   if(CUDA_COMPUTETARGET)
       foreach(CUDA_ARCH ${CUDA_COMPUTETARGET})
           set(CUDARTC_FLAGS "${CUDARTC_FLAGS} -gencode arch=compute_${CUDA_ARCH},code=sm_${CUDA_ARCH}")
diff --git a/GPU/GPUTracking/Base/hip/CMakeLists.txt b/GPU/GPUTracking/Base/hip/CMakeLists.txt
index 804191be4e224..981db2a38fe18 100644
--- a/GPU/GPUTracking/Base/hip/CMakeLists.txt
+++ b/GPU/GPUTracking/Base/hip/CMakeLists.txt
@@ -77,7 +77,7 @@ if(NOT ALIGPU_BUILD_TYPE STREQUAL "ALIROOT")
     set(HIPRTC_INCLUDES ${HIPRTC_INCLUDES} "-I$<JOIN:$<FILTER:$<TARGET_PROPERTY:O2::ITStrackingHIP,INCLUDE_DIRECTORIES>,EXCLUDE,^/usr/include/?>,$<SEMICOLON>-I>")
   endif()
 
-  set(HIPRTC_FLAGS "${CMAKE_HIP_FLAGS} ${CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE}} -std=c++${CMAKE_HIP_STANDARD}")
+  set(HIPRTC_FLAGS "${CMAKE_HIP_FLAGS} ${CMAKE_HIP_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -std=c++${CMAKE_HIP_STANDARD}")
   separate_arguments(HIPRTC_FLAGS)
 
   # convenience variables
diff --git a/GPU/GPUTracking/CMakeLists.txt b/GPU/GPUTracking/CMakeLists.txt
index 0a8e269e70dcb..9d70d24ab5119 100644
--- a/GPU/GPUTracking/CMakeLists.txt
+++ b/GPU/GPUTracking/CMakeLists.txt
@@ -11,9 +11,9 @@
 
 set(MODULE GPUTracking)
 
-# set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -O0") # to uncomment if needed, tired of typing this...
+# set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -O0") # to uncomment if needed, tired of typing this...
 
-if(NOT ${GPUCA_NO_FAST_MATH} AND NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+if(NOT ${GPUCA_NO_FAST_MATH} AND NOT $<CONFIG:Debug>)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffast-math")
 endif()
 
diff --git a/GPU/GPUTracking/Standalone/CMakeLists.txt b/GPU/GPUTracking/Standalone/CMakeLists.txt
index 643b09a332b2c..4e456c2270c30 100644
--- a/GPU/GPUTracking/Standalone/CMakeLists.txt
+++ b/GPU/GPUTracking/Standalone/CMakeLists.txt
@@ -50,7 +50,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 if(BUILD_DEBUG)
   set(CMAKE_CXX_FLAGS "-O0 -ggdb") # -fsanitize=address,undefined -fno-sanitize=vptr
-  set(CMAKE_BUILD_TYPE DEBUG)
+  set(CMAKE_BUILD_TYPE Debug)
 else()
   set(CMAKE_CXX_FLAGS "-O3 -march=native -ggdb -minline-all-stringops -funroll-loops -fno-stack-protector")
   if(DEFINED GPUCA_NO_FAST_MATH AND ${GPUCA_NO_FAST_MATH})
@@ -61,7 +61,7 @@ else()
   if (NOT CMAKE_CXX_COMPILER STREQUAL "clang++")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ftracer -fprefetch-loop-arrays")
   endif()
-  set(CMAKE_BUILD_TYPE RELEASE)
+  set(CMAKE_BUILD_TYPE Release)
   add_definitions(-DNDEBUG)
 endif()
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error -Wall -Wextra -Wshadow -Wno-unused-function -Wno-unused-parameter -Wno-unused-local-typedefs -Wno-unknown-pragmas -Wno-write-strings")
diff --git a/dependencies/FindO2GPU.cmake b/dependencies/FindO2GPU.cmake
index da0dd7b85628f..e4c427533f7a1 100644
--- a/dependencies/FindO2GPU.cmake
+++ b/dependencies/FindO2GPU.cmake
@@ -113,14 +113,14 @@ if(ENABLE_CUDA)
       string(APPEND CMAKE_CUDA_FLAGS " -Xcudafe --diag_suppress=20257")
     endif()
     set(CMAKE_CUDA_FLAGS_DEBUG "${CMAKE_CUDA_FLAGS_DEBUG} -lineinfo -Xcompiler \"${CMAKE_CXX_FLAGS_DEBUG}\" -Xptxas -O0 -Xcompiler -O0")
-    if(NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -Xcompiler \"${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}}\" -Xptxas -O4 -Xcompiler -O4")
+    if(NOT $<CONFIG:Debug>)
+      set(CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -Xcompiler \"${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>}\" -Xptxas -O4 -Xcompiler -O4")
     endif()
     set(GPUCA_CUDA_NO_FAST_MATH_FLAGS "--ftz=false --prec-div=true --prec-sqrt=true --fmad false")
     if(DEFINED GPUCA_NO_FAST_MATH AND "${GPUCA_NO_FAST_MATH}")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} ${GPUCA_CUDA_NO_FAST_MATH_FLAGS}")
-    elseif(NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -use_fast_math --ftz=true")#
+      set(CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} ${GPUCA_CUDA_NO_FAST_MATH_FLAGS}")
+    elseif(NOT $<CONFIG:Debug>)
+      set(CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -use_fast_math --ftz=true")#
     endif()
     if(CMAKE_CXX_FLAGS MATCHES "(^| )-Werror( |$)")
       set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Werror=cross-execution-space-call")
@@ -274,7 +274,7 @@ if(ENABLE_HIP)
       set(O2_HIP_CMAKE_CXX_FLAGS "${O2_HIP_CMAKE_CXX_FLAGS} --gcc-toolchain=${HIP_GCC_TOOLCHAIN_PATH}") # -ffast-math disabled, since apparently it leads to miscompilation and crashes in FollowLooper kernel
     endif()
     set(CMAKE_HIP_FLAGS "${O2_GPU_CMAKE_CXX_FLAGS_NOSTD} ${CMAKE_HIP_FLAGS} ${O2_HIP_CMAKE_CXX_FLAGS}")
-    set(CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE}}")
+    set(CMAKE_HIP_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} ${CMAKE_HIP_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>}")
   else()
     set(HIP_ENABLED OFF)
   endif()
diff --git a/dependencies/O2CompileFlags.cmake b/dependencies/O2CompileFlags.cmake
index 1157a530a2ca8..6fd27c82a261c 100644
--- a/dependencies/O2CompileFlags.cmake
+++ b/dependencies/O2CompileFlags.cmake
@@ -104,13 +104,13 @@ IF(ENABLE_CASSERT) #For the CI, we want to have <cassert> assertions enabled
 ELSE()
     set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
     set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG")
-    if (CMAKE_BUILD_TYPE STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE STREQUAL "RELWITHDEBINFO")
+    if ($<CONFIG:Release> OR $<CONFIG:RelWithDebInfo>)
       set(FAIR_MIN_SEVERITY "detail")
     endif()
 ENDIF()
 
 IF(ENABLE_THREAD_SAFETY_ANALYSIS)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -Werror=thread-safety -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS")
+  set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -Werror=thread-safety -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS")
 ENDIF()
 
 set(CMAKE_C_FLAGS_RELEASE "-O2")
@@ -122,8 +122,8 @@ set(CMAKE_CXX_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 set(CMAKE_C_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 set(CMAKE_Fortran_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 
-set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_CXX_WARNINGS}")
-set(CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_C_WARNINGS}")
+set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} ${CMAKE_CXX_WARNINGS}")
+set(CMAKE_C_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_C_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} ${CMAKE_C_WARNINGS}")
 
 if(APPLE)
 elseif(UNIX)
@@ -131,19 +131,19 @@ elseif(UNIX)
 endif()
 
 if(ENABLE_TIME_TRACE)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -ftime-trace")
+  set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -ftime-trace")
 endif()
 
 if(DEFINED ENV{O2_CXXFLAGS_OVERRIDE})
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} $ENV{O2_CXXFLAGS_OVERRIDE}")
+  set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} $ENV{O2_CXXFLAGS_OVERRIDE}")
   message(STATUS "Setting CXXFLAGS Override $ENV{O2_CXXFLAGS_OVERRIDE}")
 endif()
 
 if(GPUCA_NO_FAST_MATH_WHOLEO2)
   set(GPUCA_NO_FAST_MATH 1)
   add_definitions(-DGPUCA_NO_FAST_MATH)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -fno-fast-math -ffp-contract=off")
-  set(CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}} -fno-fast-math -ffp-contract=off")
+  set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -fno-fast-math -ffp-contract=off")
+  set(CMAKE_C_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_C_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -fno-fast-math -ffp-contract=off")
 endif()
 
-message(STATUS "Using build type: ${CMAKE_BUILD_TYPE} - CXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}}")
+message(STATUS "Using build type: ${CMAKE_BUILD_TYPE} - CXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>}")

From 2f1f8ef38d2540cce164e1b855ff2af8c5a93442 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 19 Mar 2024 10:39:13 +0100
Subject: [PATCH 455/726] Fix case sensitiveness for CMAKE_BUILD_TYPE

The way case sensitiveness works in CMake is described at:

https://cmake.org/cmake/help/v3.29/manual/cmake-buildsystem.7.html#case-sensitivity

and the "strong" convention is described a few lines below:

https://cmake.org/cmake/help/v3.29/manual/cmake-buildsystem.7.html#id38

Notice also how the variables for customising CMAKE_CXX_FLAGS (e.g. CMAKE_CXX_FLAGS_DEBUG)
are always all uppercase, regardless of the value of CMAKE_BUILD_TYPE. We use the
camel cased version, because that's what strongly suggested by CMake (probably because the
generators for some IDE / tool use that convention).
---
 GPU/GPUTracking/Base/cuda/CMakeLists.txt  |  2 +-
 GPU/GPUTracking/Base/hip/CMakeLists.txt   |  2 +-
 GPU/GPUTracking/CMakeLists.txt            |  4 ++--
 GPU/GPUTracking/Standalone/CMakeLists.txt |  4 ++--
 dependencies/FindO2GPU.cmake              | 12 ++++++------
 dependencies/O2CompileFlags.cmake         | 18 +++++++++---------
 6 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/GPU/GPUTracking/Base/cuda/CMakeLists.txt b/GPU/GPUTracking/Base/cuda/CMakeLists.txt
index 652cedeba557c..44bebc4e561a1 100644
--- a/GPU/GPUTracking/Base/cuda/CMakeLists.txt
+++ b/GPU/GPUTracking/Base/cuda/CMakeLists.txt
@@ -37,7 +37,7 @@ if(NOT ALIGPU_BUILD_TYPE STREQUAL "ALIROOT")
   endif()
   #set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -keep")
 
-  set(CUDARTC_FLAGS "${CMAKE_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -std=c++${CMAKE_CUDA_STANDARD}")
+  set(CUDARTC_FLAGS "${CMAKE_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS_$<UPPER_CASE:{CMAKE_BUILD_TYPE}>} -std=c++${CMAKE_CUDA_STANDARD}")
   if(CUDA_COMPUTETARGET)
       foreach(CUDA_ARCH ${CUDA_COMPUTETARGET})
           set(CUDARTC_FLAGS "${CUDARTC_FLAGS} -gencode arch=compute_${CUDA_ARCH},code=sm_${CUDA_ARCH}")
diff --git a/GPU/GPUTracking/Base/hip/CMakeLists.txt b/GPU/GPUTracking/Base/hip/CMakeLists.txt
index 804191be4e224..981db2a38fe18 100644
--- a/GPU/GPUTracking/Base/hip/CMakeLists.txt
+++ b/GPU/GPUTracking/Base/hip/CMakeLists.txt
@@ -77,7 +77,7 @@ if(NOT ALIGPU_BUILD_TYPE STREQUAL "ALIROOT")
     set(HIPRTC_INCLUDES ${HIPRTC_INCLUDES} "-I$<JOIN:$<FILTER:$<TARGET_PROPERTY:O2::ITStrackingHIP,INCLUDE_DIRECTORIES>,EXCLUDE,^/usr/include/?>,$<SEMICOLON>-I>")
   endif()
 
-  set(HIPRTC_FLAGS "${CMAKE_HIP_FLAGS} ${CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE}} -std=c++${CMAKE_HIP_STANDARD}")
+  set(HIPRTC_FLAGS "${CMAKE_HIP_FLAGS} ${CMAKE_HIP_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -std=c++${CMAKE_HIP_STANDARD}")
   separate_arguments(HIPRTC_FLAGS)
 
   # convenience variables
diff --git a/GPU/GPUTracking/CMakeLists.txt b/GPU/GPUTracking/CMakeLists.txt
index 0a8e269e70dcb..9d70d24ab5119 100644
--- a/GPU/GPUTracking/CMakeLists.txt
+++ b/GPU/GPUTracking/CMakeLists.txt
@@ -11,9 +11,9 @@
 
 set(MODULE GPUTracking)
 
-# set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -O0") # to uncomment if needed, tired of typing this...
+# set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -O0") # to uncomment if needed, tired of typing this...
 
-if(NOT ${GPUCA_NO_FAST_MATH} AND NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+if(NOT ${GPUCA_NO_FAST_MATH} AND NOT $<CONFIG:Debug>)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffast-math")
 endif()
 
diff --git a/GPU/GPUTracking/Standalone/CMakeLists.txt b/GPU/GPUTracking/Standalone/CMakeLists.txt
index 643b09a332b2c..4e456c2270c30 100644
--- a/GPU/GPUTracking/Standalone/CMakeLists.txt
+++ b/GPU/GPUTracking/Standalone/CMakeLists.txt
@@ -50,7 +50,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 if(BUILD_DEBUG)
   set(CMAKE_CXX_FLAGS "-O0 -ggdb") # -fsanitize=address,undefined -fno-sanitize=vptr
-  set(CMAKE_BUILD_TYPE DEBUG)
+  set(CMAKE_BUILD_TYPE Debug)
 else()
   set(CMAKE_CXX_FLAGS "-O3 -march=native -ggdb -minline-all-stringops -funroll-loops -fno-stack-protector")
   if(DEFINED GPUCA_NO_FAST_MATH AND ${GPUCA_NO_FAST_MATH})
@@ -61,7 +61,7 @@ else()
   if (NOT CMAKE_CXX_COMPILER STREQUAL "clang++")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ftracer -fprefetch-loop-arrays")
   endif()
-  set(CMAKE_BUILD_TYPE RELEASE)
+  set(CMAKE_BUILD_TYPE Release)
   add_definitions(-DNDEBUG)
 endif()
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error -Wall -Wextra -Wshadow -Wno-unused-function -Wno-unused-parameter -Wno-unused-local-typedefs -Wno-unknown-pragmas -Wno-write-strings")
diff --git a/dependencies/FindO2GPU.cmake b/dependencies/FindO2GPU.cmake
index da0dd7b85628f..e4c427533f7a1 100644
--- a/dependencies/FindO2GPU.cmake
+++ b/dependencies/FindO2GPU.cmake
@@ -113,14 +113,14 @@ if(ENABLE_CUDA)
       string(APPEND CMAKE_CUDA_FLAGS " -Xcudafe --diag_suppress=20257")
     endif()
     set(CMAKE_CUDA_FLAGS_DEBUG "${CMAKE_CUDA_FLAGS_DEBUG} -lineinfo -Xcompiler \"${CMAKE_CXX_FLAGS_DEBUG}\" -Xptxas -O0 -Xcompiler -O0")
-    if(NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -Xcompiler \"${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}}\" -Xptxas -O4 -Xcompiler -O4")
+    if(NOT $<CONFIG:Debug>)
+      set(CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -Xcompiler \"${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>}\" -Xptxas -O4 -Xcompiler -O4")
     endif()
     set(GPUCA_CUDA_NO_FAST_MATH_FLAGS "--ftz=false --prec-div=true --prec-sqrt=true --fmad false")
     if(DEFINED GPUCA_NO_FAST_MATH AND "${GPUCA_NO_FAST_MATH}")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} ${GPUCA_CUDA_NO_FAST_MATH_FLAGS}")
-    elseif(NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -use_fast_math --ftz=true")#
+      set(CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} ${GPUCA_CUDA_NO_FAST_MATH_FLAGS}")
+    elseif(NOT $<CONFIG:Debug>)
+      set(CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -use_fast_math --ftz=true")#
     endif()
     if(CMAKE_CXX_FLAGS MATCHES "(^| )-Werror( |$)")
       set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Werror=cross-execution-space-call")
@@ -274,7 +274,7 @@ if(ENABLE_HIP)
       set(O2_HIP_CMAKE_CXX_FLAGS "${O2_HIP_CMAKE_CXX_FLAGS} --gcc-toolchain=${HIP_GCC_TOOLCHAIN_PATH}") # -ffast-math disabled, since apparently it leads to miscompilation and crashes in FollowLooper kernel
     endif()
     set(CMAKE_HIP_FLAGS "${O2_GPU_CMAKE_CXX_FLAGS_NOSTD} ${CMAKE_HIP_FLAGS} ${O2_HIP_CMAKE_CXX_FLAGS}")
-    set(CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE}}")
+    set(CMAKE_HIP_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} ${CMAKE_HIP_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>}")
   else()
     set(HIP_ENABLED OFF)
   endif()
diff --git a/dependencies/O2CompileFlags.cmake b/dependencies/O2CompileFlags.cmake
index 1157a530a2ca8..6fd27c82a261c 100644
--- a/dependencies/O2CompileFlags.cmake
+++ b/dependencies/O2CompileFlags.cmake
@@ -104,13 +104,13 @@ IF(ENABLE_CASSERT) #For the CI, we want to have <cassert> assertions enabled
 ELSE()
     set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
     set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG")
-    if (CMAKE_BUILD_TYPE STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE STREQUAL "RELWITHDEBINFO")
+    if ($<CONFIG:Release> OR $<CONFIG:RelWithDebInfo>)
       set(FAIR_MIN_SEVERITY "detail")
     endif()
 ENDIF()
 
 IF(ENABLE_THREAD_SAFETY_ANALYSIS)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -Werror=thread-safety -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS")
+  set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -Werror=thread-safety -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS")
 ENDIF()
 
 set(CMAKE_C_FLAGS_RELEASE "-O2")
@@ -122,8 +122,8 @@ set(CMAKE_CXX_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 set(CMAKE_C_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 set(CMAKE_Fortran_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 
-set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_CXX_WARNINGS}")
-set(CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_C_WARNINGS}")
+set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} ${CMAKE_CXX_WARNINGS}")
+set(CMAKE_C_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_C_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} ${CMAKE_C_WARNINGS}")
 
 if(APPLE)
 elseif(UNIX)
@@ -131,19 +131,19 @@ elseif(UNIX)
 endif()
 
 if(ENABLE_TIME_TRACE)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -ftime-trace")
+  set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -ftime-trace")
 endif()
 
 if(DEFINED ENV{O2_CXXFLAGS_OVERRIDE})
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} $ENV{O2_CXXFLAGS_OVERRIDE}")
+  set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} $ENV{O2_CXXFLAGS_OVERRIDE}")
   message(STATUS "Setting CXXFLAGS Override $ENV{O2_CXXFLAGS_OVERRIDE}")
 endif()
 
 if(GPUCA_NO_FAST_MATH_WHOLEO2)
   set(GPUCA_NO_FAST_MATH 1)
   add_definitions(-DGPUCA_NO_FAST_MATH)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -fno-fast-math -ffp-contract=off")
-  set(CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}} -fno-fast-math -ffp-contract=off")
+  set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -fno-fast-math -ffp-contract=off")
+  set(CMAKE_C_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_C_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -fno-fast-math -ffp-contract=off")
 endif()
 
-message(STATUS "Using build type: ${CMAKE_BUILD_TYPE} - CXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}}")
+message(STATUS "Using build type: ${CMAKE_BUILD_TYPE} - CXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>}")

From de7f128037ae741de82a8bbe58218852b21b8fea Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 19 Mar 2024 10:47:25 +0100
Subject: [PATCH 456/726] Follow conventions for CMAKE_BUILD_TYPE and
 MinSizeRel

The way case sensitiveness works in CMake is described at:

https://cmake.org/cmake/help/v3.29/manual/cmake-buildsystem.7.html#case-sensitivity

and the "strong" convention is described a few lines below:

https://cmake.org/cmake/help/v3.29/manual/cmake-buildsystem.7.html#id38

Notice also how the variables for customising CMAKE_CXX_FLAGS (e.g. CMAKE_CXX_FLAGS_DEBUG)
are always all uppercase, regardless of the value of CMAKE_BUILD_TYPE. We use the
camel cased version, because that's what strongly suggested by CMake (probably because the
generators for some IDE / tool use that convention).

Moreover MinSizeRel is the typical spelling, not MinRelSize.
---
 Detectors/TPC/calibration/CMakeLists.txt    |  4 ++--
 Detectors/TPC/reconstruction/CMakeLists.txt |  2 +-
 Detectors/TPC/spacecharge/CMakeLists.txt    |  4 ++--
 GPU/GPUTracking/Base/cuda/CMakeLists.txt    |  2 +-
 GPU/GPUTracking/Base/hip/CMakeLists.txt     |  2 +-
 GPU/GPUTracking/CMakeLists.txt              |  4 ++--
 GPU/GPUTracking/Standalone/CMakeLists.txt   |  4 ++--
 GPU/TPCFastTransformation/CMakeLists.txt    |  2 +-
 dependencies/FindO2GPU.cmake                | 12 ++++++------
 dependencies/O2CompileFlags.cmake           | 18 +++++++++---------
 10 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/Detectors/TPC/calibration/CMakeLists.txt b/Detectors/TPC/calibration/CMakeLists.txt
index fced6bae98a8f..6e53a4af40f26 100644
--- a/Detectors/TPC/calibration/CMakeLists.txt
+++ b/Detectors/TPC/calibration/CMakeLists.txt
@@ -154,7 +154,7 @@ o2_add_test(IDCFourierTransform
             SOURCES test/testO2TPCIDCFourierTransform.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
             LABELS tpc
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 o2_add_test(IDCAverageGroup
             LABELS tpc
@@ -162,7 +162,7 @@ o2_add_test(IDCAverageGroup
             COMPONENT_NAME tpc
             SOURCES test/testO2TPCIDCAverageGroup.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 if (OpenMP_CXX_FOUND)
     target_compile_definitions(${targetName} PRIVATE WITH_OPENMP)
diff --git a/Detectors/TPC/reconstruction/CMakeLists.txt b/Detectors/TPC/reconstruction/CMakeLists.txt
index df3bb2ddb4cff..4ed65603e4712 100644
--- a/Detectors/TPC/reconstruction/CMakeLists.txt
+++ b/Detectors/TPC/reconstruction/CMakeLists.txt
@@ -112,7 +112,7 @@ o2_add_test(FastTransform
             PUBLIC_LINK_LIBRARIES O2::TPCReconstruction
             SOURCES test/testTPCFastTransform.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 # FIXME: should be moved to TPCCalibration as it requires O2::TPCCalibration
 # which is built after TPCReconstruction
diff --git a/Detectors/TPC/spacecharge/CMakeLists.txt b/Detectors/TPC/spacecharge/CMakeLists.txt
index 9af6c14eb351a..a2f4cdb51becb 100644
--- a/Detectors/TPC/spacecharge/CMakeLists.txt
+++ b/Detectors/TPC/spacecharge/CMakeLists.txt
@@ -64,7 +64,7 @@ o2_add_test(PoissonSolver
             SOURCES test/testO2TPCPoissonSolver.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
             LABELS tpc
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 o2_add_test(TriCubic
             COMPONENT_NAME spacecharge
@@ -72,7 +72,7 @@ o2_add_test(TriCubic
             SOURCES test/testO2TPCTricubic.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
             LABELS tpc
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 if (OpenMP_CXX_FOUND)
     target_compile_definitions(${targetName} PRIVATE WITH_OPENMP)
diff --git a/GPU/GPUTracking/Base/cuda/CMakeLists.txt b/GPU/GPUTracking/Base/cuda/CMakeLists.txt
index 652cedeba557c..44bebc4e561a1 100644
--- a/GPU/GPUTracking/Base/cuda/CMakeLists.txt
+++ b/GPU/GPUTracking/Base/cuda/CMakeLists.txt
@@ -37,7 +37,7 @@ if(NOT ALIGPU_BUILD_TYPE STREQUAL "ALIROOT")
   endif()
   #set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -keep")
 
-  set(CUDARTC_FLAGS "${CMAKE_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -std=c++${CMAKE_CUDA_STANDARD}")
+  set(CUDARTC_FLAGS "${CMAKE_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS_$<UPPER_CASE:{CMAKE_BUILD_TYPE}>} -std=c++${CMAKE_CUDA_STANDARD}")
   if(CUDA_COMPUTETARGET)
       foreach(CUDA_ARCH ${CUDA_COMPUTETARGET})
           set(CUDARTC_FLAGS "${CUDARTC_FLAGS} -gencode arch=compute_${CUDA_ARCH},code=sm_${CUDA_ARCH}")
diff --git a/GPU/GPUTracking/Base/hip/CMakeLists.txt b/GPU/GPUTracking/Base/hip/CMakeLists.txt
index 804191be4e224..981db2a38fe18 100644
--- a/GPU/GPUTracking/Base/hip/CMakeLists.txt
+++ b/GPU/GPUTracking/Base/hip/CMakeLists.txt
@@ -77,7 +77,7 @@ if(NOT ALIGPU_BUILD_TYPE STREQUAL "ALIROOT")
     set(HIPRTC_INCLUDES ${HIPRTC_INCLUDES} "-I$<JOIN:$<FILTER:$<TARGET_PROPERTY:O2::ITStrackingHIP,INCLUDE_DIRECTORIES>,EXCLUDE,^/usr/include/?>,$<SEMICOLON>-I>")
   endif()
 
-  set(HIPRTC_FLAGS "${CMAKE_HIP_FLAGS} ${CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE}} -std=c++${CMAKE_HIP_STANDARD}")
+  set(HIPRTC_FLAGS "${CMAKE_HIP_FLAGS} ${CMAKE_HIP_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -std=c++${CMAKE_HIP_STANDARD}")
   separate_arguments(HIPRTC_FLAGS)
 
   # convenience variables
diff --git a/GPU/GPUTracking/CMakeLists.txt b/GPU/GPUTracking/CMakeLists.txt
index 0a8e269e70dcb..9d70d24ab5119 100644
--- a/GPU/GPUTracking/CMakeLists.txt
+++ b/GPU/GPUTracking/CMakeLists.txt
@@ -11,9 +11,9 @@
 
 set(MODULE GPUTracking)
 
-# set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -O0") # to uncomment if needed, tired of typing this...
+# set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -O0") # to uncomment if needed, tired of typing this...
 
-if(NOT ${GPUCA_NO_FAST_MATH} AND NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+if(NOT ${GPUCA_NO_FAST_MATH} AND NOT $<CONFIG:Debug>)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffast-math")
 endif()
 
diff --git a/GPU/GPUTracking/Standalone/CMakeLists.txt b/GPU/GPUTracking/Standalone/CMakeLists.txt
index 643b09a332b2c..4e456c2270c30 100644
--- a/GPU/GPUTracking/Standalone/CMakeLists.txt
+++ b/GPU/GPUTracking/Standalone/CMakeLists.txt
@@ -50,7 +50,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 if(BUILD_DEBUG)
   set(CMAKE_CXX_FLAGS "-O0 -ggdb") # -fsanitize=address,undefined -fno-sanitize=vptr
-  set(CMAKE_BUILD_TYPE DEBUG)
+  set(CMAKE_BUILD_TYPE Debug)
 else()
   set(CMAKE_CXX_FLAGS "-O3 -march=native -ggdb -minline-all-stringops -funroll-loops -fno-stack-protector")
   if(DEFINED GPUCA_NO_FAST_MATH AND ${GPUCA_NO_FAST_MATH})
@@ -61,7 +61,7 @@ else()
   if (NOT CMAKE_CXX_COMPILER STREQUAL "clang++")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ftracer -fprefetch-loop-arrays")
   endif()
-  set(CMAKE_BUILD_TYPE RELEASE)
+  set(CMAKE_BUILD_TYPE Release)
   add_definitions(-DNDEBUG)
 endif()
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error -Wall -Wextra -Wshadow -Wno-unused-function -Wno-unused-parameter -Wno-unused-local-typedefs -Wno-unknown-pragmas -Wno-write-strings")
diff --git a/GPU/TPCFastTransformation/CMakeLists.txt b/GPU/TPCFastTransformation/CMakeLists.txt
index 7e14943f313e3..b338e1492cc6c 100644
--- a/GPU/TPCFastTransformation/CMakeLists.txt
+++ b/GPU/TPCFastTransformation/CMakeLists.txt
@@ -81,7 +81,7 @@ if(${ALIGPU_BUILD_TYPE} STREQUAL "O2")
               SOURCES test/testMultivarPolynomials.cxx
               ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
               LABELS gpu
-              CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+              CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
   foreach(m
           SplineDemo.C
diff --git a/dependencies/FindO2GPU.cmake b/dependencies/FindO2GPU.cmake
index da0dd7b85628f..e4c427533f7a1 100644
--- a/dependencies/FindO2GPU.cmake
+++ b/dependencies/FindO2GPU.cmake
@@ -113,14 +113,14 @@ if(ENABLE_CUDA)
       string(APPEND CMAKE_CUDA_FLAGS " -Xcudafe --diag_suppress=20257")
     endif()
     set(CMAKE_CUDA_FLAGS_DEBUG "${CMAKE_CUDA_FLAGS_DEBUG} -lineinfo -Xcompiler \"${CMAKE_CXX_FLAGS_DEBUG}\" -Xptxas -O0 -Xcompiler -O0")
-    if(NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -Xcompiler \"${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}}\" -Xptxas -O4 -Xcompiler -O4")
+    if(NOT $<CONFIG:Debug>)
+      set(CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -Xcompiler \"${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>}\" -Xptxas -O4 -Xcompiler -O4")
     endif()
     set(GPUCA_CUDA_NO_FAST_MATH_FLAGS "--ftz=false --prec-div=true --prec-sqrt=true --fmad false")
     if(DEFINED GPUCA_NO_FAST_MATH AND "${GPUCA_NO_FAST_MATH}")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} ${GPUCA_CUDA_NO_FAST_MATH_FLAGS}")
-    elseif(NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -use_fast_math --ftz=true")#
+      set(CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} ${GPUCA_CUDA_NO_FAST_MATH_FLAGS}")
+    elseif(NOT $<CONFIG:Debug>)
+      set(CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CUDA_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -use_fast_math --ftz=true")#
     endif()
     if(CMAKE_CXX_FLAGS MATCHES "(^| )-Werror( |$)")
       set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Werror=cross-execution-space-call")
@@ -274,7 +274,7 @@ if(ENABLE_HIP)
       set(O2_HIP_CMAKE_CXX_FLAGS "${O2_HIP_CMAKE_CXX_FLAGS} --gcc-toolchain=${HIP_GCC_TOOLCHAIN_PATH}") # -ffast-math disabled, since apparently it leads to miscompilation and crashes in FollowLooper kernel
     endif()
     set(CMAKE_HIP_FLAGS "${O2_GPU_CMAKE_CXX_FLAGS_NOSTD} ${CMAKE_HIP_FLAGS} ${O2_HIP_CMAKE_CXX_FLAGS}")
-    set(CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE}}")
+    set(CMAKE_HIP_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} ${CMAKE_HIP_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>}")
   else()
     set(HIP_ENABLED OFF)
   endif()
diff --git a/dependencies/O2CompileFlags.cmake b/dependencies/O2CompileFlags.cmake
index 1157a530a2ca8..6fd27c82a261c 100644
--- a/dependencies/O2CompileFlags.cmake
+++ b/dependencies/O2CompileFlags.cmake
@@ -104,13 +104,13 @@ IF(ENABLE_CASSERT) #For the CI, we want to have <cassert> assertions enabled
 ELSE()
     set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
     set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG")
-    if (CMAKE_BUILD_TYPE STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE STREQUAL "RELWITHDEBINFO")
+    if ($<CONFIG:Release> OR $<CONFIG:RelWithDebInfo>)
       set(FAIR_MIN_SEVERITY "detail")
     endif()
 ENDIF()
 
 IF(ENABLE_THREAD_SAFETY_ANALYSIS)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -Werror=thread-safety -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS")
+  set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -Werror=thread-safety -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS")
 ENDIF()
 
 set(CMAKE_C_FLAGS_RELEASE "-O2")
@@ -122,8 +122,8 @@ set(CMAKE_CXX_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 set(CMAKE_C_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 set(CMAKE_Fortran_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 
-set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_CXX_WARNINGS}")
-set(CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_C_WARNINGS}")
+set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} ${CMAKE_CXX_WARNINGS}")
+set(CMAKE_C_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_C_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} ${CMAKE_C_WARNINGS}")
 
 if(APPLE)
 elseif(UNIX)
@@ -131,19 +131,19 @@ elseif(UNIX)
 endif()
 
 if(ENABLE_TIME_TRACE)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -ftime-trace")
+  set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -ftime-trace")
 endif()
 
 if(DEFINED ENV{O2_CXXFLAGS_OVERRIDE})
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} $ENV{O2_CXXFLAGS_OVERRIDE}")
+  set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} $ENV{O2_CXXFLAGS_OVERRIDE}")
   message(STATUS "Setting CXXFLAGS Override $ENV{O2_CXXFLAGS_OVERRIDE}")
 endif()
 
 if(GPUCA_NO_FAST_MATH_WHOLEO2)
   set(GPUCA_NO_FAST_MATH 1)
   add_definitions(-DGPUCA_NO_FAST_MATH)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -fno-fast-math -ffp-contract=off")
-  set(CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}} -fno-fast-math -ffp-contract=off")
+  set(CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -fno-fast-math -ffp-contract=off")
+  set(CMAKE_C_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}> "${CMAKE_C_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>} -fno-fast-math -ffp-contract=off")
 endif()
 
-message(STATUS "Using build type: ${CMAKE_BUILD_TYPE} - CXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}}")
+message(STATUS "Using build type: ${CMAKE_BUILD_TYPE} - CXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_$<UPPER_CASE:${CMAKE_BUILD_TYPE}>}")

From beb72943b19db905673531084098b02e7cc954f8 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 19 Mar 2024 12:25:51 +0100
Subject: [PATCH 457/726] Follow conventions for CMAKE_BUILD_TYPE

The way case sensitiveness works in CMake is described at:

https://cmake.org/cmake/help/v3.29/manual/cmake-buildsystem.7.html#case-sensitivity

and the "strong" convention is described a few lines below:

https://cmake.org/cmake/help/v3.29/manual/cmake-buildsystem.7.html#id38

Notice also how the variables for customising CMAKE_CXX_FLAGS (e.g. CMAKE_CXX_FLAGS_DEBUG)
are always all uppercase, regardless of the value of CMAKE_BUILD_TYPE. We use the
camel cased version, because that's what strongly suggested by CMake (probably because the
generators for some IDE / tool use that convention).

Moreover MinSizeRel is the typical spelling, not MinRelSize.
---
 Detectors/TPC/calibration/CMakeLists.txt    |  4 ++--
 Detectors/TPC/reconstruction/CMakeLists.txt |  2 +-
 Detectors/TPC/spacecharge/CMakeLists.txt    |  4 ++--
 GPU/GPUTracking/Base/cuda/CMakeLists.txt    |  4 ++--
 GPU/GPUTracking/Base/hip/CMakeLists.txt     |  3 ++-
 GPU/GPUTracking/CMakeLists.txt              |  5 +++--
 GPU/TPCFastTransformation/CMakeLists.txt    |  2 +-
 dependencies/FindO2GPU.cmake                | 14 ++++++++------
 dependencies/O2CompileFlags.cmake           | 20 +++++++++++---------
 9 files changed, 32 insertions(+), 26 deletions(-)

diff --git a/Detectors/TPC/calibration/CMakeLists.txt b/Detectors/TPC/calibration/CMakeLists.txt
index fced6bae98a8f..6e53a4af40f26 100644
--- a/Detectors/TPC/calibration/CMakeLists.txt
+++ b/Detectors/TPC/calibration/CMakeLists.txt
@@ -154,7 +154,7 @@ o2_add_test(IDCFourierTransform
             SOURCES test/testO2TPCIDCFourierTransform.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
             LABELS tpc
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 o2_add_test(IDCAverageGroup
             LABELS tpc
@@ -162,7 +162,7 @@ o2_add_test(IDCAverageGroup
             COMPONENT_NAME tpc
             SOURCES test/testO2TPCIDCAverageGroup.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 if (OpenMP_CXX_FOUND)
     target_compile_definitions(${targetName} PRIVATE WITH_OPENMP)
diff --git a/Detectors/TPC/reconstruction/CMakeLists.txt b/Detectors/TPC/reconstruction/CMakeLists.txt
index df3bb2ddb4cff..4ed65603e4712 100644
--- a/Detectors/TPC/reconstruction/CMakeLists.txt
+++ b/Detectors/TPC/reconstruction/CMakeLists.txt
@@ -112,7 +112,7 @@ o2_add_test(FastTransform
             PUBLIC_LINK_LIBRARIES O2::TPCReconstruction
             SOURCES test/testTPCFastTransform.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 # FIXME: should be moved to TPCCalibration as it requires O2::TPCCalibration
 # which is built after TPCReconstruction
diff --git a/Detectors/TPC/spacecharge/CMakeLists.txt b/Detectors/TPC/spacecharge/CMakeLists.txt
index 9af6c14eb351a..a2f4cdb51becb 100644
--- a/Detectors/TPC/spacecharge/CMakeLists.txt
+++ b/Detectors/TPC/spacecharge/CMakeLists.txt
@@ -64,7 +64,7 @@ o2_add_test(PoissonSolver
             SOURCES test/testO2TPCPoissonSolver.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
             LABELS tpc
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 o2_add_test(TriCubic
             COMPONENT_NAME spacecharge
@@ -72,7 +72,7 @@ o2_add_test(TriCubic
             SOURCES test/testO2TPCTricubic.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
             LABELS tpc
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 if (OpenMP_CXX_FOUND)
     target_compile_definitions(${targetName} PRIVATE WITH_OPENMP)
diff --git a/GPU/GPUTracking/Base/cuda/CMakeLists.txt b/GPU/GPUTracking/Base/cuda/CMakeLists.txt
index 652cedeba557c..01a0c88c166e3 100644
--- a/GPU/GPUTracking/Base/cuda/CMakeLists.txt
+++ b/GPU/GPUTracking/Base/cuda/CMakeLists.txt
@@ -18,7 +18,7 @@ message(STATUS "Building GPUTracking with CUDA support ${TMP_TARGET}")
 
 set(SRCS GPUReconstructionCUDA.cu GPUReconstructionCUDAGenRTC.cxx GPUReconstructionCUDAKernels.cu)
 set(HDRS GPUReconstructionCUDA.h GPUReconstructionCUDAInternals.h GPUReconstructionCUDADef.h GPUReconstructionCUDAIncludes.h CUDAThrustHelpers.h)
-
+string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE_UPPER)
 # -------------------------------- Prepare RTC -------------------------------------------------------
 if(NOT ALIGPU_BUILD_TYPE STREQUAL "ALIROOT")
   enable_language(ASM)
@@ -37,7 +37,7 @@ if(NOT ALIGPU_BUILD_TYPE STREQUAL "ALIROOT")
   endif()
   #set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -keep")
 
-  set(CUDARTC_FLAGS "${CMAKE_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -std=c++${CMAKE_CUDA_STANDARD}")
+  set(CUDARTC_FLAGS "${CMAKE_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS_${BUILD_TYPE_UPPER}} -std=c++${CMAKE_CUDA_STANDARD}")
   if(CUDA_COMPUTETARGET)
       foreach(CUDA_ARCH ${CUDA_COMPUTETARGET})
           set(CUDARTC_FLAGS "${CUDARTC_FLAGS} -gencode arch=compute_${CUDA_ARCH},code=sm_${CUDA_ARCH}")
diff --git a/GPU/GPUTracking/Base/hip/CMakeLists.txt b/GPU/GPUTracking/Base/hip/CMakeLists.txt
index 804191be4e224..86212978fcdcd 100644
--- a/GPU/GPUTracking/Base/hip/CMakeLists.txt
+++ b/GPU/GPUTracking/Base/hip/CMakeLists.txt
@@ -60,6 +60,7 @@ set(SRCS ${GPUCA_HIP_SOURCE_DIR}/GPUReconstructionHIP.hip ${GPUCA_HIP_SOURCE_DIR
 set(SRCS_CXX ${GPUCA_HIP_SOURCE_DIR}/GPUReconstructionHIPGenRTC.cxx)
 set(HDRS ${GPUCA_HIP_SOURCE_DIR}/GPUReconstructionHIP.h ${GPUCA_HIP_SOURCE_DIR}/GPUReconstructionHIPInternals.h ${GPUCA_HIP_SOURCE_DIR}/GPUReconstructionHIPDef.h ${GPUCA_HIP_SOURCE_DIR}/GPUReconstructionHIPIncludes.h ${GPUCA_HIP_SOURCE_DIR}/HIPThrustHelpers.h)
 
+string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE_UPPER)
 # -------------------------------- Prepare RTC -------------------------------------------------------
 if(NOT ALIGPU_BUILD_TYPE STREQUAL "ALIROOT")
   enable_language(ASM)
@@ -77,7 +78,7 @@ if(NOT ALIGPU_BUILD_TYPE STREQUAL "ALIROOT")
     set(HIPRTC_INCLUDES ${HIPRTC_INCLUDES} "-I$<JOIN:$<FILTER:$<TARGET_PROPERTY:O2::ITStrackingHIP,INCLUDE_DIRECTORIES>,EXCLUDE,^/usr/include/?>,$<SEMICOLON>-I>")
   endif()
 
-  set(HIPRTC_FLAGS "${CMAKE_HIP_FLAGS} ${CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE}} -std=c++${CMAKE_HIP_STANDARD}")
+  set(HIPRTC_FLAGS "${CMAKE_HIP_FLAGS} ${CMAKE_HIP_FLAGS_${BUILD_TYPE_UPPER}} -std=c++${CMAKE_HIP_STANDARD}")
   separate_arguments(HIPRTC_FLAGS)
 
   # convenience variables
diff --git a/GPU/GPUTracking/CMakeLists.txt b/GPU/GPUTracking/CMakeLists.txt
index 0a8e269e70dcb..84c7bc9446822 100644
--- a/GPU/GPUTracking/CMakeLists.txt
+++ b/GPU/GPUTracking/CMakeLists.txt
@@ -11,9 +11,10 @@
 
 set(MODULE GPUTracking)
 
-# set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -O0") # to uncomment if needed, tired of typing this...
+# string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE_UPPER)
+# set(CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}} -O0") # to uncomment if needed, tired of typing this...
 
-if(NOT ${GPUCA_NO_FAST_MATH} AND NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+if(NOT ${GPUCA_NO_FAST_MATH} AND NOT BUILD_TYPE_UPPER STREQUAL "DEBUG")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffast-math")
 endif()
 
diff --git a/GPU/TPCFastTransformation/CMakeLists.txt b/GPU/TPCFastTransformation/CMakeLists.txt
index 7e14943f313e3..b338e1492cc6c 100644
--- a/GPU/TPCFastTransformation/CMakeLists.txt
+++ b/GPU/TPCFastTransformation/CMakeLists.txt
@@ -81,7 +81,7 @@ if(${ALIGPU_BUILD_TYPE} STREQUAL "O2")
               SOURCES test/testMultivarPolynomials.cxx
               ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
               LABELS gpu
-              CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+              CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
   foreach(m
           SplineDemo.C
diff --git a/dependencies/FindO2GPU.cmake b/dependencies/FindO2GPU.cmake
index da0dd7b85628f..e38a8f1608da2 100644
--- a/dependencies/FindO2GPU.cmake
+++ b/dependencies/FindO2GPU.cmake
@@ -65,6 +65,8 @@ endfunction()
 # Detect and enable CUDA
 STRING(REGEX REPLACE "\-std=[^ ]*" "" O2_GPU_CMAKE_CXX_FLAGS_NOSTD "${CMAKE_CXX_FLAGS}") # Need to strip c++17 imposed by alidist defaults
 
+string(TOUPPER ${CMAKE_BUILD_TYPE} BUILD_TYPE_UPPER)
+
 if(ENABLE_CUDA)
   set(CMAKE_CUDA_STANDARD 17)
   set(CMAKE_CUDA_STANDARD_REQUIRED TRUE)
@@ -113,14 +115,14 @@ if(ENABLE_CUDA)
       string(APPEND CMAKE_CUDA_FLAGS " -Xcudafe --diag_suppress=20257")
     endif()
     set(CMAKE_CUDA_FLAGS_DEBUG "${CMAKE_CUDA_FLAGS_DEBUG} -lineinfo -Xcompiler \"${CMAKE_CXX_FLAGS_DEBUG}\" -Xptxas -O0 -Xcompiler -O0")
-    if(NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -Xcompiler \"${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}}\" -Xptxas -O4 -Xcompiler -O4")
+    if(NOT BUILD_TYPE_UPPER STREQUAL "DEBUG")
+      set(CMAKE_CUDA_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CUDA_FLAGS_${BUILD_TYPE_UPPER}} -Xcompiler \"${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}}\" -Xptxas -O4 -Xcompiler -O4")
     endif()
     set(GPUCA_CUDA_NO_FAST_MATH_FLAGS "--ftz=false --prec-div=true --prec-sqrt=true --fmad false")
     if(DEFINED GPUCA_NO_FAST_MATH AND "${GPUCA_NO_FAST_MATH}")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} ${GPUCA_CUDA_NO_FAST_MATH_FLAGS}")
-    elseif(NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -use_fast_math --ftz=true")#
+      set(CMAKE_CUDA_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CUDA_FLAGS_${BUILD_TYPE_UPPER}} ${GPUCA_CUDA_NO_FAST_MATH_FLAGS}")
+    elseif(NOT BUILD_TYPE_UPPER STREQUAL "DEBUG")
+      set(CMAKE_CUDA_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CUDA_FLAGS_${BUILD_TYPE_UPPER}} -use_fast_math --ftz=true")#
     endif()
     if(CMAKE_CXX_FLAGS MATCHES "(^| )-Werror( |$)")
       set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Werror=cross-execution-space-call")
@@ -274,7 +276,7 @@ if(ENABLE_HIP)
       set(O2_HIP_CMAKE_CXX_FLAGS "${O2_HIP_CMAKE_CXX_FLAGS} --gcc-toolchain=${HIP_GCC_TOOLCHAIN_PATH}") # -ffast-math disabled, since apparently it leads to miscompilation and crashes in FollowLooper kernel
     endif()
     set(CMAKE_HIP_FLAGS "${O2_GPU_CMAKE_CXX_FLAGS_NOSTD} ${CMAKE_HIP_FLAGS} ${O2_HIP_CMAKE_CXX_FLAGS}")
-    set(CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE}}")
+    set(CMAKE_HIP_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}} ${CMAKE_HIP_FLAGS_${BUILD_TYPE_UPPER}}")
   else()
     set(HIP_ENABLED OFF)
   endif()
diff --git a/dependencies/O2CompileFlags.cmake b/dependencies/O2CompileFlags.cmake
index 1157a530a2ca8..844ec5d9755a4 100644
--- a/dependencies/O2CompileFlags.cmake
+++ b/dependencies/O2CompileFlags.cmake
@@ -98,19 +98,21 @@ IF (NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE Debug)
 ENDIF (NOT CMAKE_BUILD_TYPE)
 
+string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE_UPPER)
+
 IF(ENABLE_CASSERT) #For the CI, we want to have <cassert> assertions enabled
     set(CMAKE_CXX_FLAGS_RELEASE "-O2")
     set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g")
 ELSE()
     set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
     set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG")
-    if (CMAKE_BUILD_TYPE STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE STREQUAL "RELWITHDEBINFO")
+    if (BUILD_TYPE_UPPER STREQUAL "RELEASE" OR BUILD_TYPE_UPPER STREQUAL "RELWITHDEBINFO")
       set(FAIR_MIN_SEVERITY "detail")
     endif()
 ENDIF()
 
 IF(ENABLE_THREAD_SAFETY_ANALYSIS)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -Werror=thread-safety -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS")
+  set(CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}} -Werror=thread-safety -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS")
 ENDIF()
 
 set(CMAKE_C_FLAGS_RELEASE "-O2")
@@ -122,8 +124,8 @@ set(CMAKE_CXX_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 set(CMAKE_C_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 set(CMAKE_Fortran_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 
-set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_CXX_WARNINGS}")
-set(CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_C_WARNINGS}")
+set(CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}} ${CMAKE_CXX_WARNINGS}")
+set(CMAKE_C_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_C_FLAGS_${BUILD_TYPE_UPPER}} ${CMAKE_C_WARNINGS}")
 
 if(APPLE)
 elseif(UNIX)
@@ -131,19 +133,19 @@ elseif(UNIX)
 endif()
 
 if(ENABLE_TIME_TRACE)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -ftime-trace")
+  set(CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}} -ftime-trace")
 endif()
 
 if(DEFINED ENV{O2_CXXFLAGS_OVERRIDE})
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} $ENV{O2_CXXFLAGS_OVERRIDE}")
+  set(CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}} $ENV{O2_CXXFLAGS_OVERRIDE}")
   message(STATUS "Setting CXXFLAGS Override $ENV{O2_CXXFLAGS_OVERRIDE}")
 endif()
 
 if(GPUCA_NO_FAST_MATH_WHOLEO2)
   set(GPUCA_NO_FAST_MATH 1)
   add_definitions(-DGPUCA_NO_FAST_MATH)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -fno-fast-math -ffp-contract=off")
-  set(CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}} -fno-fast-math -ffp-contract=off")
+  set(CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}} -fno-fast-math -ffp-contract=off")
+  set(CMAKE_C_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_C_FLAGS_${BUILD_TYPE_UPPER}} -fno-fast-math -ffp-contract=off")
 endif()
 
-message(STATUS "Using build type: ${CMAKE_BUILD_TYPE} - CXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}}")
+message(STATUS "Using build type: ${CMAKE_BUILD_TYPE} - CXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}}")

From 49190e9284c11715276c7a4bde50abe22b0c0bd3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 19 Mar 2024 12:25:51 +0100
Subject: [PATCH 458/726] Follow conventions for CMAKE_BUILD_TYPE

The way case sensitiveness works in CMake is described at:

https://cmake.org/cmake/help/v3.29/manual/cmake-buildsystem.7.html#case-sensitivity

and the "strong" convention is described a few lines below:

https://cmake.org/cmake/help/v3.29/manual/cmake-buildsystem.7.html#id38

Notice also how the variables for customising CMAKE_CXX_FLAGS (e.g. CMAKE_CXX_FLAGS_DEBUG)
are always all uppercase, regardless of the value of CMAKE_BUILD_TYPE. We use the
camel cased version, because that's what strongly suggested by CMake (probably because the
generators for some IDE / tool use that convention).

Moreover MinSizeRel is the typical spelling, not MinRelSize.
---
 Detectors/TPC/calibration/CMakeLists.txt    |  4 ++--
 Detectors/TPC/reconstruction/CMakeLists.txt |  4 ++--
 Detectors/TPC/spacecharge/CMakeLists.txt    |  4 ++--
 GPU/GPUTracking/Base/cuda/CMakeLists.txt    |  4 ++--
 GPU/GPUTracking/Base/hip/CMakeLists.txt     |  3 ++-
 GPU/GPUTracking/CMakeLists.txt              |  5 +++--
 GPU/TPCFastTransformation/CMakeLists.txt    |  2 +-
 dependencies/FindO2GPU.cmake                | 14 ++++++++------
 dependencies/O2CompileFlags.cmake           | 20 +++++++++++---------
 9 files changed, 33 insertions(+), 27 deletions(-)

diff --git a/Detectors/TPC/calibration/CMakeLists.txt b/Detectors/TPC/calibration/CMakeLists.txt
index fced6bae98a8f..6e53a4af40f26 100644
--- a/Detectors/TPC/calibration/CMakeLists.txt
+++ b/Detectors/TPC/calibration/CMakeLists.txt
@@ -154,7 +154,7 @@ o2_add_test(IDCFourierTransform
             SOURCES test/testO2TPCIDCFourierTransform.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
             LABELS tpc
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 o2_add_test(IDCAverageGroup
             LABELS tpc
@@ -162,7 +162,7 @@ o2_add_test(IDCAverageGroup
             COMPONENT_NAME tpc
             SOURCES test/testO2TPCIDCAverageGroup.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 if (OpenMP_CXX_FOUND)
     target_compile_definitions(${targetName} PRIVATE WITH_OPENMP)
diff --git a/Detectors/TPC/reconstruction/CMakeLists.txt b/Detectors/TPC/reconstruction/CMakeLists.txt
index df3bb2ddb4cff..29e6d692968b7 100644
--- a/Detectors/TPC/reconstruction/CMakeLists.txt
+++ b/Detectors/TPC/reconstruction/CMakeLists.txt
@@ -36,7 +36,7 @@ o2_add_library(TPCReconstruction
                                      O2::TPCBase
                                      O2::GPUO2Interface
                                      O2::TPCFastTransformation
-                                     O2::DetectorsRaw 
+                                     O2::DetectorsRaw
                                      O2::DetectorsBase)
 
 o2_target_root_dictionary(
@@ -112,7 +112,7 @@ o2_add_test(FastTransform
             PUBLIC_LINK_LIBRARIES O2::TPCReconstruction
             SOURCES test/testTPCFastTransform.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 # FIXME: should be moved to TPCCalibration as it requires O2::TPCCalibration
 # which is built after TPCReconstruction
diff --git a/Detectors/TPC/spacecharge/CMakeLists.txt b/Detectors/TPC/spacecharge/CMakeLists.txt
index 9af6c14eb351a..a2f4cdb51becb 100644
--- a/Detectors/TPC/spacecharge/CMakeLists.txt
+++ b/Detectors/TPC/spacecharge/CMakeLists.txt
@@ -64,7 +64,7 @@ o2_add_test(PoissonSolver
             SOURCES test/testO2TPCPoissonSolver.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
             LABELS tpc
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 o2_add_test(TriCubic
             COMPONENT_NAME spacecharge
@@ -72,7 +72,7 @@ o2_add_test(TriCubic
             SOURCES test/testO2TPCTricubic.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
             LABELS tpc
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 if (OpenMP_CXX_FOUND)
     target_compile_definitions(${targetName} PRIVATE WITH_OPENMP)
diff --git a/GPU/GPUTracking/Base/cuda/CMakeLists.txt b/GPU/GPUTracking/Base/cuda/CMakeLists.txt
index 652cedeba557c..01a0c88c166e3 100644
--- a/GPU/GPUTracking/Base/cuda/CMakeLists.txt
+++ b/GPU/GPUTracking/Base/cuda/CMakeLists.txt
@@ -18,7 +18,7 @@ message(STATUS "Building GPUTracking with CUDA support ${TMP_TARGET}")
 
 set(SRCS GPUReconstructionCUDA.cu GPUReconstructionCUDAGenRTC.cxx GPUReconstructionCUDAKernels.cu)
 set(HDRS GPUReconstructionCUDA.h GPUReconstructionCUDAInternals.h GPUReconstructionCUDADef.h GPUReconstructionCUDAIncludes.h CUDAThrustHelpers.h)
-
+string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE_UPPER)
 # -------------------------------- Prepare RTC -------------------------------------------------------
 if(NOT ALIGPU_BUILD_TYPE STREQUAL "ALIROOT")
   enable_language(ASM)
@@ -37,7 +37,7 @@ if(NOT ALIGPU_BUILD_TYPE STREQUAL "ALIROOT")
   endif()
   #set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -keep")
 
-  set(CUDARTC_FLAGS "${CMAKE_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -std=c++${CMAKE_CUDA_STANDARD}")
+  set(CUDARTC_FLAGS "${CMAKE_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS_${BUILD_TYPE_UPPER}} -std=c++${CMAKE_CUDA_STANDARD}")
   if(CUDA_COMPUTETARGET)
       foreach(CUDA_ARCH ${CUDA_COMPUTETARGET})
           set(CUDARTC_FLAGS "${CUDARTC_FLAGS} -gencode arch=compute_${CUDA_ARCH},code=sm_${CUDA_ARCH}")
diff --git a/GPU/GPUTracking/Base/hip/CMakeLists.txt b/GPU/GPUTracking/Base/hip/CMakeLists.txt
index 804191be4e224..86212978fcdcd 100644
--- a/GPU/GPUTracking/Base/hip/CMakeLists.txt
+++ b/GPU/GPUTracking/Base/hip/CMakeLists.txt
@@ -60,6 +60,7 @@ set(SRCS ${GPUCA_HIP_SOURCE_DIR}/GPUReconstructionHIP.hip ${GPUCA_HIP_SOURCE_DIR
 set(SRCS_CXX ${GPUCA_HIP_SOURCE_DIR}/GPUReconstructionHIPGenRTC.cxx)
 set(HDRS ${GPUCA_HIP_SOURCE_DIR}/GPUReconstructionHIP.h ${GPUCA_HIP_SOURCE_DIR}/GPUReconstructionHIPInternals.h ${GPUCA_HIP_SOURCE_DIR}/GPUReconstructionHIPDef.h ${GPUCA_HIP_SOURCE_DIR}/GPUReconstructionHIPIncludes.h ${GPUCA_HIP_SOURCE_DIR}/HIPThrustHelpers.h)
 
+string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE_UPPER)
 # -------------------------------- Prepare RTC -------------------------------------------------------
 if(NOT ALIGPU_BUILD_TYPE STREQUAL "ALIROOT")
   enable_language(ASM)
@@ -77,7 +78,7 @@ if(NOT ALIGPU_BUILD_TYPE STREQUAL "ALIROOT")
     set(HIPRTC_INCLUDES ${HIPRTC_INCLUDES} "-I$<JOIN:$<FILTER:$<TARGET_PROPERTY:O2::ITStrackingHIP,INCLUDE_DIRECTORIES>,EXCLUDE,^/usr/include/?>,$<SEMICOLON>-I>")
   endif()
 
-  set(HIPRTC_FLAGS "${CMAKE_HIP_FLAGS} ${CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE}} -std=c++${CMAKE_HIP_STANDARD}")
+  set(HIPRTC_FLAGS "${CMAKE_HIP_FLAGS} ${CMAKE_HIP_FLAGS_${BUILD_TYPE_UPPER}} -std=c++${CMAKE_HIP_STANDARD}")
   separate_arguments(HIPRTC_FLAGS)
 
   # convenience variables
diff --git a/GPU/GPUTracking/CMakeLists.txt b/GPU/GPUTracking/CMakeLists.txt
index 0a8e269e70dcb..84c7bc9446822 100644
--- a/GPU/GPUTracking/CMakeLists.txt
+++ b/GPU/GPUTracking/CMakeLists.txt
@@ -11,9 +11,10 @@
 
 set(MODULE GPUTracking)
 
-# set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -O0") # to uncomment if needed, tired of typing this...
+# string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE_UPPER)
+# set(CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}} -O0") # to uncomment if needed, tired of typing this...
 
-if(NOT ${GPUCA_NO_FAST_MATH} AND NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+if(NOT ${GPUCA_NO_FAST_MATH} AND NOT BUILD_TYPE_UPPER STREQUAL "DEBUG")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffast-math")
 endif()
 
diff --git a/GPU/TPCFastTransformation/CMakeLists.txt b/GPU/TPCFastTransformation/CMakeLists.txt
index 7e14943f313e3..b338e1492cc6c 100644
--- a/GPU/TPCFastTransformation/CMakeLists.txt
+++ b/GPU/TPCFastTransformation/CMakeLists.txt
@@ -81,7 +81,7 @@ if(${ALIGPU_BUILD_TYPE} STREQUAL "O2")
               SOURCES test/testMultivarPolynomials.cxx
               ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
               LABELS gpu
-              CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+              CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
   foreach(m
           SplineDemo.C
diff --git a/dependencies/FindO2GPU.cmake b/dependencies/FindO2GPU.cmake
index da0dd7b85628f..e38a8f1608da2 100644
--- a/dependencies/FindO2GPU.cmake
+++ b/dependencies/FindO2GPU.cmake
@@ -65,6 +65,8 @@ endfunction()
 # Detect and enable CUDA
 STRING(REGEX REPLACE "\-std=[^ ]*" "" O2_GPU_CMAKE_CXX_FLAGS_NOSTD "${CMAKE_CXX_FLAGS}") # Need to strip c++17 imposed by alidist defaults
 
+string(TOUPPER ${CMAKE_BUILD_TYPE} BUILD_TYPE_UPPER)
+
 if(ENABLE_CUDA)
   set(CMAKE_CUDA_STANDARD 17)
   set(CMAKE_CUDA_STANDARD_REQUIRED TRUE)
@@ -113,14 +115,14 @@ if(ENABLE_CUDA)
       string(APPEND CMAKE_CUDA_FLAGS " -Xcudafe --diag_suppress=20257")
     endif()
     set(CMAKE_CUDA_FLAGS_DEBUG "${CMAKE_CUDA_FLAGS_DEBUG} -lineinfo -Xcompiler \"${CMAKE_CXX_FLAGS_DEBUG}\" -Xptxas -O0 -Xcompiler -O0")
-    if(NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -Xcompiler \"${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}}\" -Xptxas -O4 -Xcompiler -O4")
+    if(NOT BUILD_TYPE_UPPER STREQUAL "DEBUG")
+      set(CMAKE_CUDA_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CUDA_FLAGS_${BUILD_TYPE_UPPER}} -Xcompiler \"${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}}\" -Xptxas -O4 -Xcompiler -O4")
     endif()
     set(GPUCA_CUDA_NO_FAST_MATH_FLAGS "--ftz=false --prec-div=true --prec-sqrt=true --fmad false")
     if(DEFINED GPUCA_NO_FAST_MATH AND "${GPUCA_NO_FAST_MATH}")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} ${GPUCA_CUDA_NO_FAST_MATH_FLAGS}")
-    elseif(NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -use_fast_math --ftz=true")#
+      set(CMAKE_CUDA_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CUDA_FLAGS_${BUILD_TYPE_UPPER}} ${GPUCA_CUDA_NO_FAST_MATH_FLAGS}")
+    elseif(NOT BUILD_TYPE_UPPER STREQUAL "DEBUG")
+      set(CMAKE_CUDA_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CUDA_FLAGS_${BUILD_TYPE_UPPER}} -use_fast_math --ftz=true")#
     endif()
     if(CMAKE_CXX_FLAGS MATCHES "(^| )-Werror( |$)")
       set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Werror=cross-execution-space-call")
@@ -274,7 +276,7 @@ if(ENABLE_HIP)
       set(O2_HIP_CMAKE_CXX_FLAGS "${O2_HIP_CMAKE_CXX_FLAGS} --gcc-toolchain=${HIP_GCC_TOOLCHAIN_PATH}") # -ffast-math disabled, since apparently it leads to miscompilation and crashes in FollowLooper kernel
     endif()
     set(CMAKE_HIP_FLAGS "${O2_GPU_CMAKE_CXX_FLAGS_NOSTD} ${CMAKE_HIP_FLAGS} ${O2_HIP_CMAKE_CXX_FLAGS}")
-    set(CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE}}")
+    set(CMAKE_HIP_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}} ${CMAKE_HIP_FLAGS_${BUILD_TYPE_UPPER}}")
   else()
     set(HIP_ENABLED OFF)
   endif()
diff --git a/dependencies/O2CompileFlags.cmake b/dependencies/O2CompileFlags.cmake
index 1157a530a2ca8..844ec5d9755a4 100644
--- a/dependencies/O2CompileFlags.cmake
+++ b/dependencies/O2CompileFlags.cmake
@@ -98,19 +98,21 @@ IF (NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE Debug)
 ENDIF (NOT CMAKE_BUILD_TYPE)
 
+string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE_UPPER)
+
 IF(ENABLE_CASSERT) #For the CI, we want to have <cassert> assertions enabled
     set(CMAKE_CXX_FLAGS_RELEASE "-O2")
     set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g")
 ELSE()
     set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
     set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG")
-    if (CMAKE_BUILD_TYPE STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE STREQUAL "RELWITHDEBINFO")
+    if (BUILD_TYPE_UPPER STREQUAL "RELEASE" OR BUILD_TYPE_UPPER STREQUAL "RELWITHDEBINFO")
       set(FAIR_MIN_SEVERITY "detail")
     endif()
 ENDIF()
 
 IF(ENABLE_THREAD_SAFETY_ANALYSIS)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -Werror=thread-safety -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS")
+  set(CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}} -Werror=thread-safety -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS")
 ENDIF()
 
 set(CMAKE_C_FLAGS_RELEASE "-O2")
@@ -122,8 +124,8 @@ set(CMAKE_CXX_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 set(CMAKE_C_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 set(CMAKE_Fortran_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 
-set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_CXX_WARNINGS}")
-set(CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_C_WARNINGS}")
+set(CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}} ${CMAKE_CXX_WARNINGS}")
+set(CMAKE_C_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_C_FLAGS_${BUILD_TYPE_UPPER}} ${CMAKE_C_WARNINGS}")
 
 if(APPLE)
 elseif(UNIX)
@@ -131,19 +133,19 @@ elseif(UNIX)
 endif()
 
 if(ENABLE_TIME_TRACE)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -ftime-trace")
+  set(CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}} -ftime-trace")
 endif()
 
 if(DEFINED ENV{O2_CXXFLAGS_OVERRIDE})
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} $ENV{O2_CXXFLAGS_OVERRIDE}")
+  set(CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}} $ENV{O2_CXXFLAGS_OVERRIDE}")
   message(STATUS "Setting CXXFLAGS Override $ENV{O2_CXXFLAGS_OVERRIDE}")
 endif()
 
 if(GPUCA_NO_FAST_MATH_WHOLEO2)
   set(GPUCA_NO_FAST_MATH 1)
   add_definitions(-DGPUCA_NO_FAST_MATH)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -fno-fast-math -ffp-contract=off")
-  set(CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}} -fno-fast-math -ffp-contract=off")
+  set(CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}} -fno-fast-math -ffp-contract=off")
+  set(CMAKE_C_FLAGS_${BUILD_TYPE_UPPER} "${CMAKE_C_FLAGS_${BUILD_TYPE_UPPER}} -fno-fast-math -ffp-contract=off")
 endif()
 
-message(STATUS "Using build type: ${CMAKE_BUILD_TYPE} - CXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}}")
+message(STATUS "Using build type: ${CMAKE_BUILD_TYPE} - CXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}}")

From d11d06ec2dd0060208a35b6a22161344c1cb8a89 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 19 Mar 2024 13:34:47 +0100
Subject: [PATCH 459/726] Follow conventions for CMAKE_BUILD_TYPE

The way case sensitiveness works in CMake is described at:

https://cmake.org/cmake/help/v3.29/manual/cmake-buildsystem.7.html#case-sensitivity

and the "strong" convention is described a few lines below:

https://cmake.org/cmake/help/v3.29/manual/cmake-buildsystem.7.html#id38

Notice also how the variables for customising CMAKE_CXX_FLAGS (e.g. CMAKE_CXX_FLAGS_DEBUG)
are always all uppercase, regardless of the value of CMAKE_BUILD_TYPE. We use the
camel cased version, because that's what strongly suggested by CMake (probably because the
generators for some IDE / tool use that convention).

Moreover MinSizeRel is the typical spelling, not MinRelSize.
---
 CMakeLists.txt                              |  1 +
 Detectors/TPC/calibration/CMakeLists.txt    |  4 ++--
 Detectors/TPC/reconstruction/CMakeLists.txt |  4 ++--
 Detectors/TPC/spacecharge/CMakeLists.txt    |  4 ++--
 GPU/GPUTracking/Base/cuda/CMakeLists.txt    |  3 +--
 GPU/GPUTracking/Base/hip/CMakeLists.txt     |  2 +-
 GPU/GPUTracking/CMakeLists.txt              |  4 ++--
 GPU/TPCFastTransformation/CMakeLists.txt    |  2 +-
 cmake/O2BuildSanityChecks.cmake             |  1 +
 dependencies/FindO2GPU.cmake                | 12 +++++------
 dependencies/O2CompileFlags.cmake           | 23 ++++++++-------------
 11 files changed, 28 insertions(+), 32 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index bf1d83fbb6644..7cedd723d50ca 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -25,6 +25,7 @@ include(CTest)
 set(ALIGPU_BUILD_TYPE "O2")
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
+string(TOUPPER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_UPPER)
 set_property(GLOBAL PROPERTY REPORT_UNDEFINED_PROPERTIES)
 
 cmake_host_system_information(RESULT _totalmem QUERY TOTAL_PHYSICAL_MEMORY)
diff --git a/Detectors/TPC/calibration/CMakeLists.txt b/Detectors/TPC/calibration/CMakeLists.txt
index fced6bae98a8f..6e53a4af40f26 100644
--- a/Detectors/TPC/calibration/CMakeLists.txt
+++ b/Detectors/TPC/calibration/CMakeLists.txt
@@ -154,7 +154,7 @@ o2_add_test(IDCFourierTransform
             SOURCES test/testO2TPCIDCFourierTransform.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
             LABELS tpc
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 o2_add_test(IDCAverageGroup
             LABELS tpc
@@ -162,7 +162,7 @@ o2_add_test(IDCAverageGroup
             COMPONENT_NAME tpc
             SOURCES test/testO2TPCIDCAverageGroup.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 if (OpenMP_CXX_FOUND)
     target_compile_definitions(${targetName} PRIVATE WITH_OPENMP)
diff --git a/Detectors/TPC/reconstruction/CMakeLists.txt b/Detectors/TPC/reconstruction/CMakeLists.txt
index df3bb2ddb4cff..29e6d692968b7 100644
--- a/Detectors/TPC/reconstruction/CMakeLists.txt
+++ b/Detectors/TPC/reconstruction/CMakeLists.txt
@@ -36,7 +36,7 @@ o2_add_library(TPCReconstruction
                                      O2::TPCBase
                                      O2::GPUO2Interface
                                      O2::TPCFastTransformation
-                                     O2::DetectorsRaw 
+                                     O2::DetectorsRaw
                                      O2::DetectorsBase)
 
 o2_target_root_dictionary(
@@ -112,7 +112,7 @@ o2_add_test(FastTransform
             PUBLIC_LINK_LIBRARIES O2::TPCReconstruction
             SOURCES test/testTPCFastTransform.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 # FIXME: should be moved to TPCCalibration as it requires O2::TPCCalibration
 # which is built after TPCReconstruction
diff --git a/Detectors/TPC/spacecharge/CMakeLists.txt b/Detectors/TPC/spacecharge/CMakeLists.txt
index 9af6c14eb351a..a2f4cdb51becb 100644
--- a/Detectors/TPC/spacecharge/CMakeLists.txt
+++ b/Detectors/TPC/spacecharge/CMakeLists.txt
@@ -64,7 +64,7 @@ o2_add_test(PoissonSolver
             SOURCES test/testO2TPCPoissonSolver.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
             LABELS tpc
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 o2_add_test(TriCubic
             COMPONENT_NAME spacecharge
@@ -72,7 +72,7 @@ o2_add_test(TriCubic
             SOURCES test/testO2TPCTricubic.cxx
             ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
             LABELS tpc
-            CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+            CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
 if (OpenMP_CXX_FOUND)
     target_compile_definitions(${targetName} PRIVATE WITH_OPENMP)
diff --git a/GPU/GPUTracking/Base/cuda/CMakeLists.txt b/GPU/GPUTracking/Base/cuda/CMakeLists.txt
index 652cedeba557c..022ab0d6de667 100644
--- a/GPU/GPUTracking/Base/cuda/CMakeLists.txt
+++ b/GPU/GPUTracking/Base/cuda/CMakeLists.txt
@@ -18,7 +18,6 @@ message(STATUS "Building GPUTracking with CUDA support ${TMP_TARGET}")
 
 set(SRCS GPUReconstructionCUDA.cu GPUReconstructionCUDAGenRTC.cxx GPUReconstructionCUDAKernels.cu)
 set(HDRS GPUReconstructionCUDA.h GPUReconstructionCUDAInternals.h GPUReconstructionCUDADef.h GPUReconstructionCUDAIncludes.h CUDAThrustHelpers.h)
-
 # -------------------------------- Prepare RTC -------------------------------------------------------
 if(NOT ALIGPU_BUILD_TYPE STREQUAL "ALIROOT")
   enable_language(ASM)
@@ -37,7 +36,7 @@ if(NOT ALIGPU_BUILD_TYPE STREQUAL "ALIROOT")
   endif()
   #set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -keep")
 
-  set(CUDARTC_FLAGS "${CMAKE_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -std=c++${CMAKE_CUDA_STANDARD}")
+  set(CUDARTC_FLAGS "${CMAKE_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} -std=c++${CMAKE_CUDA_STANDARD}")
   if(CUDA_COMPUTETARGET)
       foreach(CUDA_ARCH ${CUDA_COMPUTETARGET})
           set(CUDARTC_FLAGS "${CUDARTC_FLAGS} -gencode arch=compute_${CUDA_ARCH},code=sm_${CUDA_ARCH}")
diff --git a/GPU/GPUTracking/Base/hip/CMakeLists.txt b/GPU/GPUTracking/Base/hip/CMakeLists.txt
index 804191be4e224..fa8892acd3686 100644
--- a/GPU/GPUTracking/Base/hip/CMakeLists.txt
+++ b/GPU/GPUTracking/Base/hip/CMakeLists.txt
@@ -77,7 +77,7 @@ if(NOT ALIGPU_BUILD_TYPE STREQUAL "ALIROOT")
     set(HIPRTC_INCLUDES ${HIPRTC_INCLUDES} "-I$<JOIN:$<FILTER:$<TARGET_PROPERTY:O2::ITStrackingHIP,INCLUDE_DIRECTORIES>,EXCLUDE,^/usr/include/?>,$<SEMICOLON>-I>")
   endif()
 
-  set(HIPRTC_FLAGS "${CMAKE_HIP_FLAGS} ${CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE}} -std=c++${CMAKE_HIP_STANDARD}")
+  set(HIPRTC_FLAGS "${CMAKE_HIP_FLAGS} ${CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} -std=c++${CMAKE_HIP_STANDARD}")
   separate_arguments(HIPRTC_FLAGS)
 
   # convenience variables
diff --git a/GPU/GPUTracking/CMakeLists.txt b/GPU/GPUTracking/CMakeLists.txt
index 0a8e269e70dcb..81832fe00e25d 100644
--- a/GPU/GPUTracking/CMakeLists.txt
+++ b/GPU/GPUTracking/CMakeLists.txt
@@ -11,9 +11,9 @@
 
 set(MODULE GPUTracking)
 
-# set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -O0") # to uncomment if needed, tired of typing this...
+# set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} -O0") # to uncomment if needed, tired of typing this...
 
-if(NOT ${GPUCA_NO_FAST_MATH} AND NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+if(NOT ${GPUCA_NO_FAST_MATH} AND NOT CMAKE_BUILD_TYPE_UPPER STREQUAL "DEBUG")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffast-math")
 endif()
 
diff --git a/GPU/TPCFastTransformation/CMakeLists.txt b/GPU/TPCFastTransformation/CMakeLists.txt
index 7e14943f313e3..b338e1492cc6c 100644
--- a/GPU/TPCFastTransformation/CMakeLists.txt
+++ b/GPU/TPCFastTransformation/CMakeLists.txt
@@ -81,7 +81,7 @@ if(${ALIGPU_BUILD_TYPE} STREQUAL "O2")
               SOURCES test/testMultivarPolynomials.cxx
               ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
               LABELS gpu
-              CONFIGURATIONS RelWithDebInfo Release MinRelSize)
+              CONFIGURATIONS RelWithDebInfo Release MinSizeRel)
 
   foreach(m
           SplineDemo.C
diff --git a/cmake/O2BuildSanityChecks.cmake b/cmake/O2BuildSanityChecks.cmake
index 887b490ecfb90..37bde528ffa1e 100644
--- a/cmake/O2BuildSanityChecks.cmake
+++ b/cmake/O2BuildSanityChecks.cmake
@@ -26,5 +26,6 @@ function(o2_build_sanity_checks)
   if(NOT CMAKE_BUILD_TYPE)
     message(WARNING "CMAKE_BUILD_TYPE not set : will use Debug")
     set(CMAKE_BUILD_TYPE Debug)
+    set(CMAKE_BUILD_TYPE_UPPER DEBUG)
   endif()
 endfunction()
diff --git a/dependencies/FindO2GPU.cmake b/dependencies/FindO2GPU.cmake
index da0dd7b85628f..a9c74f912c5d8 100644
--- a/dependencies/FindO2GPU.cmake
+++ b/dependencies/FindO2GPU.cmake
@@ -113,14 +113,14 @@ if(ENABLE_CUDA)
       string(APPEND CMAKE_CUDA_FLAGS " -Xcudafe --diag_suppress=20257")
     endif()
     set(CMAKE_CUDA_FLAGS_DEBUG "${CMAKE_CUDA_FLAGS_DEBUG} -lineinfo -Xcompiler \"${CMAKE_CXX_FLAGS_DEBUG}\" -Xptxas -O0 -Xcompiler -O0")
-    if(NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -Xcompiler \"${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}}\" -Xptxas -O4 -Xcompiler -O4")
+    if(NOT CMAKE_BUILD_TYPE_UPPER STREQUAL "DEBUG")
+      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE_UPPER} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} -Xcompiler \"${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}\" -Xptxas -O4 -Xcompiler -O4")
     endif()
     set(GPUCA_CUDA_NO_FAST_MATH_FLAGS "--ftz=false --prec-div=true --prec-sqrt=true --fmad false")
     if(DEFINED GPUCA_NO_FAST_MATH AND "${GPUCA_NO_FAST_MATH}")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} ${GPUCA_CUDA_NO_FAST_MATH_FLAGS}")
-    elseif(NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG")
-      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}} -use_fast_math --ftz=true")#
+      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE_UPPER} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} ${GPUCA_CUDA_NO_FAST_MATH_FLAGS}")
+    elseif(NOT CMAKE_BUILD_TYPE_UPPER STREQUAL "DEBUG")
+      set(CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE_UPPER} "${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} -use_fast_math --ftz=true")#
     endif()
     if(CMAKE_CXX_FLAGS MATCHES "(^| )-Werror( |$)")
       set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Werror=cross-execution-space-call")
@@ -274,7 +274,7 @@ if(ENABLE_HIP)
       set(O2_HIP_CMAKE_CXX_FLAGS "${O2_HIP_CMAKE_CXX_FLAGS} --gcc-toolchain=${HIP_GCC_TOOLCHAIN_PATH}") # -ffast-math disabled, since apparently it leads to miscompilation and crashes in FollowLooper kernel
     endif()
     set(CMAKE_HIP_FLAGS "${O2_GPU_CMAKE_CXX_FLAGS_NOSTD} ${CMAKE_HIP_FLAGS} ${O2_HIP_CMAKE_CXX_FLAGS}")
-    set(CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE}}")
+    set(CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} ${CMAKE_HIP_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}")
   else()
     set(HIP_ENABLED OFF)
   endif()
diff --git a/dependencies/O2CompileFlags.cmake b/dependencies/O2CompileFlags.cmake
index 1157a530a2ca8..3b4ed68732592 100644
--- a/dependencies/O2CompileFlags.cmake
+++ b/dependencies/O2CompileFlags.cmake
@@ -92,11 +92,6 @@ MARK_AS_ADVANCED(
     CMAKE_Fortran_FLAGS_THREADSANITIZER
     CMAKE_LINK_FLAGS_THREADSANITIZER)
 
-#Check the compiler and set the compile and link flags
-IF (NOT CMAKE_BUILD_TYPE)
-  Message(STATUS "Set BuildType to DEBUG")
-  set(CMAKE_BUILD_TYPE Debug)
-ENDIF (NOT CMAKE_BUILD_TYPE)
 
 IF(ENABLE_CASSERT) #For the CI, we want to have <cassert> assertions enabled
     set(CMAKE_CXX_FLAGS_RELEASE "-O2")
@@ -104,13 +99,13 @@ IF(ENABLE_CASSERT) #For the CI, we want to have <cassert> assertions enabled
 ELSE()
     set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
     set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG")
-    if (CMAKE_BUILD_TYPE STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE STREQUAL "RELWITHDEBINFO")
+    if (CMAKE_BUILD_TYPE_UPPER STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UPPER STREQUAL "RELWITHDEBINFO")
       set(FAIR_MIN_SEVERITY "detail")
     endif()
 ENDIF()
 
 IF(ENABLE_THREAD_SAFETY_ANALYSIS)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -Werror=thread-safety -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS")
+  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} -Werror=thread-safety -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS")
 ENDIF()
 
 set(CMAKE_C_FLAGS_RELEASE "-O2")
@@ -122,8 +117,8 @@ set(CMAKE_CXX_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 set(CMAKE_C_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 set(CMAKE_Fortran_FLAGS_DEBUG "-g -O0" CACHE STRING "Debug mode build flags" FORCE)
 
-set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_CXX_WARNINGS}")
-set(CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}} ${CMAKE_C_WARNINGS}")
+set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} ${CMAKE_CXX_WARNINGS}")
+set(CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UPPER} "${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} ${CMAKE_C_WARNINGS}")
 
 if(APPLE)
 elseif(UNIX)
@@ -131,19 +126,19 @@ elseif(UNIX)
 endif()
 
 if(ENABLE_TIME_TRACE)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -ftime-trace")
+  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} -ftime-trace")
 endif()
 
 if(DEFINED ENV{O2_CXXFLAGS_OVERRIDE})
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} $ENV{O2_CXXFLAGS_OVERRIDE}")
+  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} $ENV{O2_CXXFLAGS_OVERRIDE}")
   message(STATUS "Setting CXXFLAGS Override $ENV{O2_CXXFLAGS_OVERRIDE}")
 endif()
 
 if(GPUCA_NO_FAST_MATH_WHOLEO2)
   set(GPUCA_NO_FAST_MATH 1)
   add_definitions(-DGPUCA_NO_FAST_MATH)
-  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}} -fno-fast-math -ffp-contract=off")
-  set(CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE} "${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}} -fno-fast-math -ffp-contract=off")
+  set(CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER} "${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} -fno-fast-math -ffp-contract=off")
+  set(CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UPPER} "${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UPPER}} -fno-fast-math -ffp-contract=off")
 endif()
 
-message(STATUS "Using build type: ${CMAKE_BUILD_TYPE} - CXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}}")
+message(STATUS "Using build type: ${CMAKE_BUILD_TYPE} - CXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}")

From 5baaece5a873055813cb077cedffc27391942abb Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 20 Mar 2024 11:35:51 +0100
Subject: [PATCH 460/726] DPL: enable environment variables for signposts

---
 Framework/Core/src/DeviceSpecHelpers.cxx         |  2 +-
 .../Core/test/test_FrameworkDataFlowToDDS.cxx    | 16 ++++++++--------
 .../test/test_FrameworkDataFlowToO2Control.cxx   |  8 ++++++++
 3 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 969b3f37bc9eb..30197538f2dda 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1711,7 +1711,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
     ("infologger-mode", bpo::value<std::string>(), "O2_INFOLOGGER_MODE override")                                                                                    //
     ("infologger-severity", bpo::value<std::string>(), "minimun FairLogger severity which goes to info logger")                                                      //
     ("dpl-tracing-flags", bpo::value<std::string>(), "pipe separated list of events to trace")                                                                       //
-    ("signposts", bpo::value<std::string>(),                                                                                                                         //
+    ("signposts", bpo::value<std::string>()->default_value(defaultSignposts),                                                                                        //
      "comma separated list of signposts to enable (any of `completion`, `data_processor_context`, `stream_context`, `device`, `monitoring_service`)")                //
     ("child-driver", bpo::value<std::string>(), "external driver to start childs with (e.g. valgrind)");                                                             //
 
diff --git a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
index 410abe7d6d640..3c08bbae74475 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
@@ -355,19 +355,19 @@ TEST_CASE("TestDDS")
 }"/>
    <decltask name="A">
        <assets><name>dpl_json</name></assets>
-       <exe reachable="true">cat ${DDS_LOCATION}/dpl_json.asset | foo --id A_dds%TaskIndex%_%CollectionIndex% --shm-monitor false --log-color false --batch --color false --channel-config "name=from_A_to_B,type=push,method=bind,address=ipc://@localhostworkflow-id_22000,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --channel-config "name=from_A_to_C,type=push,method=bind,address=ipc://@localhostworkflow-id_22001,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --bad-alloc-attempt-interval 50 --bad-alloc-max-attempts 1 --early-forward-policy never --io-threads 1 --jobs 4 --severity info --shm-allocation rbtree_best_fit --shm-metadata-msg-size 0 --shm-mlock-segment false --shm-mlock-segment-on-creation false --shm-no-cleanup false --shm-segment-id 0 --shm-throw-bad-alloc true --shm-zero-segment false --stacktrace-on-signal simple --timeframes-rate-limit 0 --session dpl_workflow-id --plugin odc</exe>
+       <exe reachable="true">cat ${DDS_LOCATION}/dpl_json.asset | foo --id A_dds%TaskIndex%_%CollectionIndex% --shm-monitor false --log-color false --batch --color false --channel-config "name=from_A_to_B,type=push,method=bind,address=ipc://@localhostworkflow-id_22000,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --channel-config "name=from_A_to_C,type=push,method=bind,address=ipc://@localhostworkflow-id_22001,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --bad-alloc-attempt-interval 50 --bad-alloc-max-attempts 1 --early-forward-policy never --io-threads 1 --jobs 4 --severity info --shm-allocation rbtree_best_fit --shm-metadata-msg-size 0 --shm-mlock-segment false --shm-mlock-segment-on-creation false --shm-no-cleanup false --shm-segment-id 0 --shm-throw-bad-alloc true --shm-zero-segment false --signposts "" --stacktrace-on-signal simple --timeframes-rate-limit 0 --session dpl_workflow-id --plugin odc</exe>
    </decltask>
    <decltask name="B">
        <assets><name>dpl_json</name></assets>
-       <exe reachable="true">cat ${DDS_LOCATION}/dpl_json.asset | foo --id B_dds%TaskIndex%_%CollectionIndex% --shm-monitor false --log-color false --batch --color false --channel-config "name=from_B_to_D,type=push,method=bind,address=ipc://@localhostworkflow-id_22002,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --channel-config "name=from_A_to_B,type=pull,method=connect,address=ipc://@localhostworkflow-id_22000,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --bad-alloc-attempt-interval 50 --bad-alloc-max-attempts 1 --early-forward-policy never --io-threads 1 --jobs 4 --severity info --shm-allocation rbtree_best_fit --shm-metadata-msg-size 0 --shm-mlock-segment false --shm-mlock-segment-on-creation false --shm-no-cleanup false --shm-segment-id 0 --shm-throw-bad-alloc true --shm-zero-segment false --stacktrace-on-signal simple --timeframes-rate-limit 0 --session dpl_workflow-id --plugin odc</exe>
+       <exe reachable="true">cat ${DDS_LOCATION}/dpl_json.asset | foo --id B_dds%TaskIndex%_%CollectionIndex% --shm-monitor false --log-color false --batch --color false --channel-config "name=from_B_to_D,type=push,method=bind,address=ipc://@localhostworkflow-id_22002,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --channel-config "name=from_A_to_B,type=pull,method=connect,address=ipc://@localhostworkflow-id_22000,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --bad-alloc-attempt-interval 50 --bad-alloc-max-attempts 1 --early-forward-policy never --io-threads 1 --jobs 4 --severity info --shm-allocation rbtree_best_fit --shm-metadata-msg-size 0 --shm-mlock-segment false --shm-mlock-segment-on-creation false --shm-no-cleanup false --shm-segment-id 0 --shm-throw-bad-alloc true --shm-zero-segment false --signposts "" --stacktrace-on-signal simple --timeframes-rate-limit 0 --session dpl_workflow-id --plugin odc</exe>
    </decltask>
    <decltask name="C">
        <assets><name>dpl_json</name></assets>
-       <exe reachable="true">cat ${DDS_LOCATION}/dpl_json.asset | foo --id C_dds%TaskIndex%_%CollectionIndex% --shm-monitor false --log-color false --batch --color false --channel-config "name=from_C_to_D,type=push,method=bind,address=ipc://@localhostworkflow-id_22003,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --channel-config "name=from_A_to_C,type=pull,method=connect,address=ipc://@localhostworkflow-id_22001,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --bad-alloc-attempt-interval 50 --bad-alloc-max-attempts 1 --early-forward-policy never --io-threads 1 --jobs 4 --severity info --shm-allocation rbtree_best_fit --shm-metadata-msg-size 0 --shm-mlock-segment false --shm-mlock-segment-on-creation false --shm-no-cleanup false --shm-segment-id 0 --shm-throw-bad-alloc true --shm-zero-segment false --stacktrace-on-signal simple --timeframes-rate-limit 0 --session dpl_workflow-id --plugin odc</exe>
+       <exe reachable="true">cat ${DDS_LOCATION}/dpl_json.asset | foo --id C_dds%TaskIndex%_%CollectionIndex% --shm-monitor false --log-color false --batch --color false --channel-config "name=from_C_to_D,type=push,method=bind,address=ipc://@localhostworkflow-id_22003,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --channel-config "name=from_A_to_C,type=pull,method=connect,address=ipc://@localhostworkflow-id_22001,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --bad-alloc-attempt-interval 50 --bad-alloc-max-attempts 1 --early-forward-policy never --io-threads 1 --jobs 4 --severity info --shm-allocation rbtree_best_fit --shm-metadata-msg-size 0 --shm-mlock-segment false --shm-mlock-segment-on-creation false --shm-no-cleanup false --shm-segment-id 0 --shm-throw-bad-alloc true --shm-zero-segment false --signposts "" --stacktrace-on-signal simple --timeframes-rate-limit 0 --session dpl_workflow-id --plugin odc</exe>
    </decltask>
    <decltask name="D">
        <assets><name>dpl_json</name></assets>
-       <exe reachable="true">cat ${DDS_LOCATION}/dpl_json.asset | foo --id D_dds%TaskIndex%_%CollectionIndex% --shm-monitor false --log-color false --batch --color false --channel-config "name=from_B_to_D,type=pull,method=connect,address=ipc://@localhostworkflow-id_22002,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --channel-config "name=from_C_to_D,type=pull,method=connect,address=ipc://@localhostworkflow-id_22003,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --bad-alloc-attempt-interval 50 --bad-alloc-max-attempts 1 --early-forward-policy never --io-threads 1 --jobs 4 --severity info --shm-allocation rbtree_best_fit --shm-metadata-msg-size 0 --shm-mlock-segment false --shm-mlock-segment-on-creation false --shm-no-cleanup false --shm-segment-id 0 --shm-throw-bad-alloc true --shm-zero-segment false --stacktrace-on-signal simple --timeframes-rate-limit 0 --a-param 1 --b-param "" --c-param "foo;bar" --session dpl_workflow-id --plugin odc</exe>
+       <exe reachable="true">cat ${DDS_LOCATION}/dpl_json.asset | foo --id D_dds%TaskIndex%_%CollectionIndex% --shm-monitor false --log-color false --batch --color false --channel-config "name=from_B_to_D,type=pull,method=connect,address=ipc://@localhostworkflow-id_22002,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --channel-config "name=from_C_to_D,type=pull,method=connect,address=ipc://@localhostworkflow-id_22003,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --bad-alloc-attempt-interval 50 --bad-alloc-max-attempts 1 --early-forward-policy never --io-threads 1 --jobs 4 --severity info --shm-allocation rbtree_best_fit --shm-metadata-msg-size 0 --shm-mlock-segment false --shm-mlock-segment-on-creation false --shm-no-cleanup false --shm-segment-id 0 --shm-throw-bad-alloc true --shm-zero-segment false --signposts "" --stacktrace-on-signal simple --timeframes-rate-limit 0 --a-param 1 --b-param "" --c-param "foo;bar" --session dpl_workflow-id --plugin odc</exe>
    </decltask>
    <declcollection name="DPL">
        <tasks>
@@ -622,19 +622,19 @@ TEST_CASE("TestDDSExpendable")
 }"/>
    <decltask name="A">
        <assets><name>dpl_json</name></assets>
-       <exe reachable="true">cat ${DDS_LOCATION}/dpl_json.asset | foo --id A_dds%TaskIndex%_%CollectionIndex% --shm-monitor false --log-color false --batch --color false --channel-config "name=from_A_to_B,type=push,method=bind,address=ipc://@localhostworkflow-id_22000,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --channel-config "name=from_A_to_C,type=push,method=bind,address=ipc://@localhostworkflow-id_22001,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --bad-alloc-attempt-interval 50 --bad-alloc-max-attempts 1 --early-forward-policy never --io-threads 1 --jobs 4 --severity info --shm-allocation rbtree_best_fit --shm-metadata-msg-size 0 --shm-mlock-segment false --shm-mlock-segment-on-creation false --shm-no-cleanup false --shm-segment-id 0 --shm-throw-bad-alloc true --shm-zero-segment false --stacktrace-on-signal simple --timeframes-rate-limit 0 --session dpl_workflow-id --plugin odc</exe>
+       <exe reachable="true">cat ${DDS_LOCATION}/dpl_json.asset | foo --id A_dds%TaskIndex%_%CollectionIndex% --shm-monitor false --log-color false --batch --color false --channel-config "name=from_A_to_B,type=push,method=bind,address=ipc://@localhostworkflow-id_22000,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --channel-config "name=from_A_to_C,type=push,method=bind,address=ipc://@localhostworkflow-id_22001,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --bad-alloc-attempt-interval 50 --bad-alloc-max-attempts 1 --early-forward-policy never --io-threads 1 --jobs 4 --severity info --shm-allocation rbtree_best_fit --shm-metadata-msg-size 0 --shm-mlock-segment false --shm-mlock-segment-on-creation false --shm-no-cleanup false --shm-segment-id 0 --shm-throw-bad-alloc true --shm-zero-segment false --signposts "" --stacktrace-on-signal simple --timeframes-rate-limit 0 --session dpl_workflow-id --plugin odc</exe>
    </decltask>
    <decltask name="B">
        <assets><name>dpl_json</name></assets>
-       <exe reachable="true">cat ${DDS_LOCATION}/dpl_json.asset | foo --id B_dds%TaskIndex%_%CollectionIndex% --shm-monitor false --log-color false --batch --color false --channel-config "name=from_B_to_D,type=push,method=bind,address=ipc://@localhostworkflow-id_22002,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --channel-config "name=from_A_to_B,type=pull,method=connect,address=ipc://@localhostworkflow-id_22000,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --bad-alloc-attempt-interval 50 --bad-alloc-max-attempts 1 --early-forward-policy never --io-threads 1 --jobs 4 --severity info --shm-allocation rbtree_best_fit --shm-metadata-msg-size 0 --shm-mlock-segment false --shm-mlock-segment-on-creation false --shm-no-cleanup false --shm-segment-id 0 --shm-throw-bad-alloc true --shm-zero-segment false --stacktrace-on-signal simple --timeframes-rate-limit 0 --session dpl_workflow-id --plugin odc</exe>
+       <exe reachable="true">cat ${DDS_LOCATION}/dpl_json.asset | foo --id B_dds%TaskIndex%_%CollectionIndex% --shm-monitor false --log-color false --batch --color false --channel-config "name=from_B_to_D,type=push,method=bind,address=ipc://@localhostworkflow-id_22002,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --channel-config "name=from_A_to_B,type=pull,method=connect,address=ipc://@localhostworkflow-id_22000,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --bad-alloc-attempt-interval 50 --bad-alloc-max-attempts 1 --early-forward-policy never --io-threads 1 --jobs 4 --severity info --shm-allocation rbtree_best_fit --shm-metadata-msg-size 0 --shm-mlock-segment false --shm-mlock-segment-on-creation false --shm-no-cleanup false --shm-segment-id 0 --shm-throw-bad-alloc true --shm-zero-segment false --signposts "" --stacktrace-on-signal simple --timeframes-rate-limit 0 --session dpl_workflow-id --plugin odc</exe>
    </decltask>
    <decltask name="C">
        <assets><name>dpl_json</name></assets>
-       <exe reachable="true">cat ${DDS_LOCATION}/dpl_json.asset | foo --id C_dds%TaskIndex%_%CollectionIndex% --shm-monitor false --log-color false --batch --color false --channel-config "name=from_C_to_D,type=push,method=bind,address=ipc://@localhostworkflow-id_22003,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --channel-config "name=from_A_to_C,type=pull,method=connect,address=ipc://@localhostworkflow-id_22001,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --bad-alloc-attempt-interval 50 --bad-alloc-max-attempts 1 --early-forward-policy never --io-threads 1 --jobs 4 --severity info --shm-allocation rbtree_best_fit --shm-metadata-msg-size 0 --shm-mlock-segment false --shm-mlock-segment-on-creation false --shm-no-cleanup false --shm-segment-id 0 --shm-throw-bad-alloc true --shm-zero-segment false --stacktrace-on-signal simple --timeframes-rate-limit 0 --session dpl_workflow-id --plugin odc</exe>
+       <exe reachable="true">cat ${DDS_LOCATION}/dpl_json.asset | foo --id C_dds%TaskIndex%_%CollectionIndex% --shm-monitor false --log-color false --batch --color false --channel-config "name=from_C_to_D,type=push,method=bind,address=ipc://@localhostworkflow-id_22003,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --channel-config "name=from_A_to_C,type=pull,method=connect,address=ipc://@localhostworkflow-id_22001,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --bad-alloc-attempt-interval 50 --bad-alloc-max-attempts 1 --early-forward-policy never --io-threads 1 --jobs 4 --severity info --shm-allocation rbtree_best_fit --shm-metadata-msg-size 0 --shm-mlock-segment false --shm-mlock-segment-on-creation false --shm-no-cleanup false --shm-segment-id 0 --shm-throw-bad-alloc true --shm-zero-segment false --signposts "" --stacktrace-on-signal simple --timeframes-rate-limit 0 --session dpl_workflow-id --plugin odc</exe>
    </decltask>
    <decltask name="D">
        <assets><name>dpl_json</name></assets>
-       <exe reachable="true">cat ${DDS_LOCATION}/dpl_json.asset | foo --id D_dds%TaskIndex%_%CollectionIndex% --shm-monitor false --log-color false --batch --color false --channel-config "name=from_B_to_D,type=pull,method=connect,address=ipc://@localhostworkflow-id_22002,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --channel-config "name=from_C_to_D,type=pull,method=connect,address=ipc://@localhostworkflow-id_22003,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --bad-alloc-attempt-interval 50 --bad-alloc-max-attempts 1 --early-forward-policy never --io-threads 1 --jobs 4 --severity info --shm-allocation rbtree_best_fit --shm-metadata-msg-size 0 --shm-mlock-segment false --shm-mlock-segment-on-creation false --shm-no-cleanup false --shm-segment-id 0 --shm-throw-bad-alloc true --shm-zero-segment false --stacktrace-on-signal simple --timeframes-rate-limit 0 --a-param 1 --b-param "" --c-param "foo;bar" --session dpl_workflow-id --plugin odc</exe>
+       <exe reachable="true">cat ${DDS_LOCATION}/dpl_json.asset | foo --id D_dds%TaskIndex%_%CollectionIndex% --shm-monitor false --log-color false --batch --color false --channel-config "name=from_B_to_D,type=pull,method=connect,address=ipc://@localhostworkflow-id_22002,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --channel-config "name=from_C_to_D,type=pull,method=connect,address=ipc://@localhostworkflow-id_22003,transport=shmem,rateLogging=0,rcvBufSize=1,sndBufSize=1" --bad-alloc-attempt-interval 50 --bad-alloc-max-attempts 1 --early-forward-policy never --io-threads 1 --jobs 4 --severity info --shm-allocation rbtree_best_fit --shm-metadata-msg-size 0 --shm-mlock-segment false --shm-mlock-segment-on-creation false --shm-no-cleanup false --shm-segment-id 0 --shm-throw-bad-alloc true --shm-zero-segment false --signposts "" --stacktrace-on-signal simple --timeframes-rate-limit 0 --a-param 1 --b-param "" --c-param "foo;bar" --session dpl_workflow-id --plugin odc</exe>
        <requirements>
            <name>odc_expendable_task</name>
        </requirements>
diff --git a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
index 7e26ffd8c27a4..bbd5b78fbbb68 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
@@ -223,6 +223,8 @@ const std::vector expectedTasks{
     - "'0'"
     - "--shm-zero-segment"
     - "'false'"
+    - "--signposts"
+    - "''"
     - "--stacktrace-on-signal"
     - "'simple'"
     - "--timeframes-rate-limit"
@@ -318,6 +320,8 @@ const std::vector expectedTasks{
     - "'0'"
     - "--shm-zero-segment"
     - "'false'"
+    - "--signposts"
+    - "''"
     - "--stacktrace-on-signal"
     - "'simple'"
     - "--timeframes-rate-limit"
@@ -413,6 +417,8 @@ const std::vector expectedTasks{
     - "'0'"
     - "--shm-zero-segment"
     - "'false'"
+    - "--signposts"
+    - "''"
     - "--stacktrace-on-signal"
     - "'simple'"
     - "--timeframes-rate-limit"
@@ -505,6 +511,8 @@ const std::vector expectedTasks{
     - "'0'"
     - "--shm-zero-segment"
     - "'false'"
+    - "--signposts"
+    - "''"
     - "--stacktrace-on-signal"
     - "'simple'"
     - "--timeframes-rate-limit"

From 99d8906b4018e3c7b2a200037efee63402665d70 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 25 Mar 2024 00:12:51 +0100
Subject: [PATCH 461/726] DPL: add reproducer for dropped consecutive
 timeframes issue

---
 Framework/Core/CMakeLists.txt                 |  5 ++
 .../Core/test/test_ConsumeWhenAllOrdered.cxx  | 89 +++++++++++++++++++
 2 files changed, 94 insertions(+)
 create mode 100644 Framework/Core/test/test_ConsumeWhenAllOrdered.cxx

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index 05bcb1bd53083..f2f8de670d8fd 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -263,6 +263,11 @@ o2_add_test(Timers NAME test_Framework_test_Timers
             LABELS framework
             PUBLIC_LINK_LIBRARIES O2::Framework)
 
+# FIXME: make this a proper test, when it actually does not hang.
+o2_add_executable(test-framework-ConsumeWhenAllOrdered
+            SOURCES test/test_ConsumeWhenAllOrdered.cxx
+            PUBLIC_LINK_LIBRARIES O2::Framework)
+
 o2_add_test(SuppressionGenerator NAME test_Framework_test_SuppressionGenerator
             SOURCES test/test_SuppressionGenerator.cxx
             COMPONENT_NAME Framework
diff --git a/Framework/Core/test/test_ConsumeWhenAllOrdered.cxx b/Framework/Core/test/test_ConsumeWhenAllOrdered.cxx
new file mode 100644
index 0000000000000..8be6d539e6781
--- /dev/null
+++ b/Framework/Core/test/test_ConsumeWhenAllOrdered.cxx
@@ -0,0 +1,89 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/DataTakingContext.h"
+#include "Framework/CompletionPolicyHelpers.h"
+#include "Framework/DeviceSpec.h"
+#include "Framework/ControlService.h"
+#include "Framework/Configurable.h"
+#include "Framework/RunningWorkflowInfo.h"
+#include "Framework/CallbackService.h"
+#include "Framework/EndOfStreamContext.h"
+#include "Framework/CompletionPolicyHelpers.h"
+#include <fairmq/Device.h>
+
+void customize(std::vector<o2::framework::CompletionPolicy>& policies)
+{
+  policies.push_back(o2::framework::CompletionPolicyHelpers::consumeWhenAllOrdered("fake-output-proxy"));
+}
+
+#include <iostream>
+#include <vector>
+
+using namespace o2::framework;
+
+#include "Framework/runDataProcessing.h"
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& specs)
+{
+  DataProcessorSpec producer{
+    .name = "producer",
+    .outputs = {OutputSpec{{"counter"}, "TST", "A1"}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, ProcessingContext& pcx) {
+        static int counter = 0;
+        auto& aData = outputs.make<int>(OutputRef{"counter"});
+        aData = counter++;
+        if (counter == 100) {
+          pcx.services().get<ControlService>().endOfStream();
+        }
+      })},
+  };
+
+  DataProcessorSpec producerSkipping{
+    .name = "producerSkipping",
+    .outputs = {OutputSpec{{"counter"}, "TST", "A2"}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, ProcessingContext& pcx) {
+        static int counter = -1;
+        counter++;
+        if (((counter % 10) == 4) || ((counter % 10) == 5)) {
+          return;
+        }
+        auto& aData = outputs.make<int>(OutputRef{"counter"});
+        aData = counter;
+        if (counter == 100) {
+          pcx.services().get<ControlService>().endOfStream();
+        }
+      })},
+  };
+
+  DataProcessorSpec outputProxy{
+    .name = "fake-output-proxy",
+    .inputs = {
+      InputSpec{"x", "TST", "A1", Lifetime::Timeframe},
+      InputSpec{"y", "TST", "A2", Lifetime::Timeframe}},
+    .algorithm = adaptStateful([](CallbackService& callbacks) {
+        static int count = 0;
+        auto eosCallback = [](EndOfStreamContext &ctx) {
+          if (count != 80) {
+            LOGP(fatal, "Wrong number of timeframes seen: {} != 80", count);
+          }
+        };
+        callbacks.set<CallbackService::Id::EndOfStream>(eosCallback);
+        return adaptStateless([](Input<"x", int> const& x)
+          {
+            std::cout << "See: " << count++  << " with contents " << (int)x << std::endl;
+        }); })};
+
+  return WorkflowSpec{producer, producerSkipping, outputProxy};
+}

From 354dd4f822678e4b3998edb94de8196823d6b31d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 25 Mar 2024 12:07:30 +0100
Subject: [PATCH 462/726] DPL: use requires rather than call_if_defined

---
 .../Core/include/Framework/DataAllocator.h    | 21 +++++++------------
 1 file changed, 8 insertions(+), 13 deletions(-)

diff --git a/Framework/Core/include/Framework/DataAllocator.h b/Framework/Core/include/Framework/DataAllocator.h
index 029e922aeb90b..d746031ad58c5 100644
--- a/Framework/Core/include/Framework/DataAllocator.h
+++ b/Framework/Core/include/Framework/DataAllocator.h
@@ -23,7 +23,6 @@
 #include "Framework/TypeTraits.h"
 #include "Framework/Traits.h"
 #include "Framework/SerializationMethods.h"
-#include "Framework/CheckTypes.h"
 #include "Framework/ServiceRegistry.h"
 #include "Framework/RuntimeError.h"
 #include "Framework/RouteState.h"
@@ -211,18 +210,14 @@ class DataAllocator
       auto* s = new std::string(args...);
       adopt(spec, s);
       return *s;
-    } else if constexpr (std::is_base_of_v<struct TableBuilder, T>) {
-      return call_if_defined_forward<LifetimeHolder<struct TableBuilder>>([&](auto* p) {
-        auto tb = std::move(LifetimeHolder<TableBuilder>(new typename std::decay_t<decltype(*p)>::type(args...)));
-        adopt(spec, tb);
-        return std::move(tb);
-      });
-    } else if constexpr (std::is_base_of_v<struct TreeToTable, T>) {
-      return call_if_defined_forward<LifetimeHolder<struct TreeToTable>>([&](auto* p) {
-        auto t2t = std::move(LifetimeHolder<TreeToTable>(new typename std::decay_t<decltype(*p)>::type(args...)));
-        adopt(spec, t2t);
-        return std::move(t2t);
-      });
+    } else if constexpr (requires { static_cast<struct TableBuilder>(std::declval<std::decay_t<T>>()); }) {
+      auto tb = std::move(LifetimeHolder<TableBuilder>(new std::decay_t<T>(args...)));
+      adopt(spec, tb);
+      return tb;
+    } else if constexpr (requires { static_cast<struct TreeToTable>(std::declval<std::decay_t<T>>()); }) {
+      auto t2t = std::move(LifetimeHolder<TreeToTable>(new std::decay_t<T>(args...)));
+      adopt(spec, t2t);
+      return t2t;
     } else if constexpr (sizeof...(Args) == 0) {
       if constexpr (is_messageable<T>::value == true) {
         return *reinterpret_cast<T*>(newChunk(spec, sizeof(T)).data());

From 4907830eadcb34ad01e8929277598ea212df0c4f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 25 Mar 2024 13:31:50 +0100
Subject: [PATCH 463/726] DPL GUI: fix issues with backspace

Move to use the new ImGUI API for keys / mouse events.
---
 Framework/GUISupport/src/FrameworkGUIDebugger.cxx | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
index 005f7082d77b4..4b51a429daa6b 100644
--- a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
@@ -1216,22 +1216,22 @@ std::function<void(void)> getGUIDebugger(std::vector<DeviceInfo> const& infos,
 void updateMousePos(float x, float y)
 {
   ImGuiIO& io = ImGui::GetIO();
-  io.MousePos = ImVec2(x, y);
+  io.AddMousePosEvent(x, y);
 }
 
 void updateMouseButton(bool clicked)
 {
   ImGuiIO& io = ImGui::GetIO();
-  io.MouseDown[0] = clicked;
+  io.AddMouseButtonEvent(0, clicked);
 }
 
 void updateMouseWheel(int direction)
 {
   ImGuiIO& io = ImGui::GetIO();
   if (direction > 0) {
-    io.MouseWheel++;
+    io.AddMouseWheelEvent(0, 1.0);
   } else {
-    io.MouseWheel--;
+    io.AddMouseWheelEvent(0, -1.0);
   }
 }
 
@@ -1244,13 +1244,13 @@ void updateWindowSize(int x, int y)
 void keyDown(char key)
 {
   ImGuiIO& io = ImGui::GetIO();
-  io.KeysDown[io.KeyMap[(int)key]] = true;
+  io.AddKeyEvent((ImGuiKey)key , true);
 }
 
 void keyUp(char key)
 {
   ImGuiIO& io = ImGui::GetIO();
-  io.KeysDown[io.KeyMap[(int)key]] = false;
+  io.AddKeyEvent((ImGuiKey)key , false);
 }
 
 void charIn(char key)

From fc8c6a9a9b24b8cfd0c0d1f4c76301fb68404502 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 25 Mar 2024 14:01:42 +0100
Subject: [PATCH 464/726] DPL GUI: fix issues with backspace

Move to use the new ImGUI API for keys / mouse events.
---
 Framework/GUISupport/src/FrameworkGUIDebugger.cxx | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
index 005f7082d77b4..7342f83645772 100644
--- a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
@@ -1216,22 +1216,22 @@ std::function<void(void)> getGUIDebugger(std::vector<DeviceInfo> const& infos,
 void updateMousePos(float x, float y)
 {
   ImGuiIO& io = ImGui::GetIO();
-  io.MousePos = ImVec2(x, y);
+  io.AddMousePosEvent(x, y);
 }
 
 void updateMouseButton(bool clicked)
 {
   ImGuiIO& io = ImGui::GetIO();
-  io.MouseDown[0] = clicked;
+  io.AddMouseButtonEvent(0, clicked);
 }
 
 void updateMouseWheel(int direction)
 {
   ImGuiIO& io = ImGui::GetIO();
   if (direction > 0) {
-    io.MouseWheel++;
+    io.AddMouseWheelEvent(0, 1.0);
   } else {
-    io.MouseWheel--;
+    io.AddMouseWheelEvent(0, -1.0);
   }
 }
 
@@ -1244,13 +1244,13 @@ void updateWindowSize(int x, int y)
 void keyDown(char key)
 {
   ImGuiIO& io = ImGui::GetIO();
-  io.KeysDown[io.KeyMap[(int)key]] = true;
+  io.AddKeyEvent((ImGuiKey)key, true);
 }
 
 void keyUp(char key)
 {
   ImGuiIO& io = ImGui::GetIO();
-  io.KeysDown[io.KeyMap[(int)key]] = false;
+  io.AddKeyEvent((ImGuiKey)key, false);
 }
 
 void charIn(char key)

From c850ca55c597041e97328bcc9acdb72e2981a72e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 25 Mar 2024 14:01:42 +0100
Subject: [PATCH 465/726] DPL GUI: fix issues with backspace

Move to use the new ImGUI API for keys / mouse events.
---
 Framework/Core/include/Framework/DebugGUI.h   |  3 +-
 Framework/Core/src/DPLWebSocket.cxx           |  4 +-
 .../GUISupport/src/FrameworkGUIDebugger.cxx   | 42 +++++++++++++------
 .../GUISupport/src/FrameworkGUIDebugger.h     |  3 +-
 Framework/GUISupport/src/Plugin.cxx           |  8 +---
 5 files changed, 36 insertions(+), 24 deletions(-)

diff --git a/Framework/Core/include/Framework/DebugGUI.h b/Framework/Core/include/Framework/DebugGUI.h
index c7a611d50a1d4..6b92859e42c0f 100644
--- a/Framework/Core/include/Framework/DebugGUI.h
+++ b/Framework/Core/include/Framework/DebugGUI.h
@@ -40,8 +40,7 @@ struct DebugGUI {
   virtual void updateMouseButton(bool isClicked) = 0;
   virtual void updateMouseWheel(int direction) = 0;
   virtual void updateWindowSize(int x, int y) = 0;
-  virtual void keyDown(char key) = 0;
-  virtual void keyUp(char key) = 0;
+  virtual void keyEvent(char key, bool down) = 0;
   virtual void charIn(char key) = 0;
 
   virtual void* initGUI(char const* windowTitle, ServiceRegistry& registry) = 0;
diff --git a/Framework/Core/src/DPLWebSocket.cxx b/Framework/Core/src/DPLWebSocket.cxx
index 146a13206b55a..77018f8cb44b4 100644
--- a/Framework/Core/src/DPLWebSocket.cxx
+++ b/Framework/Core/src/DPLWebSocket.cxx
@@ -152,12 +152,12 @@ struct GUIWebSocketHandler : public WebSocketHandler {
       }
       case GUIOpcodes::Keydown: {
         char key = *frame;
-        mContext.gui->plugin->keyDown(key);
+        mContext.gui->plugin->keyEvent(key, true);
         break;
       }
       case GUIOpcodes::Keyup: {
         char key = *frame;
-        mContext.gui->plugin->keyUp(key);
+        mContext.gui->plugin->keyEvent(key, false);
         break;
       }
       case GUIOpcodes::Charin: {
diff --git a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
index 005f7082d77b4..bd8dc8877f2d1 100644
--- a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
@@ -1216,22 +1216,22 @@ std::function<void(void)> getGUIDebugger(std::vector<DeviceInfo> const& infos,
 void updateMousePos(float x, float y)
 {
   ImGuiIO& io = ImGui::GetIO();
-  io.MousePos = ImVec2(x, y);
+  io.AddMousePosEvent(x, y);
 }
 
 void updateMouseButton(bool clicked)
 {
   ImGuiIO& io = ImGui::GetIO();
-  io.MouseDown[0] = clicked;
+  io.AddMouseButtonEvent(0, clicked);
 }
 
 void updateMouseWheel(int direction)
 {
   ImGuiIO& io = ImGui::GetIO();
   if (direction > 0) {
-    io.MouseWheel++;
+    io.AddMouseWheelEvent(0, 1.0);
   } else {
-    io.MouseWheel--;
+    io.AddMouseWheelEvent(0, -1.0);
   }
 }
 
@@ -1241,16 +1241,34 @@ void updateWindowSize(int x, int y)
   io.DisplaySize = ImVec2(x, y);
 }
 
-void keyDown(char key)
+void keyEvent(char key, bool down)
 {
   ImGuiIO& io = ImGui::GetIO();
-  io.KeysDown[io.KeyMap[(int)key]] = true;
-}
-
-void keyUp(char key)
-{
-  ImGuiIO& io = ImGui::GetIO();
-  io.KeysDown[io.KeyMap[(int)key]] = false;
+  switch (key) {
+    case 1:
+      io.AddKeyEvent((ImGuiKey)ImGuiKey_LeftArrow, down);
+      break;
+    case 2:
+      io.AddKeyEvent((ImGuiKey)ImGuiKey_RightArrow, down);
+      break;
+    case 3:
+      io.AddKeyEvent((ImGuiKey)ImGuiKey_UpArrow, down);
+      break;
+    case 4:
+      io.AddKeyEvent((ImGuiKey)ImGuiKey_DownArrow, down);
+      break;
+    case 10:
+      io.AddKeyEvent((ImGuiKey)ImGuiKey_Delete, down);
+      break;
+    case 11:
+      io.AddKeyEvent((ImGuiKey)ImGuiKey_Backspace, down);
+      break;
+    case 14:
+      io.AddKeyEvent((ImGuiKey)ImGuiKey_Escape, down);
+      break;
+    default:
+      io.AddKeyEvent((ImGuiKey)key, down);
+  }
 }
 
 void charIn(char key)
diff --git a/Framework/GUISupport/src/FrameworkGUIDebugger.h b/Framework/GUISupport/src/FrameworkGUIDebugger.h
index 5de5e350ddf8d..d352f9899353b 100644
--- a/Framework/GUISupport/src/FrameworkGUIDebugger.h
+++ b/Framework/GUISupport/src/FrameworkGUIDebugger.h
@@ -43,8 +43,7 @@ void updateMousePos(float x, float y);
 void updateMouseButton(bool clicked);
 void updateMouseWheel(int direction);
 void updateWindowSize(int x, int y);
-void keyDown(char key);
-void keyUp(char key);
+void keyEvent(char key, bool pressed);
 void charIn(char key);
 
 } // namespace gui
diff --git a/Framework/GUISupport/src/Plugin.cxx b/Framework/GUISupport/src/Plugin.cxx
index 7a356f6e55d9b..17d13fbf56cd3 100644
--- a/Framework/GUISupport/src/Plugin.cxx
+++ b/Framework/GUISupport/src/Plugin.cxx
@@ -57,13 +57,9 @@ struct ImGUIDebugGUI : o2::framework::DebugGUI {
   {
     o2::framework::gui::updateWindowSize(x, y);
   }
-  void keyDown(char key) override
+  void keyEvent(char key, bool down) override
   {
-    o2::framework::gui::keyDown(key);
-  }
-  void keyUp(char key) override
-  {
-    o2::framework::gui::keyUp(key);
+    o2::framework::gui::keyEvent(key, down);
   }
   void charIn(char key) override
   {

From cc11b02f5a85f1df89fd5a681c941541c800061f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 25 Mar 2024 22:17:03 +0100
Subject: [PATCH 466/726] DPL GUI: fix keyboard support in remote GUI

Move to use the new ImGUI API for keys / mouse events.
Map events from the web page to the correct enum value.
---
 Framework/Core/include/Framework/DebugGUI.h   |  3 +-
 Framework/Core/src/DPLWebSocket.cxx           |  4 +-
 .../GUISupport/src/FrameworkGUIDebugger.cxx   | 60 +++++++++++++++----
 .../GUISupport/src/FrameworkGUIDebugger.h     |  3 +-
 Framework/GUISupport/src/Plugin.cxx           |  8 +--
 5 files changed, 54 insertions(+), 24 deletions(-)

diff --git a/Framework/Core/include/Framework/DebugGUI.h b/Framework/Core/include/Framework/DebugGUI.h
index c7a611d50a1d4..6b92859e42c0f 100644
--- a/Framework/Core/include/Framework/DebugGUI.h
+++ b/Framework/Core/include/Framework/DebugGUI.h
@@ -40,8 +40,7 @@ struct DebugGUI {
   virtual void updateMouseButton(bool isClicked) = 0;
   virtual void updateMouseWheel(int direction) = 0;
   virtual void updateWindowSize(int x, int y) = 0;
-  virtual void keyDown(char key) = 0;
-  virtual void keyUp(char key) = 0;
+  virtual void keyEvent(char key, bool down) = 0;
   virtual void charIn(char key) = 0;
 
   virtual void* initGUI(char const* windowTitle, ServiceRegistry& registry) = 0;
diff --git a/Framework/Core/src/DPLWebSocket.cxx b/Framework/Core/src/DPLWebSocket.cxx
index 146a13206b55a..77018f8cb44b4 100644
--- a/Framework/Core/src/DPLWebSocket.cxx
+++ b/Framework/Core/src/DPLWebSocket.cxx
@@ -152,12 +152,12 @@ struct GUIWebSocketHandler : public WebSocketHandler {
       }
       case GUIOpcodes::Keydown: {
         char key = *frame;
-        mContext.gui->plugin->keyDown(key);
+        mContext.gui->plugin->keyEvent(key, true);
         break;
       }
       case GUIOpcodes::Keyup: {
         char key = *frame;
-        mContext.gui->plugin->keyUp(key);
+        mContext.gui->plugin->keyEvent(key, false);
         break;
       }
       case GUIOpcodes::Charin: {
diff --git a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
index 005f7082d77b4..2fc8c7623c857 100644
--- a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
@@ -1216,22 +1216,22 @@ std::function<void(void)> getGUIDebugger(std::vector<DeviceInfo> const& infos,
 void updateMousePos(float x, float y)
 {
   ImGuiIO& io = ImGui::GetIO();
-  io.MousePos = ImVec2(x, y);
+  io.AddMousePosEvent(x, y);
 }
 
 void updateMouseButton(bool clicked)
 {
   ImGuiIO& io = ImGui::GetIO();
-  io.MouseDown[0] = clicked;
+  io.AddMouseButtonEvent(0, clicked);
 }
 
 void updateMouseWheel(int direction)
 {
   ImGuiIO& io = ImGui::GetIO();
   if (direction > 0) {
-    io.MouseWheel++;
+    io.AddMouseWheelEvent(0, 1.0);
   } else {
-    io.MouseWheel--;
+    io.AddMouseWheelEvent(0, -1.0);
   }
 }
 
@@ -1241,16 +1241,52 @@ void updateWindowSize(int x, int y)
   io.DisplaySize = ImVec2(x, y);
 }
 
-void keyDown(char key)
+void keyEvent(char key, bool down)
 {
   ImGuiIO& io = ImGui::GetIO();
-  io.KeysDown[io.KeyMap[(int)key]] = true;
-}
-
-void keyUp(char key)
-{
-  ImGuiIO& io = ImGui::GetIO();
-  io.KeysDown[io.KeyMap[(int)key]] = false;
+  switch (key) {
+    case 0:
+      io.AddKeyEvent(ImGuiKey_Tab, down);
+      break;
+    case 1:
+      io.AddKeyEvent(ImGuiKey_LeftArrow, down);
+      break;
+    case 2:
+      io.AddKeyEvent(ImGuiKey_RightArrow, down);
+      break;
+    case 3:
+      io.AddKeyEvent(ImGuiKey_UpArrow, down);
+      break;
+    case 4:
+      io.AddKeyEvent(ImGuiKey_DownArrow, down);
+      break;
+    case 5:
+      io.AddKeyEvent(ImGuiKey_PageUp, down);
+      break;
+    case 6:
+      io.AddKeyEvent(ImGuiKey_PageDown, down);
+      break;
+    case 7:
+      io.AddKeyEvent(ImGuiKey_Home, down);
+      break;
+    case 8:
+      io.AddKeyEvent(ImGuiKey_End, down);
+      break;
+    case 10:
+      io.AddKeyEvent(ImGuiKey_Delete, down);
+      break;
+    case 11:
+      io.AddKeyEvent(ImGuiKey_Backspace, down);
+      break;
+    case 13:
+      io.AddKeyEvent(ImGuiKey_Enter, down);
+      break;
+    case 14:
+      io.AddKeyEvent(ImGuiKey_Escape, down);
+      break;
+    default:
+      io.AddKeyEvent((ImGuiKey)key, down);
+  }
 }
 
 void charIn(char key)
diff --git a/Framework/GUISupport/src/FrameworkGUIDebugger.h b/Framework/GUISupport/src/FrameworkGUIDebugger.h
index 5de5e350ddf8d..d352f9899353b 100644
--- a/Framework/GUISupport/src/FrameworkGUIDebugger.h
+++ b/Framework/GUISupport/src/FrameworkGUIDebugger.h
@@ -43,8 +43,7 @@ void updateMousePos(float x, float y);
 void updateMouseButton(bool clicked);
 void updateMouseWheel(int direction);
 void updateWindowSize(int x, int y);
-void keyDown(char key);
-void keyUp(char key);
+void keyEvent(char key, bool pressed);
 void charIn(char key);
 
 } // namespace gui
diff --git a/Framework/GUISupport/src/Plugin.cxx b/Framework/GUISupport/src/Plugin.cxx
index 7a356f6e55d9b..17d13fbf56cd3 100644
--- a/Framework/GUISupport/src/Plugin.cxx
+++ b/Framework/GUISupport/src/Plugin.cxx
@@ -57,13 +57,9 @@ struct ImGUIDebugGUI : o2::framework::DebugGUI {
   {
     o2::framework::gui::updateWindowSize(x, y);
   }
-  void keyDown(char key) override
+  void keyEvent(char key, bool down) override
   {
-    o2::framework::gui::keyDown(key);
-  }
-  void keyUp(char key) override
-  {
-    o2::framework::gui::keyUp(key);
+    o2::framework::gui::keyEvent(key, down);
   }
   void charIn(char key) override
   {

From a7e9e2d34f37dd9da749277710c26731fe41a0d5 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 27 Mar 2024 23:05:51 +0100
Subject: [PATCH 467/726] DPL: replace LOG with Signposts

* Fix AsyncQueue
* Initial fixes for TimeSliceIndex
---
 Framework/Core/src/AsyncQueue.cxx     | 23 ++++++++++++---------
 Framework/Core/src/TimesliceIndex.cxx | 29 ++++++++++++++++++---------
 2 files changed, 33 insertions(+), 19 deletions(-)

diff --git a/Framework/Core/src/AsyncQueue.cxx b/Framework/Core/src/AsyncQueue.cxx
index 44115d4985e91..c1e7274fafc23 100644
--- a/Framework/Core/src/AsyncQueue.cxx
+++ b/Framework/Core/src/AsyncQueue.cxx
@@ -10,9 +10,11 @@
 // or submit itself to any jurisdiction.
 
 #include "Framework/AsyncQueue.h"
-#include "Framework/Logger.h"
+#include "Framework/Signpost.h"
 #include <numeric>
 
+O2_DECLARE_DYNAMIC_LOG(async_queue);
+
 namespace o2::framework
 {
 auto AsyncQueueHelpers::create(AsyncQueue& queue, AsyncTaskSpec spec) -> AsyncTaskId
@@ -38,7 +40,8 @@ auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> vo
   if (queue.tasks.empty()) {
     return;
   }
-  LOGP(debug, "Attempting at running {} tasks", queue.tasks.size());
+  O2_SIGNPOST_ID_GENERATE(opid, async_queue);
+  O2_SIGNPOST_START(async_queue, opid, "run", "Attempting at running %zu tasks with oldestPossible timeframe %zu", queue.tasks.size(), oldestPossible.value);
   std::vector<int> order;
   order.resize(queue.tasks.size());
   std::iota(order.begin(), order.end(), 0);
@@ -71,9 +74,9 @@ auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> vo
   });
   for (auto i : order) {
     if (queue.tasks[i].runnable) {
-      LOGP(debug, "AsyncQueue: Running task {}, timeslice {}, score {}, debounce {}", queue.tasks[i].id.value, queue.tasks[i].timeslice.value, queue.prototypes[queue.tasks[i].id.value].score, queue.tasks[i].debounce);
+      O2_SIGNPOST_EVENT_EMIT(async_queue, opid, "run", "Running task %d, score %d, debounce %d", queue.tasks[i].id.value, queue.prototypes[queue.tasks[i].id.value].score, queue.tasks[i].debounce);
     } else {
-      LOGP(debug, "AsyncQueue: Skipping task {}, timeslice {}, score {}, debounce {}", queue.tasks[i].id.value, queue.tasks[i].timeslice.value, queue.prototypes[queue.tasks[i].id.value].score, queue.tasks[i].debounce);
+      O2_SIGNPOST_EVENT_EMIT(async_queue, opid, "run", "Skipping task %d (timeslice %zu), score %d, debounce %d", queue.tasks[i].id.value, queue.tasks[i].timeslice.value, queue.prototypes[queue.tasks[i].id.value].score, queue.tasks[i].debounce);
     }
   }
   // Keep only the tasks with the highest debounce value for a given id
@@ -83,20 +86,21 @@ auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> vo
   order.erase(newEnd, order.end());
 
   if (order.empty() && queue.tasks.size() > 0) {
-    LOGP(debug, "AsyncQueue: not running iteration {} timeslice {} pending {}.", order.size(), queue.iteration, oldestPossible.value, queue.tasks.size());
+    O2_SIGNPOST_END(async_queue, opid, "run", "Not running iteration %zu pending %zu.",
+                    queue.iteration, queue.tasks.size());
     return;
   } else if (order.empty()) {
+    O2_SIGNPOST_END(async_queue, opid, "run", "Not running iteration %zu. No tasks.", queue.iteration);
     return;
   }
-  LOGP(debug, "AsyncQueue: Running {} tasks in iteration {} timeslice {}", order.size(), queue.iteration, oldestPossible.value);
-  bool obsolete = true;
+  O2_SIGNPOST_EVENT_EMIT(async_queue, opid, "run", "Running %zu tasks in iteration %zu", order.size(), queue.iteration);
 
   for (auto i : order) {
     if (queue.tasks[i].runnable) {
       // If a task is runable, we can run the task and remove it from the queue
-      LOGP(debug, "Running task {} ({})", queue.prototypes[queue.tasks[i].id.value].name, i);
+      O2_SIGNPOST_EVENT_EMIT(async_queue, opid, "run", "Running task %{public}s (%d)", queue.prototypes[queue.tasks[i].id.value].name.c_str(), i);
       queue.tasks[i].task();
-      LOGP(debug, "Done running {}", i);
+      O2_SIGNPOST_EVENT_EMIT(async_queue, opid, "run", "Done running %d", i);
     }
   }
   // Remove all runnable tasks regardless  they actually
@@ -105,6 +109,7 @@ auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> vo
                       return task.runnable;
                     }),
                     queue.tasks.end());
+  O2_SIGNPOST_END(async_queue, opid, "run", "Done running");
 }
 
 auto AsyncQueueHelpers::reset(AsyncQueue& queue) -> void
diff --git a/Framework/Core/src/TimesliceIndex.cxx b/Framework/Core/src/TimesliceIndex.cxx
index 4d5cf443e5385..451636fbcef37 100644
--- a/Framework/Core/src/TimesliceIndex.cxx
+++ b/Framework/Core/src/TimesliceIndex.cxx
@@ -9,7 +9,9 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 #include "Framework/TimesliceIndex.h"
-#include "Framework/Logger.h"
+#include "Framework/Signpost.h"
+
+O2_DECLARE_DYNAMIC_LOG(timeslice_index);
 
 namespace o2::framework
 {
@@ -124,9 +126,12 @@ bool TimesliceIndex::didReceiveData() const
 
 TimesliceIndex::OldestInputInfo TimesliceIndex::setOldestPossibleInput(TimesliceId timestamp, ChannelIndex channel)
 {
+  O2_SIGNPOST_ID_GENERATE(tid, timeslice_index);
   // Each channel oldest possible input must be monotoically increasing.
   if (timestamp.value < mChannels[channel.value].oldestForChannel.value) {
-    LOG(error) << "Received bogus oldest possible timeslice " << timestamp.value << " for channel " << channel.value << ". Expected >= " << mChannels[channel.value].oldestForChannel.value;
+    O2_SIGNPOST_EVENT_EMIT_ERROR(timeslice_index, tid, "setOldestPossibleInput",
+                                 "Received bogus oldest possible timeslice %zu for channel %d. Expected >= %zu.",
+                                 timestamp.value, channel.value, mChannels[channel.value].oldestForChannel.value);
   }
   mChannels[channel.value].oldestForChannel = timestamp;
   OldestInputInfo result{timestamp, channel};
@@ -144,11 +149,13 @@ TimesliceIndex::OldestInputInfo TimesliceIndex::setOldestPossibleInput(Timeslice
     }
   }
   if (changed && mOldestPossibleInput.timeslice.value != result.timeslice.value) {
-    LOG(debug) << "Success: Oldest possible input is " << result.timeslice.value << " due to channel " << result.channel.value;
+    O2_SIGNPOST_EVENT_EMIT(timeslice_index, tid, "setOldestPossibleInput", "Success: Oldest possible input is %zu due to channel %d",
+                           result.timeslice.value, result.channel.value);
   } else if (mOldestPossibleInput.timeslice.value != result.timeslice.value) {
-    LOG(debug) << "Oldest possible input updated from timestamp: " << mOldestPossibleInput.timeslice.value << " --> " << result.timeslice.value;
+    O2_SIGNPOST_EVENT_EMIT(timeslice_index, tid, "setOldestPossibleInput", "Oldest possible input updated from timestamp: %zu --> %zu",
+                           mOldestPossibleInput.timeslice.value, result.timeslice.value);
   } else {
-    LOG(debug) << "No change in oldest possible input";
+    O2_SIGNPOST_EVENT_EMIT(timeslice_index, tid, "setOldestPossibleInput", "No change in oldest possible input");
   }
   mOldestPossibleInput = result;
   return mOldestPossibleInput;
@@ -187,13 +194,15 @@ TimesliceIndex::OldestOutputInfo TimesliceIndex::updateOldestPossibleOutput()
       result.channel = {(int)-1};
     }
   }
+  O2_SIGNPOST_ID_GENERATE(tid, timeslice_index);
   if (changed && mOldestPossibleOutput.timeslice.value != result.timeslice.value) {
-    LOGP(debug, "Oldest possible output {} due to {} {}",
-         result.timeslice.value,
-         result.channel.value == -1 ? "slot" : "channel",
-         result.channel.value == -1 ? mOldestPossibleOutput.slot.index : mOldestPossibleOutput.channel.value);
+    O2_SIGNPOST_EVENT_EMIT(timeslice_index, tid, "updateOldestPossibleOutput", "Oldest possible output %zu due to %{public}s %zu",
+                           result.timeslice.value,
+                           result.channel.value == -1 ? "slot" : "channel",
+                           result.channel.value == -1 ? mOldestPossibleOutput.slot.index : mOldestPossibleOutput.channel.value);
   } else if (mOldestPossibleOutput.timeslice.value != result.timeslice.value) {
-    LOG(debug) << "Oldest possible output updated from oldest Input : " << mOldestPossibleOutput.timeslice.value << " --> " << result.timeslice.value;
+    O2_SIGNPOST_EVENT_EMIT(timeslice_index, tid, "updateOldestPossibleOutput", "Oldest possible output updated from oldest Input : %zu --> %zu",
+                           mOldestPossibleOutput.timeslice.value, result.timeslice.value);
   }
   mOldestPossibleOutput = result;
 

From 38213d55c1ee973562f5c74dd1350c2e4beecc85 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 28 Mar 2024 11:47:28 +0100
Subject: [PATCH 468/726] DPL: replace LOG with Signposts

Move a bunch of log messages to use signposts for:

* AsyncQueue
* TimesliceIndex
* CommonServices
* DataRelayer

Also modified the async queue to get the oldest possible timeslice as index, so
that we can use that to track executions.
---
 Framework/Core/include/Framework/AsyncQueue.h |  8 ++--
 Framework/Core/src/AsyncQueue.cxx             | 39 +++++++++++-----
 Framework/Core/src/CommonServices.cxx         | 24 ++++++----
 Framework/Core/src/DataProcessingDevice.cxx   | 20 ++++++--
 Framework/Core/src/DataRelayer.cxx            | 12 +++++
 Framework/Core/src/TimesliceIndex.cxx         | 46 ++++++++++++++-----
 Framework/Core/test/test_AsyncQueue.cxx       | 28 +++++------
 7 files changed, 123 insertions(+), 54 deletions(-)

diff --git a/Framework/Core/include/Framework/AsyncQueue.h b/Framework/Core/include/Framework/AsyncQueue.h
index 37971fbe03bfb..6945c2a5ab1da 100644
--- a/Framework/Core/include/Framework/AsyncQueue.h
+++ b/Framework/Core/include/Framework/AsyncQueue.h
@@ -32,8 +32,9 @@ struct AsyncTaskId {
 
 /// An actuatual task to be executed
 struct AsyncTask {
-  // The task to be executed
-  std::function<void()> task;
+  // The task to be executed. Id can be used as unique
+  // id for the signpost in the async_queue stream.
+  std::function<void(size_t id)> task;
   // The associated task spec
   AsyncTaskId id = {-1};
   TimesliceId timeslice = {TimesliceId::INVALID};
@@ -49,10 +50,11 @@ struct AsyncQueue {
 };
 
 struct AsyncQueueHelpers {
+  using AsyncCallback = std::function<void(size_t)>;
   static AsyncTaskId create(AsyncQueue& queue, AsyncTaskSpec spec);
   // Schedule a task with @a taskId to be executed whenever the timeslice
   // is past timeslice. If debounce is provided, only execute the task
-  static void post(AsyncQueue& queue, AsyncTaskId taskId, std::function<void()> task, TimesliceId timeslice, int64_t debounce = 0);
+  static void post(AsyncQueue& queue, AsyncTaskId taskId, AsyncCallback task, TimesliceId timeslice, int64_t debounce = 0);
   /// Run all the tasks which are older than the oldestPossible timeslice
   /// executing them by:
   /// 1. sorting the tasks by timeslice
diff --git a/Framework/Core/src/AsyncQueue.cxx b/Framework/Core/src/AsyncQueue.cxx
index 44115d4985e91..36c0c17009f82 100644
--- a/Framework/Core/src/AsyncQueue.cxx
+++ b/Framework/Core/src/AsyncQueue.cxx
@@ -10,9 +10,11 @@
 // or submit itself to any jurisdiction.
 
 #include "Framework/AsyncQueue.h"
-#include "Framework/Logger.h"
+#include "Framework/Signpost.h"
 #include <numeric>
 
+O2_DECLARE_DYNAMIC_LOG(async_queue);
+
 namespace o2::framework
 {
 auto AsyncQueueHelpers::create(AsyncQueue& queue, AsyncTaskSpec spec) -> AsyncTaskId
@@ -23,7 +25,7 @@ auto AsyncQueueHelpers::create(AsyncQueue& queue, AsyncTaskSpec spec) -> AsyncTa
   return id;
 }
 
-auto AsyncQueueHelpers::post(AsyncQueue& queue, AsyncTaskId id, std::function<void()> task, TimesliceId timeslice, int64_t debounce) -> void
+auto AsyncQueueHelpers::post(AsyncQueue& queue, AsyncTaskId id, AsyncCallback task, TimesliceId timeslice, int64_t debounce) -> void
 {
   AsyncTask taskToPost;
   taskToPost.task = task;
@@ -38,7 +40,8 @@ auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> vo
   if (queue.tasks.empty()) {
     return;
   }
-  LOGP(debug, "Attempting at running {} tasks", queue.tasks.size());
+  O2_SIGNPOST_ID_GENERATE(opid, async_queue);
+  O2_SIGNPOST_START(async_queue, opid, "run", "Attempting at running %zu tasks with oldestPossible timeframe %zu", queue.tasks.size(), oldestPossible.value);
   std::vector<int> order;
   order.resize(queue.tasks.size());
   std::iota(order.begin(), order.end(), 0);
@@ -47,6 +50,10 @@ auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> vo
     if (task.timeslice.value <= oldestPossible.value) {
       task.runnable = true;
     }
+    O2_SIGNPOST_EVENT_EMIT(async_queue, opid, "run",
+                           "Task %d (timeslice %zu), score %d, debounce %d is %{public}s when oldestPossible timeframe is %zu",
+                           task.id.value, task.timeslice.value, queue.prototypes[task.id.value].score, task.debounce,
+                           task.runnable ? "runnable" : "not runnable", oldestPossible.value);
   }
 
   // Sort by runnable, timeslice, then priority and finally debounce
@@ -69,34 +76,43 @@ auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> vo
       return queue.tasks[a].timeslice.value > queue.tasks[b].timeslice.value;
     }
   });
+
   for (auto i : order) {
     if (queue.tasks[i].runnable) {
-      LOGP(debug, "AsyncQueue: Running task {}, timeslice {}, score {}, debounce {}", queue.tasks[i].id.value, queue.tasks[i].timeslice.value, queue.prototypes[queue.tasks[i].id.value].score, queue.tasks[i].debounce);
+      O2_SIGNPOST_EVENT_EMIT(async_queue, opid, "run", "Running task %d (%d), (timeslice %zu), score %d, debounce %d", queue.tasks[i].id.value, i, queue.tasks[i].timeslice.value, queue.prototypes[queue.tasks[i].id.value].score, queue.tasks[i].debounce);
     } else {
-      LOGP(debug, "AsyncQueue: Skipping task {}, timeslice {}, score {}, debounce {}", queue.tasks[i].id.value, queue.tasks[i].timeslice.value, queue.prototypes[queue.tasks[i].id.value].score, queue.tasks[i].debounce);
+      O2_SIGNPOST_EVENT_EMIT(async_queue, opid, "run", "Skipping task %d (%d) (timeslice %zu), score %d, debounce %d", queue.tasks[i].id.value, i, queue.tasks[i].timeslice.value, queue.prototypes[queue.tasks[i].id.value].score, queue.tasks[i].debounce);
     }
   }
   // Keep only the tasks with the highest debounce value for a given id
   auto newEnd = std::unique(order.begin(), order.end(), [&queue](int a, int b) {
     return queue.tasks[a].runnable == queue.tasks[b].runnable && queue.tasks[a].id.value == queue.tasks[b].id.value && queue.tasks[a].debounce >= 0 && queue.tasks[b].debounce >= 0;
   });
+  for (auto ii = newEnd; ii != order.end(); ii++) {
+    O2_SIGNPOST_EVENT_EMIT(async_queue, opid, "dropping", "Dropping task %d for timeslice %zu", queue.tasks[*ii].id.value, queue.tasks[*ii].timeslice.value);
+  }
   order.erase(newEnd, order.end());
 
   if (order.empty() && queue.tasks.size() > 0) {
-    LOGP(debug, "AsyncQueue: not running iteration {} timeslice {} pending {}.", order.size(), queue.iteration, oldestPossible.value, queue.tasks.size());
+    O2_SIGNPOST_END(async_queue, opid, "run", "Not running iteration %zu pending %zu.",
+                    queue.iteration, queue.tasks.size());
     return;
   } else if (order.empty()) {
+    O2_SIGNPOST_END(async_queue, opid, "run", "Not running iteration %zu. No tasks.", queue.iteration);
     return;
   }
-  LOGP(debug, "AsyncQueue: Running {} tasks in iteration {} timeslice {}", order.size(), queue.iteration, oldestPossible.value);
-  bool obsolete = true;
+  O2_SIGNPOST_EVENT_EMIT(async_queue, opid, "run", "Running %zu tasks in iteration %zu", order.size(), queue.iteration);
 
+  int runCount = 0;
   for (auto i : order) {
     if (queue.tasks[i].runnable) {
+      runCount++;
       // If a task is runable, we can run the task and remove it from the queue
-      LOGP(debug, "Running task {} ({})", queue.prototypes[queue.tasks[i].id.value].name, i);
-      queue.tasks[i].task();
-      LOGP(debug, "Done running {}", i);
+      O2_SIGNPOST_EVENT_EMIT(async_queue, opid, "run", "Running task %{public}s (%d) for timeslice %zu",
+                             queue.prototypes[queue.tasks[i].id.value].name.c_str(), i,
+                             queue.tasks[i].timeslice.value);
+      queue.tasks[i].task(opid.value);
+      O2_SIGNPOST_EVENT_EMIT(async_queue, opid, "run", "Done running %d", i);
     }
   }
   // Remove all runnable tasks regardless  they actually
@@ -105,6 +121,7 @@ auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> vo
                       return task.runnable;
                     }),
                     queue.tasks.end());
+  O2_SIGNPOST_END(async_queue, opid, "run", "Done running %d/%zu tasks", runCount, order.size());
 }
 
 auto AsyncQueueHelpers::reset(AsyncQueue& queue) -> void
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 0d0828729ef42..7e16c03ef69e1 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -82,6 +82,7 @@ using Value = o2::monitoring::tags::Value;
 
 O2_DECLARE_DYNAMIC_LOG(data_processor_context);
 O2_DECLARE_DYNAMIC_LOG(stream_context);
+O2_DECLARE_DYNAMIC_LOG(async_queue);
 
 namespace o2::framework
 {
@@ -580,11 +581,14 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
                              (uint64_t)oldestPossibleOutput.timeslice.value,
                              oldestPossibleOutput.slot.index == -1 ? "channel" : "slot",
                              (uint64_t)(oldestPossibleOutput.slot.index == -1 ? oldestPossibleOutput.channel.value : oldestPossibleOutput.slot.index));
+      O2_SIGNPOST_EVENT_EMIT_ERROR(data_processor_context, cid, "oldest_possible_timeslice", "Ordered active %d", decongestion->orderedCompletionPolicyActive);
       if (decongestion->orderedCompletionPolicyActive) {
         auto oldNextTimeslice = decongestion->nextTimeslice;
         decongestion->nextTimeslice = std::max(decongestion->nextTimeslice, (int64_t)oldestPossibleOutput.timeslice.value);
+        timesliceIndex.updateOldestPossibleOutput();
+        O2_SIGNPOST_EVENT_EMIT_ERROR(data_processor_context, cid, "oldest_possible_timeslice", "Next timeslice %" PRIi64, decongestion->nextTimeslice);
         if (oldNextTimeslice != decongestion->nextTimeslice) {
-          LOGP(error, "Some Lifetime::Timeframe data got dropped starting at {}", oldNextTimeslice);
+          O2_SIGNPOST_EVENT_EMIT_ERROR(data_processor_context, cid, "oldest_possible_timeslice", "Some Lifetime::Timeframe data got dropped starting at %" PRIi64, oldNextTimeslice);
           timesliceIndex.rescan();
         }
       }
@@ -630,7 +634,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
       auto oldestPossibleOutput = relayer.getOldestPossibleOutput();
 
       if (oldestPossibleOutput.timeslice.value == decongestion.lastTimeslice) {
-        O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Not sending already sent value: %" PRIu64, (uint64_t)oldestPossibleOutput.timeslice.value);
+        O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Synchronous: Not sending already sent value: %" PRIu64, (uint64_t)oldestPossibleOutput.timeslice.value);
         return;
       }
       if (oldestPossibleOutput.timeslice.value < decongestion.lastTimeslice) {
@@ -646,14 +650,15 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
       O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Queueing oldest possible timeslice %" PRIu64 " propagation for execution.",
                              (uint64_t)oldestPossibleOutput.timeslice.value);
       AsyncQueueHelpers::post(
-        queue, decongestion.oldestPossibleTimesliceTask, [ref = services, oldestPossibleOutput, &decongestion, &proxy, &spec, device, &timesliceIndex]() {
-          O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, &decongestion);
+        queue, decongestion.oldestPossibleTimesliceTask, [ref = services, oldestPossibleOutput, &decongestion, &proxy, &spec, device, &timesliceIndex](size_t id) {
+          O2_SIGNPOST_ID_GENERATE(cid, async_queue);
+          cid.value = id;
           if (decongestion.lastTimeslice >= oldestPossibleOutput.timeslice.value) {
-            O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Not sending already sent value: %" PRIu64 "> %" PRIu64,
+            O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice", "Not sending already sent value: %" PRIu64 "> %" PRIu64,
                 decongestion.lastTimeslice, (uint64_t)oldestPossibleOutput.timeslice.value);
             return;
           }
-          O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Running oldest possible timeslice %" PRIu64 " propagation.",
+          O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice", "Running oldest possible timeslice %" PRIu64 " propagation.",
                                  (uint64_t)oldestPossibleOutput.timeslice.value);
           DataProcessingHelpers::broadcastOldestPossibleTimeslice(ref, oldestPossibleOutput.timeslice.value);
 
@@ -662,21 +667,22 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
             auto& state = proxy.getForwardChannelState(ChannelIndex{fi});
             // TODO: this we could cache in the proxy at the bind moment.
             if (info.channelType != ChannelAccountingType::DPL) {
-              O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
+              O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
               continue;
             }
             if (DataProcessingHelpers::sendOldestPossibleTimeframe(ref, info, state, oldestPossibleOutput.timeslice.value)) {
-              O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice",
+              O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice",
                                      "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
                                      info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
             }
           }
           decongestion.lastTimeslice = oldestPossibleOutput.timeslice.value;
           if (decongestion.orderedCompletionPolicyActive) {
+
             int64_t oldNextTimeslice = decongestion.nextTimeslice;
             decongestion.nextTimeslice = std::max(decongestion.nextTimeslice, (int64_t)oldestPossibleOutput.timeslice.value);
             if (oldNextTimeslice != decongestion.nextTimeslice) {
-              LOGP(error, "Some Lifetime::Timeframe data got dropped starting at {}", oldNextTimeslice);
+              O2_SIGNPOST_EVENT_EMIT_ERROR(async_queue, cid, "oldest_possible_timeslice", "Some Lifetime::Timeframe data got dropped starting at %" PRIi64, oldNextTimeslice);
               timesliceIndex.rescan();
             }
           }
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 70654bc43e84b..35cc2524ed98b 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -92,6 +92,8 @@ struct formatter<o2::framework::CompletionPolicy::CompletionOp> : ostream_format
 O2_DECLARE_DYNAMIC_LOG(device);
 // Special log to keep track of the lifetime of the parts
 O2_DECLARE_DYNAMIC_LOG(parts);
+// Special log to track the async queue behavior
+O2_DECLARE_DYNAMIC_LOG(async_queue);
 
 using namespace o2::framework;
 using ConfigurationInterface = o2::configuration::ConfigurationInterface;
@@ -681,9 +683,11 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
 
   auto& asyncQueue = registry.get<AsyncQueue>();
   auto& decongestion = registry.get<DecongestionService>();
-  LOG(debug) << "Queuing forwarding oldestPossible " << oldestTimeslice.timeslice.value;
+  auto& timesliceIndex = registry.get<TimesliceIndex>();
+  O2_SIGNPOST_ID_GENERATE(aid, async_queue);
+  O2_SIGNPOST_EVENT_EMIT(async_queue, aid, "forwardInputs", "Queuing forwarding oldestPossible %zu", oldestTimeslice.timeslice.value);
   AsyncQueueHelpers::post(
-    asyncQueue, decongestion.oldestPossibleTimesliceTask, [&proxy, &decongestion, registry, oldestTimeslice]() {
+    asyncQueue, decongestion.oldestPossibleTimesliceTask, [&proxy, &decongestion, registry, oldestTimeslice, &timesliceIndex](size_t aid) {
       // DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, oldestTimeslice.timeslice.value);
       if (oldestTimeslice.timeslice.value <= decongestion.lastTimeslice) {
         LOG(debug) << "Not sending already sent oldest possible timeslice " << oldestTimeslice.timeslice.value;
@@ -692,13 +696,17 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
       for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
         auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
         auto& state = proxy.getForwardChannelState(ChannelIndex{fi});
+        O2_SIGNPOST_ID_GENERATE(aid, async_queue);
         // TODO: this we could cache in the proxy at the bind moment.
         if (info.channelType != ChannelAccountingType::DPL) {
-          LOG(debug) << "Skipping channel";
+          O2_SIGNPOST_EVENT_EMIT(async_queue, aid, "forwardInputsCallback", "Skipping channel %{public}s because it's not a DPL channel",
+                                 info.name.c_str());
+
           continue;
         }
         if (DataProcessingHelpers::sendOldestPossibleTimeframe(registry, info, state, oldestTimeslice.timeslice.value)) {
-          LOGP(debug, "Forwarding to channel {} oldest possible timeslice {}, prio 20", info.name, oldestTimeslice.timeslice.value);
+          O2_SIGNPOST_EVENT_EMIT(async_queue, aid, "forwardInputsCallback", "Forwarding to channel %{public}s oldest possible timeslice %zu, prio 20",
+                                 info.name.c_str(), oldestTimeslice.timeslice.value);
         }
       }
     },
@@ -1900,7 +1908,9 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
             ii += (nMessages / 2) - 1;
           }
           auto onDrop = [ref](TimesliceSlot slot, std::vector<MessageSet>& dropped, TimesliceIndex::OldestOutputInfo oldestOutputInfo) {
-            LOGP(debug, "Dropping message from slot {}. Forwarding as needed. Timeslice {}", slot.index, oldestOutputInfo.timeslice.value);
+            O2_SIGNPOST_ID_GENERATE(cid, async_queue);
+            O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "onDrop", "Dropping message from slot %zu. Forwarding as needed. Timeslice %zu",
+                                   slot.index, oldestOutputInfo.timeslice.value);
             ref.get<AsyncQueue>();
             ref.get<DecongestionService>();
             ref.get<DataRelayer>();
diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index 1daf4bbd2a20b..bcb5f2a0d9d74 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -54,6 +54,8 @@ using DataHeader = o2::header::DataHeader;
 using DataProcessingHeader = o2::framework::DataProcessingHeader;
 using Verbosity = o2::monitoring::Verbosity;
 
+O2_DECLARE_DYNAMIC_LOG(data_relayer);
+
 namespace o2::framework
 {
 
@@ -369,6 +371,8 @@ void DataRelayer::pruneCache(TimesliceSlot slot, OnDropCallback onDrop)
       }
       bool anyDropped = std::any_of(dropped.begin(), dropped.end(), [](auto& m) { return m.size(); });
       if (anyDropped) {
+        O2_SIGNPOST_ID_GENERATE(aid, data_relayer);
+        O2_SIGNPOST_EVENT_EMIT(data_relayer, aid, "pruneCache", "Dropping stuff from slot %zu %zu", slot.index, oldestPossibleTimeslice.timeslice.value);
         onDrop(slot, dropped, oldestPossibleTimeslice);
       }
     }
@@ -440,6 +444,8 @@ DataRelayer::RelayChoice
                      &nPayloads,
                      &cache = mCache,
                      numInputTypes = mDistinctRoutesIndex.size()](TimesliceId timeslice, int input, TimesliceSlot slot) {
+    O2_SIGNPOST_ID_GENERATE(aid, data_relayer);
+    O2_SIGNPOST_EVENT_EMIT(data_relayer, aid, "saveInSlot", "saving %zu in slot %zu", timeslice.value, slot.index);
     auto cacheIdx = numInputTypes * slot.index + input;
     MessageSet& target = cache[cacheIdx];
     cachedStateMetrics[cacheIdx] = CacheEntryStatus::PENDING;
@@ -573,8 +579,14 @@ DataRelayer::RelayChoice
     return RelayChoice{.type = RelayChoice::Type::Invalid, .timeslice = timeslice};
   }
 
+  O2_SIGNPOST_ID_GENERATE(aid, data_relayer);
   TimesliceIndex::ActionTaken action;
   std::tie(action, slot) = index.replaceLRUWith(pristineContext, timeslice);
+  uint64_t const* debugTimestamp = std::get_if<uint64_t>(&pristineContext.get(0));
+  if (action != TimesliceIndex::ActionTaken::Wait) {
+    O2_SIGNPOST_EVENT_EMIT(data_relayer, aid, "saveInSlot",
+                           "Slot %zu updated with %zu using action %d, %" PRIu64, slot.index, timeslice.value, (int)action, *debugTimestamp);
+  }
 
   updateStatistics(action);
 
diff --git a/Framework/Core/src/TimesliceIndex.cxx b/Framework/Core/src/TimesliceIndex.cxx
index 4d5cf443e5385..843ef55ead060 100644
--- a/Framework/Core/src/TimesliceIndex.cxx
+++ b/Framework/Core/src/TimesliceIndex.cxx
@@ -9,7 +9,9 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 #include "Framework/TimesliceIndex.h"
-#include "Framework/Logger.h"
+#include "Framework/Signpost.h"
+
+O2_DECLARE_DYNAMIC_LOG(timeslice_index);
 
 namespace o2::framework
 {
@@ -33,6 +35,8 @@ void TimesliceIndex::associate(TimesliceId timestamp, TimesliceSlot slot)
   mVariables[slot.index].put({0, static_cast<uint64_t>(timestamp.value)});
   mVariables[slot.index].commit();
   mDirty[slot.index] = true;
+  O2_SIGNPOST_ID_GENERATE(tid, timeslice_index);
+  O2_SIGNPOST_EVENT_EMIT(timeslice_index, tid, "associate", "Associating timestamp %zu to slot %zu", timestamp.value, slot.index);
 }
 
 TimesliceSlot TimesliceIndex::findOldestSlot(TimesliceId timestamp) const
@@ -63,13 +67,18 @@ TimesliceSlot TimesliceIndex::findOldestSlot(TimesliceId timestamp) const
 std::tuple<TimesliceIndex::ActionTaken, TimesliceSlot> TimesliceIndex::replaceLRUWith(data_matcher::VariableContext& newContext, TimesliceId timestamp)
 {
   auto oldestSlot = findOldestSlot(timestamp);
+  O2_SIGNPOST_ID_GENERATE(tid, timeslice_index);
   if (TimesliceIndex::isValid(oldestSlot) == false) {
     mVariables[oldestSlot.index] = newContext;
+    auto debugTimestamp = std::get_if<uint64_t>(&mVariables[oldestSlot.index].get(0));
+    O2_SIGNPOST_EVENT_EMIT(timeslice_index, tid, "replaceLRUWith", "slot %zu timeslice %zu (%" PRIu64 ")", oldestSlot.index, timestamp.value, *debugTimestamp);
     return std::make_tuple(ActionTaken::ReplaceUnused, oldestSlot);
   }
   auto oldTimestamp = std::get_if<uint64_t>(&mVariables[oldestSlot.index].get(0));
   if (oldTimestamp == nullptr) {
     mVariables[oldestSlot.index] = newContext;
+    auto debugTimestamp = std::get_if<uint64_t>(&mVariables[oldestSlot.index].get(0));
+    O2_SIGNPOST_EVENT_EMIT(timeslice_index, tid, "replaceLRUWith", "slot %zu timeslice %zu (%" PRIu64 ")", oldestSlot.index, timestamp.value, *debugTimestamp);
     return std::make_tuple(ActionTaken::ReplaceUnused, oldestSlot);
   }
 
@@ -80,9 +89,12 @@ std::tuple<TimesliceIndex::ActionTaken, TimesliceSlot> TimesliceIndex::replaceLR
 
   if (*newTimestamp > *oldTimestamp) {
     switch (mBackpressurePolicy) {
-      case BackpressureOp::DropAncient:
+      case BackpressureOp::DropAncient: {
         mVariables[oldestSlot.index] = newContext;
+        auto debugTimestamp = std::get_if<uint64_t>(&mVariables[oldestSlot.index].get(0));
+        O2_SIGNPOST_EVENT_EMIT(timeslice_index, tid, "replaceLRUWith", "slot %zu timeslice %zu (%" PRIu64 ")", oldestSlot.index, timestamp.value, *debugTimestamp);
         return std::make_tuple(ActionTaken::ReplaceObsolete, oldestSlot);
+      }
       case BackpressureOp::DropRecent:
         return std::make_tuple(ActionTaken::DropObsolete, TimesliceSlot{TimesliceSlot::INVALID});
       case BackpressureOp::Wait:
@@ -90,9 +102,12 @@ std::tuple<TimesliceIndex::ActionTaken, TimesliceSlot> TimesliceIndex::replaceLR
     }
   } else {
     switch (mBackpressurePolicy) {
-      case BackpressureOp::DropRecent:
+      case BackpressureOp::DropRecent: {
         mVariables[oldestSlot.index] = newContext;
+        auto debugTimestamp = std::get_if<uint64_t>(&mVariables[oldestSlot.index].get(0));
+        O2_SIGNPOST_EVENT_EMIT(timeslice_index, tid, "replaceLRUWith", "slot %zu timeslice %zu (%" PRIu64 ")", oldestSlot.index, timestamp.value, *debugTimestamp);
         return std::make_tuple(ActionTaken::ReplaceObsolete, oldestSlot);
+      }
       case BackpressureOp::DropAncient:
         return std::make_tuple(ActionTaken::DropObsolete, TimesliceSlot{TimesliceSlot::INVALID});
       case BackpressureOp::Wait:
@@ -124,9 +139,12 @@ bool TimesliceIndex::didReceiveData() const
 
 TimesliceIndex::OldestInputInfo TimesliceIndex::setOldestPossibleInput(TimesliceId timestamp, ChannelIndex channel)
 {
+  O2_SIGNPOST_ID_GENERATE(tid, timeslice_index);
   // Each channel oldest possible input must be monotoically increasing.
   if (timestamp.value < mChannels[channel.value].oldestForChannel.value) {
-    LOG(error) << "Received bogus oldest possible timeslice " << timestamp.value << " for channel " << channel.value << ". Expected >= " << mChannels[channel.value].oldestForChannel.value;
+    O2_SIGNPOST_EVENT_EMIT_ERROR(timeslice_index, tid, "setOldestPossibleInput",
+                                 "Received bogus oldest possible timeslice %zu for channel %d. Expected >= %zu.",
+                                 timestamp.value, channel.value, mChannels[channel.value].oldestForChannel.value);
   }
   mChannels[channel.value].oldestForChannel = timestamp;
   OldestInputInfo result{timestamp, channel};
@@ -144,11 +162,13 @@ TimesliceIndex::OldestInputInfo TimesliceIndex::setOldestPossibleInput(Timeslice
     }
   }
   if (changed && mOldestPossibleInput.timeslice.value != result.timeslice.value) {
-    LOG(debug) << "Success: Oldest possible input is " << result.timeslice.value << " due to channel " << result.channel.value;
+    O2_SIGNPOST_EVENT_EMIT(timeslice_index, tid, "setOldestPossibleInput", "Success: Oldest possible input is %zu due to channel %d",
+                           result.timeslice.value, result.channel.value);
   } else if (mOldestPossibleInput.timeslice.value != result.timeslice.value) {
-    LOG(debug) << "Oldest possible input updated from timestamp: " << mOldestPossibleInput.timeslice.value << " --> " << result.timeslice.value;
+    O2_SIGNPOST_EVENT_EMIT(timeslice_index, tid, "setOldestPossibleInput", "Oldest possible input updated from timestamp: %zu --> %zu",
+                           mOldestPossibleInput.timeslice.value, result.timeslice.value);
   } else {
-    LOG(debug) << "No change in oldest possible input";
+    O2_SIGNPOST_EVENT_EMIT(timeslice_index, tid, "setOldestPossibleInput", "No change in oldest possible input");
   }
   mOldestPossibleInput = result;
   return mOldestPossibleInput;
@@ -187,13 +207,15 @@ TimesliceIndex::OldestOutputInfo TimesliceIndex::updateOldestPossibleOutput()
       result.channel = {(int)-1};
     }
   }
+  O2_SIGNPOST_ID_GENERATE(tid, timeslice_index);
   if (changed && mOldestPossibleOutput.timeslice.value != result.timeslice.value) {
-    LOGP(debug, "Oldest possible output {} due to {} {}",
-         result.timeslice.value,
-         result.channel.value == -1 ? "slot" : "channel",
-         result.channel.value == -1 ? mOldestPossibleOutput.slot.index : mOldestPossibleOutput.channel.value);
+    O2_SIGNPOST_EVENT_EMIT(timeslice_index, tid, "updateOldestPossibleOutput", "Oldest possible output %zu due to %{public}s %zu",
+                           result.timeslice.value,
+                           result.channel.value == -1 ? "slot" : "channel",
+                           result.channel.value == -1 ? mOldestPossibleOutput.slot.index : mOldestPossibleOutput.channel.value);
   } else if (mOldestPossibleOutput.timeslice.value != result.timeslice.value) {
-    LOG(debug) << "Oldest possible output updated from oldest Input : " << mOldestPossibleOutput.timeslice.value << " --> " << result.timeslice.value;
+    O2_SIGNPOST_EVENT_EMIT(timeslice_index, tid, "updateOldestPossibleOutput", "Oldest possible output updated from oldest Input : %zu --> %zu",
+                           mOldestPossibleOutput.timeslice.value, result.timeslice.value);
   }
   mOldestPossibleOutput = result;
 
diff --git a/Framework/Core/test/test_AsyncQueue.cxx b/Framework/Core/test/test_AsyncQueue.cxx
index c3ac4ea06bd82..635eb131cc666 100644
--- a/Framework/Core/test/test_AsyncQueue.cxx
+++ b/Framework/Core/test/test_AsyncQueue.cxx
@@ -22,9 +22,9 @@ TEST_CASE("TestDebouncing")
   // Push two tasks on the queue with the same id
   auto count = 0;
   AsyncQueueHelpers::post(
-    queue, taskId, [&count]() { count += 1; }, TimesliceId{0}, 10);
+    queue, taskId, [&count](size_t) { count += 1; }, TimesliceId{0}, 10);
   AsyncQueueHelpers::post(
-    queue, taskId, [&count]() { count += 2; }, TimesliceId{1}, 20);
+    queue, taskId, [&count](size_t) { count += 2; }, TimesliceId{1}, 20);
   AsyncQueueHelpers::run(queue, TimesliceId{2});
   REQUIRE(count == 2);
 }
@@ -39,9 +39,9 @@ TEST_CASE("TestPriority")
   // Push two tasks on the queue with the same id
   auto count = 0;
   AsyncQueueHelpers::post(
-    queue, taskId1, [&count]() { count += 10; }, TimesliceId{0});
+    queue, taskId1, [&count](size_t) { count += 10; }, TimesliceId{0});
   AsyncQueueHelpers::post(
-    queue, taskId2, [&count]() { count /= 10; }, TimesliceId{0});
+    queue, taskId2, [&count](size_t) { count /= 10; }, TimesliceId{0});
   AsyncQueueHelpers::run(queue, TimesliceId{2});
   REQUIRE(count == 10);
 }
@@ -56,9 +56,9 @@ TEST_CASE("TestOldestTimeslice")
   // Push two tasks on the queue with the same id
   auto count = 0;
   AsyncQueueHelpers::post(
-    queue, taskId1, [&count]() { count += 10; }, TimesliceId{1});
+    queue, taskId1, [&count](size_t) { count += 10; }, TimesliceId{1});
   AsyncQueueHelpers::post(
-    queue, taskId2, [&count]() { count += 20; }, TimesliceId{0});
+    queue, taskId2, [&count](size_t) { count += 20; }, TimesliceId{0});
   AsyncQueueHelpers::run(queue, TimesliceId{0});
   REQUIRE(count == 20);
   AsyncQueueHelpers::run(queue, TimesliceId{0});
@@ -77,11 +77,11 @@ TEST_CASE("TestOldestTimesliceWithBounce")
   // Push two tasks on the queue with the same id
   auto count = 0;
   AsyncQueueHelpers::post(
-    queue, taskId1, [&count]() { count += 10; }, TimesliceId{2});
+    queue, taskId1, [&count](size_t) { count += 10; }, TimesliceId{2});
   AsyncQueueHelpers::post(
-    queue, taskId2, [&count]() { count += 20; }, TimesliceId{1}, 10);
+    queue, taskId2, [&count](size_t) { count += 20; }, TimesliceId{1}, 10);
   AsyncQueueHelpers::post(
-    queue, taskId2, [&count]() { count += 30; }, TimesliceId{1}, 20);
+    queue, taskId2, [&count](size_t) { count += 30; }, TimesliceId{1}, 20);
   AsyncQueueHelpers::run(queue, TimesliceId{0});
   REQUIRE(count == 0);
   REQUIRE(queue.tasks.size() == 3);
@@ -103,11 +103,11 @@ TEST_CASE("TestOldestTimesliceWithNegativeBounce")
   // Push two tasks on the queue with the same id
   auto count = 0;
   AsyncQueueHelpers::post(
-    queue, taskId1, [&count]() { count += 10; }, TimesliceId{2});
+    queue, taskId1, [&count](size_t) { count += 10; }, TimesliceId{2});
   AsyncQueueHelpers::post(
-    queue, taskId2, [&count]() { count += 20; }, TimesliceId{1}, -10);
+    queue, taskId2, [&count](size_t) { count += 20; }, TimesliceId{1}, -10);
   AsyncQueueHelpers::post(
-    queue, taskId2, [&count]() { count += 30; }, TimesliceId{1}, -20);
+    queue, taskId2, [&count](size_t) { count += 30; }, TimesliceId{1}, -20);
   AsyncQueueHelpers::run(queue, TimesliceId{0});
   REQUIRE(count == 0);
   REQUIRE(queue.tasks.size() == 3);
@@ -128,13 +128,13 @@ TEST_CASE("TestOldestTimeslicePerTimeslice")
   // Push two tasks on the queue with the same id
   auto count = 0;
   AsyncQueueHelpers::post(
-    queue, taskId1, [&count]() { count += 10; }, TimesliceId{1});
+    queue, taskId1, [&count](size_t) { count += 10; }, TimesliceId{1});
   REQUIRE(queue.tasks.size() == 1);
   AsyncQueueHelpers::run(queue, TimesliceId{0});
   REQUIRE(queue.tasks.size() == 1);
   REQUIRE(count == 0);
   AsyncQueueHelpers::post(
-    queue, taskId1, [&count]() { count += 20; }, TimesliceId{2});
+    queue, taskId1, [&count](size_t) { count += 20; }, TimesliceId{2});
   REQUIRE(queue.tasks.size() == 2);
   AsyncQueueHelpers::run(queue, TimesliceId{1});
   REQUIRE(queue.tasks.size() == 1);

From ab3b803f5cce7fdc7b31f3af5b9b8b4e3db5147a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 28 Mar 2024 16:29:59 +0100
Subject: [PATCH 469/726] DPL: fix spurios error message and wrong update.

---
 Framework/Core/src/CommonServices.cxx | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 7e16c03ef69e1..d7f6538310cb4 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -581,12 +581,11 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
                              (uint64_t)oldestPossibleOutput.timeslice.value,
                              oldestPossibleOutput.slot.index == -1 ? "channel" : "slot",
                              (uint64_t)(oldestPossibleOutput.slot.index == -1 ? oldestPossibleOutput.channel.value : oldestPossibleOutput.slot.index));
-      O2_SIGNPOST_EVENT_EMIT_ERROR(data_processor_context, cid, "oldest_possible_timeslice", "Ordered active %d", decongestion->orderedCompletionPolicyActive);
+      O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Ordered active %d", decongestion->orderedCompletionPolicyActive);
       if (decongestion->orderedCompletionPolicyActive) {
         auto oldNextTimeslice = decongestion->nextTimeslice;
         decongestion->nextTimeslice = std::max(decongestion->nextTimeslice, (int64_t)oldestPossibleOutput.timeslice.value);
-        timesliceIndex.updateOldestPossibleOutput();
-        O2_SIGNPOST_EVENT_EMIT_ERROR(data_processor_context, cid, "oldest_possible_timeslice", "Next timeslice %" PRIi64, decongestion->nextTimeslice);
+        O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Next timeslice %" PRIi64, decongestion->nextTimeslice);
         if (oldNextTimeslice != decongestion->nextTimeslice) {
           O2_SIGNPOST_EVENT_EMIT_ERROR(data_processor_context, cid, "oldest_possible_timeslice", "Some Lifetime::Timeframe data got dropped starting at %" PRIi64, oldNextTimeslice);
           timesliceIndex.rescan();

From c1c8ce9a5bd4b0df46def50546c8ee4ba49956a3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 2 Apr 2024 20:41:20 -0400
Subject: [PATCH 470/726] DPL: disable cpu_usage_fraction for proxies

---
 Framework/Core/src/CommonServices.cxx | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index d7f6538310cb4..a87ea384697fd 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -43,6 +43,7 @@
 #include "Framework/DeviceState.h"
 #include "Framework/DeviceConfig.h"
 #include "Framework/DefaultsHelpers.h"
+#include "Framework/Signpost.h"
 
 #include "TextDriverClient.h"
 #include "WSDriverClient.h"
@@ -83,6 +84,7 @@ using Value = o2::monitoring::tags::Value;
 O2_DECLARE_DYNAMIC_LOG(data_processor_context);
 O2_DECLARE_DYNAMIC_LOG(stream_context);
 O2_DECLARE_DYNAMIC_LOG(async_queue);
+O2_DECLARE_DYNAMIC_LOG(policies);
 
 namespace o2::framework
 {
@@ -845,6 +847,15 @@ o2::framework::ServiceSpec CommonServices::dataProcessingStats()
       if (deploymentMode != DeploymentMode::OnlineDDS && deploymentMode != DeploymentMode::OnlineECS && deploymentMode != DeploymentMode::OnlineAUX && deploymentMode != DeploymentMode::FST) {
         arrowAndResourceLimitingMetrics = true;
       }
+      // Input proxies should not report cpu_usage_fraction,
+      // because of the rate limiting which biases the measurement.
+      auto& spec = services.get<DeviceSpec const>();
+      bool enableCPUUsageFraction = true;
+      if (spec.name.find("proxy") == std::string::npos) {
+        O2_SIGNPOST_ID_GENERATE(mid, policies);
+        O2_SIGNPOST_EVENT_EMIT(policies, mid, "metrics", "Disabling cpu_usage_fraction metric for proxy %{public}s", spec.name.c_str());
+        enableCPUUsageFraction = false;
+      }
 
       std::vector<DataProcessingStats::MetricSpec> metrics = {
         MetricSpec{.name = "errors",
@@ -928,6 +939,7 @@ o2::framework::ServiceSpec CommonServices::dataProcessingStats()
                    .maxRefreshLatency = onlineRefreshLatency,
                    .sendInitialValue = true},
         MetricSpec{.name = "cpu_usage_fraction",
+                   .enabled = enableCPUUsageFraction,
                    .metricId = (int)ProcessingStatsId::CPU_USAGE_FRACTION,
                    .kind = Kind::Rate,
                    .scope = Scope::Online,

From 7de19658e360958522b66e99aee29978bade7ae9 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 11 Apr 2024 13:18:05 +0200
Subject: [PATCH 471/726] Use C++14 namespaces

---
 Detectors/TRD/pid/include/TRDPID/ML.h                     | 5 +----
 Detectors/ZDC/fastsimulation/include/Processors.h         | 2 +-
 Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h | 5 +----
 3 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/Detectors/TRD/pid/include/TRDPID/ML.h b/Detectors/TRD/pid/include/TRDPID/ML.h
index 7c72edc30ae3c..5165996d24803 100644
--- a/Detectors/TRD/pid/include/TRDPID/ML.h
+++ b/Detectors/TRD/pid/include/TRDPID/ML.h
@@ -27,9 +27,7 @@
 #include <array>
 #include <string>
 
-namespace o2
-{
-namespace trd
+namespace o2::trd
 {
 
 /// This is the ML Base class which defines the interface all machine learning
@@ -123,7 +121,6 @@ class PY final : public ML
   ClassDefNV(PY, 1);
 };
 
-} // namespace trd
 } // namespace o2
 
 #endif
diff --git a/Detectors/ZDC/fastsimulation/include/Processors.h b/Detectors/ZDC/fastsimulation/include/Processors.h
index e389bcd42c271..b9836484fe871 100644
--- a/Detectors/ZDC/fastsimulation/include/Processors.h
+++ b/Detectors/ZDC/fastsimulation/include/Processors.h
@@ -81,4 +81,4 @@ std::vector<int> readClassifier(const Ort::Value& value, size_t batchSize);
 std::vector<std::array<long, 5>> calculateChannels(const Ort::Value& value, size_t batchSize);
 
 } // namespace o2::zdc::fastsim::processors
-#endif // O2_ZDC_FAST_SIMULATIONS_PROCESSORS_H
\ No newline at end of file
+#endif // O2_ZDC_FAST_SIMULATIONS_PROCESSORS_H
diff --git a/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h b/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
index a0a9722a9d2f9..a150f8b78fe8b 100644
--- a/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
+++ b/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
@@ -42,9 +42,7 @@ class StandardScaler;
 
 class FairVolume;
 
-namespace o2
-{
-namespace zdc
+namespace o2::zdc
 {
 
 class Detector : public o2::base::DetImpl<Detector>
@@ -273,7 +271,6 @@ class Detector : public o2::base::DetImpl<Detector>
   friend class o2::base::DetImpl;
   ClassDefOverride(Detector, 1);
 };
-} // namespace zdc
 } // namespace o2
 
 #ifdef USESHM

From e6a674f7164bde674303e7fb3f1de22d7f5995f7 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 11 Apr 2024 13:18:51 +0200
Subject: [PATCH 472/726] Add support for ONNXRuntime 1.17.1

---
 Detectors/TRD/pid/include/TRDPID/ML.h                     | 4 ++++
 Detectors/TRD/pid/src/ML.cxx                              | 4 ++++
 Detectors/ZDC/fastsimulation/include/FastSimulations.h    | 4 ++++
 Detectors/ZDC/fastsimulation/include/Processors.h         | 4 ++++
 Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h | 6 ++++++
 5 files changed, 22 insertions(+)

diff --git a/Detectors/TRD/pid/include/TRDPID/ML.h b/Detectors/TRD/pid/include/TRDPID/ML.h
index 5165996d24803..5b1248418e010 100644
--- a/Detectors/TRD/pid/include/TRDPID/ML.h
+++ b/Detectors/TRD/pid/include/TRDPID/ML.h
@@ -21,7 +21,11 @@
 #include "DataFormatsTRD/PID.h"
 #include "Framework/ProcessingContext.h"
 #include "Framework/InputRecord.h"
+#if __has_include (<onnxruntime/core/session/experimental_onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/experimental_onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
 #include <memory>
 #include <vector>
 #include <array>
diff --git a/Detectors/TRD/pid/src/ML.cxx b/Detectors/TRD/pid/src/ML.cxx
index 488bf055a7f6c..2633e63820bb0 100644
--- a/Detectors/TRD/pid/src/ML.cxx
+++ b/Detectors/TRD/pid/src/ML.cxx
@@ -28,7 +28,11 @@
 #include "DetectorsBase/Propagator.h"
 
 #include <fmt/format.h>
+#if __has_include (<onnxruntime/core/session/experimental_onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/experimental_onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
 #include <boost/range.hpp>
 
 #include <array>
diff --git a/Detectors/ZDC/fastsimulation/include/FastSimulations.h b/Detectors/ZDC/fastsimulation/include/FastSimulations.h
index 9244ceb894b0a..987b68efc8102 100644
--- a/Detectors/ZDC/fastsimulation/include/FastSimulations.h
+++ b/Detectors/ZDC/fastsimulation/include/FastSimulations.h
@@ -17,7 +17,11 @@
 #ifndef O2_ZDC_FAST_SIMULATIONS_H
 #define O2_ZDC_FAST_SIMULATIONS_H
 
+#if __has_include(<onnxruntime/core/sessions/onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
 #include <optional>
 #include <mutex>
 
diff --git a/Detectors/ZDC/fastsimulation/include/Processors.h b/Detectors/ZDC/fastsimulation/include/Processors.h
index b9836484fe871..1a0b4f609fc58 100644
--- a/Detectors/ZDC/fastsimulation/include/Processors.h
+++ b/Detectors/ZDC/fastsimulation/include/Processors.h
@@ -17,7 +17,11 @@
 #ifndef O2_ZDC_FAST_SIMULATIONS_PROCESSORS_H
 #define O2_ZDC_FAST_SIMULATIONS_PROCESSORS_H
 
+#if __has_include (<onnxruntime/core/session/onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
 #include <optional>
 #include <vector>
 
diff --git a/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h b/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
index a150f8b78fe8b..6e0bf0667a95b 100644
--- a/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
+++ b/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
@@ -26,7 +26,13 @@
 
 // inclusions and forward decl for fast sim
 #ifdef ZDC_FASTSIM_ONNX
+
+#if __has_include(<onnxruntime/core/session/onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
+
 namespace o2::zdc
 {
 namespace fastsim

From 0cbc0a6034ced9a087f1def6d0cd0b8a8a31474c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 11 Apr 2024 13:20:22 +0200
Subject: [PATCH 473/726] Add support for ONNXRuntime 1.17.1

---
 Detectors/TRD/pid/include/TRDPID/ML.h                     | 4 ++++
 Detectors/TRD/pid/src/ML.cxx                              | 4 ++++
 Detectors/ZDC/fastsimulation/include/FastSimulations.h    | 4 ++++
 Detectors/ZDC/fastsimulation/include/Processors.h         | 4 ++++
 Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h | 6 ++++++
 5 files changed, 22 insertions(+)

diff --git a/Detectors/TRD/pid/include/TRDPID/ML.h b/Detectors/TRD/pid/include/TRDPID/ML.h
index 5165996d24803..011d8c9ee9c9b 100644
--- a/Detectors/TRD/pid/include/TRDPID/ML.h
+++ b/Detectors/TRD/pid/include/TRDPID/ML.h
@@ -21,7 +21,11 @@
 #include "DataFormatsTRD/PID.h"
 #include "Framework/ProcessingContext.h"
 #include "Framework/InputRecord.h"
+#if __has_include(<onnxruntime/core/session/experimental_onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/experimental_onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
 #include <memory>
 #include <vector>
 #include <array>
diff --git a/Detectors/TRD/pid/src/ML.cxx b/Detectors/TRD/pid/src/ML.cxx
index 488bf055a7f6c..785f3b05cc112 100644
--- a/Detectors/TRD/pid/src/ML.cxx
+++ b/Detectors/TRD/pid/src/ML.cxx
@@ -28,7 +28,11 @@
 #include "DetectorsBase/Propagator.h"
 
 #include <fmt/format.h>
+#if __has_include(<onnxruntime/core/session/experimental_onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/experimental_onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
 #include <boost/range.hpp>
 
 #include <array>
diff --git a/Detectors/ZDC/fastsimulation/include/FastSimulations.h b/Detectors/ZDC/fastsimulation/include/FastSimulations.h
index 9244ceb894b0a..987b68efc8102 100644
--- a/Detectors/ZDC/fastsimulation/include/FastSimulations.h
+++ b/Detectors/ZDC/fastsimulation/include/FastSimulations.h
@@ -17,7 +17,11 @@
 #ifndef O2_ZDC_FAST_SIMULATIONS_H
 #define O2_ZDC_FAST_SIMULATIONS_H
 
+#if __has_include(<onnxruntime/core/sessions/onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
 #include <optional>
 #include <mutex>
 
diff --git a/Detectors/ZDC/fastsimulation/include/Processors.h b/Detectors/ZDC/fastsimulation/include/Processors.h
index b9836484fe871..4ae4d0786bdd9 100644
--- a/Detectors/ZDC/fastsimulation/include/Processors.h
+++ b/Detectors/ZDC/fastsimulation/include/Processors.h
@@ -17,7 +17,11 @@
 #ifndef O2_ZDC_FAST_SIMULATIONS_PROCESSORS_H
 #define O2_ZDC_FAST_SIMULATIONS_PROCESSORS_H
 
+#if __has_include(<onnxruntime/core/session/onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
 #include <optional>
 #include <vector>
 
diff --git a/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h b/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
index a150f8b78fe8b..6e0bf0667a95b 100644
--- a/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
+++ b/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
@@ -26,7 +26,13 @@
 
 // inclusions and forward decl for fast sim
 #ifdef ZDC_FASTSIM_ONNX
+
+#if __has_include(<onnxruntime/core/session/onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
+
 namespace o2::zdc
 {
 namespace fastsim

From 98a87bfe926d21634cf537ce21b014c671f93b16 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 11 Apr 2024 13:20:23 +0200
Subject: [PATCH 474/726] Add support for ONNXRuntime 1.17.1

---
 Detectors/TRD/pid/include/TRDPID/ML.h                     | 4 ++++
 Detectors/TRD/pid/src/ML.cxx                              | 4 ++++
 Detectors/ZDC/fastsimulation/include/FastSimulations.h    | 4 ++++
 Detectors/ZDC/fastsimulation/include/Processors.h         | 4 ++++
 Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h | 6 ++++++
 5 files changed, 22 insertions(+)

diff --git a/Detectors/TRD/pid/include/TRDPID/ML.h b/Detectors/TRD/pid/include/TRDPID/ML.h
index 367c33bef682b..c43de8921630a 100644
--- a/Detectors/TRD/pid/include/TRDPID/ML.h
+++ b/Detectors/TRD/pid/include/TRDPID/ML.h
@@ -21,7 +21,11 @@
 #include "DataFormatsTRD/PID.h"
 #include "Framework/ProcessingContext.h"
 #include "Framework/InputRecord.h"
+#if __has_include(<onnxruntime/core/session/experimental_onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/experimental_onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
 #include <memory>
 #include <vector>
 #include <array>
diff --git a/Detectors/TRD/pid/src/ML.cxx b/Detectors/TRD/pid/src/ML.cxx
index 488bf055a7f6c..785f3b05cc112 100644
--- a/Detectors/TRD/pid/src/ML.cxx
+++ b/Detectors/TRD/pid/src/ML.cxx
@@ -28,7 +28,11 @@
 #include "DetectorsBase/Propagator.h"
 
 #include <fmt/format.h>
+#if __has_include(<onnxruntime/core/session/experimental_onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/experimental_onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
 #include <boost/range.hpp>
 
 #include <array>
diff --git a/Detectors/ZDC/fastsimulation/include/FastSimulations.h b/Detectors/ZDC/fastsimulation/include/FastSimulations.h
index 9244ceb894b0a..987b68efc8102 100644
--- a/Detectors/ZDC/fastsimulation/include/FastSimulations.h
+++ b/Detectors/ZDC/fastsimulation/include/FastSimulations.h
@@ -17,7 +17,11 @@
 #ifndef O2_ZDC_FAST_SIMULATIONS_H
 #define O2_ZDC_FAST_SIMULATIONS_H
 
+#if __has_include(<onnxruntime/core/sessions/onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
 #include <optional>
 #include <mutex>
 
diff --git a/Detectors/ZDC/fastsimulation/include/Processors.h b/Detectors/ZDC/fastsimulation/include/Processors.h
index b9836484fe871..4ae4d0786bdd9 100644
--- a/Detectors/ZDC/fastsimulation/include/Processors.h
+++ b/Detectors/ZDC/fastsimulation/include/Processors.h
@@ -17,7 +17,11 @@
 #ifndef O2_ZDC_FAST_SIMULATIONS_PROCESSORS_H
 #define O2_ZDC_FAST_SIMULATIONS_PROCESSORS_H
 
+#if __has_include(<onnxruntime/core/session/onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
 #include <optional>
 #include <vector>
 
diff --git a/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h b/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
index 0f00f53d9f217..40e4babe8d760 100644
--- a/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
+++ b/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
@@ -26,7 +26,13 @@
 
 // inclusions and forward decl for fast sim
 #ifdef ZDC_FASTSIM_ONNX
+
+#if __has_include(<onnxruntime/core/session/onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
+
 namespace o2::zdc
 {
 namespace fastsim

From 5ab474e69d98cbcb0e3bfcee41468a80a4751237 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 11 Apr 2024 13:18:42 +0200
Subject: [PATCH 475/726] Use C++14 namespaces

---
 Detectors/TRD/pid/include/TRDPID/ML.h         |  7 ++-----
 .../ZDC/fastsimulation/include/Processors.h   |  2 +-
 .../include/ZDCSimulation/Detector.h          | 19 ++++++++-----------
 3 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/Detectors/TRD/pid/include/TRDPID/ML.h b/Detectors/TRD/pid/include/TRDPID/ML.h
index 7c72edc30ae3c..367c33bef682b 100644
--- a/Detectors/TRD/pid/include/TRDPID/ML.h
+++ b/Detectors/TRD/pid/include/TRDPID/ML.h
@@ -27,9 +27,7 @@
 #include <array>
 #include <string>
 
-namespace o2
-{
-namespace trd
+namespace o2::trd
 {
 
 /// This is the ML Base class which defines the interface all machine learning
@@ -123,7 +121,6 @@ class PY final : public ML
   ClassDefNV(PY, 1);
 };
 
-} // namespace trd
-} // namespace o2
+} // namespace o2::trd
 
 #endif
diff --git a/Detectors/ZDC/fastsimulation/include/Processors.h b/Detectors/ZDC/fastsimulation/include/Processors.h
index e389bcd42c271..b9836484fe871 100644
--- a/Detectors/ZDC/fastsimulation/include/Processors.h
+++ b/Detectors/ZDC/fastsimulation/include/Processors.h
@@ -81,4 +81,4 @@ std::vector<int> readClassifier(const Ort::Value& value, size_t batchSize);
 std::vector<std::array<long, 5>> calculateChannels(const Ort::Value& value, size_t batchSize);
 
 } // namespace o2::zdc::fastsim::processors
-#endif // O2_ZDC_FAST_SIMULATIONS_PROCESSORS_H
\ No newline at end of file
+#endif // O2_ZDC_FAST_SIMULATIONS_PROCESSORS_H
diff --git a/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h b/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
index a0a9722a9d2f9..0f00f53d9f217 100644
--- a/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
+++ b/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
@@ -42,9 +42,7 @@ class StandardScaler;
 
 class FairVolume;
 
-namespace o2
-{
-namespace zdc
+namespace o2::zdc
 {
 
 class Detector : public o2::base::DetImpl<Detector>
@@ -197,10 +195,10 @@ class Detector : public o2::base::DetImpl<Detector>
   /// Container for hit data
   std::vector<o2::zdc::Hit>* mHits;
 
-  float mLumiLength = 0;         //TODO: make part of configurable params
-  float mTCLIAAPERTURE = 3.5;    //TODO: make part of configurable params
-  float mTCLIAAPERTURENEG = 3.5; //TODO: make part of configurable params
-  float mVCollSideCCentreY = 0.; //TODO: make part of configurable params
+  float mLumiLength = 0;         // TODO: make part of configurable params
+  float mTCLIAAPERTURE = 3.5;    // TODO: make part of configurable params
+  float mTCLIAAPERTURENEG = 3.5; // TODO: make part of configurable params
+  float mVCollSideCCentreY = 0.; // TODO: make part of configurable params
 
   int mZNENVVolID = -1; // the volume id for the neutron det envelope volume
   int mZPENVVolID = -1; // the volume id for the proton det envelope volume
@@ -240,12 +238,12 @@ class Detector : public o2::base::DetImpl<Detector>
 
 // fastsim model wrapper
 #ifdef ZDC_FASTSIM_ONNX
-  fastsim::NeuralFastSimulation* mFastSimClassifier = nullptr; //! no ROOT serialization
+  fastsim::NeuralFastSimulation* mFastSimClassifier = nullptr;   //! no ROOT serialization
   fastsim::NeuralFastSimulation* mFastSimModelNeutron = nullptr; //!
   fastsim::NeuralFastSimulation* mFastSimModelProton = nullptr;  //!
 
   // Scalers for models inputs
-  fastsim::processors::StandardScaler* mClassifierScaler = nullptr; //!
+  fastsim::processors::StandardScaler* mClassifierScaler = nullptr;   //!
   fastsim::processors::StandardScaler* mModelScalerNeutron = nullptr; //!
   fastsim::processors::StandardScaler* mModelScalerProton = nullptr;  //!
 
@@ -273,8 +271,7 @@ class Detector : public o2::base::DetImpl<Detector>
   friend class o2::base::DetImpl;
   ClassDefOverride(Detector, 1);
 };
-} // namespace zdc
-} // namespace o2
+} // namespace o2::zdc
 
 #ifdef USESHM
 namespace o2

From 60232bb28047a47471ad2c1f4c7734ec6b888d80 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 11 Apr 2024 14:34:19 +0200
Subject: [PATCH 476/726] Add support for ONNXRuntime 1.17.1

Notice that this includes a fix to give precedence to the old ONNXRuntime.

Without this, it will currently prefer (non-working) local installations.
---
 Detectors/TRD/pid/include/TRDPID/ML.h             |  4 ++++
 Detectors/TRD/pid/src/ML.cxx                      |  4 ++++
 .../ZDC/fastsimulation/include/FastSimulations.h  |  4 ++++
 Detectors/ZDC/fastsimulation/include/Processors.h |  4 ++++
 .../simulation/include/ZDCSimulation/Detector.h   |  6 ++++++
 dependencies/FindONNXRuntime.cmake                | 15 ++++++++-------
 6 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/Detectors/TRD/pid/include/TRDPID/ML.h b/Detectors/TRD/pid/include/TRDPID/ML.h
index 367c33bef682b..c43de8921630a 100644
--- a/Detectors/TRD/pid/include/TRDPID/ML.h
+++ b/Detectors/TRD/pid/include/TRDPID/ML.h
@@ -21,7 +21,11 @@
 #include "DataFormatsTRD/PID.h"
 #include "Framework/ProcessingContext.h"
 #include "Framework/InputRecord.h"
+#if __has_include(<onnxruntime/core/session/experimental_onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/experimental_onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
 #include <memory>
 #include <vector>
 #include <array>
diff --git a/Detectors/TRD/pid/src/ML.cxx b/Detectors/TRD/pid/src/ML.cxx
index 488bf055a7f6c..785f3b05cc112 100644
--- a/Detectors/TRD/pid/src/ML.cxx
+++ b/Detectors/TRD/pid/src/ML.cxx
@@ -28,7 +28,11 @@
 #include "DetectorsBase/Propagator.h"
 
 #include <fmt/format.h>
+#if __has_include(<onnxruntime/core/session/experimental_onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/experimental_onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
 #include <boost/range.hpp>
 
 #include <array>
diff --git a/Detectors/ZDC/fastsimulation/include/FastSimulations.h b/Detectors/ZDC/fastsimulation/include/FastSimulations.h
index 9244ceb894b0a..9fd89c8fb5838 100644
--- a/Detectors/ZDC/fastsimulation/include/FastSimulations.h
+++ b/Detectors/ZDC/fastsimulation/include/FastSimulations.h
@@ -17,7 +17,11 @@
 #ifndef O2_ZDC_FAST_SIMULATIONS_H
 #define O2_ZDC_FAST_SIMULATIONS_H
 
+#if __has_include(<onnxruntime/core/session/onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
 #include <optional>
 #include <mutex>
 
diff --git a/Detectors/ZDC/fastsimulation/include/Processors.h b/Detectors/ZDC/fastsimulation/include/Processors.h
index b9836484fe871..4ae4d0786bdd9 100644
--- a/Detectors/ZDC/fastsimulation/include/Processors.h
+++ b/Detectors/ZDC/fastsimulation/include/Processors.h
@@ -17,7 +17,11 @@
 #ifndef O2_ZDC_FAST_SIMULATIONS_PROCESSORS_H
 #define O2_ZDC_FAST_SIMULATIONS_PROCESSORS_H
 
+#if __has_include(<onnxruntime/core/session/onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
 #include <optional>
 #include <vector>
 
diff --git a/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h b/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
index 0f00f53d9f217..40e4babe8d760 100644
--- a/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
+++ b/Detectors/ZDC/simulation/include/ZDCSimulation/Detector.h
@@ -26,7 +26,13 @@
 
 // inclusions and forward decl for fast sim
 #ifdef ZDC_FASTSIM_ONNX
+
+#if __has_include(<onnxruntime/core/session/onnxruntime_cxx_api.h>)
 #include <onnxruntime/core/session/onnxruntime_cxx_api.h>
+#else
+#include <onnxruntime_cxx_api.h>
+#endif
+
 namespace o2::zdc
 {
 namespace fastsim
diff --git a/dependencies/FindONNXRuntime.cmake b/dependencies/FindONNXRuntime.cmake
index b34782e99050f..0983d44644a6c 100644
--- a/dependencies/FindONNXRuntime.cmake
+++ b/dependencies/FindONNXRuntime.cmake
@@ -9,11 +9,12 @@
 # granted to it by virtue of its status as an Intergovernmental Organization
 # or submit itself to any jurisdiction.
 
-find_package(onnxruntime CONFIG)
-if (NOT onnxruntime_FOUND)
-  find_package(ONNXRuntime::ONNXRuntime CONFIG)
-  if (ONNXRuntime::ONNXRuntime_FOUND)
-    set(onnxruntime_FOUND 1)
-    add_library(onnxruntime::onnxruntime ALIAS ONNXRuntime::ONNXRuntime)
-  endif()
+find_package(ONNXRuntime::ONNXRuntime CONFIG)
+if (ONNXRuntime::ONNXRuntime_FOUND)
+  set(onnxruntime_FOUND 1)
+  add_library(onnxruntime::onnxruntime ALIAS ONNXRuntime::ONNXRuntime)
+endif()
+
+if (NOT ONNXRuntime::ONNXRuntime_FOUND)
+  find_package(onnxruntime CONFIG)
 endif()

From 226a5ae540497445fc67499dbbc7f0b932923b3d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 19 Apr 2024 12:57:08 +0200
Subject: [PATCH 477/726] DPL: disable cpu_usage_fraction for proxies

---
 Framework/Core/src/CommonServices.cxx | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 028ed6e8e6806..4f54cc389590d 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -43,6 +43,7 @@
 #include "Framework/DeviceState.h"
 #include "Framework/DeviceConfig.h"
 #include "Framework/DefaultsHelpers.h"
+#include "Framework/Signpost.h"
 
 #include "TextDriverClient.h"
 #include "WSDriverClient.h"
@@ -83,6 +84,7 @@ using Value = o2::monitoring::tags::Value;
 O2_DECLARE_DYNAMIC_LOG(data_processor_context);
 O2_DECLARE_DYNAMIC_LOG(stream_context);
 O2_DECLARE_DYNAMIC_LOG(async_queue);
+O2_DECLARE_DYNAMIC_LOG(policies);
 
 namespace o2::framework
 {
@@ -849,6 +851,16 @@ o2::framework::ServiceSpec CommonServices::dataProcessingStats()
       if (deploymentMode != DeploymentMode::OnlineDDS && deploymentMode != DeploymentMode::OnlineECS && deploymentMode != DeploymentMode::OnlineAUX && deploymentMode != DeploymentMode::FST) {
         arrowAndResourceLimitingMetrics = true;
       }
+      // Input proxies should not report cpu_usage_fraction,
+      // because of the rate limiting which biases the measurement.
+      auto& spec = services.get<DeviceSpec const>();
+      bool enableCPUUsageFraction = true;
+      auto isProxy = [](DataProcessorLabel const& label) -> bool { return label == DataProcessorLabel{"input-proxy"} || label == DataProcessorLabel{"output-proxy"}; };
+      if (std::find_if(spec.labels.begin(), spec.labels.end(), isProxy) != spec.labels.end()) {
+        O2_SIGNPOST_ID_GENERATE(mid, policies);
+        O2_SIGNPOST_EVENT_EMIT(policies, mid, "metrics", "Disabling cpu_usage_fraction metric for proxy %{public}s", spec.name.c_str());
+        enableCPUUsageFraction = false;
+      }
 
       std::vector<DataProcessingStats::MetricSpec> metrics = {
         MetricSpec{.name = "errors",
@@ -932,6 +944,7 @@ o2::framework::ServiceSpec CommonServices::dataProcessingStats()
                    .maxRefreshLatency = onlineRefreshLatency,
                    .sendInitialValue = true},
         MetricSpec{.name = "cpu_usage_fraction",
+                   .enabled = enableCPUUsageFraction,
                    .metricId = (int)ProcessingStatsId::CPU_USAGE_FRACTION,
                    .kind = Kind::Rate,
                    .scope = Scope::Online,

From 69b5961c1ceba53e2222c35728208ebe15c92d12 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 19 Apr 2024 14:57:58 +0200
Subject: [PATCH 478/726] DPL: Fix memory leak in variant_helper

---
 Framework/Core/include/Framework/Variant.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/include/Framework/Variant.h b/Framework/Core/include/Framework/Variant.h
index b8be3a780547e..54e4b54be9606 100644
--- a/Framework/Core/include/Framework/Variant.h
+++ b/Framework/Core/include/Framework/Variant.h
@@ -285,7 +285,7 @@ struct variant_helper<S, std::string_view> {
 
 template <typename S>
 struct variant_helper<S, std::string> {
-  static std::string get(const S* store) { return std::string(strdup(*reinterpret_cast<const char* const*>(store))); }
+  static std::string get(const S* store) { return std::string(*reinterpret_cast<const char* const*>(store)); }
 
   static void set(S* store, std::string value) { *reinterpret_cast<char**>(store) = strdup(value.data()); }
 };

From b9c3b61efece3891a77c6376da6cebc89e7f5a7d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 22 Apr 2024 12:36:41 +0200
Subject: [PATCH 479/726] CCDBApi: avoid leaking curl handle.

---
 CCDB/src/CCDBDownloader.cxx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CCDB/src/CCDBDownloader.cxx b/CCDB/src/CCDBDownloader.cxx
index bd2bf22d0add9..af0a95e3864b7 100644
--- a/CCDB/src/CCDBDownloader.cxx
+++ b/CCDB/src/CCDBDownloader.cxx
@@ -537,6 +537,7 @@ void CCDBDownloader::transferFinished(CURL* easy_handle, CURLcode curlCode)
         --(*performData->requestsLeft);
         delete requestData;
         delete performData->codeDestination;
+        curl_easy_cleanup(easy_handle);
       }
     } break;
   }

From 45661cbd5594818063991c94b8c0e14d464f98c2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 22 Apr 2024 22:12:29 +0200
Subject: [PATCH 480/726] CCDBApi: avoid leaking curl handle.

---
 CCDB/include/CCDB/CCDBDownloader.h  | 4 +++-
 CCDB/src/CCDBDownloader.cxx         | 6 +++++-
 CCDB/src/CcdbApi.cxx                | 1 +
 CCDB/test/testCcdbApiDownloader.cxx | 4 +++-
 4 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/CCDB/include/CCDB/CCDBDownloader.h b/CCDB/include/CCDB/CCDBDownloader.h
index 17b65deb06dc2..e53421dcc26fc 100644
--- a/CCDB/include/CCDB/CCDBDownloader.h
+++ b/CCDB/include/CCDB/CCDBDownloader.h
@@ -52,6 +52,7 @@ typedef struct DownloaderRequestData {
   HeaderObjectPair_t hoPair;
   std::map<std::string, std::string>* headers;
   std::string userAgent;
+  curl_slist* optionsList;
 
   std::function<bool(std::string)> localContentCallback;
 } DownloaderRequestData;
@@ -296,6 +297,7 @@ class CCDBDownloader
     int hostInd;
     int locInd;
     DownloaderRequestData* requestData;
+    curl_slist** options;
   } PerformData;
 #endif
 
@@ -421,6 +423,6 @@ typedef struct DataForClosingSocket {
   curl_socket_t socket;
 } DataForClosingSocket;
 
-} // namespace o2
+} // namespace o2::ccdb
 
 #endif // O2_CCDB_CCDBDOWNLOADER_H
diff --git a/CCDB/src/CCDBDownloader.cxx b/CCDB/src/CCDBDownloader.cxx
index bd2bf22d0add9..8d13368688cb7 100644
--- a/CCDB/src/CCDBDownloader.cxx
+++ b/CCDB/src/CCDBDownloader.cxx
@@ -535,8 +535,10 @@ void CCDBDownloader::transferFinished(CURL* easy_handle, CURLcode curlCode)
           }
         }
         --(*performData->requestsLeft);
+        curl_slist_free_all(*performData->options);
         delete requestData;
         delete performData->codeDestination;
+        curl_easy_cleanup(easy_handle);
       }
     } break;
   }
@@ -681,6 +683,7 @@ void CCDBDownloader::asynchSchedule(CURL* handle, size_t* requestCounter)
   curl_easy_getinfo(handle, CURLINFO_PRIVATE, &requestData);
   headerMap = &(requestData->hoPair.header);
   hostsPool = &(requestData->hosts);
+  auto* options = &(requestData->optionsList);
 
   // Prepare temporary data about transfer
   auto* data = new CCDBDownloader::PerformData(); // Freed in transferFinished
@@ -692,6 +695,7 @@ void CCDBDownloader::asynchSchedule(CURL* handle, size_t* requestCounter)
   data->hostInd = 0;
   data->locInd = 0;
   data->requestData = requestData;
+  data->options = options;
 
   // Prepare handle and schedule download
   setHandleOptions(handle, data);
@@ -719,4 +723,4 @@ std::string CCDBDownloader::prepareLogMessage(std::string host_url, std::string
   return fmt::format("CcdbDownloader finished transfer {}{}{} for {} (agent_id: {}) with http code: {}", host_url, (host_url.back() == '/') ? "" : "/", upath, (ts < 0) ? getCurrentTimestamp() : ts, userAgent, httpCode);
 }
 
-} // namespace o2
+} // namespace o2::ccdb
diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx
index 46d95bae477d4..4c4aa6ca74a80 100644
--- a/CCDB/src/CcdbApi.cxx
+++ b/CCDB/src/CcdbApi.cxx
@@ -1656,6 +1656,7 @@ void CcdbApi::scheduleDownload(RequestContext& requestContext, size_t* requestCo
   data->timestamp = requestContext.timestamp;
   data->localContentCallback = localContentCallback;
   data->userAgent = mUniqueAgentID;
+  data->optionsList = options_list;
 
   curl_easy_setopt(curl_handle, CURLOPT_URL, fullUrl.c_str());
   initCurlOptionsForRetrieve(curl_handle, (void*)(&data->hoPair), writeCallback, false);
diff --git a/CCDB/test/testCcdbApiDownloader.cxx b/CCDB/test/testCcdbApiDownloader.cxx
index 68c8333f46ffa..6bbc2d5dca288 100644
--- a/CCDB/test/testCcdbApiDownloader.cxx
+++ b/CCDB/test/testCcdbApiDownloader.cxx
@@ -172,7 +172,9 @@ BOOST_AUTO_TEST_CASE(asynch_schedule_test)
     curl_easy_getinfo(handles[i], CURLINFO_HTTP_CODE, &httpCode);
     BOOST_CHECK(httpCode == 200);
     BOOST_CHECK(dests[i]->size() != 0);
-    curl_easy_cleanup(handles[i]);
+    // I suspect the following was needed simply because the rest of the
+    // code was leaking.
+    //    curl_easy_cleanup(handles[i]);
     delete dests[i];
   }
   curl_global_cleanup();

From 5cc937cf973f7b06a8095be1c336190f537ef955 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 22 Apr 2024 22:12:29 +0200
Subject: [PATCH 481/726] CCDBApi: avoid leaking curl handle.

---
 CCDB/include/CCDB/CCDBDownloader.h  |  4 +-
 CCDB/src/CCDBDownloader.cxx         |  6 ++-
 CCDB/src/CcdbApi.cxx                |  1 +
 CCDB/test/testCcdbApiDownloader.cxx | 67 +++++++++++------------------
 4 files changed, 33 insertions(+), 45 deletions(-)

diff --git a/CCDB/include/CCDB/CCDBDownloader.h b/CCDB/include/CCDB/CCDBDownloader.h
index 17b65deb06dc2..e53421dcc26fc 100644
--- a/CCDB/include/CCDB/CCDBDownloader.h
+++ b/CCDB/include/CCDB/CCDBDownloader.h
@@ -52,6 +52,7 @@ typedef struct DownloaderRequestData {
   HeaderObjectPair_t hoPair;
   std::map<std::string, std::string>* headers;
   std::string userAgent;
+  curl_slist* optionsList;
 
   std::function<bool(std::string)> localContentCallback;
 } DownloaderRequestData;
@@ -296,6 +297,7 @@ class CCDBDownloader
     int hostInd;
     int locInd;
     DownloaderRequestData* requestData;
+    curl_slist** options;
   } PerformData;
 #endif
 
@@ -421,6 +423,6 @@ typedef struct DataForClosingSocket {
   curl_socket_t socket;
 } DataForClosingSocket;
 
-} // namespace o2
+} // namespace o2::ccdb
 
 #endif // O2_CCDB_CCDBDOWNLOADER_H
diff --git a/CCDB/src/CCDBDownloader.cxx b/CCDB/src/CCDBDownloader.cxx
index bd2bf22d0add9..8d13368688cb7 100644
--- a/CCDB/src/CCDBDownloader.cxx
+++ b/CCDB/src/CCDBDownloader.cxx
@@ -535,8 +535,10 @@ void CCDBDownloader::transferFinished(CURL* easy_handle, CURLcode curlCode)
           }
         }
         --(*performData->requestsLeft);
+        curl_slist_free_all(*performData->options);
         delete requestData;
         delete performData->codeDestination;
+        curl_easy_cleanup(easy_handle);
       }
     } break;
   }
@@ -681,6 +683,7 @@ void CCDBDownloader::asynchSchedule(CURL* handle, size_t* requestCounter)
   curl_easy_getinfo(handle, CURLINFO_PRIVATE, &requestData);
   headerMap = &(requestData->hoPair.header);
   hostsPool = &(requestData->hosts);
+  auto* options = &(requestData->optionsList);
 
   // Prepare temporary data about transfer
   auto* data = new CCDBDownloader::PerformData(); // Freed in transferFinished
@@ -692,6 +695,7 @@ void CCDBDownloader::asynchSchedule(CURL* handle, size_t* requestCounter)
   data->hostInd = 0;
   data->locInd = 0;
   data->requestData = requestData;
+  data->options = options;
 
   // Prepare handle and schedule download
   setHandleOptions(handle, data);
@@ -719,4 +723,4 @@ std::string CCDBDownloader::prepareLogMessage(std::string host_url, std::string
   return fmt::format("CcdbDownloader finished transfer {}{}{} for {} (agent_id: {}) with http code: {}", host_url, (host_url.back() == '/') ? "" : "/", upath, (ts < 0) ? getCurrentTimestamp() : ts, userAgent, httpCode);
 }
 
-} // namespace o2
+} // namespace o2::ccdb
diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx
index 46d95bae477d4..4c4aa6ca74a80 100644
--- a/CCDB/src/CcdbApi.cxx
+++ b/CCDB/src/CcdbApi.cxx
@@ -1656,6 +1656,7 @@ void CcdbApi::scheduleDownload(RequestContext& requestContext, size_t* requestCo
   data->timestamp = requestContext.timestamp;
   data->localContentCallback = localContentCallback;
   data->userAgent = mUniqueAgentID;
+  data->optionsList = options_list;
 
   curl_easy_setopt(curl_handle, CURLOPT_URL, fullUrl.c_str());
   initCurlOptionsForRetrieve(curl_handle, (void*)(&data->hoPair), writeCallback, false);
diff --git a/CCDB/test/testCcdbApiDownloader.cxx b/CCDB/test/testCcdbApiDownloader.cxx
index 68c8333f46ffa..63b0434484719 100644
--- a/CCDB/test/testCcdbApiDownloader.cxx
+++ b/CCDB/test/testCcdbApiDownloader.cxx
@@ -23,7 +23,6 @@
 #include <fairlogger/Logger.h>
 
 #include <boost/test/unit_test.hpp>
-#include <boost/optional/optional.hpp>
 #include <boost/asio/ip/host_name.hpp>
 #include <uv.h>
 
@@ -129,7 +128,7 @@ std::vector<CURL*> prepareAsyncHandles(size_t num, std::vector<o2::pmr::vector<c
 
     auto data = new DownloaderRequestData();
     data->hoPair.object = dest;
-    data->hosts.push_back("http://ccdb-test.cern.ch:8080");
+    data->hosts.emplace_back("http://ccdb-test.cern.ch:8080");
     data->path = "Analysis/ALICE3/Centrality";
     data->timestamp = 1646729604010;
     data->localContentCallback = nullptr;
@@ -164,15 +163,17 @@ BOOST_AUTO_TEST_CASE(asynch_schedule_test)
   }
 
   while (transfersLeft > 0) {
-    downloader.runLoop(0);
+    downloader.runLoop(false);
   }
 
   for (int i = 0; i < TRANSFERS; i++) {
     long httpCode;
     curl_easy_getinfo(handles[i], CURLINFO_HTTP_CODE, &httpCode);
-    BOOST_CHECK(httpCode == 200);
-    BOOST_CHECK(dests[i]->size() != 0);
-    curl_easy_cleanup(handles[i]);
+    BOOST_CHECK_EQUAL(httpCode,  200);
+    BOOST_CHECK_NE(dests[i]->size(), 0);
+    // I suspect the following was needed simply because the rest of the
+    // code was leaking.
+    //    curl_easy_cleanup(handles[i]);
     delete dests[i];
   }
   curl_global_cleanup();
@@ -191,13 +192,11 @@ BOOST_AUTO_TEST_CASE(perform_test)
 
   CURLcode curlCode = downloader.perform(handle);
 
-  BOOST_CHECK(curlCode == CURLE_OK);
-  std::cout << "CURL code: " << curlCode << "\n";
+  BOOST_CHECK_EQUAL(curlCode, CURLE_OK);
 
   long httpCode;
   curl_easy_getinfo(handle, CURLINFO_HTTP_CODE, &httpCode);
-  BOOST_CHECK(httpCode == 200);
-  std::cout << "HTTP code: " << httpCode << "\n";
+  BOOST_CHECK_EQUAL(httpCode, 200);
 
   curl_easy_cleanup(handle);
   curl_global_cleanup();
@@ -220,19 +219,13 @@ BOOST_AUTO_TEST_CASE(blocking_batch_test)
 
   auto curlCodes = downloader.batchBlockingPerform(handleVector);
   for (CURLcode code : curlCodes) {
-    BOOST_CHECK(code == CURLE_OK);
-    if (code != CURLE_OK) {
-      std::cout << "CURL Code: " << code << "\n";
-    }
+    BOOST_CHECK_EQUAL(code, CURLE_OK);
   }
 
   for (CURL* handle : handleVector) {
     long httpCode;
     curl_easy_getinfo(handle, CURLINFO_HTTP_CODE, &httpCode);
-    BOOST_CHECK(httpCode == 200);
-    if (httpCode != 200) {
-      std::cout << "HTTP Code: " << httpCode << "\n";
-    }
+    BOOST_CHECK_EQUAL(httpCode, 200);
     curl_easy_cleanup(handle);
   }
 
@@ -261,19 +254,13 @@ BOOST_AUTO_TEST_CASE(test_with_break)
   auto curlCodes = downloader.batchBlockingPerform(handleVector);
 
   for (CURLcode code : curlCodes) {
-    BOOST_CHECK(code == CURLE_OK);
-    if (code != CURLE_OK) {
-      std::cout << "CURL Code: " << code << "\n";
-    }
+    BOOST_CHECK_EQUAL(code, CURLE_OK);
   }
 
   for (CURL* handle : handleVector) {
     long httpCode;
     curl_easy_getinfo(handle, CURLINFO_HTTP_CODE, &httpCode);
-    BOOST_CHECK(httpCode == 200);
-    if (httpCode != 200) {
-      std::cout << "HTTP Code: " << httpCode << "\n";
-    }
+    BOOST_CHECK_EQUAL(httpCode, 200);
     curl_easy_cleanup(handle);
   }
 
@@ -292,19 +279,13 @@ BOOST_AUTO_TEST_CASE(test_with_break)
 
   auto curlCodes2 = downloader.batchBlockingPerform(handleVector2);
   for (CURLcode code : curlCodes2) {
-    BOOST_CHECK(code == CURLE_OK);
-    if (code != CURLE_OK) {
-      std::cout << "CURL Code: " << code << "\n";
-    }
+    BOOST_CHECK_EQUAL(code, CURLE_OK);
   }
 
   for (CURL* handle : handleVector2) {
     long httpCode;
     curl_easy_getinfo(handle, CURLINFO_HTTP_CODE, &httpCode);
-    BOOST_CHECK(httpCode == 200);
-    if (httpCode != 200) {
-      std::cout << "HTTP Code: " << httpCode << "\n";
-    }
+    BOOST_CHECK_EQUAL(httpCode, 200);
     curl_easy_cleanup(handle);
   }
 
@@ -357,18 +338,18 @@ BOOST_AUTO_TEST_CASE(external_loop_test)
 
   CURLcode curlCode = downloader->perform(handle);
 
-  BOOST_CHECK(curlCode == CURLE_OK);
+  BOOST_CHECK_EQUAL(curlCode, CURLE_OK);
 
   long httpCode;
   curl_easy_getinfo(handle, CURLINFO_HTTP_CODE, &httpCode);
-  BOOST_CHECK(httpCode == 200);
+  BOOST_CHECK_EQUAL(httpCode, 200);
 
   curl_easy_cleanup(handle);
   curl_global_cleanup();
 
   // Check if test timer and external loop are still alive
-  BOOST_CHECK(uv_is_active((uv_handle_t*)testTimer) != 0);
-  BOOST_CHECK(uv_loop_alive(uvLoop) != 0);
+  BOOST_CHECK_NE(uv_is_active((uv_handle_t*)testTimer), 0);
+  BOOST_CHECK_NE(uv_loop_alive(uvLoop), 0);
 
   // Downloader must be closed before uv_loop.
   // The reason for that are the uv_poll handles attached to the curl multi handle.
@@ -384,11 +365,11 @@ BOOST_AUTO_TEST_CASE(external_loop_test)
 BOOST_AUTO_TEST_CASE(trim_host_url_test)
 {
   CCDBDownloader downloader;
-  BOOST_CHECK(downloader.trimHostUrl("http://localhost:8080") == "http://localhost:8080");
-  BOOST_CHECK(downloader.trimHostUrl("http://localhost") == "http://localhost");
-  BOOST_CHECK(downloader.trimHostUrl("http://localhost:8080/some/path") == "http://localhost:8080");
-  BOOST_CHECK(downloader.trimHostUrl("http://localhost/some/path") == "http://localhost");
-  BOOST_CHECK(downloader.trimHostUrl("http://localhost:8080/Task/Detector/1?HTTPOnly=true") == "http://localhost:8080");
+  BOOST_CHECK_EQUAL(downloader.trimHostUrl("http://localhost:8080"), "http://localhost:8080");
+  BOOST_CHECK_EQUAL(downloader.trimHostUrl("http://localhost"), "http://localhost");
+  BOOST_CHECK_EQUAL(downloader.trimHostUrl("http://localhost:8080/some/path"), "http://localhost:8080");
+  BOOST_CHECK_EQUAL(downloader.trimHostUrl("http://localhost/some/path"), "http://localhost");
+  BOOST_CHECK_EQUAL(downloader.trimHostUrl("http://localhost:8080/Task/Detector/1?HTTPOnly=true"), "http://localhost:8080");
 }
 
 } // namespace ccdb

From 62a7e49820b1d962b9895f6ff116e12e95793903 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 23 Apr 2024 10:43:56 +0200
Subject: [PATCH 482/726] CCDBApi: avoid leaking curl handle.

---
 CCDB/include/CCDB/CCDBDownloader.h  |  4 +-
 CCDB/src/CCDBDownloader.cxx         |  6 ++-
 CCDB/src/CcdbApi.cxx                |  1 +
 CCDB/test/testCcdbApiDownloader.cxx | 74 +++++++++++------------------
 4 files changed, 38 insertions(+), 47 deletions(-)

diff --git a/CCDB/include/CCDB/CCDBDownloader.h b/CCDB/include/CCDB/CCDBDownloader.h
index 17b65deb06dc2..e53421dcc26fc 100644
--- a/CCDB/include/CCDB/CCDBDownloader.h
+++ b/CCDB/include/CCDB/CCDBDownloader.h
@@ -52,6 +52,7 @@ typedef struct DownloaderRequestData {
   HeaderObjectPair_t hoPair;
   std::map<std::string, std::string>* headers;
   std::string userAgent;
+  curl_slist* optionsList;
 
   std::function<bool(std::string)> localContentCallback;
 } DownloaderRequestData;
@@ -296,6 +297,7 @@ class CCDBDownloader
     int hostInd;
     int locInd;
     DownloaderRequestData* requestData;
+    curl_slist** options;
   } PerformData;
 #endif
 
@@ -421,6 +423,6 @@ typedef struct DataForClosingSocket {
   curl_socket_t socket;
 } DataForClosingSocket;
 
-} // namespace o2
+} // namespace o2::ccdb
 
 #endif // O2_CCDB_CCDBDOWNLOADER_H
diff --git a/CCDB/src/CCDBDownloader.cxx b/CCDB/src/CCDBDownloader.cxx
index bd2bf22d0add9..8d13368688cb7 100644
--- a/CCDB/src/CCDBDownloader.cxx
+++ b/CCDB/src/CCDBDownloader.cxx
@@ -535,8 +535,10 @@ void CCDBDownloader::transferFinished(CURL* easy_handle, CURLcode curlCode)
           }
         }
         --(*performData->requestsLeft);
+        curl_slist_free_all(*performData->options);
         delete requestData;
         delete performData->codeDestination;
+        curl_easy_cleanup(easy_handle);
       }
     } break;
   }
@@ -681,6 +683,7 @@ void CCDBDownloader::asynchSchedule(CURL* handle, size_t* requestCounter)
   curl_easy_getinfo(handle, CURLINFO_PRIVATE, &requestData);
   headerMap = &(requestData->hoPair.header);
   hostsPool = &(requestData->hosts);
+  auto* options = &(requestData->optionsList);
 
   // Prepare temporary data about transfer
   auto* data = new CCDBDownloader::PerformData(); // Freed in transferFinished
@@ -692,6 +695,7 @@ void CCDBDownloader::asynchSchedule(CURL* handle, size_t* requestCounter)
   data->hostInd = 0;
   data->locInd = 0;
   data->requestData = requestData;
+  data->options = options;
 
   // Prepare handle and schedule download
   setHandleOptions(handle, data);
@@ -719,4 +723,4 @@ std::string CCDBDownloader::prepareLogMessage(std::string host_url, std::string
   return fmt::format("CcdbDownloader finished transfer {}{}{} for {} (agent_id: {}) with http code: {}", host_url, (host_url.back() == '/') ? "" : "/", upath, (ts < 0) ? getCurrentTimestamp() : ts, userAgent, httpCode);
 }
 
-} // namespace o2
+} // namespace o2::ccdb
diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx
index 46d95bae477d4..4c4aa6ca74a80 100644
--- a/CCDB/src/CcdbApi.cxx
+++ b/CCDB/src/CcdbApi.cxx
@@ -1656,6 +1656,7 @@ void CcdbApi::scheduleDownload(RequestContext& requestContext, size_t* requestCo
   data->timestamp = requestContext.timestamp;
   data->localContentCallback = localContentCallback;
   data->userAgent = mUniqueAgentID;
+  data->optionsList = options_list;
 
   curl_easy_setopt(curl_handle, CURLOPT_URL, fullUrl.c_str());
   initCurlOptionsForRetrieve(curl_handle, (void*)(&data->hoPair), writeCallback, false);
diff --git a/CCDB/test/testCcdbApiDownloader.cxx b/CCDB/test/testCcdbApiDownloader.cxx
index 68c8333f46ffa..264eff8fdb848 100644
--- a/CCDB/test/testCcdbApiDownloader.cxx
+++ b/CCDB/test/testCcdbApiDownloader.cxx
@@ -23,7 +23,6 @@
 #include <fairlogger/Logger.h>
 
 #include <boost/test/unit_test.hpp>
-#include <boost/optional/optional.hpp>
 #include <boost/asio/ip/host_name.hpp>
 #include <uv.h>
 
@@ -129,7 +128,7 @@ std::vector<CURL*> prepareAsyncHandles(size_t num, std::vector<o2::pmr::vector<c
 
     auto data = new DownloaderRequestData();
     data->hoPair.object = dest;
-    data->hosts.push_back("http://ccdb-test.cern.ch:8080");
+    data->hosts.emplace_back("http://ccdb-test.cern.ch:8080");
     data->path = "Analysis/ALICE3/Centrality";
     data->timestamp = 1646729604010;
     data->localContentCallback = nullptr;
@@ -164,15 +163,20 @@ BOOST_AUTO_TEST_CASE(asynch_schedule_test)
   }
 
   while (transfersLeft > 0) {
-    downloader.runLoop(0);
+    downloader.runLoop(false);
   }
 
   for (int i = 0; i < TRANSFERS; i++) {
-    long httpCode;
-    curl_easy_getinfo(handles[i], CURLINFO_HTTP_CODE, &httpCode);
-    BOOST_CHECK(httpCode == 200);
-    BOOST_CHECK(dests[i]->size() != 0);
-    curl_easy_cleanup(handles[i]);
+    // I would claim that accessing the handles after they are complete
+    // is actually not supported by the current API, because it was
+    // previously relying on leaking the handles. Disabling the whole
+    // thing until we verify that's actually the case.
+    //
+    // long httpCode;
+    // curl_easy_getinfo(handles[i], CURLINFO_HTTP_CODE, &httpCode);
+    // BOOST_CHECK_EQUAL(httpCode, 200);
+    // BOOST_CHECK_NE(dests[i]->size(), 0);
+    //    curl_easy_cleanup(handles[i]);
     delete dests[i];
   }
   curl_global_cleanup();
@@ -191,13 +195,11 @@ BOOST_AUTO_TEST_CASE(perform_test)
 
   CURLcode curlCode = downloader.perform(handle);
 
-  BOOST_CHECK(curlCode == CURLE_OK);
-  std::cout << "CURL code: " << curlCode << "\n";
+  BOOST_CHECK_EQUAL(curlCode, CURLE_OK);
 
   long httpCode;
   curl_easy_getinfo(handle, CURLINFO_HTTP_CODE, &httpCode);
-  BOOST_CHECK(httpCode == 200);
-  std::cout << "HTTP code: " << httpCode << "\n";
+  BOOST_CHECK_EQUAL(httpCode, 200);
 
   curl_easy_cleanup(handle);
   curl_global_cleanup();
@@ -220,19 +222,13 @@ BOOST_AUTO_TEST_CASE(blocking_batch_test)
 
   auto curlCodes = downloader.batchBlockingPerform(handleVector);
   for (CURLcode code : curlCodes) {
-    BOOST_CHECK(code == CURLE_OK);
-    if (code != CURLE_OK) {
-      std::cout << "CURL Code: " << code << "\n";
-    }
+    BOOST_CHECK_EQUAL(code, CURLE_OK);
   }
 
   for (CURL* handle : handleVector) {
     long httpCode;
     curl_easy_getinfo(handle, CURLINFO_HTTP_CODE, &httpCode);
-    BOOST_CHECK(httpCode == 200);
-    if (httpCode != 200) {
-      std::cout << "HTTP Code: " << httpCode << "\n";
-    }
+    BOOST_CHECK_EQUAL(httpCode, 200);
     curl_easy_cleanup(handle);
   }
 
@@ -261,19 +257,13 @@ BOOST_AUTO_TEST_CASE(test_with_break)
   auto curlCodes = downloader.batchBlockingPerform(handleVector);
 
   for (CURLcode code : curlCodes) {
-    BOOST_CHECK(code == CURLE_OK);
-    if (code != CURLE_OK) {
-      std::cout << "CURL Code: " << code << "\n";
-    }
+    BOOST_CHECK_EQUAL(code, CURLE_OK);
   }
 
   for (CURL* handle : handleVector) {
     long httpCode;
     curl_easy_getinfo(handle, CURLINFO_HTTP_CODE, &httpCode);
-    BOOST_CHECK(httpCode == 200);
-    if (httpCode != 200) {
-      std::cout << "HTTP Code: " << httpCode << "\n";
-    }
+    BOOST_CHECK_EQUAL(httpCode, 200);
     curl_easy_cleanup(handle);
   }
 
@@ -292,19 +282,13 @@ BOOST_AUTO_TEST_CASE(test_with_break)
 
   auto curlCodes2 = downloader.batchBlockingPerform(handleVector2);
   for (CURLcode code : curlCodes2) {
-    BOOST_CHECK(code == CURLE_OK);
-    if (code != CURLE_OK) {
-      std::cout << "CURL Code: " << code << "\n";
-    }
+    BOOST_CHECK_EQUAL(code, CURLE_OK);
   }
 
   for (CURL* handle : handleVector2) {
     long httpCode;
     curl_easy_getinfo(handle, CURLINFO_HTTP_CODE, &httpCode);
-    BOOST_CHECK(httpCode == 200);
-    if (httpCode != 200) {
-      std::cout << "HTTP Code: " << httpCode << "\n";
-    }
+    BOOST_CHECK_EQUAL(httpCode, 200);
     curl_easy_cleanup(handle);
   }
 
@@ -357,18 +341,18 @@ BOOST_AUTO_TEST_CASE(external_loop_test)
 
   CURLcode curlCode = downloader->perform(handle);
 
-  BOOST_CHECK(curlCode == CURLE_OK);
+  BOOST_CHECK_EQUAL(curlCode, CURLE_OK);
 
   long httpCode;
   curl_easy_getinfo(handle, CURLINFO_HTTP_CODE, &httpCode);
-  BOOST_CHECK(httpCode == 200);
+  BOOST_CHECK_EQUAL(httpCode, 200);
 
   curl_easy_cleanup(handle);
   curl_global_cleanup();
 
   // Check if test timer and external loop are still alive
-  BOOST_CHECK(uv_is_active((uv_handle_t*)testTimer) != 0);
-  BOOST_CHECK(uv_loop_alive(uvLoop) != 0);
+  BOOST_CHECK_NE(uv_is_active((uv_handle_t*)testTimer), 0);
+  BOOST_CHECK_NE(uv_loop_alive(uvLoop), 0);
 
   // Downloader must be closed before uv_loop.
   // The reason for that are the uv_poll handles attached to the curl multi handle.
@@ -384,11 +368,11 @@ BOOST_AUTO_TEST_CASE(external_loop_test)
 BOOST_AUTO_TEST_CASE(trim_host_url_test)
 {
   CCDBDownloader downloader;
-  BOOST_CHECK(downloader.trimHostUrl("http://localhost:8080") == "http://localhost:8080");
-  BOOST_CHECK(downloader.trimHostUrl("http://localhost") == "http://localhost");
-  BOOST_CHECK(downloader.trimHostUrl("http://localhost:8080/some/path") == "http://localhost:8080");
-  BOOST_CHECK(downloader.trimHostUrl("http://localhost/some/path") == "http://localhost");
-  BOOST_CHECK(downloader.trimHostUrl("http://localhost:8080/Task/Detector/1?HTTPOnly=true") == "http://localhost:8080");
+  BOOST_CHECK_EQUAL(downloader.trimHostUrl("http://localhost:8080"), "http://localhost:8080");
+  BOOST_CHECK_EQUAL(downloader.trimHostUrl("http://localhost"), "http://localhost");
+  BOOST_CHECK_EQUAL(downloader.trimHostUrl("http://localhost:8080/some/path"), "http://localhost:8080");
+  BOOST_CHECK_EQUAL(downloader.trimHostUrl("http://localhost/some/path"), "http://localhost");
+  BOOST_CHECK_EQUAL(downloader.trimHostUrl("http://localhost:8080/Task/Detector/1?HTTPOnly=true"), "http://localhost:8080");
 }
 
 } // namespace ccdb

From 56aa6f555f5f0a59c2fd665e9ee0f173f452fae0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 24 Apr 2024 10:45:04 +0200
Subject: [PATCH 483/726] DPL: disable cpu_usage_fraction for proxies

---
 Framework/Core/src/CommonServices.cxx | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 028ed6e8e6806..ddaaa81090769 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -43,6 +43,7 @@
 #include "Framework/DeviceState.h"
 #include "Framework/DeviceConfig.h"
 #include "Framework/DefaultsHelpers.h"
+#include "Framework/Signpost.h"
 
 #include "TextDriverClient.h"
 #include "WSDriverClient.h"
@@ -83,6 +84,7 @@ using Value = o2::monitoring::tags::Value;
 O2_DECLARE_DYNAMIC_LOG(data_processor_context);
 O2_DECLARE_DYNAMIC_LOG(stream_context);
 O2_DECLARE_DYNAMIC_LOG(async_queue);
+O2_DECLARE_DYNAMIC_LOG(policies);
 
 namespace o2::framework
 {
@@ -849,6 +851,16 @@ o2::framework::ServiceSpec CommonServices::dataProcessingStats()
       if (deploymentMode != DeploymentMode::OnlineDDS && deploymentMode != DeploymentMode::OnlineECS && deploymentMode != DeploymentMode::OnlineAUX && deploymentMode != DeploymentMode::FST) {
         arrowAndResourceLimitingMetrics = true;
       }
+      // Input proxies should not report cpu_usage_fraction,
+      // because of the rate limiting which biases the measurement.
+      auto& spec = services.get<DeviceSpec const>();
+      bool enableCPUUsageFraction = true;
+      auto isProxy = [](DataProcessorLabel const& label) -> bool { return label == DataProcessorLabel{"input-proxy"}; };
+      if (std::find_if(spec.labels.begin(), spec.labels.end(), isProxy) != spec.labels.end()) {
+        O2_SIGNPOST_ID_GENERATE(mid, policies);
+        O2_SIGNPOST_EVENT_EMIT(policies, mid, "metrics", "Disabling cpu_usage_fraction metric for proxy %{public}s", spec.name.c_str());
+        enableCPUUsageFraction = false;
+      }
 
       std::vector<DataProcessingStats::MetricSpec> metrics = {
         MetricSpec{.name = "errors",
@@ -932,6 +944,7 @@ o2::framework::ServiceSpec CommonServices::dataProcessingStats()
                    .maxRefreshLatency = onlineRefreshLatency,
                    .sendInitialValue = true},
         MetricSpec{.name = "cpu_usage_fraction",
+                   .enabled = enableCPUUsageFraction,
                    .metricId = (int)ProcessingStatsId::CPU_USAGE_FRACTION,
                    .kind = Kind::Rate,
                    .scope = Scope::Online,

From 43774be4ce2b59569377aa9b79fc3531e7a743c2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 25 Apr 2024 14:43:01 +0200
Subject: [PATCH 484/726] DPL: force flushing of the metrics at the end of
 processing

---
 Framework/Core/src/CommonServices.cxx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index ddaaa81090769..7f0e710d96b2e 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -131,6 +131,7 @@ o2::framework::ServiceSpec CommonServices::monitoringSpec()
       } },
     .exit = [](ServiceRegistryRef registry, void* service) {
                        auto* monitoring = reinterpret_cast<Monitoring*>(service);
+                       monitoring->flushBuffer();
                        delete monitoring; },
     .kind = ServiceKind::Serial};
 }

From d92764e283662c2024290ffda0b0b807daa798ec Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 26 Apr 2024 11:58:16 +0200
Subject: [PATCH 485/726] DPL: improve messages on caught exceptions

- Provide a bit of context in the printout
- Use signposts
---
 Framework/Core/src/DataProcessingDevice.cxx | 32 +++++++++++++++++----
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index d6b73bc7476f8..1cbabfc8a2f57 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -372,6 +372,9 @@ void DataProcessingDevice::Init()
   auto ref = ServiceRegistryRef{mServiceRegistry};
   auto& context = ref.get<DataProcessorContext>();
   auto& spec = getRunningDevice(mRunningDevice, ref);
+
+  O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
+  O2_SIGNPOST_START(device, cid, "Init", "Entering Init callback.");
   context.statelessProcess = spec.algorithm.onProcess;
   context.statefulProcess = nullptr;
   context.error = spec.algorithm.onError;
@@ -412,8 +415,10 @@ void DataProcessingDevice::Init()
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
       ServiceRegistryRef ref{serviceRegistry, ServiceRegistry::globalDeviceSalt()};
+      auto& context = ref.get<DataProcessorContext>();
       auto& err = error_from_ref(e);
-      LOGP(error, "Exception caught: {} ", err.what);
+      O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
+      O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "Init", "Exception caught while in Init: %{public}s. Invoking errorCallback.", err.what);
       demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
       auto& stats = ref.get<DataProcessingStats>();
       stats.updateStats({(int)ProcessingStatsId::EXCEPTION_COUNT, DataProcessingStats::Op::Add, 1});
@@ -425,8 +430,10 @@ void DataProcessingDevice::Init()
       auto& err = error_from_ref(e);
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
-      LOGP(error, "Exception caught: {} ", err.what);
       ServiceRegistryRef ref{serviceRegistry, ServiceRegistry::globalDeviceSalt()};
+      auto& context = ref.get<DataProcessorContext>();
+      O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
+      O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "Init", "Exception caught while in Init: %{public}s. Exiting with 1.", err.what);
       demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
       auto& stats = ref.get<DataProcessingStats>();
       stats.updateStats({(int)ProcessingStatsId::EXCEPTION_COUNT, DataProcessingStats::Op::Add, 1});
@@ -491,6 +498,7 @@ void DataProcessingDevice::Init()
     ServiceRegistry::Salt streamSalt = ServiceRegistry::streamSalt(si + 1, ServiceRegistry::globalDeviceSalt().dataProcessorId);
     mServiceRegistry.lateBindStreamServices(state, *options, streamSalt);
   }
+  O2_SIGNPOST_END(device, cid, "Init", "Exiting Init callback.");
 }
 
 void on_signal_callback(uv_signal_t* handle, int signum)
@@ -1079,7 +1087,9 @@ void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceCont
       ///        can access information which were stored in the stream.
       ServiceRegistryRef ref{serviceRegistry, ServiceRegistry::globalDeviceSalt()};
       auto& err = error_from_ref(e);
-      LOGP(error, "Exception caught: {} ", err.what);
+      auto& context = ref.get<DataProcessorContext>();
+      O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
+      O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "Run", "Exception while running: %{public}s. Invoking callback.", err.what);
       demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
       auto& stats = ref.get<DataProcessingStats>();
       stats.updateStats({(int)ProcessingStatsId::EXCEPTION_COUNT, DataProcessingStats::Op::Add, 1});
@@ -1092,15 +1102,18 @@ void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceCont
       auto& err = error_from_ref(e);
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
-      LOGP(error, "Exception caught: {} ", err.what);
       ServiceRegistryRef ref{serviceRegistry, ServiceRegistry::globalDeviceSalt()};
+      auto& context = ref.get<DataProcessorContext>();
+      O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
       demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
       auto& stats = ref.get<DataProcessingStats>();
       stats.updateStats({(int)ProcessingStatsId::EXCEPTION_COUNT, DataProcessingStats::Op::Add, 1});
       switch (errorPolicy) {
         case TerminationPolicy::QUIT:
+          O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "Run", "Exception while running: %{public}s. Rethrowing.", err.what);
           throw e;
         default:
+          O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "Run", "Exception while running: %{public}s. Skipping to next timeframe.", err.what);
           break;
       }
     };
@@ -1148,6 +1161,9 @@ void DataProcessingDevice::PreRun()
 {
   auto ref = ServiceRegistryRef{mServiceRegistry};
   auto& state = ref.get<DeviceState>();
+
+  O2_SIGNPOST_ID_FROM_POINTER(cid, device, state.loop);
+  O2_SIGNPOST_START(device, cid, "PreRun", "Entering PreRun callback.");
   state.quitRequested = false;
   state.streaming = StreamingState::Streaming;
   for (auto& info : state.inputChannelInfos) {
@@ -1168,13 +1184,16 @@ void DataProcessingDevice::PreRun()
       context.preStartStreamCallbacks(streamRef);
     }
   } catch (std::exception& e) {
-    LOGP(error, "Exception caught: {} ", e.what());
+    O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "PreRun", "Exception of type std::exception caught in PreRun: %{public}s. Rethrowing.", e.what());
+    O2_SIGNPOST_END(device, cid, "PreRun", "Exiting PreRun due to exception thrown.");
     throw;
   } catch (o2::framework::RuntimeErrorRef& e) {
     auto& err = error_from_ref(e);
-    LOGP(error, "Exception caught: {} ", err.what);
+    O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "PreRun", "Exception of type o2::framework::RuntimeErrorRef caught in PreRun: %{public}s. Rethrowing.", err.what);
+    O2_SIGNPOST_END(device, cid, "PreRun", "Exiting PreRun due to exception thrown.");
     throw;
   } catch (...) {
+    O2_SIGNPOST_END(device, cid, "PreRun", "Unknown exception being thrown. Rethrowing.");
     throw;
   }
 
@@ -1189,6 +1208,7 @@ void DataProcessingDevice::PreRun()
 
   auto& monitoring = ref.get<Monitoring>();
   monitoring.send(Metric{(uint64_t)1, "device_state"}.addTag(Key::Subsystem, Value::DPL));
+  O2_SIGNPOST_END(device, cid, "PreRun", "Exiting PreRun callback.");
 }
 
 void DataProcessingDevice::PostRun()

From 149d36257f487c096c69852a98e4766d26325c6f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 29 Apr 2024 20:14:56 +0200
Subject: [PATCH 486/726] DPL: improve Signposts implementation

- make remove_engineering_type helper function consteval
- Remove engineering type when invoking the RAW function
---
 Framework/Foundation/include/Framework/Signpost.h | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index fc22147415792..664e0569354f7 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -26,9 +26,8 @@ struct o2_log_handle_t {
 
 // Helper function which replaces engineering types with a printf
 // compatible format string.
-// FIXME: make this consteval when available in C++20
 template <auto N>
-constexpr auto remove_engineering_type(char const (&src)[N])
+consteval auto remove_engineering_type(char const (&src)[N])
 {
   std::array<char, N> res = {};
   // do whatever string manipulation you want in res.
@@ -501,7 +500,7 @@ void o2_debug_log_set_stacktrace(_o2_log_t* log, int stacktrace)
   } else if (O2_BUILTIN_UNLIKELY(private_o2_log_##log->stacktrace)) {                                               \
     _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
   } else {                                                                                                          \
-    O2_LOG_MACRO_RAW(info, format, ##__VA_ARGS__);                                                                  \
+    O2_LOG_MACRO_RAW(info, remove_engineering_type(format), ##__VA_ARGS__);                                         \
   }                                                                                                                 \
 })
 
@@ -512,7 +511,7 @@ void o2_debug_log_set_stacktrace(_o2_log_t* log, int stacktrace)
   } else if (O2_BUILTIN_UNLIKELY(private_o2_log_##log->stacktrace)) {                                               \
     _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
   }                                                                                                                 \
-  O2_LOG_MACRO_RAW(error, format, ##__VA_ARGS__);                                                                   \
+  O2_LOG_MACRO_RAW(error, remove_engineering_type(format), ##__VA_ARGS__);                                          \
 })
 
 // Similar to the above, however it will also print a normal warning message regardless of the signpost being enabled or not.
@@ -522,7 +521,7 @@ void o2_debug_log_set_stacktrace(_o2_log_t* log, int stacktrace)
   } else if (O2_BUILTIN_UNLIKELY(private_o2_log_##log->stacktrace)) {                                               \
     _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
   }                                                                                                                 \
-  O2_LOG_MACRO_RAW(warn, format, ##__VA_ARGS__);                                                                    \
+  O2_LOG_MACRO_RAW(warn, remove_engineering_type(format), ##__VA_ARGS__);                                           \
 })
 
 #define O2_SIGNPOST_START(log, id, name, format, ...)                                                                   \

From 183e5c527ffeb6504a5bcc07aeeac449143dd5d7 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 29 Apr 2024 22:16:06 +0200
Subject: [PATCH 487/726] DPL: improve Signposts implementation

- make remove_engineering_type helper function consteval
- Remove engineering type when invoking the RAW function
---
 Framework/Foundation/include/Framework/Signpost.h | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index fc22147415792..1ddaaa6fb793f 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -26,9 +26,8 @@ struct o2_log_handle_t {
 
 // Helper function which replaces engineering types with a printf
 // compatible format string.
-// FIXME: make this consteval when available in C++20
 template <auto N>
-constexpr auto remove_engineering_type(char const (&src)[N])
+consteval auto remove_engineering_type(char const (&src)[N])
 {
   std::array<char, N> res = {};
   // do whatever string manipulation you want in res.
@@ -501,7 +500,7 @@ void o2_debug_log_set_stacktrace(_o2_log_t* log, int stacktrace)
   } else if (O2_BUILTIN_UNLIKELY(private_o2_log_##log->stacktrace)) {                                               \
     _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
   } else {                                                                                                          \
-    O2_LOG_MACRO_RAW(info, format, ##__VA_ARGS__);                                                                  \
+    O2_LOG_MACRO_RAW(info, remove_engineering_type(format).data(), ##__VA_ARGS__);                                  \
   }                                                                                                                 \
 })
 
@@ -512,7 +511,7 @@ void o2_debug_log_set_stacktrace(_o2_log_t* log, int stacktrace)
   } else if (O2_BUILTIN_UNLIKELY(private_o2_log_##log->stacktrace)) {                                               \
     _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
   }                                                                                                                 \
-  O2_LOG_MACRO_RAW(error, format, ##__VA_ARGS__);                                                                   \
+  O2_LOG_MACRO_RAW(error, remove_engineering_type(format).data(), ##__VA_ARGS__);                                   \
 })
 
 // Similar to the above, however it will also print a normal warning message regardless of the signpost being enabled or not.
@@ -522,7 +521,7 @@ void o2_debug_log_set_stacktrace(_o2_log_t* log, int stacktrace)
   } else if (O2_BUILTIN_UNLIKELY(private_o2_log_##log->stacktrace)) {                                               \
     _o2_signpost_event_emit(private_o2_log_##log, id, name, remove_engineering_type(format).data(), ##__VA_ARGS__); \
   }                                                                                                                 \
-  O2_LOG_MACRO_RAW(warn, format, ##__VA_ARGS__);                                                                    \
+  O2_LOG_MACRO_RAW(warn, remove_engineering_type(format).data(), ##__VA_ARGS__);                                    \
 })
 
 #define O2_SIGNPOST_START(log, id, name, format, ...)                                                                   \

From 2cd5b2e82f256982959a20a3ed9dc5fabc62aa90 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 1 May 2024 16:59:39 +0200
Subject: [PATCH 488/726] DPL: add test to verify crashing workflows

---
 Framework/Core/CMakeLists.txt                 |  6 ++
 Framework/Core/test/test_AllCrashTypes.sh     | 26 ++++++
 Framework/Core/test/test_CrashingWorkflow.cxx | 88 +++++++++++++++++++
 3 files changed, 120 insertions(+)
 create mode 100644 Framework/Core/test/test_AllCrashTypes.sh
 create mode 100644 Framework/Core/test/test_CrashingWorkflow.cxx

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index d153fe9b4184e..6667a3788074a 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -281,6 +281,11 @@ o2_add_test(O2DatabasePDG NAME test_Framework_test_O2DatabasePDG
             LABELS framework
             PUBLIC_LINK_LIBRARIES O2::Framework O2::FrameworkPhysicsSupport)
 
+o2_add_executable(crashing-workflow
+            SOURCES test/test_CrashingWorkflow.cxx
+            COMPONENT_NAME Framework
+            PUBLIC_LINK_LIBRARIES O2::Framework)
+
 # All the tests which require ROOT to work
 add_executable(o2-test-framework-root
                test/test_Root2ArrowTable.cxx
@@ -290,6 +295,7 @@ target_link_libraries(o2-test-framework-root PRIVATE O2::Catch2)
 target_link_libraries(o2-test-framework-root PRIVATE ROOT::ROOTDataFrame)
 set_property(TARGET o2-test-framework-root PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
 add_test(NAME framework:root COMMAND o2-test-framework-root --skip-benchmarks)
+add_test(NAME framework:crash COMMAND sh -e ${CMAKE_CURRENT_LIST_DIR}/test/test_AllCrashTypes.sh)
 
 o2_add_test(InfoLogger NAME test_Framework_test_InfoLogger
             SOURCES test/test_InfoLogger.cxx
diff --git a/Framework/Core/test/test_AllCrashTypes.sh b/Framework/Core/test/test_AllCrashTypes.sh
new file mode 100644
index 0000000000000..3724d5e25e937
--- /dev/null
+++ b/Framework/Core/test/test_AllCrashTypes.sh
@@ -0,0 +1,26 @@
+#!/bin/sh -e
+printf "Testing abort-init..."
+o2-framework-test-crashing-workflow --crash-type=abort-init --completion-policy=quit -b --run | grep -q "Abort trap" || { echo "abort not found in init"; exit 1; }
+printf "ok\nTesting runtime-init..."
+o2-framework-test-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a std::runtime_error" || { printf "runtime error not found" ; exit 1; }
+printf "ok\nTesting framework-init..."
+o2-framework-test-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a o2::framework::runtime_error" || { printf "framework error not found" ;  exit 1; }
+printf "ok\nTesting abort-run..."
+o2-framework-test-crashing-workflow --crash-type=abort-run --completion-policy=quit -b --run | grep -q "Program crashed (Abort trap: 6)" || { printf "abort not found" ; exit 1; }
+printf "ok\nTesting framework-run..."
+o2-framework-test-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-test-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
+printf "ok\nTesting runtime-run..."
+o2-framework-test-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-test-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found" ; exit 1; }
+printf "ok\n"
+
+export O2_NO_CATCHALL_EXCEPTIONS=1
+echo O2_NO_CATCHALL_EXCEPTIONS enabled
+printf "ok\nTesting runtime-init..."
+o2-framework-test-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -v -q "Exception caught while in Init: This is a std::runtime_error. Exiting with 1." || { printf "runtime error not found" ; exit 1; }
+printf "ok\nTesting framework-init..."
+o2-framework-test-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -v -q "Exception caught while in Init: This is a o2::framework::runtime_error. Exiting with 1" || { printf "framework error not found" ;  exit 1; }
+printf "ok\nTesting framework-run..."
+o2-framework-test-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -v -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-test-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
+printf "ok\nTesting runtime-run..."
+o2-framework-test-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -v -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-test-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found" ; exit 1; }
+printf "ok"
diff --git a/Framework/Core/test/test_CrashingWorkflow.cxx b/Framework/Core/test/test_CrashingWorkflow.cxx
new file mode 100644
index 0000000000000..396b48a85c166
--- /dev/null
+++ b/Framework/Core/test/test_CrashingWorkflow.cxx
@@ -0,0 +1,88 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/AlgorithmSpec.h"
+#include "Framework/Configurable.h"
+#include "Framework/Logger.h"
+#include "Framework/CallbackService.h"
+#include "Framework/Signpost.h"
+
+O2_DECLARE_DYNAMIC_LOG(crash_test);
+
+using namespace o2::framework;
+
+struct WorkflowOptions {
+  Configurable<std::string> crashType{"crash-type", "fatal-init", {"how should this crash? (fatal-init, fatal-run, runtime-init, runtime-fail, abort-init, abort-run)"}};
+};
+
+#include "Framework/runDataProcessing.h"
+
+AlgorithmSpec simpleCrashingSource(std::string const& what)
+{
+  return AlgorithmSpec{adaptStateful([what](InitContext& ctx) {
+    O2_SIGNPOST_ID_FROM_POINTER(ii, crash_test, &ctx);
+    O2_SIGNPOST_START(crash_test, ii, "Init", "Starting Init");
+    O2_SIGNPOST_EVENT_EMIT(crash_test, ii, "Init", "%{public}s selected", what.c_str());
+
+    if (what == "fatal-init") {
+      LOG(fatal) << "This should have a fatal";
+    } else if (what == "runtime-init") {
+      throw std::runtime_error("This is a std::runtime_error");
+    } else if (what == "abort-init") {
+      abort();
+    } else if (what == "framework-init") {
+      throw o2::framework::runtime_error("This is a o2::framework::runtime_error");
+    } else if (what == "framework-prerun") {
+      ctx.services().get<CallbackService>().set<CallbackService::Id::PreProcessing>([](ServiceRegistryRef, int) {
+        throw o2::framework::runtime_error("This is o2::framework::runtime_error in PreProcessing");
+      });
+    } else if (what == "runtime-prerun") {
+      ctx.services().get<CallbackService>().set<CallbackService::Id::PreProcessing>([](ServiceRegistryRef, int) {
+        throw std::runtime_error("This is std::runtime_error in PreProcessing");
+      });
+    }
+    O2_SIGNPOST_END(crash_test, ii, "Init", "Init Done");
+    return adaptStateless([what](ProcessingContext& pCtx) {
+      O2_SIGNPOST_ID_FROM_POINTER(ri, crash_test, &pCtx);
+      O2_SIGNPOST_START(crash_test, ri, "Run", "Starting Run");
+      O2_SIGNPOST_EVENT_EMIT(crash_test, ri, "Run", "%{public}s selected", what.c_str());
+      if (what == "fatal-run") {
+        LOG(fatal) << "This should have a fatal";
+      } else if (what == "runtime-run") {
+        throw std::runtime_error("This is a std::runtime_error");
+      } else if (what == "abort-run") {
+        abort();
+      } else if (what == "framework-run") {
+        throw o2::framework::runtime_error("This is a o2::framework::runtime_error");
+      }
+      O2_SIGNPOST_EVENT_EMIT_ERROR(crash_test, ri, "Run", "Unknown option for crash-type: %{public}s.", what.c_str());
+      O2_SIGNPOST_END(crash_test, ri, "Init", "Run Done");
+      exit(1);
+    });
+  })};
+}
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& config)
+{
+  auto crashType = config.options().get<std::string>("crash-type");
+  DataProcessorSpec a{
+    .name = "deliberately-crashing",
+    .outputs = {OutputSpec{{"a1"}, "TST", "A1"}},
+    .algorithm = AlgorithmSpec{simpleCrashingSource(crashType)}};
+  DataProcessorSpec b{
+    .name = "B",
+    .inputs = {InputSpec{"x", "TST", "A1", Lifetime::Timeframe}},
+    .algorithm = AlgorithmSpec{adaptStateless([](ProcessingContext&) {})}};
+
+  return workflow::concat(WorkflowSpec{a, b});
+}
+

From fefb5ca5a2c0815cba4e9580307a2764fcb6e89a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 1 May 2024 21:17:36 +0200
Subject: [PATCH 489/726] DPL: add test to verify crashing workflows

---
 Framework/Core/CMakeLists.txt                 |  6 ++
 Framework/Core/test/test_AllCrashTypes.sh     | 26 ++++++
 Framework/Core/test/test_CrashingWorkflow.cxx | 88 +++++++++++++++++++
 3 files changed, 120 insertions(+)
 create mode 100644 Framework/Core/test/test_AllCrashTypes.sh
 create mode 100644 Framework/Core/test/test_CrashingWorkflow.cxx

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index d153fe9b4184e..6667a3788074a 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -281,6 +281,11 @@ o2_add_test(O2DatabasePDG NAME test_Framework_test_O2DatabasePDG
             LABELS framework
             PUBLIC_LINK_LIBRARIES O2::Framework O2::FrameworkPhysicsSupport)
 
+o2_add_executable(crashing-workflow
+            SOURCES test/test_CrashingWorkflow.cxx
+            COMPONENT_NAME Framework
+            PUBLIC_LINK_LIBRARIES O2::Framework)
+
 # All the tests which require ROOT to work
 add_executable(o2-test-framework-root
                test/test_Root2ArrowTable.cxx
@@ -290,6 +295,7 @@ target_link_libraries(o2-test-framework-root PRIVATE O2::Catch2)
 target_link_libraries(o2-test-framework-root PRIVATE ROOT::ROOTDataFrame)
 set_property(TARGET o2-test-framework-root PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
 add_test(NAME framework:root COMMAND o2-test-framework-root --skip-benchmarks)
+add_test(NAME framework:crash COMMAND sh -e ${CMAKE_CURRENT_LIST_DIR}/test/test_AllCrashTypes.sh)
 
 o2_add_test(InfoLogger NAME test_Framework_test_InfoLogger
             SOURCES test/test_InfoLogger.cxx
diff --git a/Framework/Core/test/test_AllCrashTypes.sh b/Framework/Core/test/test_AllCrashTypes.sh
new file mode 100644
index 0000000000000..48933e42c9083
--- /dev/null
+++ b/Framework/Core/test/test_AllCrashTypes.sh
@@ -0,0 +1,26 @@
+#!/bin/sh -e
+printf "Testing abort-init..."
+o2-framework-crashing-workflow --crash-type=abort-init --completion-policy=quit -b --run | grep -q "Abort trap" || { echo "abort not found in init"; exit 1; }
+printf "ok\nTesting runtime-init..."
+o2-framework-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a std::runtime_error" || { printf "runtime error not found" ; exit 1; }
+printf "ok\nTesting framework-init..."
+o2-framework-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a o2::framework::runtime_error" || { printf "framework error not found" ;  exit 1; }
+printf "ok\nTesting abort-run..."
+o2-framework-crashing-workflow --crash-type=abort-run --completion-policy=quit -b --run | grep -q "Program crashed (Abort trap: 6)" || { printf "abort not found" ; exit 1; }
+printf "ok\nTesting framework-run..."
+o2-framework-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
+printf "ok\nTesting runtime-run..."
+o2-framework-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found" ; exit 1; }
+printf "ok\n"
+
+export O2_NO_CATCHALL_EXCEPTIONS=1
+echo O2_NO_CATCHALL_EXCEPTIONS enabled
+printf "ok\nTesting runtime-init..."
+o2-framework-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -v -q "Exception caught while in Init: This is a std::runtime_error. Exiting with 1." || { printf "runtime error not found" ; exit 1; }
+printf "ok\nTesting framework-init..."
+o2-framework-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -v -q "Exception caught while in Init: This is a o2::framework::runtime_error. Exiting with 1" || { printf "framework error not found" ;  exit 1; }
+printf "ok\nTesting framework-run..."
+o2-framework-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -v -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
+printf "ok\nTesting runtime-run..."
+o2-framework-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -v -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found" ; exit 1; }
+printf "ok"
diff --git a/Framework/Core/test/test_CrashingWorkflow.cxx b/Framework/Core/test/test_CrashingWorkflow.cxx
new file mode 100644
index 0000000000000..396b48a85c166
--- /dev/null
+++ b/Framework/Core/test/test_CrashingWorkflow.cxx
@@ -0,0 +1,88 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/AlgorithmSpec.h"
+#include "Framework/Configurable.h"
+#include "Framework/Logger.h"
+#include "Framework/CallbackService.h"
+#include "Framework/Signpost.h"
+
+O2_DECLARE_DYNAMIC_LOG(crash_test);
+
+using namespace o2::framework;
+
+struct WorkflowOptions {
+  Configurable<std::string> crashType{"crash-type", "fatal-init", {"how should this crash? (fatal-init, fatal-run, runtime-init, runtime-fail, abort-init, abort-run)"}};
+};
+
+#include "Framework/runDataProcessing.h"
+
+AlgorithmSpec simpleCrashingSource(std::string const& what)
+{
+  return AlgorithmSpec{adaptStateful([what](InitContext& ctx) {
+    O2_SIGNPOST_ID_FROM_POINTER(ii, crash_test, &ctx);
+    O2_SIGNPOST_START(crash_test, ii, "Init", "Starting Init");
+    O2_SIGNPOST_EVENT_EMIT(crash_test, ii, "Init", "%{public}s selected", what.c_str());
+
+    if (what == "fatal-init") {
+      LOG(fatal) << "This should have a fatal";
+    } else if (what == "runtime-init") {
+      throw std::runtime_error("This is a std::runtime_error");
+    } else if (what == "abort-init") {
+      abort();
+    } else if (what == "framework-init") {
+      throw o2::framework::runtime_error("This is a o2::framework::runtime_error");
+    } else if (what == "framework-prerun") {
+      ctx.services().get<CallbackService>().set<CallbackService::Id::PreProcessing>([](ServiceRegistryRef, int) {
+        throw o2::framework::runtime_error("This is o2::framework::runtime_error in PreProcessing");
+      });
+    } else if (what == "runtime-prerun") {
+      ctx.services().get<CallbackService>().set<CallbackService::Id::PreProcessing>([](ServiceRegistryRef, int) {
+        throw std::runtime_error("This is std::runtime_error in PreProcessing");
+      });
+    }
+    O2_SIGNPOST_END(crash_test, ii, "Init", "Init Done");
+    return adaptStateless([what](ProcessingContext& pCtx) {
+      O2_SIGNPOST_ID_FROM_POINTER(ri, crash_test, &pCtx);
+      O2_SIGNPOST_START(crash_test, ri, "Run", "Starting Run");
+      O2_SIGNPOST_EVENT_EMIT(crash_test, ri, "Run", "%{public}s selected", what.c_str());
+      if (what == "fatal-run") {
+        LOG(fatal) << "This should have a fatal";
+      } else if (what == "runtime-run") {
+        throw std::runtime_error("This is a std::runtime_error");
+      } else if (what == "abort-run") {
+        abort();
+      } else if (what == "framework-run") {
+        throw o2::framework::runtime_error("This is a o2::framework::runtime_error");
+      }
+      O2_SIGNPOST_EVENT_EMIT_ERROR(crash_test, ri, "Run", "Unknown option for crash-type: %{public}s.", what.c_str());
+      O2_SIGNPOST_END(crash_test, ri, "Init", "Run Done");
+      exit(1);
+    });
+  })};
+}
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& config)
+{
+  auto crashType = config.options().get<std::string>("crash-type");
+  DataProcessorSpec a{
+    .name = "deliberately-crashing",
+    .outputs = {OutputSpec{{"a1"}, "TST", "A1"}},
+    .algorithm = AlgorithmSpec{simpleCrashingSource(crashType)}};
+  DataProcessorSpec b{
+    .name = "B",
+    .inputs = {InputSpec{"x", "TST", "A1", Lifetime::Timeframe}},
+    .algorithm = AlgorithmSpec{adaptStateless([](ProcessingContext&) {})}};
+
+  return workflow::concat(WorkflowSpec{a, b});
+}
+

From 0c567b40907a2b2de9d18d1d46a01e735edbcc00 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 1 May 2024 23:51:44 +0200
Subject: [PATCH 490/726] DPL: add test to verify crashing workflows

---
 Framework/Core/CMakeLists.txt                 |  6 ++
 Framework/Core/test/test_AllCrashTypes.sh     | 27 ++++++
 Framework/Core/test/test_CrashingWorkflow.cxx | 88 +++++++++++++++++++
 3 files changed, 121 insertions(+)
 create mode 100644 Framework/Core/test/test_AllCrashTypes.sh
 create mode 100644 Framework/Core/test/test_CrashingWorkflow.cxx

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index d153fe9b4184e..fabb62eb6530d 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -281,6 +281,11 @@ o2_add_test(O2DatabasePDG NAME test_Framework_test_O2DatabasePDG
             LABELS framework
             PUBLIC_LINK_LIBRARIES O2::Framework O2::FrameworkPhysicsSupport)
 
+o2_add_executable(crashing-workflow
+            SOURCES test/test_CrashingWorkflow.cxx
+            COMPONENT_NAME Framework
+            PUBLIC_LINK_LIBRARIES O2::Framework)
+
 # All the tests which require ROOT to work
 add_executable(o2-test-framework-root
                test/test_Root2ArrowTable.cxx
@@ -290,6 +295,7 @@ target_link_libraries(o2-test-framework-root PRIVATE O2::Catch2)
 target_link_libraries(o2-test-framework-root PRIVATE ROOT::ROOTDataFrame)
 set_property(TARGET o2-test-framework-root PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
 add_test(NAME framework:root COMMAND o2-test-framework-root --skip-benchmarks)
+add_test(NAME framework:crash COMMAND  sh -e -c "PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:$PATH ${CMAKE_CURRENT_LIST_DIR}/test/test_AllCrashTypes.sh")
 
 o2_add_test(InfoLogger NAME test_Framework_test_InfoLogger
             SOURCES test/test_InfoLogger.cxx
diff --git a/Framework/Core/test/test_AllCrashTypes.sh b/Framework/Core/test/test_AllCrashTypes.sh
new file mode 100644
index 0000000000000..28ea3567e1f8c
--- /dev/null
+++ b/Framework/Core/test/test_AllCrashTypes.sh
@@ -0,0 +1,27 @@
+#!/bin/sh -e
+echo $PATH
+printf "Testing abort-init..."
+o2-framework-crashing-workflow --crash-type=abort-init --completion-policy=quit -b --run | grep -q "Abort trap" || { echo "abort not found in init"; exit 1; }
+printf "ok\nTesting runtime-init..."
+o2-framework-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a std::runtime_error" || { printf "runtime error not found" ; exit 1; }
+printf "ok\nTesting framework-init..."
+o2-framework-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a o2::framework::runtime_error" || { printf "framework error not found" ;  exit 1; }
+printf "ok\nTesting abort-run..."
+o2-framework-crashing-workflow --crash-type=abort-run --completion-policy=quit -b --run | grep -q "Program crashed (Abort trap: 6)" || { printf "abort not found" ; exit 1; }
+printf "ok\nTesting framework-run..."
+o2-framework-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
+printf "ok\nTesting runtime-run..."
+o2-framework-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found" ; exit 1; }
+printf "ok\n"
+
+export O2_NO_CATCHALL_EXCEPTIONS=1
+echo O2_NO_CATCHALL_EXCEPTIONS enabled
+printf "ok\nTesting runtime-init..."
+o2-framework-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -v -q "Exception caught while in Init: This is a std::runtime_error. Exiting with 1." || { printf "runtime error not found" ; exit 1; }
+printf "ok\nTesting framework-init..."
+o2-framework-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -v -q "Exception caught while in Init: This is a o2::framework::runtime_error. Exiting with 1" || { printf "framework error not found" ;  exit 1; }
+printf "ok\nTesting framework-run..."
+o2-framework-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -v -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
+printf "ok\nTesting runtime-run..."
+o2-framework-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -v -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found" ; exit 1; }
+printf "ok"
diff --git a/Framework/Core/test/test_CrashingWorkflow.cxx b/Framework/Core/test/test_CrashingWorkflow.cxx
new file mode 100644
index 0000000000000..396b48a85c166
--- /dev/null
+++ b/Framework/Core/test/test_CrashingWorkflow.cxx
@@ -0,0 +1,88 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/AlgorithmSpec.h"
+#include "Framework/Configurable.h"
+#include "Framework/Logger.h"
+#include "Framework/CallbackService.h"
+#include "Framework/Signpost.h"
+
+O2_DECLARE_DYNAMIC_LOG(crash_test);
+
+using namespace o2::framework;
+
+struct WorkflowOptions {
+  Configurable<std::string> crashType{"crash-type", "fatal-init", {"how should this crash? (fatal-init, fatal-run, runtime-init, runtime-fail, abort-init, abort-run)"}};
+};
+
+#include "Framework/runDataProcessing.h"
+
+AlgorithmSpec simpleCrashingSource(std::string const& what)
+{
+  return AlgorithmSpec{adaptStateful([what](InitContext& ctx) {
+    O2_SIGNPOST_ID_FROM_POINTER(ii, crash_test, &ctx);
+    O2_SIGNPOST_START(crash_test, ii, "Init", "Starting Init");
+    O2_SIGNPOST_EVENT_EMIT(crash_test, ii, "Init", "%{public}s selected", what.c_str());
+
+    if (what == "fatal-init") {
+      LOG(fatal) << "This should have a fatal";
+    } else if (what == "runtime-init") {
+      throw std::runtime_error("This is a std::runtime_error");
+    } else if (what == "abort-init") {
+      abort();
+    } else if (what == "framework-init") {
+      throw o2::framework::runtime_error("This is a o2::framework::runtime_error");
+    } else if (what == "framework-prerun") {
+      ctx.services().get<CallbackService>().set<CallbackService::Id::PreProcessing>([](ServiceRegistryRef, int) {
+        throw o2::framework::runtime_error("This is o2::framework::runtime_error in PreProcessing");
+      });
+    } else if (what == "runtime-prerun") {
+      ctx.services().get<CallbackService>().set<CallbackService::Id::PreProcessing>([](ServiceRegistryRef, int) {
+        throw std::runtime_error("This is std::runtime_error in PreProcessing");
+      });
+    }
+    O2_SIGNPOST_END(crash_test, ii, "Init", "Init Done");
+    return adaptStateless([what](ProcessingContext& pCtx) {
+      O2_SIGNPOST_ID_FROM_POINTER(ri, crash_test, &pCtx);
+      O2_SIGNPOST_START(crash_test, ri, "Run", "Starting Run");
+      O2_SIGNPOST_EVENT_EMIT(crash_test, ri, "Run", "%{public}s selected", what.c_str());
+      if (what == "fatal-run") {
+        LOG(fatal) << "This should have a fatal";
+      } else if (what == "runtime-run") {
+        throw std::runtime_error("This is a std::runtime_error");
+      } else if (what == "abort-run") {
+        abort();
+      } else if (what == "framework-run") {
+        throw o2::framework::runtime_error("This is a o2::framework::runtime_error");
+      }
+      O2_SIGNPOST_EVENT_EMIT_ERROR(crash_test, ri, "Run", "Unknown option for crash-type: %{public}s.", what.c_str());
+      O2_SIGNPOST_END(crash_test, ri, "Init", "Run Done");
+      exit(1);
+    });
+  })};
+}
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& config)
+{
+  auto crashType = config.options().get<std::string>("crash-type");
+  DataProcessorSpec a{
+    .name = "deliberately-crashing",
+    .outputs = {OutputSpec{{"a1"}, "TST", "A1"}},
+    .algorithm = AlgorithmSpec{simpleCrashingSource(crashType)}};
+  DataProcessorSpec b{
+    .name = "B",
+    .inputs = {InputSpec{"x", "TST", "A1", Lifetime::Timeframe}},
+    .algorithm = AlgorithmSpec{adaptStateless([](ProcessingContext&) {})}};
+
+  return workflow::concat(WorkflowSpec{a, b});
+}
+

From de302d5ec579e26b2e5c5bff62b000e56732c978 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 2 May 2024 09:17:56 +0200
Subject: [PATCH 491/726] DPL: improve instrumentation for the
 preProcessingCallbacks

---
 Framework/Core/src/DataProcessingDevice.cxx | 1 +
 Framework/Core/src/StreamContext.cxx        | 7 ++++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index d6b73bc7476f8..bb3858a9d3ed2 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -2321,6 +2321,7 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
                        *context.registry};
     ProcessingContext processContext{record, ref, ref.get<DataAllocator>()};
     {
+      O2_SIGNPOST_EVENT_EMIT(device, aid, "device", "Invoking preProcessingCallbacks");
       // Notice this should be thread safe and reentrant
       // as it is called from many threads.
       streamContext.preProcessingCallbacks(processContext);
diff --git a/Framework/Core/src/StreamContext.cxx b/Framework/Core/src/StreamContext.cxx
index 2ce0c1c427cbd..0712025849447 100644
--- a/Framework/Core/src/StreamContext.cxx
+++ b/Framework/Core/src/StreamContext.cxx
@@ -31,12 +31,17 @@ void StreamContext::preStartStreamCallbacks(ServiceRegistryRef ref)
 /// Invoke callbacks to be executed before every process method invokation
 void StreamContext::preProcessingCallbacks(ProcessingContext& pcx)
 {
+  O2_SIGNPOST_ID_FROM_POINTER(dpid, stream_context, &pcx);
+  O2_SIGNPOST_START(stream_context, dpid, "callbacks", "Starting StreamContext preProcessingCallbacks");
   for (auto& handle : preProcessingHandles) {
-    LOG(debug) << "Invoking preProcessingCallbacks for" << handle.service;
+    O2_SIGNPOST_ID_FROM_POINTER(cid, stream_context, handle.service);
+    O2_SIGNPOST_START(stream_context, cid, "callbacks", "Starting StreamContext::preProcessingCallbacks for service %{public}s", handle.spec.name.c_str());
     assert(handle.service);
     assert(handle.callback);
     handle.callback(pcx, handle.service);
+    O2_SIGNPOST_END(stream_context, cid, "callbacks", "Ending StreamContext::preProcessingCallbacks for service %{public}s", handle.spec.name.c_str());
   }
+  O2_SIGNPOST_END(stream_context, dpid, "callbacks", "Ending StreamContext preProcessingCallbacks");
 }
 
 /// Invoke callbacks to be executed after every process method invokation

From 918946b6610e2f9009c1e27a62931b3494a5ed97 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 1 May 2024 23:51:44 +0200
Subject: [PATCH 492/726] DPL: add test to verify crashing workflows

---
 Framework/Core/CMakeLists.txt                 |  6 ++
 Framework/Core/test/test_AllCrashTypes.sh     | 27 ++++++
 Framework/Core/test/test_CrashingWorkflow.cxx | 88 +++++++++++++++++++
 3 files changed, 121 insertions(+)
 create mode 100755 Framework/Core/test/test_AllCrashTypes.sh
 create mode 100644 Framework/Core/test/test_CrashingWorkflow.cxx

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index d153fe9b4184e..fabb62eb6530d 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -281,6 +281,11 @@ o2_add_test(O2DatabasePDG NAME test_Framework_test_O2DatabasePDG
             LABELS framework
             PUBLIC_LINK_LIBRARIES O2::Framework O2::FrameworkPhysicsSupport)
 
+o2_add_executable(crashing-workflow
+            SOURCES test/test_CrashingWorkflow.cxx
+            COMPONENT_NAME Framework
+            PUBLIC_LINK_LIBRARIES O2::Framework)
+
 # All the tests which require ROOT to work
 add_executable(o2-test-framework-root
                test/test_Root2ArrowTable.cxx
@@ -290,6 +295,7 @@ target_link_libraries(o2-test-framework-root PRIVATE O2::Catch2)
 target_link_libraries(o2-test-framework-root PRIVATE ROOT::ROOTDataFrame)
 set_property(TARGET o2-test-framework-root PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
 add_test(NAME framework:root COMMAND o2-test-framework-root --skip-benchmarks)
+add_test(NAME framework:crash COMMAND  sh -e -c "PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:$PATH ${CMAKE_CURRENT_LIST_DIR}/test/test_AllCrashTypes.sh")
 
 o2_add_test(InfoLogger NAME test_Framework_test_InfoLogger
             SOURCES test/test_InfoLogger.cxx
diff --git a/Framework/Core/test/test_AllCrashTypes.sh b/Framework/Core/test/test_AllCrashTypes.sh
new file mode 100755
index 0000000000000..28ea3567e1f8c
--- /dev/null
+++ b/Framework/Core/test/test_AllCrashTypes.sh
@@ -0,0 +1,27 @@
+#!/bin/sh -e
+echo $PATH
+printf "Testing abort-init..."
+o2-framework-crashing-workflow --crash-type=abort-init --completion-policy=quit -b --run | grep -q "Abort trap" || { echo "abort not found in init"; exit 1; }
+printf "ok\nTesting runtime-init..."
+o2-framework-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a std::runtime_error" || { printf "runtime error not found" ; exit 1; }
+printf "ok\nTesting framework-init..."
+o2-framework-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a o2::framework::runtime_error" || { printf "framework error not found" ;  exit 1; }
+printf "ok\nTesting abort-run..."
+o2-framework-crashing-workflow --crash-type=abort-run --completion-policy=quit -b --run | grep -q "Program crashed (Abort trap: 6)" || { printf "abort not found" ; exit 1; }
+printf "ok\nTesting framework-run..."
+o2-framework-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
+printf "ok\nTesting runtime-run..."
+o2-framework-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found" ; exit 1; }
+printf "ok\n"
+
+export O2_NO_CATCHALL_EXCEPTIONS=1
+echo O2_NO_CATCHALL_EXCEPTIONS enabled
+printf "ok\nTesting runtime-init..."
+o2-framework-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -v -q "Exception caught while in Init: This is a std::runtime_error. Exiting with 1." || { printf "runtime error not found" ; exit 1; }
+printf "ok\nTesting framework-init..."
+o2-framework-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -v -q "Exception caught while in Init: This is a o2::framework::runtime_error. Exiting with 1" || { printf "framework error not found" ;  exit 1; }
+printf "ok\nTesting framework-run..."
+o2-framework-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -v -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
+printf "ok\nTesting runtime-run..."
+o2-framework-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -v -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found" ; exit 1; }
+printf "ok"
diff --git a/Framework/Core/test/test_CrashingWorkflow.cxx b/Framework/Core/test/test_CrashingWorkflow.cxx
new file mode 100644
index 0000000000000..396b48a85c166
--- /dev/null
+++ b/Framework/Core/test/test_CrashingWorkflow.cxx
@@ -0,0 +1,88 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/AlgorithmSpec.h"
+#include "Framework/Configurable.h"
+#include "Framework/Logger.h"
+#include "Framework/CallbackService.h"
+#include "Framework/Signpost.h"
+
+O2_DECLARE_DYNAMIC_LOG(crash_test);
+
+using namespace o2::framework;
+
+struct WorkflowOptions {
+  Configurable<std::string> crashType{"crash-type", "fatal-init", {"how should this crash? (fatal-init, fatal-run, runtime-init, runtime-fail, abort-init, abort-run)"}};
+};
+
+#include "Framework/runDataProcessing.h"
+
+AlgorithmSpec simpleCrashingSource(std::string const& what)
+{
+  return AlgorithmSpec{adaptStateful([what](InitContext& ctx) {
+    O2_SIGNPOST_ID_FROM_POINTER(ii, crash_test, &ctx);
+    O2_SIGNPOST_START(crash_test, ii, "Init", "Starting Init");
+    O2_SIGNPOST_EVENT_EMIT(crash_test, ii, "Init", "%{public}s selected", what.c_str());
+
+    if (what == "fatal-init") {
+      LOG(fatal) << "This should have a fatal";
+    } else if (what == "runtime-init") {
+      throw std::runtime_error("This is a std::runtime_error");
+    } else if (what == "abort-init") {
+      abort();
+    } else if (what == "framework-init") {
+      throw o2::framework::runtime_error("This is a o2::framework::runtime_error");
+    } else if (what == "framework-prerun") {
+      ctx.services().get<CallbackService>().set<CallbackService::Id::PreProcessing>([](ServiceRegistryRef, int) {
+        throw o2::framework::runtime_error("This is o2::framework::runtime_error in PreProcessing");
+      });
+    } else if (what == "runtime-prerun") {
+      ctx.services().get<CallbackService>().set<CallbackService::Id::PreProcessing>([](ServiceRegistryRef, int) {
+        throw std::runtime_error("This is std::runtime_error in PreProcessing");
+      });
+    }
+    O2_SIGNPOST_END(crash_test, ii, "Init", "Init Done");
+    return adaptStateless([what](ProcessingContext& pCtx) {
+      O2_SIGNPOST_ID_FROM_POINTER(ri, crash_test, &pCtx);
+      O2_SIGNPOST_START(crash_test, ri, "Run", "Starting Run");
+      O2_SIGNPOST_EVENT_EMIT(crash_test, ri, "Run", "%{public}s selected", what.c_str());
+      if (what == "fatal-run") {
+        LOG(fatal) << "This should have a fatal";
+      } else if (what == "runtime-run") {
+        throw std::runtime_error("This is a std::runtime_error");
+      } else if (what == "abort-run") {
+        abort();
+      } else if (what == "framework-run") {
+        throw o2::framework::runtime_error("This is a o2::framework::runtime_error");
+      }
+      O2_SIGNPOST_EVENT_EMIT_ERROR(crash_test, ri, "Run", "Unknown option for crash-type: %{public}s.", what.c_str());
+      O2_SIGNPOST_END(crash_test, ri, "Init", "Run Done");
+      exit(1);
+    });
+  })};
+}
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& config)
+{
+  auto crashType = config.options().get<std::string>("crash-type");
+  DataProcessorSpec a{
+    .name = "deliberately-crashing",
+    .outputs = {OutputSpec{{"a1"}, "TST", "A1"}},
+    .algorithm = AlgorithmSpec{simpleCrashingSource(crashType)}};
+  DataProcessorSpec b{
+    .name = "B",
+    .inputs = {InputSpec{"x", "TST", "A1", Lifetime::Timeframe}},
+    .algorithm = AlgorithmSpec{adaptStateless([](ProcessingContext&) {})}};
+
+  return workflow::concat(WorkflowSpec{a, b});
+}
+

From c424436af59c9409deb02f25ccd23fa95c3b93a2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 2 May 2024 10:48:42 +0200
Subject: [PATCH 493/726] DPL: add test to verify crashing workflows

---
 Framework/Core/CMakeLists.txt                 |  6 ++
 Framework/Core/test/test_AllCrashTypes.sh     | 27 ++++++
 Framework/Core/test/test_CrashingWorkflow.cxx | 88 +++++++++++++++++++
 3 files changed, 121 insertions(+)
 create mode 100755 Framework/Core/test/test_AllCrashTypes.sh
 create mode 100644 Framework/Core/test/test_CrashingWorkflow.cxx

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index d153fe9b4184e..fabb62eb6530d 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -281,6 +281,11 @@ o2_add_test(O2DatabasePDG NAME test_Framework_test_O2DatabasePDG
             LABELS framework
             PUBLIC_LINK_LIBRARIES O2::Framework O2::FrameworkPhysicsSupport)
 
+o2_add_executable(crashing-workflow
+            SOURCES test/test_CrashingWorkflow.cxx
+            COMPONENT_NAME Framework
+            PUBLIC_LINK_LIBRARIES O2::Framework)
+
 # All the tests which require ROOT to work
 add_executable(o2-test-framework-root
                test/test_Root2ArrowTable.cxx
@@ -290,6 +295,7 @@ target_link_libraries(o2-test-framework-root PRIVATE O2::Catch2)
 target_link_libraries(o2-test-framework-root PRIVATE ROOT::ROOTDataFrame)
 set_property(TARGET o2-test-framework-root PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
 add_test(NAME framework:root COMMAND o2-test-framework-root --skip-benchmarks)
+add_test(NAME framework:crash COMMAND  sh -e -c "PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:$PATH ${CMAKE_CURRENT_LIST_DIR}/test/test_AllCrashTypes.sh")
 
 o2_add_test(InfoLogger NAME test_Framework_test_InfoLogger
             SOURCES test/test_InfoLogger.cxx
diff --git a/Framework/Core/test/test_AllCrashTypes.sh b/Framework/Core/test/test_AllCrashTypes.sh
new file mode 100755
index 0000000000000..a748eda773279
--- /dev/null
+++ b/Framework/Core/test/test_AllCrashTypes.sh
@@ -0,0 +1,27 @@
+#!/bin/sh -e
+echo $PATH
+printf "Testing abort-init..."
+o2-framework-crashing-workflow --crash-type=abort-init --completion-policy=quit -b --run | grep -q "Abort trap" || { echo "abort not found in init"; exit 1; }
+printf "ok\nTesting runtime-init..."
+o2-framework-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a std::runtime_error" || { printf "runtime error not found" ; exit 1; }
+printf "ok\nTesting framework-init..."
+o2-framework-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a o2::framework::runtime_error" || { printf "framework error not found" ;  exit 1; }
+printf "ok\nTesting abort-run..."
+o2-framework-crashing-workflow --crash-type=abort-run --completion-policy=quit -b --run | grep -q "Program crashed (Abort trap: 6)" || { printf "abort not found" ; exit 1; }
+printf "ok\nTesting framework-run..."
+o2-framework-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
+printf "ok\nTesting runtime-run..."
+o2-framework-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found" ; exit 1; }
+printf "ok\n"
+
+export O2_NO_CATCHALL_EXCEPTIONS=1
+echo O2_NO_CATCHALL_EXCEPTIONS enabled
+printf "ok\nTesting runtime-init..."
+o2-framework-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -v -q "Exception caught: This is a std::runtime_error" || { printf "runtime error not found" ; exit 1; }
+printf "ok\nTesting framework-init..."
+o2-framework-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -v -q "Exception caught: This is a o2::framework::runtime_error" || { printf "framework error not found" ;  exit 1; }
+printf "ok\nTesting framework-run..."
+o2-framework-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -v -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
+printf "ok\nTesting runtime-run..."
+o2-framework-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -v -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found" ; exit 1; }
+printf "ok"
diff --git a/Framework/Core/test/test_CrashingWorkflow.cxx b/Framework/Core/test/test_CrashingWorkflow.cxx
new file mode 100644
index 0000000000000..396b48a85c166
--- /dev/null
+++ b/Framework/Core/test/test_CrashingWorkflow.cxx
@@ -0,0 +1,88 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/AlgorithmSpec.h"
+#include "Framework/Configurable.h"
+#include "Framework/Logger.h"
+#include "Framework/CallbackService.h"
+#include "Framework/Signpost.h"
+
+O2_DECLARE_DYNAMIC_LOG(crash_test);
+
+using namespace o2::framework;
+
+struct WorkflowOptions {
+  Configurable<std::string> crashType{"crash-type", "fatal-init", {"how should this crash? (fatal-init, fatal-run, runtime-init, runtime-fail, abort-init, abort-run)"}};
+};
+
+#include "Framework/runDataProcessing.h"
+
+AlgorithmSpec simpleCrashingSource(std::string const& what)
+{
+  return AlgorithmSpec{adaptStateful([what](InitContext& ctx) {
+    O2_SIGNPOST_ID_FROM_POINTER(ii, crash_test, &ctx);
+    O2_SIGNPOST_START(crash_test, ii, "Init", "Starting Init");
+    O2_SIGNPOST_EVENT_EMIT(crash_test, ii, "Init", "%{public}s selected", what.c_str());
+
+    if (what == "fatal-init") {
+      LOG(fatal) << "This should have a fatal";
+    } else if (what == "runtime-init") {
+      throw std::runtime_error("This is a std::runtime_error");
+    } else if (what == "abort-init") {
+      abort();
+    } else if (what == "framework-init") {
+      throw o2::framework::runtime_error("This is a o2::framework::runtime_error");
+    } else if (what == "framework-prerun") {
+      ctx.services().get<CallbackService>().set<CallbackService::Id::PreProcessing>([](ServiceRegistryRef, int) {
+        throw o2::framework::runtime_error("This is o2::framework::runtime_error in PreProcessing");
+      });
+    } else if (what == "runtime-prerun") {
+      ctx.services().get<CallbackService>().set<CallbackService::Id::PreProcessing>([](ServiceRegistryRef, int) {
+        throw std::runtime_error("This is std::runtime_error in PreProcessing");
+      });
+    }
+    O2_SIGNPOST_END(crash_test, ii, "Init", "Init Done");
+    return adaptStateless([what](ProcessingContext& pCtx) {
+      O2_SIGNPOST_ID_FROM_POINTER(ri, crash_test, &pCtx);
+      O2_SIGNPOST_START(crash_test, ri, "Run", "Starting Run");
+      O2_SIGNPOST_EVENT_EMIT(crash_test, ri, "Run", "%{public}s selected", what.c_str());
+      if (what == "fatal-run") {
+        LOG(fatal) << "This should have a fatal";
+      } else if (what == "runtime-run") {
+        throw std::runtime_error("This is a std::runtime_error");
+      } else if (what == "abort-run") {
+        abort();
+      } else if (what == "framework-run") {
+        throw o2::framework::runtime_error("This is a o2::framework::runtime_error");
+      }
+      O2_SIGNPOST_EVENT_EMIT_ERROR(crash_test, ri, "Run", "Unknown option for crash-type: %{public}s.", what.c_str());
+      O2_SIGNPOST_END(crash_test, ri, "Init", "Run Done");
+      exit(1);
+    });
+  })};
+}
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& config)
+{
+  auto crashType = config.options().get<std::string>("crash-type");
+  DataProcessorSpec a{
+    .name = "deliberately-crashing",
+    .outputs = {OutputSpec{{"a1"}, "TST", "A1"}},
+    .algorithm = AlgorithmSpec{simpleCrashingSource(crashType)}};
+  DataProcessorSpec b{
+    .name = "B",
+    .inputs = {InputSpec{"x", "TST", "A1", Lifetime::Timeframe}},
+    .algorithm = AlgorithmSpec{adaptStateless([](ProcessingContext&) {})}};
+
+  return workflow::concat(WorkflowSpec{a, b});
+}
+

From 7b11ac649e68fdf1d0c60e9e49a12580da968a29 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 2 May 2024 13:16:39 +0200
Subject: [PATCH 494/726] DPL: add test to verify crashing workflows

---
 Framework/Core/CMakeLists.txt                 |  6 ++
 Framework/Core/test/test_AllCrashTypes.sh     | 25 ++++++
 Framework/Core/test/test_CrashingWorkflow.cxx | 88 +++++++++++++++++++
 3 files changed, 119 insertions(+)
 create mode 100755 Framework/Core/test/test_AllCrashTypes.sh
 create mode 100644 Framework/Core/test/test_CrashingWorkflow.cxx

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index d153fe9b4184e..fabb62eb6530d 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -281,6 +281,11 @@ o2_add_test(O2DatabasePDG NAME test_Framework_test_O2DatabasePDG
             LABELS framework
             PUBLIC_LINK_LIBRARIES O2::Framework O2::FrameworkPhysicsSupport)
 
+o2_add_executable(crashing-workflow
+            SOURCES test/test_CrashingWorkflow.cxx
+            COMPONENT_NAME Framework
+            PUBLIC_LINK_LIBRARIES O2::Framework)
+
 # All the tests which require ROOT to work
 add_executable(o2-test-framework-root
                test/test_Root2ArrowTable.cxx
@@ -290,6 +295,7 @@ target_link_libraries(o2-test-framework-root PRIVATE O2::Catch2)
 target_link_libraries(o2-test-framework-root PRIVATE ROOT::ROOTDataFrame)
 set_property(TARGET o2-test-framework-root PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
 add_test(NAME framework:root COMMAND o2-test-framework-root --skip-benchmarks)
+add_test(NAME framework:crash COMMAND  sh -e -c "PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:$PATH ${CMAKE_CURRENT_LIST_DIR}/test/test_AllCrashTypes.sh")
 
 o2_add_test(InfoLogger NAME test_Framework_test_InfoLogger
             SOURCES test/test_InfoLogger.cxx
diff --git a/Framework/Core/test/test_AllCrashTypes.sh b/Framework/Core/test/test_AllCrashTypes.sh
new file mode 100755
index 0000000000000..d735e55e23b30
--- /dev/null
+++ b/Framework/Core/test/test_AllCrashTypes.sh
@@ -0,0 +1,25 @@
+#!/bin/sh -e
+echo $PATH
+printf "Testing abort-init..."
+o2-framework-crashing-workflow --crash-type=abort-init --completion-policy=quit -b --run | grep -q "Abort trap" || { echo "abort not found in init"; exit 1; }
+printf "ok\nTesting runtime-init..."
+o2-framework-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a std::runtime_error" || { printf "runtime error not found" ; exit 1; }
+printf "ok\nTesting framework-init..."
+o2-framework-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a o2::framework::runtime_error" || { printf "framework error not found" ;  exit 1; }
+printf "ok\nTesting framework-run..."
+o2-framework-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
+printf "ok\nTesting runtime-run..."
+o2-framework-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found" ; exit 1; }
+printf "ok\n"
+
+export O2_NO_CATCHALL_EXCEPTIONS=1
+echo O2_NO_CATCHALL_EXCEPTIONS enabled
+printf "ok\nTesting runtime-init..."
+o2-framework-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -v -q "Exception caught: This is a std::runtime_error" || { printf "runtime error not found" ; exit 1; }
+printf "ok\nTesting framework-init..."
+o2-framework-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -v -q "Exception caught: This is a o2::framework::runtime_error" || { printf "framework error not found" ;  exit 1; }
+printf "ok\nTesting framework-run..."
+o2-framework-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -v -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
+printf "ok\nTesting runtime-run..."
+o2-framework-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -v -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found" ; exit 1; }
+printf "ok"
diff --git a/Framework/Core/test/test_CrashingWorkflow.cxx b/Framework/Core/test/test_CrashingWorkflow.cxx
new file mode 100644
index 0000000000000..396b48a85c166
--- /dev/null
+++ b/Framework/Core/test/test_CrashingWorkflow.cxx
@@ -0,0 +1,88 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/AlgorithmSpec.h"
+#include "Framework/Configurable.h"
+#include "Framework/Logger.h"
+#include "Framework/CallbackService.h"
+#include "Framework/Signpost.h"
+
+O2_DECLARE_DYNAMIC_LOG(crash_test);
+
+using namespace o2::framework;
+
+struct WorkflowOptions {
+  Configurable<std::string> crashType{"crash-type", "fatal-init", {"how should this crash? (fatal-init, fatal-run, runtime-init, runtime-fail, abort-init, abort-run)"}};
+};
+
+#include "Framework/runDataProcessing.h"
+
+AlgorithmSpec simpleCrashingSource(std::string const& what)
+{
+  return AlgorithmSpec{adaptStateful([what](InitContext& ctx) {
+    O2_SIGNPOST_ID_FROM_POINTER(ii, crash_test, &ctx);
+    O2_SIGNPOST_START(crash_test, ii, "Init", "Starting Init");
+    O2_SIGNPOST_EVENT_EMIT(crash_test, ii, "Init", "%{public}s selected", what.c_str());
+
+    if (what == "fatal-init") {
+      LOG(fatal) << "This should have a fatal";
+    } else if (what == "runtime-init") {
+      throw std::runtime_error("This is a std::runtime_error");
+    } else if (what == "abort-init") {
+      abort();
+    } else if (what == "framework-init") {
+      throw o2::framework::runtime_error("This is a o2::framework::runtime_error");
+    } else if (what == "framework-prerun") {
+      ctx.services().get<CallbackService>().set<CallbackService::Id::PreProcessing>([](ServiceRegistryRef, int) {
+        throw o2::framework::runtime_error("This is o2::framework::runtime_error in PreProcessing");
+      });
+    } else if (what == "runtime-prerun") {
+      ctx.services().get<CallbackService>().set<CallbackService::Id::PreProcessing>([](ServiceRegistryRef, int) {
+        throw std::runtime_error("This is std::runtime_error in PreProcessing");
+      });
+    }
+    O2_SIGNPOST_END(crash_test, ii, "Init", "Init Done");
+    return adaptStateless([what](ProcessingContext& pCtx) {
+      O2_SIGNPOST_ID_FROM_POINTER(ri, crash_test, &pCtx);
+      O2_SIGNPOST_START(crash_test, ri, "Run", "Starting Run");
+      O2_SIGNPOST_EVENT_EMIT(crash_test, ri, "Run", "%{public}s selected", what.c_str());
+      if (what == "fatal-run") {
+        LOG(fatal) << "This should have a fatal";
+      } else if (what == "runtime-run") {
+        throw std::runtime_error("This is a std::runtime_error");
+      } else if (what == "abort-run") {
+        abort();
+      } else if (what == "framework-run") {
+        throw o2::framework::runtime_error("This is a o2::framework::runtime_error");
+      }
+      O2_SIGNPOST_EVENT_EMIT_ERROR(crash_test, ri, "Run", "Unknown option for crash-type: %{public}s.", what.c_str());
+      O2_SIGNPOST_END(crash_test, ri, "Init", "Run Done");
+      exit(1);
+    });
+  })};
+}
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& config)
+{
+  auto crashType = config.options().get<std::string>("crash-type");
+  DataProcessorSpec a{
+    .name = "deliberately-crashing",
+    .outputs = {OutputSpec{{"a1"}, "TST", "A1"}},
+    .algorithm = AlgorithmSpec{simpleCrashingSource(crashType)}};
+  DataProcessorSpec b{
+    .name = "B",
+    .inputs = {InputSpec{"x", "TST", "A1", Lifetime::Timeframe}},
+    .algorithm = AlgorithmSpec{adaptStateless([](ProcessingContext&) {})}};
+
+  return workflow::concat(WorkflowSpec{a, b});
+}
+

From 14556a1e02ddf3f6643e54d1134f79da1f68f35f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 2 May 2024 15:33:06 +0200
Subject: [PATCH 495/726] DPL: add test to verify crashing workflows

---
 Framework/Core/CMakeLists.txt                 |  6 ++
 Framework/Core/test/test_AllCrashTypes.sh     | 23 +++++
 Framework/Core/test/test_CrashingWorkflow.cxx | 88 +++++++++++++++++++
 3 files changed, 117 insertions(+)
 create mode 100755 Framework/Core/test/test_AllCrashTypes.sh
 create mode 100644 Framework/Core/test/test_CrashingWorkflow.cxx

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index d153fe9b4184e..fabb62eb6530d 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -281,6 +281,11 @@ o2_add_test(O2DatabasePDG NAME test_Framework_test_O2DatabasePDG
             LABELS framework
             PUBLIC_LINK_LIBRARIES O2::Framework O2::FrameworkPhysicsSupport)
 
+o2_add_executable(crashing-workflow
+            SOURCES test/test_CrashingWorkflow.cxx
+            COMPONENT_NAME Framework
+            PUBLIC_LINK_LIBRARIES O2::Framework)
+
 # All the tests which require ROOT to work
 add_executable(o2-test-framework-root
                test/test_Root2ArrowTable.cxx
@@ -290,6 +295,7 @@ target_link_libraries(o2-test-framework-root PRIVATE O2::Catch2)
 target_link_libraries(o2-test-framework-root PRIVATE ROOT::ROOTDataFrame)
 set_property(TARGET o2-test-framework-root PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
 add_test(NAME framework:root COMMAND o2-test-framework-root --skip-benchmarks)
+add_test(NAME framework:crash COMMAND  sh -e -c "PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:$PATH ${CMAKE_CURRENT_LIST_DIR}/test/test_AllCrashTypes.sh")
 
 o2_add_test(InfoLogger NAME test_Framework_test_InfoLogger
             SOURCES test/test_InfoLogger.cxx
diff --git a/Framework/Core/test/test_AllCrashTypes.sh b/Framework/Core/test/test_AllCrashTypes.sh
new file mode 100755
index 0000000000000..52c8e62ee8469
--- /dev/null
+++ b/Framework/Core/test/test_AllCrashTypes.sh
@@ -0,0 +1,23 @@
+#!/bin/sh -e
+echo $PATH
+printf "ok\nTesting runtime-init..."
+o2-framework-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a std::runtime_error" || { printf "runtime error not found" ; exit 1; }
+printf "ok\nTesting framework-init..."
+o2-framework-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a o2::framework::runtime_error" || { printf "framework error not found" ;  exit 1; }
+printf "ok\nTesting framework-run..."
+o2-framework-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
+printf "ok\nTesting runtime-run..."
+o2-framework-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found" ; exit 1; }
+printf "ok\n"
+
+export O2_NO_CATCHALL_EXCEPTIONS=1
+echo O2_NO_CATCHALL_EXCEPTIONS enabled
+printf "ok\nTesting runtime-init..."
+o2-framework-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -v -q "Exception caught: This is a std::runtime_error" || { printf "runtime error not found" ; exit 1; }
+printf "ok\nTesting framework-init..."
+o2-framework-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -v -q "Exception caught: This is a o2::framework::runtime_error" || { printf "framework error not found" ;  exit 1; }
+printf "ok\nTesting framework-run..."
+o2-framework-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -v -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
+printf "ok\nTesting runtime-run..."
+o2-framework-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -v -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found" ; exit 1; }
+printf "ok"
diff --git a/Framework/Core/test/test_CrashingWorkflow.cxx b/Framework/Core/test/test_CrashingWorkflow.cxx
new file mode 100644
index 0000000000000..396b48a85c166
--- /dev/null
+++ b/Framework/Core/test/test_CrashingWorkflow.cxx
@@ -0,0 +1,88 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/AlgorithmSpec.h"
+#include "Framework/Configurable.h"
+#include "Framework/Logger.h"
+#include "Framework/CallbackService.h"
+#include "Framework/Signpost.h"
+
+O2_DECLARE_DYNAMIC_LOG(crash_test);
+
+using namespace o2::framework;
+
+struct WorkflowOptions {
+  Configurable<std::string> crashType{"crash-type", "fatal-init", {"how should this crash? (fatal-init, fatal-run, runtime-init, runtime-fail, abort-init, abort-run)"}};
+};
+
+#include "Framework/runDataProcessing.h"
+
+AlgorithmSpec simpleCrashingSource(std::string const& what)
+{
+  return AlgorithmSpec{adaptStateful([what](InitContext& ctx) {
+    O2_SIGNPOST_ID_FROM_POINTER(ii, crash_test, &ctx);
+    O2_SIGNPOST_START(crash_test, ii, "Init", "Starting Init");
+    O2_SIGNPOST_EVENT_EMIT(crash_test, ii, "Init", "%{public}s selected", what.c_str());
+
+    if (what == "fatal-init") {
+      LOG(fatal) << "This should have a fatal";
+    } else if (what == "runtime-init") {
+      throw std::runtime_error("This is a std::runtime_error");
+    } else if (what == "abort-init") {
+      abort();
+    } else if (what == "framework-init") {
+      throw o2::framework::runtime_error("This is a o2::framework::runtime_error");
+    } else if (what == "framework-prerun") {
+      ctx.services().get<CallbackService>().set<CallbackService::Id::PreProcessing>([](ServiceRegistryRef, int) {
+        throw o2::framework::runtime_error("This is o2::framework::runtime_error in PreProcessing");
+      });
+    } else if (what == "runtime-prerun") {
+      ctx.services().get<CallbackService>().set<CallbackService::Id::PreProcessing>([](ServiceRegistryRef, int) {
+        throw std::runtime_error("This is std::runtime_error in PreProcessing");
+      });
+    }
+    O2_SIGNPOST_END(crash_test, ii, "Init", "Init Done");
+    return adaptStateless([what](ProcessingContext& pCtx) {
+      O2_SIGNPOST_ID_FROM_POINTER(ri, crash_test, &pCtx);
+      O2_SIGNPOST_START(crash_test, ri, "Run", "Starting Run");
+      O2_SIGNPOST_EVENT_EMIT(crash_test, ri, "Run", "%{public}s selected", what.c_str());
+      if (what == "fatal-run") {
+        LOG(fatal) << "This should have a fatal";
+      } else if (what == "runtime-run") {
+        throw std::runtime_error("This is a std::runtime_error");
+      } else if (what == "abort-run") {
+        abort();
+      } else if (what == "framework-run") {
+        throw o2::framework::runtime_error("This is a o2::framework::runtime_error");
+      }
+      O2_SIGNPOST_EVENT_EMIT_ERROR(crash_test, ri, "Run", "Unknown option for crash-type: %{public}s.", what.c_str());
+      O2_SIGNPOST_END(crash_test, ri, "Init", "Run Done");
+      exit(1);
+    });
+  })};
+}
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& config)
+{
+  auto crashType = config.options().get<std::string>("crash-type");
+  DataProcessorSpec a{
+    .name = "deliberately-crashing",
+    .outputs = {OutputSpec{{"a1"}, "TST", "A1"}},
+    .algorithm = AlgorithmSpec{simpleCrashingSource(crashType)}};
+  DataProcessorSpec b{
+    .name = "B",
+    .inputs = {InputSpec{"x", "TST", "A1", Lifetime::Timeframe}},
+    .algorithm = AlgorithmSpec{adaptStateless([](ProcessingContext&) {})}};
+
+  return workflow::concat(WorkflowSpec{a, b});
+}
+

From 6090b340efcf8895392ffb261293c27e9d56b6ac Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 3 May 2024 21:46:29 +0200
Subject: [PATCH 496/726] DPL: refactor plugin manager

- Move PluginManager to a separate header so that we do not include the
  code to load plugin in the plugin declarations.
- Make the PluginManager code generic to use it for the upcoming
  metadata discovery.
---
 Framework/Core/CMakeLists.txt                 |   2 +-
 .../Core/include/Framework/AnalysisManagers.h |   6 +-
 .../Core/include/Framework/PluginManager.h    | 137 ++++++++++++++++++
 Framework/Core/include/Framework/Plugins.h    |  41 ------
 .../Core/include/Framework/ServiceSpec.h      |   5 -
 Framework/Core/src/CommonDataProcessors.cxx   |   2 +-
 Framework/Core/src/CommonDriverServices.cxx   |   7 +-
 Framework/Core/src/CommonServices.cxx         |   8 +-
 .../src/{Plugins.cxx => PluginManager.cxx}    |  51 ++++++-
 Framework/Core/src/ServiceSpec.cxx            | 120 ---------------
 Framework/Core/src/WorkflowHelpers.cxx        |   2 +-
 Framework/Core/src/runDataProcessing.cxx      |   2 +-
 Framework/Core/test/test_AlgorithmWrapper.cxx |   2 +-
 13 files changed, 203 insertions(+), 182 deletions(-)
 create mode 100644 Framework/Core/include/Framework/PluginManager.h
 rename Framework/Core/src/{Plugins.cxx => PluginManager.cxx} (72%)

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index d153fe9b4184e..7963e1d3745af 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -109,7 +109,7 @@ o2_add_library(Framework
                        src/OptionsHelpers.cxx
                        src/PropertyTreeHelpers.cxx
                        src/ProcessingContext.cxx
-                       src/Plugins.cxx
+                       src/PluginManager.cxx
                        src/RateLimiter.cxx
                        src/ResourcesMonitoringHelper.cxx
                        src/ResourcePolicy.cxx
diff --git a/Framework/Core/include/Framework/AnalysisManagers.h b/Framework/Core/include/Framework/AnalysisManagers.h
index c8daac1c5edb5..f341f60392e9f 100644
--- a/Framework/Core/include/Framework/AnalysisManagers.h
+++ b/Framework/Core/include/Framework/AnalysisManagers.h
@@ -28,7 +28,7 @@
 #include "Framework/RootConfigParamHelpers.h"
 #include "Framework/ExpressionHelpers.h"
 #include "Framework/CommonServices.h"
-#include "Framework/Plugins.h"
+#include "Framework/PluginManager.h"
 #include "Framework/RootMessageContext.h"
 #include "Framework/DeviceSpec.h"
 
@@ -465,8 +465,8 @@ struct ServiceManager<Service<T>> {
   {
     if constexpr (o2::framework::is_base_of_template_v<LoadableServicePlugin, T>) {
       T p = T{};
-      auto loadableServices = ServiceHelpers::parseServiceSpecString(p.loadSpec.c_str());
-      ServiceHelpers::loadFromPlugin(loadableServices, specs);
+      auto loadableServices = PluginManager::parsePluginSpecString(p.loadSpec.c_str());
+      PluginManager::loadFromPlugin<ServiceSpec, ServicePlugin>(loadableServices, specs);
     }
     return true;
   }
diff --git a/Framework/Core/include/Framework/PluginManager.h b/Framework/Core/include/Framework/PluginManager.h
new file mode 100644
index 0000000000000..38966c5a969a9
--- /dev/null
+++ b/Framework/Core/include/Framework/PluginManager.h
@@ -0,0 +1,137 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef O2_FRAMEWORK_PLUGIN_MANAGER_H_
+#define O2_FRAMEWORK_PLUGIN_MANAGER_H_
+
+#include "Framework/Plugins.h"
+#include <cstring>
+#include <uv.h>
+#include <functional>
+
+// Struct to hold live plugin information which the plugin itself cannot
+// know and that is owned by the framework.
+struct PluginInfo {
+  uv_lib_t* dso = nullptr;
+  std::string name;
+  DPLPluginHandle* instance = nullptr;
+};
+
+// Struct to hold information about the location of a plugin
+struct LoadablePlugin {
+  std::string name;
+  std::string library;
+};
+
+namespace o2::framework
+{
+struct PluginManager {
+  using WrapperProcessCallback = std::function<void(AlgorithmSpec::ProcessCallback&, ProcessingContext&)>;
+
+  template <typename T>
+  static T* getByName(DPLPluginHandle* handle, char const* name)
+  {
+    while (handle != nullptr) {
+      if (strncmp(handle->name, name, strlen(name)) == 0) {
+        return reinterpret_cast<T*>(handle->instance);
+      }
+      handle = handle->previous;
+    }
+    return nullptr;
+  }
+  /// Load a DSO called @a dso and insert its handle in @a infos
+  /// On successfull completion @a onSuccess is called passing
+  /// the DPLPluginHandle provided by the library.
+  static void load(std::vector<PluginInfo>& infos, const char* dso, std::function<void(DPLPluginHandle*)>& onSuccess);
+  /// Load an called @plugin from a library called @a library and
+  /// return the associtated AlgorithmSpec.
+  static auto loadAlgorithmFromPlugin(std::string library, std::string plugin) -> AlgorithmSpec;
+  /// Wrap an algorithm with some lambda @wrapper which will be called
+  /// with the original callback and the ProcessingContext.
+  static auto wrapAlgorithm(AlgorithmSpec const& spec, WrapperProcessCallback&& wrapper) -> AlgorithmSpec;
+
+  /// Parse a comma separated list of <library>:<plugin-name> plugin declarations.
+  static std::vector<LoadablePlugin> parsePluginSpecString(char const* str);
+
+  template <typename CONCRETE, typename PLUGIN>
+  static void loadFromPlugin(std::vector<LoadablePlugin> const& loadablePlugins, std::vector<CONCRETE>& specs)
+  {
+    struct LoadedDSO {
+      std::string library;
+      uv_lib_t handle;
+    };
+
+    struct LoadedPlugin {
+      std::string name;
+      ServicePlugin* factory;
+    };
+    std::vector<LoadedDSO> loadedDSOs;
+    std::vector<LoadedPlugin> loadedPlugins;
+    for (auto& loadablePlugin : loadablePlugins) {
+      auto loadedDSO = std::find_if(loadedDSOs.begin(), loadedDSOs.end(), [&loadablePlugin](auto& dso) {
+        return dso.library == loadablePlugin.library;
+      });
+
+      if (loadedDSO == loadedDSOs.end()) {
+        uv_lib_t handle;
+#ifdef __APPLE__
+        auto libraryName = fmt::format("lib{}.dylib", loadablePlugin.library);
+#else
+        auto libraryName = fmt::format("lib{}.so", loadablePlugin.library);
+#endif
+        auto ret = uv_dlopen(libraryName.c_str(), &handle);
+        if (ret != 0) {
+          LOGP(error, "Could not load library {}", loadablePlugin.library);
+          LOG(error) << uv_dlerror(&handle);
+          continue;
+        }
+        loadedDSOs.push_back({loadablePlugin.library, handle});
+        loadedDSO = loadedDSOs.end() - 1;
+      }
+      int result = 0;
+
+      auto loadedPlugin = std::find_if(loadedPlugins.begin(), loadedPlugins.end(), [&loadablePlugin](auto& plugin) {
+        return plugin.name == loadablePlugin.name;
+      });
+
+      if (loadedPlugin == loadedPlugins.end()) {
+        DPLPluginHandle* (*dpl_plugin_callback)(DPLPluginHandle*);
+        result = uv_dlsym(&loadedDSO->handle, "dpl_plugin_callback", (void**)&dpl_plugin_callback);
+        if (result == -1) {
+          LOG(error) << uv_dlerror(&loadedDSO->handle);
+          continue;
+        }
+
+        DPLPluginHandle* pluginInstance = dpl_plugin_callback(nullptr);
+        PLUGIN* factory = PluginManager::getByName<PLUGIN>(pluginInstance, loadablePlugin.name.c_str());
+        if (factory == nullptr) {
+          LOGP(error, "Could not find service {} in library {}", loadablePlugin.name, loadablePlugin.library);
+          continue;
+        }
+
+        loadedPlugins.push_back({loadablePlugin.name, factory});
+        loadedPlugin = loadedPlugins.begin() + loadedPlugins.size() - 1;
+      }
+      assert(loadedPlugin != loadedPlugins.end());
+      assert(loadedPlugin->factory != nullptr);
+
+      CONCRETE* spec = loadedPlugin->factory->create();
+      if (!spec) {
+        LOG(error) << "Plugin " << loadablePlugin.name << " could not be created";
+        continue;
+      }
+      LOGP(debug, "Loading service {} from {}", loadablePlugin.name, loadablePlugin.library);
+      specs.push_back(*spec);
+    }
+  }
+};
+
+} // namespace o2::framework
+#endif // O2_FRAMEWORK_PLUGIN_MANAGER_H_
diff --git a/Framework/Core/include/Framework/Plugins.h b/Framework/Core/include/Framework/Plugins.h
index 788470b472f2a..c29d96f7e0781 100644
--- a/Framework/Core/include/Framework/Plugins.h
+++ b/Framework/Core/include/Framework/Plugins.h
@@ -12,10 +12,7 @@
 #define O2_FRAMEWORK_PLUGINS_H_
 
 #include "Framework/AlgorithmSpec.h"
-#include <cstring>
 #include <string>
-#include <functional>
-#include <uv.h>
 
 namespace o2::framework
 {
@@ -82,14 +79,6 @@ struct DPLPluginHandle {
   DPLPluginHandle* previous = nullptr;
 };
 
-// Struct to hold live plugin information which the plugin itself cannot
-// know and that is owned by the framework.
-struct PluginInfo {
-  uv_lib_t* dso = nullptr;
-  std::string name;
-  DPLPluginHandle* instance = nullptr;
-};
-
 #define DEFINE_DPL_PLUGIN(NAME, KIND)                                                                    \
   extern "C" {                                                                                           \
   DPLPluginHandle* dpl_plugin_callback(DPLPluginHandle* previous)                                        \
@@ -111,34 +100,4 @@ struct PluginInfo {
   }                            \
   }
 
-namespace o2::framework
-{
-struct PluginManager {
-  using WrapperProcessCallback = std::function<void(AlgorithmSpec::ProcessCallback&, ProcessingContext&)>;
-
-  template <typename T>
-  static T* getByName(DPLPluginHandle* handle, char const* name)
-  {
-    while (handle != nullptr) {
-      if (strncmp(handle->name, name, strlen(name)) == 0) {
-        return reinterpret_cast<T*>(handle->instance);
-      }
-      handle = handle->previous;
-    }
-    return nullptr;
-  }
-  /// Load a DSO called @a dso and insert its handle in @a infos
-  /// On successfull completion @a onSuccess is called passing
-  /// the DPLPluginHandle provided by the library.
-  static void load(std::vector<PluginInfo>& infos, const char* dso, std::function<void(DPLPluginHandle*)>& onSuccess);
-  /// Load an called @plugin from a library called @a library and
-  /// return the associtated AlgorithmSpec.
-  static auto loadAlgorithmFromPlugin(std::string library, std::string plugin) -> AlgorithmSpec;
-  /// Wrap an algorithm with some lambda @wrapper which will be called
-  /// with the original callback and the ProcessingContext.
-  static auto wrapAlgorithm(AlgorithmSpec const& spec, WrapperProcessCallback&& wrapper) -> AlgorithmSpec;
-};
-
-} // namespace o2::framework
-
 #endif // O2_FRAMEWORK_PLUGINS_H_
diff --git a/Framework/Core/include/Framework/ServiceSpec.h b/Framework/Core/include/Framework/ServiceSpec.h
index 910620fcd0655..abac3ad05914c 100644
--- a/Framework/Core/include/Framework/ServiceSpec.h
+++ b/Framework/Core/include/Framework/ServiceSpec.h
@@ -327,11 +327,6 @@ struct LoadableService {
   std::string library;
 };
 
-struct ServiceHelpers {
-  static std::vector<LoadableService> parseServiceSpecString(char const* str);
-  static void loadFromPlugin(std::vector<LoadableService> const& loadableServices, std::vector<ServiceSpec>& specs);
-};
-
 } // namespace o2::framework
 
 #endif // O2_FRAMEWORK_SERVICESPEC_H_
diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 593cb8d1ac1f8..9817a57734c1d 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -39,7 +39,7 @@
 #include "Framework/ExternalFairMQDeviceProxy.h"
 #include "Framework/RuntimeError.h"
 #include "Framework/RateLimiter.h"
-#include "Framework/Plugins.h"
+#include "Framework/PluginManager.h"
 #include "Framework/DeviceSpec.h"
 #include "WorkflowHelpers.h"
 #include <Monitoring/Monitoring.h>
diff --git a/Framework/Core/src/CommonDriverServices.cxx b/Framework/Core/src/CommonDriverServices.cxx
index 893087d555210..6fd27b700e047 100644
--- a/Framework/Core/src/CommonDriverServices.cxx
+++ b/Framework/Core/src/CommonDriverServices.cxx
@@ -11,6 +11,7 @@
 
 #include "CommonDriverServices.h"
 #include "Framework/CommonServices.h"
+#include "Framework/PluginManager.h"
 
 // Make sure we can use aggregated initialisers.
 #pragma GCC diagnostic push
@@ -24,14 +25,14 @@ std::vector<ServiceSpec> o2::framework::CommonDriverServices::defaultServices()
   std::vector<ServiceSpec> specs{
     CommonServices::configurationSpec()};
   // Load plugins depending on the environment
-  std::vector<LoadableService> loadableServices = {};
+  std::vector<LoadablePlugin> loadableServices = {};
   char* loadableServicesEnv = getenv("DPL_LOAD_DRIVER_SERVICES");
   // String to define the services to load is:
   //
   // library1:name1,library2:name2,...
   if (loadableServicesEnv) {
-    loadableServices = ServiceHelpers::parseServiceSpecString(loadableServicesEnv);
-    ServiceHelpers::loadFromPlugin(loadableServices, specs);
+    loadableServices = PluginManager::parsePluginSpecString(loadableServicesEnv);
+    PluginManager::loadFromPlugin<ServiceSpec, ServicePlugin>(loadableServices, specs);
   }
   return specs;
 }
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 7f0e710d96b2e..7f145fe4318ff 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -36,7 +36,7 @@
 #include "Framework/Tracing.h"
 #include "Framework/Monitoring.h"
 #include "Framework/AsyncQueue.h"
-#include "Framework/Plugins.h"
+#include "Framework/PluginManager.h"
 #include "Framework/DeviceContext.h"
 #include "Framework/DataProcessingContext.h"
 #include "Framework/StreamContext.h"
@@ -1260,7 +1260,7 @@ std::vector<ServiceSpec> CommonServices::defaultServices(std::string extraPlugin
     loadableServicesStr += "O2FrameworkDataTakingSupport:InfoLoggerContext,O2FrameworkDataTakingSupport:InfoLogger";
   }
   // Load plugins depending on the environment
-  std::vector<LoadableService> loadableServices = {};
+  std::vector<LoadablePlugin> loadablePlugins = {};
   char* loadableServicesEnv = getenv("DPL_LOAD_SERVICES");
   // String to define the services to load is:
   //
@@ -1271,8 +1271,8 @@ std::vector<ServiceSpec> CommonServices::defaultServices(std::string extraPlugin
     }
     loadableServicesStr += loadableServicesEnv;
   }
-  loadableServices = ServiceHelpers::parseServiceSpecString(loadableServicesStr.c_str());
-  ServiceHelpers::loadFromPlugin(loadableServices, specs);
+  loadablePlugins = PluginManager::parsePluginSpecString(loadableServicesStr.c_str());
+  PluginManager::loadFromPlugin<ServiceSpec, ServicePlugin>(loadablePlugins, specs);
   // I should make it optional depending wether the GUI is there or not...
   specs.push_back(CommonServices::guiMetricsSpec());
   if (numThreads) {
diff --git a/Framework/Core/src/Plugins.cxx b/Framework/Core/src/PluginManager.cxx
similarity index 72%
rename from Framework/Core/src/Plugins.cxx
rename to Framework/Core/src/PluginManager.cxx
index c4a0ee7fd6dc5..96666722fc169 100644
--- a/Framework/Core/src/Plugins.cxx
+++ b/Framework/Core/src/PluginManager.cxx
@@ -8,7 +8,7 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
-#include "Framework/Plugins.h"
+#include "Framework/PluginManager.h"
 #include "Framework/Logger.h"
 #include <uv.h>
 #include <functional>
@@ -16,6 +16,55 @@
 
 namespace o2::framework
 {
+std::vector<LoadablePlugin> PluginManager::parsePluginSpecString(char const* str)
+{
+  std::vector<LoadablePlugin> loadablePlugins;
+  enum struct ParserState : int {
+    IN_LIBRARY,
+    IN_NAME,
+    IN_END,
+    IN_ERROR,
+  };
+  const char* cur = str;
+  const char* next = cur;
+  ParserState state = ParserState::IN_LIBRARY;
+  std::string_view library;
+  std::string_view name;
+  while (cur && *cur != '\0') {
+    ParserState previousState = state;
+    state = ParserState::IN_ERROR;
+    switch (previousState) {
+      case ParserState::IN_LIBRARY:
+        next = strchr(cur, ':');
+        if (next != nullptr) {
+          state = ParserState::IN_NAME;
+          library = std::string_view(cur, next - cur);
+        }
+        break;
+      case ParserState::IN_NAME:
+        next = strchr(cur, ',');
+        if (next == nullptr) {
+          state = ParserState::IN_END;
+          name = std::string_view(cur, strlen(cur));
+        } else {
+          name = std::string_view(cur, next - cur);
+          state = ParserState::IN_LIBRARY;
+        }
+        loadablePlugins.push_back({std::string(name), std::string(library)});
+        break;
+      case ParserState::IN_END:
+        break;
+      case ParserState::IN_ERROR:
+        LOG(error) << "Error while parsing DPL_LOAD_SERVICES";
+        break;
+    }
+    if (!next) {
+      break;
+    }
+    cur = next + 1;
+  };
+  return loadablePlugins;
+}
 
 void PluginManager::load(std::vector<PluginInfo>& libs, const char* dso, std::function<void(DPLPluginHandle*)>& onSuccess)
 {
diff --git a/Framework/Core/src/ServiceSpec.cxx b/Framework/Core/src/ServiceSpec.cxx
index fdbe0c0c4dd4b..4f38651b68924 100644
--- a/Framework/Core/src/ServiceSpec.cxx
+++ b/Framework/Core/src/ServiceSpec.cxx
@@ -104,124 +104,4 @@ ServiceSpecs ServiceSpecHelpers::filterDisabled(ServiceSpecs originals, Override
   return result;
 }
 
-std::vector<LoadableService> ServiceHelpers::parseServiceSpecString(char const* str)
-{
-  std::vector<LoadableService> loadableServices;
-  enum struct ParserState : int {
-    IN_LIBRARY,
-    IN_NAME,
-    IN_END,
-    IN_ERROR,
-  };
-  const char* cur = str;
-  const char* next = cur;
-  ParserState state = ParserState::IN_LIBRARY;
-  std::string_view library;
-  std::string_view name;
-  while (cur && *cur != '\0') {
-    ParserState previousState = state;
-    state = ParserState::IN_ERROR;
-    switch (previousState) {
-      case ParserState::IN_LIBRARY:
-        next = strchr(cur, ':');
-        if (next != nullptr) {
-          state = ParserState::IN_NAME;
-          library = std::string_view(cur, next - cur);
-        }
-        break;
-      case ParserState::IN_NAME:
-        next = strchr(cur, ',');
-        if (next == nullptr) {
-          state = ParserState::IN_END;
-          name = std::string_view(cur, strlen(cur));
-        } else {
-          name = std::string_view(cur, next - cur);
-          state = ParserState::IN_LIBRARY;
-        }
-        loadableServices.push_back({std::string(name), std::string(library)});
-        break;
-      case ParserState::IN_END:
-        break;
-      case ParserState::IN_ERROR:
-        LOG(error) << "Error while parsing DPL_LOAD_SERVICES";
-        break;
-    }
-    if (!next) {
-      break;
-    }
-    cur = next + 1;
-  };
-  return loadableServices;
-}
-
-void ServiceHelpers::loadFromPlugin(std::vector<LoadableService> const& loadableServices, std::vector<ServiceSpec>& specs)
-{
-  struct LoadedDSO {
-    std::string library;
-    uv_lib_t handle;
-  };
-
-  struct LoadedPlugin {
-    std::string name;
-    ServicePlugin* factory;
-  };
-  std::vector<LoadedDSO> loadedDSOs;
-  std::vector<LoadedPlugin> loadedPlugins;
-  for (auto& loadableService : loadableServices) {
-    auto loadedDSO = std::find_if(loadedDSOs.begin(), loadedDSOs.end(), [&loadableService](auto& dso) {
-      return dso.library == loadableService.library;
-    });
-
-    if (loadedDSO == loadedDSOs.end()) {
-      uv_lib_t handle;
-#ifdef __APPLE__
-      auto libraryName = fmt::format("lib{}.dylib", loadableService.library);
-#else
-      auto libraryName = fmt::format("lib{}.so", loadableService.library);
-#endif
-      auto ret = uv_dlopen(libraryName.c_str(), &handle);
-      if (ret != 0) {
-        LOGP(error, "Could not load library {}", loadableService.library);
-        LOG(error) << uv_dlerror(&handle);
-        continue;
-      }
-      loadedDSOs.push_back({loadableService.library, handle});
-      loadedDSO = loadedDSOs.end() - 1;
-    }
-    int result = 0;
-
-    auto loadedPlugin = std::find_if(loadedPlugins.begin(), loadedPlugins.end(), [&loadableService](auto& plugin) {
-      return plugin.name == loadableService.name;
-    });
-
-    if (loadedPlugin == loadedPlugins.end()) {
-      DPLPluginHandle* (*dpl_plugin_callback)(DPLPluginHandle*);
-      result = uv_dlsym(&loadedDSO->handle, "dpl_plugin_callback", (void**)&dpl_plugin_callback);
-      if (result == -1) {
-        LOG(error) << uv_dlerror(&loadedDSO->handle);
-        continue;
-      }
-
-      DPLPluginHandle* pluginInstance = dpl_plugin_callback(nullptr);
-      ServicePlugin* factory = PluginManager::getByName<ServicePlugin>(pluginInstance, loadableService.name.c_str());
-      if (factory == nullptr) {
-        LOGP(error, "Could not find service {} in library {}", loadableService.name, loadableService.library);
-        continue;
-      }
-
-      loadedPlugins.push_back({loadableService.name, factory});
-      loadedPlugin = loadedPlugins.begin() + loadedPlugins.size() - 1;
-    }
-    assert(loadedPlugin != loadedPlugins.end());
-    assert(loadedPlugin->factory != nullptr);
-
-    ServiceSpec* spec = loadedPlugin->factory->create();
-    if (!spec) {
-      LOG(error) << "Plugin " << loadableService.name << " could not be created";
-      continue;
-    }
-    LOGP(debug, "Loading service {} from {}", loadableService.name, loadableService.library);
-    specs.push_back(*spec);
-  }
-}
 } // namespace o2::framework
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index f5a66722aa713..bd875ad421f6e 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -21,7 +21,7 @@
 #include "Framework/RawDeviceService.h"
 #include "Framework/StringHelpers.h"
 #include "Framework/ChannelSpecHelpers.h"
-#include "Framework/Plugins.h"
+#include "Framework/PluginManager.h"
 #include "Framework/DataTakingContext.h"
 #include "Framework/DefaultsHelpers.h"
 
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index c72b4da73a45a..f91716d28cfd7 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -22,7 +22,7 @@
 #include "Framework/DataProcessingDevice.h"
 #include "Framework/DataProcessingContext.h"
 #include "Framework/DataProcessorSpec.h"
-#include "Framework/Plugins.h"
+#include "Framework/PluginManager.h"
 #include "Framework/DeviceControl.h"
 #include "Framework/DeviceExecution.h"
 #include "Framework/DeviceInfo.h"
diff --git a/Framework/Core/test/test_AlgorithmWrapper.cxx b/Framework/Core/test/test_AlgorithmWrapper.cxx
index 7671b6afa194b..6a02aea62c528 100644
--- a/Framework/Core/test/test_AlgorithmWrapper.cxx
+++ b/Framework/Core/test/test_AlgorithmWrapper.cxx
@@ -14,7 +14,7 @@
 #include "Framework/ServiceRegistry.h"
 #include "Framework/RawDeviceService.h"
 #include "Framework/runDataProcessing.h"
-#include "Framework/Plugins.h"
+#include "Framework/PluginManager.h"
 #include <Monitoring/Monitoring.h>
 
 using namespace o2::framework;

From 2b900a5858a854b375187c8d8a14152fbf8dc0f1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 4 May 2024 09:33:41 +0200
Subject: [PATCH 497/726] DPL: improve messages on caught exceptions

- Provide a bit of context in the printout
- Use signposts
---
 Framework/Core/src/DataProcessingDevice.cxx | 32 +++++++++++++++++----
 Framework/Core/test/test_AllCrashTypes.sh   |  4 +--
 2 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index bb3858a9d3ed2..602fe7cc29548 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -372,6 +372,9 @@ void DataProcessingDevice::Init()
   auto ref = ServiceRegistryRef{mServiceRegistry};
   auto& context = ref.get<DataProcessorContext>();
   auto& spec = getRunningDevice(mRunningDevice, ref);
+
+  O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
+  O2_SIGNPOST_START(device, cid, "Init", "Entering Init callback.");
   context.statelessProcess = spec.algorithm.onProcess;
   context.statefulProcess = nullptr;
   context.error = spec.algorithm.onError;
@@ -412,8 +415,10 @@ void DataProcessingDevice::Init()
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
       ServiceRegistryRef ref{serviceRegistry, ServiceRegistry::globalDeviceSalt()};
+      auto& context = ref.get<DataProcessorContext>();
       auto& err = error_from_ref(e);
-      LOGP(error, "Exception caught: {} ", err.what);
+      O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
+      O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "Init", "Exception caught while in Init: %{public}s. Invoking errorCallback.", err.what);
       demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
       auto& stats = ref.get<DataProcessingStats>();
       stats.updateStats({(int)ProcessingStatsId::EXCEPTION_COUNT, DataProcessingStats::Op::Add, 1});
@@ -425,8 +430,10 @@ void DataProcessingDevice::Init()
       auto& err = error_from_ref(e);
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
-      LOGP(error, "Exception caught: {} ", err.what);
       ServiceRegistryRef ref{serviceRegistry, ServiceRegistry::globalDeviceSalt()};
+      auto& context = ref.get<DataProcessorContext>();
+      O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
+      O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "Init", "Exception caught while in Init: %{public}s. Exiting with 1.", err.what);
       demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
       auto& stats = ref.get<DataProcessingStats>();
       stats.updateStats({(int)ProcessingStatsId::EXCEPTION_COUNT, DataProcessingStats::Op::Add, 1});
@@ -491,6 +498,7 @@ void DataProcessingDevice::Init()
     ServiceRegistry::Salt streamSalt = ServiceRegistry::streamSalt(si + 1, ServiceRegistry::globalDeviceSalt().dataProcessorId);
     mServiceRegistry.lateBindStreamServices(state, *options, streamSalt);
   }
+  O2_SIGNPOST_END(device, cid, "Init", "Exiting Init callback.");
 }
 
 void on_signal_callback(uv_signal_t* handle, int signum)
@@ -1079,7 +1087,9 @@ void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceCont
       ///        can access information which were stored in the stream.
       ServiceRegistryRef ref{serviceRegistry, ServiceRegistry::globalDeviceSalt()};
       auto& err = error_from_ref(e);
-      LOGP(error, "Exception caught: {} ", err.what);
+      auto& context = ref.get<DataProcessorContext>();
+      O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
+      O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "Run", "Exception while running: %{public}s. Invoking callback.", err.what);
       demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
       auto& stats = ref.get<DataProcessingStats>();
       stats.updateStats({(int)ProcessingStatsId::EXCEPTION_COUNT, DataProcessingStats::Op::Add, 1});
@@ -1092,15 +1102,18 @@ void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceCont
       auto& err = error_from_ref(e);
       /// FIXME: we should pass the salt in, so that the message
       ///        can access information which were stored in the stream.
-      LOGP(error, "Exception caught: {} ", err.what);
       ServiceRegistryRef ref{serviceRegistry, ServiceRegistry::globalDeviceSalt()};
+      auto& context = ref.get<DataProcessorContext>();
+      O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
       demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
       auto& stats = ref.get<DataProcessingStats>();
       stats.updateStats({(int)ProcessingStatsId::EXCEPTION_COUNT, DataProcessingStats::Op::Add, 1});
       switch (errorPolicy) {
         case TerminationPolicy::QUIT:
+          O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "Run", "Exception while running: %{public}s. Rethrowing.", err.what);
           throw e;
         default:
+          O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "Run", "Exception while running: %{public}s. Skipping to next timeframe.", err.what);
           break;
       }
     };
@@ -1148,6 +1161,9 @@ void DataProcessingDevice::PreRun()
 {
   auto ref = ServiceRegistryRef{mServiceRegistry};
   auto& state = ref.get<DeviceState>();
+
+  O2_SIGNPOST_ID_FROM_POINTER(cid, device, state.loop);
+  O2_SIGNPOST_START(device, cid, "PreRun", "Entering PreRun callback.");
   state.quitRequested = false;
   state.streaming = StreamingState::Streaming;
   for (auto& info : state.inputChannelInfos) {
@@ -1168,13 +1184,16 @@ void DataProcessingDevice::PreRun()
       context.preStartStreamCallbacks(streamRef);
     }
   } catch (std::exception& e) {
-    LOGP(error, "Exception caught: {} ", e.what());
+    O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "PreRun", "Exception of type std::exception caught in PreRun: %{public}s. Rethrowing.", e.what());
+    O2_SIGNPOST_END(device, cid, "PreRun", "Exiting PreRun due to exception thrown.");
     throw;
   } catch (o2::framework::RuntimeErrorRef& e) {
     auto& err = error_from_ref(e);
-    LOGP(error, "Exception caught: {} ", err.what);
+    O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "PreRun", "Exception of type o2::framework::RuntimeErrorRef caught in PreRun: %{public}s. Rethrowing.", err.what);
+    O2_SIGNPOST_END(device, cid, "PreRun", "Exiting PreRun due to exception thrown.");
     throw;
   } catch (...) {
+    O2_SIGNPOST_END(device, cid, "PreRun", "Unknown exception being thrown. Rethrowing.");
     throw;
   }
 
@@ -1189,6 +1208,7 @@ void DataProcessingDevice::PreRun()
 
   auto& monitoring = ref.get<Monitoring>();
   monitoring.send(Metric{(uint64_t)1, "device_state"}.addTag(Key::Subsystem, Value::DPL));
+  O2_SIGNPOST_END(device, cid, "PreRun", "Exiting PreRun callback.");
 }
 
 void DataProcessingDevice::PostRun()
diff --git a/Framework/Core/test/test_AllCrashTypes.sh b/Framework/Core/test/test_AllCrashTypes.sh
index 52c8e62ee8469..54898fd9c4c5d 100755
--- a/Framework/Core/test/test_AllCrashTypes.sh
+++ b/Framework/Core/test/test_AllCrashTypes.sh
@@ -1,9 +1,9 @@
 #!/bin/sh -e
 echo $PATH
 printf "ok\nTesting runtime-init..."
-o2-framework-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a std::runtime_error" || { printf "runtime error not found" ; exit 1; }
+o2-framework-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -q "Exception caught while in Init: This is a std::runtime_error. Exiting with 1." || { printf "runtime error not found" ; exit 1; }
 printf "ok\nTesting framework-init..."
-o2-framework-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a o2::framework::runtime_error" || { printf "framework error not found" ;  exit 1; }
+o2-framework-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -q "Exception caught while in Init: This is a o2::framework::runtime_error. Exiting with 1." || { printf "framework error not found" ;  exit 1; }
 printf "ok\nTesting framework-run..."
 o2-framework-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
 printf "ok\nTesting runtime-run..."

From 87f58def427e5736823d9d144b3a961a5e7d5c06 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 7 May 2024 08:50:21 +0200
Subject: [PATCH 498/726] DPL: mark o2-dpl-input-proxy as a proxy

---
 Framework/Utils/src/raw-proxy.cxx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Framework/Utils/src/raw-proxy.cxx b/Framework/Utils/src/raw-proxy.cxx
index e04e9879e1e0c..fe33b4b4c8ab8 100644
--- a/Framework/Utils/src/raw-proxy.cxx
+++ b/Framework/Utils/src/raw-proxy.cxx
@@ -77,6 +77,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
     std::move(readoutProxyOutput),
     "type=pair,method=connect,address=ipc:///tmp/readout-pipe-0,rateLogging=1,transport=shmem",
     dplModelAdaptor(filterSpecs, throwOnUnmatched), minSHM, false, injectMissingData, printSizes);
+  readoutProxy.labels.emplace_back(DataProcessorLabel{"input-proxy"});
 
   WorkflowSpec workflow;
   workflow.emplace_back(readoutProxy);

From ed2bbf4c228f7bcaf5a074408581f80f59b1bb65 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 10 May 2024 11:47:53 +0200
Subject: [PATCH 499/726] DPL: mark input proxies with the correct label

---
 Detectors/CTP/workflowScalers/src/ctp-proxy.cxx           | 1 +
 Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx        | 3 ++-
 Detectors/DCS/testWorkflow/src/dcs-proxy.cxx              | 1 +
 Framework/Core/test/test_ExternalFairMQDeviceProxy.cxx    | 1 +
 Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx | 1 +
 Utilities/Mergers/test/multinodeBenchmarkMergers.cxx      | 1 +
 6 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/Detectors/CTP/workflowScalers/src/ctp-proxy.cxx b/Detectors/CTP/workflowScalers/src/ctp-proxy.cxx
index 3634df6d30bda..1b90eb8c23d9d 100644
--- a/Detectors/CTP/workflowScalers/src/ctp-proxy.cxx
+++ b/Detectors/CTP/workflowScalers/src/ctp-proxy.cxx
@@ -110,6 +110,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
     // this is just default, can be overriden by --ctp-config-proxy '--channel-config..'
     chan.c_str(),
     dcs2dpl(ccdbhost));
+  ctpProxy.labels.emplace_back(DataProcessorLabel{"input-proxy"});
   LOG(info) << "===> Proxy done";
   WorkflowSpec workflow;
   workflow.emplace_back(ctpProxy);
diff --git a/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx b/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx
index 2372ba07a28f7..0a4eeb64cf3af 100644
--- a/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx
+++ b/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx
@@ -47,7 +47,7 @@ InjectorFunction dcs2dpl()
 // InjectorFunction dcs2dpl()
 {
   return [](TimingInfo&, ServiceRegistryRef const& services, fair::mq::Parts& parts, ChannelRetriever channelRetriever, size_t newTimesliceId, bool&) -> bool {
-    auto *device = services.get<RawDeviceService>().device();
+    auto* device = services.get<RawDeviceService>().device();
     std::string messageHeader{static_cast<const char*>(parts.At(0)->GetData()), parts.At(0)->GetSize()};
     size_t dataSize = parts.At(1)->GetSize();
     std::string messageData{static_cast<const char*>(parts.At(1)->GetData()), parts.At(1)->GetSize()};
@@ -135,6 +135,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
     // this is just default, can be overriden by --ctp-config-proxy '--channel-config..'
     chan.c_str(),
     dcs2dpl());
+  ctpProxy.labels.emplace_back(DataProcessorLabel{"input-proxy"});
   LOG(info) << "===> Proxy done";
   WorkflowSpec workflow;
   workflow.emplace_back(ctpProxy);
diff --git a/Detectors/DCS/testWorkflow/src/dcs-proxy.cxx b/Detectors/DCS/testWorkflow/src/dcs-proxy.cxx
index f8f0dba8fd8f6..bfe91a946d13b 100644
--- a/Detectors/DCS/testWorkflow/src/dcs-proxy.cxx
+++ b/Detectors/DCS/testWorkflow/src/dcs-proxy.cxx
@@ -122,6 +122,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
     std::move(dcsOutputs),
     "type=pull,method=connect,address=tcp://aldcsadaposactor:60000,rateLogging=1,transport=zeromq",
     dcs2dpl(dpid2DataDesc, fbiFirst, verbose, repRate));
+  dcsProxy.labels.emplace_back(DataProcessorLabel{"input-proxy"});
 
   WorkflowSpec workflow;
   workflow.emplace_back(dcsProxy);
diff --git a/Framework/Core/test/test_ExternalFairMQDeviceProxy.cxx b/Framework/Core/test/test_ExternalFairMQDeviceProxy.cxx
index cf33da8cd6dd6..4ca725c1551e5 100644
--- a/Framework/Core/test/test_ExternalFairMQDeviceProxy.cxx
+++ b/Framework/Core/test/test_ExternalFairMQDeviceProxy.cxx
@@ -20,6 +20,7 @@ TEST_CASE("ExternalFairMQDeviceProxy")
   InjectorFunction f;
   DataProcessorSpec spec = specifyExternalFairMQDeviceProxy("testSource",
                                                             {}, "type=sub,method=connect,address=tcp://localhost:10000,rateLogging=1", f);
+  spec.labels.emplace_back(DataProcessorLabel{"input-proxy"});
   REQUIRE(spec.name == "testSource");
   REQUIRE(spec.inputs.size() == 0);
   REQUIRE(spec.options.size() == 2);
diff --git a/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx b/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx
index d4d204f206e0a..6c991aba7fff5 100644
--- a/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx
+++ b/Framework/Core/test/test_ExternalFairMQDeviceWorkflow.cxx
@@ -441,6 +441,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
       channelConfig.c_str(),
       o2::framework::dplModelAdaptor(filterSpecs, true)));
   }
+  workflow.back().labels.emplace_back(DataProcessorLabel{"input-proxy"});
 
   return workflow;
 }
diff --git a/Utilities/Mergers/test/multinodeBenchmarkMergers.cxx b/Utilities/Mergers/test/multinodeBenchmarkMergers.cxx
index bc8c5162a6100..319bfacea8aa0 100644
--- a/Utilities/Mergers/test/multinodeBenchmarkMergers.cxx
+++ b/Utilities/Mergers/test/multinodeBenchmarkMergers.cxx
@@ -67,6 +67,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
     {{{"histo"}, {"TST", "HISTO"}}},
     inputChannelConfig.c_str(),
     dplModelAdaptor())));
+  specs.back().labels.emplace_back(DataProcessorLabel{"input-proxy"});
 
   MergerInfrastructureBuilder mergersBuilder;
   mergersBuilder.setInfrastructureName("histos");

From 9d48607fcaaf7a183625175a991158a7210a10f2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 13 May 2024 09:39:09 +0200
Subject: [PATCH 500/726] DPL Analysis: provide metadata to workflow
 construction

---
 .../src/AODJAlienReaderHelpers.cxx            |   6 +-
 Framework/AnalysisSupport/src/Plugin.cxx      | 121 ++++++++++++++++++
 Framework/Core/CMakeLists.txt                 |   1 +
 Framework/Core/include/Framework/Capability.h |  42 ++++++
 .../include/Framework/ConfigParamDiscovery.h  |  37 ++++++
 .../include/Framework/ConfigParamRegistry.h   |   6 +
 .../Core/include/Framework/ConfigParamStore.h |   4 +-
 .../Core/include/Framework/PluginManager.h    |   2 +-
 Framework/Core/include/Framework/Plugins.h    |   9 ++
 .../Core/include/Framework/ServiceSpec.h      |   5 -
 .../include/Framework/runDataProcessing.h     |   8 ++
 Framework/Core/src/ConfigParamDiscovery.cxx   |  81 ++++++++++++
 Framework/Core/src/ConfigParamStore.cxx       |  11 ++
 .../Core/src/WorkflowCustomizationHelpers.cxx |   4 +
 Framework/Core/src/WorkflowHelpers.cxx        |   2 +-
 Framework/TestWorkflows/CMakeLists.txt        |   4 +
 .../TestWorkflows/src/o2AnalysisWorkflow.cxx  |  13 +-
 .../TestWorkflows/src/o2TestHistograms.cxx    |  62 +++++++++
 18 files changed, 404 insertions(+), 14 deletions(-)
 create mode 100644 Framework/Core/include/Framework/Capability.h
 create mode 100644 Framework/Core/include/Framework/ConfigParamDiscovery.h
 create mode 100644 Framework/Core/src/ConfigParamDiscovery.cxx
 create mode 100644 Framework/TestWorkflows/src/o2TestHistograms.cxx

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 092857d585a02..5ce2eca49c07b 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -133,12 +133,12 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_MESSAGES_DESTROYED), DataProcessingStats::Op::Set, 0});
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_BYTES_EXPIRED), DataProcessingStats::Op::Set, 0});
 
-    if (!options.isSet("aod-file")) {
+    if (!options.isSet("aod-file-private")) {
       LOGP(fatal, "No input file defined!");
       throw std::runtime_error("Processing is stopped!");
     }
 
-    auto filename = options.get<std::string>("aod-file");
+    auto filename = options.get<std::string>("aod-file-private");
 
     std::string parentFileReplacement;
     if (options.isSet("aod-parent-base-path-replacement")) {
@@ -306,7 +306,7 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
           control.readyToQuit(QuitRequest::Me);
           return;
         }
-      } 
+      }
     });
   })};
 
diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index 8f2e489086afb..8e358408bad4e 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -12,7 +12,17 @@
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/ServiceSpec.h"
 #include "Framework/ServiceMetricsInfo.h"
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/Capability.h"
+#include "Framework/Signpost.h"
 #include "AODJAlienReaderHelpers.h"
+#include <TFile.h>
+#include <TMap.h>
+#include <TObjString.h>
+#include <TString.h>
+#include <fmt/format.h>
+
+O2_DECLARE_DYNAMIC_LOG(analysis_support);
 
 struct ROOTFileReader : o2::framework::AlgorithmPlugin {
   o2::framework::AlgorithmSpec create() override
@@ -54,7 +64,118 @@ struct RunSummary : o2::framework::ServicePlugin {
   }
 };
 
+auto lookForAodFile = [](ConfigParamRegistry& registry) -> bool {
+  if (registry.hasOption("aod-file") && registry.isSet("aod-file")) {
+    // Do not load the metadata again if we find it already in the registry.
+    // This should avoid initialising ROOT too many times
+    if (registry.hasOption("aod-metadata-run")) {
+      return false;
+    }
+    return true;
+  }
+  return false;
+};
+
+struct DiscoverMetadataInAODCapability : o2::framework::CapabilityPlugin {
+  Capability* create() override
+  {
+    return new Capability{
+      .checkIfNeeded = lookForAodFile,
+      .requiredPlugin = "O2FrameworkAnalysisSupport:DiscoverMetadataInAOD"};
+  }
+};
+
+struct DiscoverMetadataInEnvironmentCapability : o2::framework::CapabilityPlugin {
+  Capability* create() override
+  {
+    return new Capability{
+      .checkIfNeeded = lookForAodFile,
+      .requiredPlugin = "O2FrameworkAnalysisSupport:DiscoverMetadataInEnvironment"};
+  }
+};
+
+struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
+  ConfigDiscovery* create() override
+  {
+    return new ConfigDiscovery{
+      .init = []() {},
+      .discover = [](ConfigParamRegistry& registry) -> std::vector<ConfigParamSpec> {
+        auto filename = registry.get<std::string>("aod-file");
+        if (filename.empty()) {
+          return {};
+        }
+        LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
+        std::vector<ConfigParamSpec> results;
+        TFile* currentFile = nullptr;
+        if (filename.at(0) == '@') {
+          filename.erase(0, 1);
+          // read the text file and set filename to the contents of the first line
+          std::ifstream file(filename);
+          if (!file.is_open()) {
+            LOGP(fatal, "Couldn't open file \"{}\"!", filename);
+          }
+          std::getline(file, filename);
+          file.close();
+          currentFile = TFile::Open(filename.c_str());
+        } else {
+          currentFile = TFile::Open(filename.c_str());
+        }
+        if (!currentFile) {
+          LOGP(fatal, "Couldn't open file \"{}\"!", filename);
+        }
+
+        // Get the metadata, if any
+        auto m = (TMap*)currentFile->Get("metaData");
+        if (!m) {
+          LOGP(warning, "No metadata found in file \"{}\"", filename);
+          return results;
+        }
+        auto it = m->MakeIterator();
+
+        // Serialise metadata into a ; separated string with : separating key and value
+        bool first = true;
+        while (auto obj = it->Next()) {
+          if (first) {
+            LOGP(info, "Metadata for file \"{}\":", filename);
+            first = false;
+          }
+          auto objString = (TObjString*)m->GetValue(obj);
+          LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
+          std::string key = "aod-metadata-" + std::string(obj->GetName());
+          char const* value = strdup(objString->String());
+          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
+        }
+        return results;
+      }};
+  }
+};
+
+struct DiscoverMetadataInEnvironment : o2::framework::ConfigDiscoveryPlugin {
+  ConfigDiscovery* create() override
+  {
+    return new ConfigDiscovery{
+      .init = []() {},
+      .discover = [](ConfigParamRegistry& registry) -> std::vector<ConfigParamSpec> {
+        O2_SIGNPOST_ID_GENERATE(sid, analysis_support);
+        O2_SIGNPOST_EVENT_EMIT(analysis_support, sid, "DiscoverMetadataInEnvironment",
+                               "Discovering metadata for analysis from well known environment variables.");
+        std::vector<ConfigParamSpec> results;
+        char* runType = getenv("DPL_ANALYSIS_RUN_TYPE");
+        if (runType) {
+          O2_SIGNPOST_EVENT_EMIT(analysis_support, sid, "DiscoverMetadataInEnvironment",
+                                 "Found DPL_ANALYSIS_RUN_TYPE with value %{public}s.", runType);
+          results.push_back(ConfigParamSpec{"aod-metadata-run-type", VariantType::String, runType, {"Metadata in environment"}});
+        }
+        return results;
+      }};
+  }
+};
+
 DEFINE_DPL_PLUGINS_BEGIN
 DEFINE_DPL_PLUGIN_INSTANCE(ROOTFileReader, CustomAlgorithm);
 DEFINE_DPL_PLUGIN_INSTANCE(RunSummary, CustomService);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAODCapability, Capability);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAOD, ConfigDiscovery);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInEnvironmentCapability, Capability);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInEnvironment, ConfigDiscovery);
 DEFINE_DPL_PLUGINS_END
diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index d392ba42fe8ac..723fa2c081a9a 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -47,6 +47,7 @@ o2_add_library(Framework
                        src/DefaultsHelpers.cxx
                        src/DomainInfoHeader.cxx
                        src/ProcessingPoliciesHelpers.cxx
+                       src/ConfigParamDiscovery.cxx
                        src/ConfigParamStore.cxx
                        src/ConfigParamsHelper.cxx
                        src/ChannelParamSpec.cxx
diff --git a/Framework/Core/include/Framework/Capability.h b/Framework/Core/include/Framework/Capability.h
new file mode 100644
index 0000000000000..f3ec6f97bf278
--- /dev/null
+++ b/Framework/Core/include/Framework/Capability.h
@@ -0,0 +1,42 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_CAPABILITY_H_
+#define O2_FRAMEWORK_CAPABILITY_H_
+
+#include <string_view>
+#include <functional>
+
+namespace o2::framework
+{
+
+struct ConfigParamRegistry;
+
+/// A plugin which is able to discover more option then the ones
+/// provided on the command line. The actual loading is in two parts,
+/// the first one decides if the options are enough to actually perform
+/// the discovery, the second part will do the discovery itself.
+///
+/// Its a good idea to have the Loader part in a standalone library to
+/// minimize dependency on unneed thir party code, e.g. ROOT.
+struct Capability {
+  // Wether or not this capability is required.
+  std::function<bool(ConfigParamRegistry&)> checkIfNeeded;
+  char const* requiredPlugin;
+};
+
+struct CapabilityPlugin {
+  virtual Capability* create() = 0;
+};
+
+} // namespace o2::framework
+
+#endif // O2_FRAMEWORK_CAPABILITY_H_
diff --git a/Framework/Core/include/Framework/ConfigParamDiscovery.h b/Framework/Core/include/Framework/ConfigParamDiscovery.h
new file mode 100644
index 0000000000000..5cded1a2eb284
--- /dev/null
+++ b/Framework/Core/include/Framework/ConfigParamDiscovery.h
@@ -0,0 +1,37 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
+#define O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
+
+#include "Framework/ConfigParamRegistry.h"
+
+namespace o2::framework
+{
+// A plugin which can be used to inject extra configuration
+// options.
+struct ConfigParamDiscovery {
+  static std::vector<ConfigParamSpec> discover(ConfigParamRegistry&);
+};
+
+struct ConfigDiscovery {
+  std::function<void()> init;
+  /// A function which uses the arguments available so far to discover more
+  /// @return the extra ConfigParamSpecs it derives.
+  std::function<std::vector<ConfigParamSpec>(ConfigParamRegistry&)> discover;
+};
+
+struct ConfigDiscoveryPlugin {
+  virtual ConfigDiscovery* create() = 0;
+};
+
+} // namespace o2::framework
+#endif // O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
diff --git a/Framework/Core/include/Framework/ConfigParamRegistry.h b/Framework/Core/include/Framework/ConfigParamRegistry.h
index 8f2d80cea0948..397a491be9f9c 100644
--- a/Framework/Core/include/Framework/ConfigParamRegistry.h
+++ b/Framework/Core/include/Framework/ConfigParamRegistry.h
@@ -125,6 +125,12 @@ class ConfigParamRegistry
     }
   }
 
+  // Load extra parameters discovered while we process data
+  void loadExtra(std::vector<ConfigParamSpec>& extras)
+  {
+    mStore->load(extras);
+  }
+
  private:
   std::unique_ptr<ConfigParamStore> mStore;
 };
diff --git a/Framework/Core/include/Framework/ConfigParamStore.h b/Framework/Core/include/Framework/ConfigParamStore.h
index 37442a6cf7f41..b3ae76aa47ad1 100644
--- a/Framework/Core/include/Framework/ConfigParamStore.h
+++ b/Framework/Core/include/Framework/ConfigParamStore.h
@@ -34,12 +34,14 @@ class ConfigParamStore
   /// the configuration.
   void preload();
 
+  void load(std::vector<ConfigParamSpec>& specs);
+
   /// Get the store
   boost::property_tree::ptree& store() { return *mStore; };
   boost::property_tree::ptree& provenanceTree() { return *mProvenance; };
 
   /// Get the specs
-  std::vector<ConfigParamSpec> const& specs() const
+  [[nodiscard]] std::vector<ConfigParamSpec> const& specs() const
   {
     return mSpecs;
   }
diff --git a/Framework/Core/include/Framework/PluginManager.h b/Framework/Core/include/Framework/PluginManager.h
index 38966c5a969a9..4c6e965502500 100644
--- a/Framework/Core/include/Framework/PluginManager.h
+++ b/Framework/Core/include/Framework/PluginManager.h
@@ -70,7 +70,7 @@ struct PluginManager {
 
     struct LoadedPlugin {
       std::string name;
-      ServicePlugin* factory;
+      PLUGIN* factory;
     };
     std::vector<LoadedDSO> loadedDSOs;
     std::vector<LoadedPlugin> loadedPlugins;
diff --git a/Framework/Core/include/Framework/Plugins.h b/Framework/Core/include/Framework/Plugins.h
index c29d96f7e0781..23d55a512e1fa 100644
--- a/Framework/Core/include/Framework/Plugins.h
+++ b/Framework/Core/include/Framework/Plugins.h
@@ -27,6 +27,15 @@ enum struct DplPluginKind : int {
   // A plugin which implements a custom Services. Needs to return
   // an object of the kind o2::framework::ServiceSpec
   CustomService,
+  // A plugin which implements a new way to discover extra configuration.
+  // parameters. E.g. it can be used to read metadata from a file or a service
+  // if a certain parameter is available.
+  ConfigDiscovery,
+  // A capability plugin is a plugin used to discover other viable plugins.
+  // For example, if you find out that you have the --aod-file option
+  // set, you might want to load metadata from it and attach it to the
+  // configuration.
+  Capability,
   // A plugin which was not initialised properly.
   Unknown
 };
diff --git a/Framework/Core/include/Framework/ServiceSpec.h b/Framework/Core/include/Framework/ServiceSpec.h
index abac3ad05914c..5684889e85376 100644
--- a/Framework/Core/include/Framework/ServiceSpec.h
+++ b/Framework/Core/include/Framework/ServiceSpec.h
@@ -322,11 +322,6 @@ struct ServicePlugin {
   virtual ServiceSpec* create() = 0;
 };
 
-struct LoadableService {
-  std::string name;
-  std::string library;
-};
-
 } // namespace o2::framework
 
 #endif // O2_FRAMEWORK_SERVICESPEC_H_
diff --git a/Framework/Core/include/Framework/runDataProcessing.h b/Framework/Core/include/Framework/runDataProcessing.h
index 77a4c9f5ba8d6..ec8098e7aca0e 100644
--- a/Framework/Core/include/Framework/runDataProcessing.h
+++ b/Framework/Core/include/Framework/runDataProcessing.h
@@ -29,6 +29,7 @@
 #include "Framework/Logger.h"
 #include "Framework/CheckTypes.h"
 #include "Framework/StructToTuple.h"
+#include "Framework/ConfigParamDiscovery.h"
 
 #include <vector>
 
@@ -192,6 +193,13 @@ int mainNoCatch(int argc, char** argv)
   workflowOptionsStore->preload();
   workflowOptionsStore->activate();
   ConfigParamRegistry workflowOptionsRegistry(std::move(workflowOptionsStore));
+
+  auto extraOptions = o2::framework::ConfigParamDiscovery::discover(workflowOptionsRegistry);
+  for (auto& extra : extraOptions) {
+    workflowOptions.push_back(extra);
+  }
+  workflowOptionsRegistry.loadExtra(extraOptions);
+
   ConfigContext configContext(workflowOptionsRegistry, argc, argv);
   o2::framework::WorkflowSpec specs = defineDataProcessing(configContext);
   overrideCloning(configContext, specs);
diff --git a/Framework/Core/src/ConfigParamDiscovery.cxx b/Framework/Core/src/ConfigParamDiscovery.cxx
new file mode 100644
index 0000000000000..d68e04013c20f
--- /dev/null
+++ b/Framework/Core/src/ConfigParamDiscovery.cxx
@@ -0,0 +1,81 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/PluginManager.h"
+#include "Framework/Signpost.h"
+#include "Framework/Capability.h"
+#include <TFile.h>
+#include <TMap.h>
+#include <TObjString.h>
+
+O2_DECLARE_DYNAMIC_LOG(config_parameter_discovery);
+
+namespace o2::framework
+{
+
+std::vector<ConfigParamSpec> ConfigParamDiscovery::discover(ConfigParamRegistry& registry)
+{
+  std::vector<char const*> capabilitiesSpecs = {
+    "O2FrameworkAnalysisSupport:DiscoverMetadataInAODCapability",
+    "O2FrameworkAnalysisSupport:DiscoverMetadataInEnvironmentCapability",
+  };
+
+  // Load all the requested plugins and discover what we can do.
+  O2_SIGNPOST_ID_GENERATE(sid, config_parameter_discovery);
+  std::vector<LoadablePlugin> plugins;
+  for (auto spec : capabilitiesSpecs) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "Parsing Capability plugin: %{public}s", spec);
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+  }
+
+  std::vector<Capability> availableCapabilities;
+  std::vector<char const*> configDiscoverySpec = {};
+  PluginManager::loadFromPlugin<Capability, CapabilityPlugin>(plugins, availableCapabilities);
+  for (auto& capability : availableCapabilities) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- Using Capability of concrete type: %{public}s",
+                           typeid(&capability).name());
+
+    if (capability.checkIfNeeded(registry)) {
+      O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- Required: %{}s",
+                             capability.requiredPlugin);
+      configDiscoverySpec.push_back(capability.requiredPlugin);
+    } else {
+      O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- No requirements added");
+    }
+  }
+
+  std::vector<LoadablePlugin> configDiscoveryPlugins;
+  for (auto spec : configDiscoverySpec) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "discovery", "Parsing ConfigDiscovery plugin: %{public}s", spec);
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      configDiscoveryPlugins.push_back(extra);
+    }
+  }
+
+  std::vector<ConfigDiscovery> availableConfigDiscovery;
+  PluginManager::loadFromPlugin<ConfigDiscovery, ConfigDiscoveryPlugin>(configDiscoveryPlugins, availableConfigDiscovery);
+  std::vector<ConfigParamSpec> result;
+  for (auto& configDiscovery : availableConfigDiscovery) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "discovery", "Parsing extra configuration as needed");
+    auto extras = configDiscovery.discover(registry);
+    for (auto& extra : extras) {
+      result.push_back(extra);
+    }
+  }
+  return result;
+}
+
+} // namespace o2::framework
diff --git a/Framework/Core/src/ConfigParamStore.cxx b/Framework/Core/src/ConfigParamStore.cxx
index 249a8b0052a67..19e123a23e5d0 100644
--- a/Framework/Core/src/ConfigParamStore.cxx
+++ b/Framework/Core/src/ConfigParamStore.cxx
@@ -41,6 +41,17 @@ void ConfigParamStore::preload()
   }
 }
 
+void ConfigParamStore::load(std::vector<ConfigParamSpec>& specs)
+{
+  mNextStore->clear();
+  mNextProvenance->clear();
+  // By default we populate with code.
+  PropertyTreeHelpers::populateDefaults(specs, *mNextStore, *mNextProvenance);
+  for (auto& retriever : mRetrievers) {
+    retriever->update(specs, *mStore, *mProvenance);
+  }
+}
+
 /// Activate the next store
 void ConfigParamStore::activate()
 {
diff --git a/Framework/Core/src/WorkflowCustomizationHelpers.cxx b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
index ffad8a8720019..05abb5dab98cd 100644
--- a/Framework/Core/src/WorkflowCustomizationHelpers.cxx
+++ b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
@@ -54,6 +54,10 @@ std::vector<ConfigParamSpec> WorkflowCustomizationHelpers::requiredWorkflowOptio
            // options for AOD rate limiting
            {"aod-memory-rate-limit", VariantType::Int64, 0LL, {"Rate limit AOD processing based on memory"}},
 
+           // options for the AOD reader
+           // aod-file needs to be available as workflow option, because we
+           // can configure the workflow based on the contents of the first file.
+           {"aod-file", VariantType::String, "", {"Input AOD file"}},
            // options for AOD writer
            {"aod-writer-json", VariantType::String, "", {"Name of the json configuration file"}},
            {"aod-writer-resdir", VariantType::String, "", {"Name of the output directory"}},
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index bd875ad421f6e..da60bdbc6961e 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -292,7 +292,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
                          static_cast<DataAllocator::SubSpecificationType>(compile_time_hash("internal-dpl-aod-reader")),
                          aodLifetime}},
     .algorithm = AlgorithmSpec::dummyAlgorithm(),
-    .options = {ConfigParamSpec{"aod-file", VariantType::String, {"Input AOD file"}},
+    .options = {ConfigParamSpec{"aod-file-private", VariantType::String, ctx.options().get<std::string>("aod-file"), {"AOD file"}},
                 ConfigParamSpec{"aod-reader-json", VariantType::String, {"json configuration file"}},
                 ConfigParamSpec{"aod-parent-access-level", VariantType::String, {"Allow parent file access up to specified level. Default: no (0)"}},
                 ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}},
diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index bfbdcfc6f169b..21af45422e0b1 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -33,6 +33,10 @@ o2_add_dpl_workflow(diamond-workflow
                   SOURCES src/o2DiamondWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
 
+o2_add_dpl_workflow(analysis-histograms
+                  SOURCES src/o2TestHistograms.cxx
+                  COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(two-timers
                   SOURCES src/o2TwoTimers.cxx
                   COMPONENT_NAME TestWorkflows)
diff --git a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
index 05bf3aabaceab..43d71a9b40528 100644
--- a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
+++ b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
@@ -49,7 +49,14 @@ struct EtaAndClsHistograms {
 
 WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
 {
-  return WorkflowSpec{
-    adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
-  };
+  // For the sake of running without an option, we do not throw an exception
+  // in case the option is not present.
+  if (cfgc.options().hasOption("aod-metadata-Run") == false ||
+      cfgc.options().get<std::string>("aod-metadata-Run") == "2") {
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
+    };
+  } else {
+    throw std::runtime_error("Unsupported run type");
+  }
 }
diff --git a/Framework/TestWorkflows/src/o2TestHistograms.cxx b/Framework/TestWorkflows/src/o2TestHistograms.cxx
new file mode 100644
index 0000000000000..331ee828e1c7b
--- /dev/null
+++ b/Framework/TestWorkflows/src/o2TestHistograms.cxx
@@ -0,0 +1,62 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+///
+/// \brief FullTracks is a join of Tracks, TracksCov, and TracksExtra.
+/// \author
+/// \since
+
+#include "Framework/runDataProcessing.h"
+#include "Framework/AnalysisTask.h"
+#include <TH2F.h>
+#include <cmath>
+
+using namespace o2;
+using namespace o2::framework;
+using namespace o2::framework::expressions;
+
+struct EtaAndClsHistograms {
+  OutputObj<TH3F> etaClsH{TH3F("eta_vs_cls_vs_sigmapT", "#eta vs N_{cls} vs sigma_{1/pT}", 102, -2.01, 2.01, 160, -0.5, 159.5, 100, 0, 10)};
+
+  void process(soa::Join<aod::FullTracks, aod::TracksCov> const& tracks)
+  {
+    for (auto& track : tracks) {
+      etaClsH->Fill(track.eta(), track.tpcNClsFindable(), track.sigma1Pt());
+    }
+  }
+};
+
+struct EtaAndClsHistogramsSimple {
+  OutputObj<TH2F> etaClsH{TH2F("eta_vs_pt", "#eta vs pT", 102, -2.01, 2.01, 100, 0, 10)};
+
+  void process(aod::Tracks const& tracks)
+  {
+    for (auto& track : tracks) {
+      etaClsH->Fill(track.eta(), track.pt(), 0);
+    }
+  }
+};
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  std::string runType = "3";
+  if (cfgc.options().hasOption("aod-metadata-Run")) {
+    runType = cfgc.options().get<std::string>("aod-metadata-Run");
+  }
+  if (runType == "2") {
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistogramsSimple>(cfgc),
+    };
+  } else {
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
+    };
+  }
+}

From 96f9f2ab392578b32556a70c9f6e09ee75933f6b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 13 May 2024 09:39:09 +0200
Subject: [PATCH 501/726] DPL Analysis: provide metadata to workflow
 construction

---
 .../src/AODJAlienReaderHelpers.cxx            |   6 +-
 Framework/AnalysisSupport/src/Plugin.cxx      | 142 ++++++++++++++++++
 Framework/Core/CMakeLists.txt                 |   1 +
 Framework/Core/include/Framework/Capability.h |  44 ++++++
 .../include/Framework/ConfigParamDiscovery.h  |  37 +++++
 .../include/Framework/ConfigParamRegistry.h   |   6 +
 .../Core/include/Framework/ConfigParamStore.h |   4 +-
 .../Core/include/Framework/PluginManager.h    |   2 +-
 Framework/Core/include/Framework/Plugins.h    |   9 ++
 .../Core/include/Framework/ServiceSpec.h      |   5 -
 .../include/Framework/runDataProcessing.h     |   7 +
 Framework/Core/src/ConfigParamDiscovery.cxx   |  81 ++++++++++
 Framework/Core/src/ConfigParamStore.cxx       |  11 ++
 .../Core/src/WorkflowCustomizationHelpers.cxx |   4 +
 Framework/Core/src/WorkflowHelpers.cxx        |   2 +-
 Framework/TestWorkflows/CMakeLists.txt        |   4 +
 .../TestWorkflows/src/o2AnalysisWorkflow.cxx  |  13 +-
 .../TestWorkflows/src/o2TestHistograms.cxx    |  62 ++++++++
 18 files changed, 426 insertions(+), 14 deletions(-)
 create mode 100644 Framework/Core/include/Framework/Capability.h
 create mode 100644 Framework/Core/include/Framework/ConfigParamDiscovery.h
 create mode 100644 Framework/Core/src/ConfigParamDiscovery.cxx
 create mode 100644 Framework/TestWorkflows/src/o2TestHistograms.cxx

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 092857d585a02..5ce2eca49c07b 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -133,12 +133,12 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_MESSAGES_DESTROYED), DataProcessingStats::Op::Set, 0});
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_BYTES_EXPIRED), DataProcessingStats::Op::Set, 0});
 
-    if (!options.isSet("aod-file")) {
+    if (!options.isSet("aod-file-private")) {
       LOGP(fatal, "No input file defined!");
       throw std::runtime_error("Processing is stopped!");
     }
 
-    auto filename = options.get<std::string>("aod-file");
+    auto filename = options.get<std::string>("aod-file-private");
 
     std::string parentFileReplacement;
     if (options.isSet("aod-parent-base-path-replacement")) {
@@ -306,7 +306,7 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
           control.readyToQuit(QuitRequest::Me);
           return;
         }
-      } 
+      }
     });
   })};
 
diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index 8f2e489086afb..8dbba8271a6b9 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -12,7 +12,17 @@
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/ServiceSpec.h"
 #include "Framework/ServiceMetricsInfo.h"
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/Capability.h"
+#include "Framework/Signpost.h"
 #include "AODJAlienReaderHelpers.h"
+#include <TFile.h>
+#include <TMap.h>
+#include <TObjString.h>
+#include <TString.h>
+#include <fmt/format.h>
+
+O2_DECLARE_DYNAMIC_LOG(analysis_support);
 
 struct ROOTFileReader : o2::framework::AlgorithmPlugin {
   o2::framework::AlgorithmSpec create() override
@@ -54,7 +64,139 @@ struct RunSummary : o2::framework::ServicePlugin {
   }
 };
 
+auto lookForAodFile = [](ConfigParamRegistry& registry, int argc, char** argv) -> bool {
+  if (registry.hasOption("aod-file") && registry.isSet("aod-file")) {
+    for (size_t i = 0; i < argc; i++) {
+      std::string_view arg = argv[i];
+      if (arg.starts_with("--aod-metadata-")) {
+        return false;
+      }
+    }
+    LOGP(info, "Will read metadata from AOD file.");
+    return true;
+  }
+  return false;
+};
+
+auto lookForCommandLineOptions = [](ConfigParamRegistry& registry, int argc, char** argv) -> bool {
+  for (size_t i = 0; i < argc; i++) {
+    std::string_view arg = argv[i];
+    if (arg.starts_with("--aod-metadata-")) {
+      LOGP(info, "Metadata found in arguments. Populating from them.");
+      return true;
+    }
+  }
+  return false;
+};
+
+struct DiscoverMetadataInAODCapability : o2::framework::CapabilityPlugin {
+  Capability* create() override
+  {
+    return new Capability{
+      .name = "DiscoverMetadataInAODCapability",
+      .checkIfNeeded = lookForAodFile,
+      .requiredPlugin = "O2FrameworkAnalysisSupport:DiscoverMetadataInAOD"};
+  }
+};
+
+// Trigger discovery of metadata from command line, if needed.
+struct DiscoverMetadataInCommandLineCapability : o2::framework::CapabilityPlugin {
+  Capability* create() override
+  {
+    return new Capability{
+      .name = "DiscoverMetadataInCommandLineCapability",
+      .checkIfNeeded = lookForCommandLineOptions,
+      .requiredPlugin = "O2FrameworkAnalysisSupport:DiscoverMetadataInCommandLine"};
+  }
+};
+
+struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
+  ConfigDiscovery* create() override
+  {
+    return new ConfigDiscovery{
+      .init = []() {},
+      .discover = [](ConfigParamRegistry& registry, int argc, char** argv) -> std::vector<ConfigParamSpec> {
+        auto filename = registry.get<std::string>("aod-file");
+        if (filename.empty()) {
+          return {};
+        }
+        LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
+        std::vector<ConfigParamSpec> results;
+        TFile* currentFile = nullptr;
+        if (filename.at(0) == '@') {
+          filename.erase(0, 1);
+          // read the text file and set filename to the contents of the first line
+          std::ifstream file(filename);
+          if (!file.is_open()) {
+            LOGP(fatal, "Couldn't open file \"{}\"!", filename);
+          }
+          std::getline(file, filename);
+          file.close();
+          currentFile = TFile::Open(filename.c_str());
+        } else {
+          currentFile = TFile::Open(filename.c_str());
+        }
+        if (!currentFile) {
+          LOGP(fatal, "Couldn't open file \"{}\"!", filename);
+        }
+
+        // Get the metadata, if any
+        auto m = (TMap*)currentFile->Get("metaData");
+        if (!m) {
+          LOGP(warning, "No metadata found in file \"{}\"", filename);
+          return results;
+        }
+        auto it = m->MakeIterator();
+
+        // Serialise metadata into a ; separated string with : separating key and value
+        bool first = true;
+        while (auto obj = it->Next()) {
+          if (first) {
+            LOGP(info, "Metadata for file \"{}\":", filename);
+            first = false;
+          }
+          auto objString = (TObjString*)m->GetValue(obj);
+          LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
+          std::string key = "aod-metadata-" + std::string(obj->GetName());
+          char const* value = strdup(objString->String());
+          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
+        }
+        return results;
+      }};
+  }
+};
+
+struct DiscoverMetadataInCommandLine : o2::framework::ConfigDiscoveryPlugin {
+  ConfigDiscovery* create() override
+  {
+    return new ConfigDiscovery{
+      .init = []() {},
+      .discover = [](ConfigParamRegistry& registry, int argc, char** argv) -> std::vector<ConfigParamSpec> {
+        O2_SIGNPOST_ID_GENERATE(sid, analysis_support);
+        O2_SIGNPOST_EVENT_EMIT(analysis_support, sid, "DiscoverMetadataInCommandLine",
+                               "Discovering metadata for analysis from well known environment variables.");
+        std::vector<ConfigParamSpec> results;
+        for (size_t i = 0; i < argc; i++) {
+          std::string_view arg = argv[i];
+          if (!arg.starts_with("--aod-metadata")) {
+            continue;
+          }
+          std::string key = arg.data() + 2;
+          std::string value = argv[i + 1];
+          O2_SIGNPOST_EVENT_EMIT(analysis_support, sid, "DiscoverMetadataInCommandLine",
+                                 "Found %{public}s with value %{public}s.", key.c_str(), value.c_str());
+          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in command line"}});
+        }
+        return results;
+      }};
+  }
+};
+
 DEFINE_DPL_PLUGINS_BEGIN
 DEFINE_DPL_PLUGIN_INSTANCE(ROOTFileReader, CustomAlgorithm);
 DEFINE_DPL_PLUGIN_INSTANCE(RunSummary, CustomService);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAODCapability, Capability);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAOD, ConfigDiscovery);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLineCapability, Capability);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLine, ConfigDiscovery);
 DEFINE_DPL_PLUGINS_END
diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index d392ba42fe8ac..723fa2c081a9a 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -47,6 +47,7 @@ o2_add_library(Framework
                        src/DefaultsHelpers.cxx
                        src/DomainInfoHeader.cxx
                        src/ProcessingPoliciesHelpers.cxx
+                       src/ConfigParamDiscovery.cxx
                        src/ConfigParamStore.cxx
                        src/ConfigParamsHelper.cxx
                        src/ChannelParamSpec.cxx
diff --git a/Framework/Core/include/Framework/Capability.h b/Framework/Core/include/Framework/Capability.h
new file mode 100644
index 0000000000000..23ef7b7d9bf17
--- /dev/null
+++ b/Framework/Core/include/Framework/Capability.h
@@ -0,0 +1,44 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_CAPABILITY_H_
+#define O2_FRAMEWORK_CAPABILITY_H_
+
+#include <string_view>
+#include <functional>
+
+namespace o2::framework
+{
+
+struct ConfigParamRegistry;
+
+/// A plugin which is able to discover more option then the ones
+/// provided on the command line. The actual loading is in two parts,
+/// the first one decides if the options are enough to actually perform
+/// the discovery, the second part will do the discovery itself.
+///
+/// Its a good idea to have the Loader part in a standalone library to
+/// minimize dependency on unneed thir party code, e.g. ROOT.
+struct Capability {
+  // Wether or not this capability is required.
+  // FIXME: for now let's pass the arguments to get the metadata to work
+  std::string name;
+  std::function<bool(ConfigParamRegistry&, int argc, char** argv)> checkIfNeeded;
+  char const* requiredPlugin;
+};
+
+struct CapabilityPlugin {
+  virtual Capability* create() = 0;
+};
+
+} // namespace o2::framework
+
+#endif // O2_FRAMEWORK_CAPABILITY_H_
diff --git a/Framework/Core/include/Framework/ConfigParamDiscovery.h b/Framework/Core/include/Framework/ConfigParamDiscovery.h
new file mode 100644
index 0000000000000..3d13d3662ceb1
--- /dev/null
+++ b/Framework/Core/include/Framework/ConfigParamDiscovery.h
@@ -0,0 +1,37 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
+#define O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
+
+#include "Framework/ConfigParamRegistry.h"
+
+namespace o2::framework
+{
+// A plugin which can be used to inject extra configuration
+// options.
+struct ConfigParamDiscovery {
+  static std::vector<ConfigParamSpec> discover(ConfigParamRegistry&, int, char**);
+};
+
+struct ConfigDiscovery {
+  std::function<void()> init;
+  /// A function which uses the arguments available so far to discover more
+  /// @return the extra ConfigParamSpecs it derives.
+  std::function<std::vector<ConfigParamSpec>(ConfigParamRegistry&, int, char**)> discover;
+};
+
+struct ConfigDiscoveryPlugin {
+  virtual ConfigDiscovery* create() = 0;
+};
+
+} // namespace o2::framework
+#endif // O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
diff --git a/Framework/Core/include/Framework/ConfigParamRegistry.h b/Framework/Core/include/Framework/ConfigParamRegistry.h
index 8f2d80cea0948..397a491be9f9c 100644
--- a/Framework/Core/include/Framework/ConfigParamRegistry.h
+++ b/Framework/Core/include/Framework/ConfigParamRegistry.h
@@ -125,6 +125,12 @@ class ConfigParamRegistry
     }
   }
 
+  // Load extra parameters discovered while we process data
+  void loadExtra(std::vector<ConfigParamSpec>& extras)
+  {
+    mStore->load(extras);
+  }
+
  private:
   std::unique_ptr<ConfigParamStore> mStore;
 };
diff --git a/Framework/Core/include/Framework/ConfigParamStore.h b/Framework/Core/include/Framework/ConfigParamStore.h
index 37442a6cf7f41..b3ae76aa47ad1 100644
--- a/Framework/Core/include/Framework/ConfigParamStore.h
+++ b/Framework/Core/include/Framework/ConfigParamStore.h
@@ -34,12 +34,14 @@ class ConfigParamStore
   /// the configuration.
   void preload();
 
+  void load(std::vector<ConfigParamSpec>& specs);
+
   /// Get the store
   boost::property_tree::ptree& store() { return *mStore; };
   boost::property_tree::ptree& provenanceTree() { return *mProvenance; };
 
   /// Get the specs
-  std::vector<ConfigParamSpec> const& specs() const
+  [[nodiscard]] std::vector<ConfigParamSpec> const& specs() const
   {
     return mSpecs;
   }
diff --git a/Framework/Core/include/Framework/PluginManager.h b/Framework/Core/include/Framework/PluginManager.h
index 38966c5a969a9..4c6e965502500 100644
--- a/Framework/Core/include/Framework/PluginManager.h
+++ b/Framework/Core/include/Framework/PluginManager.h
@@ -70,7 +70,7 @@ struct PluginManager {
 
     struct LoadedPlugin {
       std::string name;
-      ServicePlugin* factory;
+      PLUGIN* factory;
     };
     std::vector<LoadedDSO> loadedDSOs;
     std::vector<LoadedPlugin> loadedPlugins;
diff --git a/Framework/Core/include/Framework/Plugins.h b/Framework/Core/include/Framework/Plugins.h
index c29d96f7e0781..23d55a512e1fa 100644
--- a/Framework/Core/include/Framework/Plugins.h
+++ b/Framework/Core/include/Framework/Plugins.h
@@ -27,6 +27,15 @@ enum struct DplPluginKind : int {
   // A plugin which implements a custom Services. Needs to return
   // an object of the kind o2::framework::ServiceSpec
   CustomService,
+  // A plugin which implements a new way to discover extra configuration.
+  // parameters. E.g. it can be used to read metadata from a file or a service
+  // if a certain parameter is available.
+  ConfigDiscovery,
+  // A capability plugin is a plugin used to discover other viable plugins.
+  // For example, if you find out that you have the --aod-file option
+  // set, you might want to load metadata from it and attach it to the
+  // configuration.
+  Capability,
   // A plugin which was not initialised properly.
   Unknown
 };
diff --git a/Framework/Core/include/Framework/ServiceSpec.h b/Framework/Core/include/Framework/ServiceSpec.h
index abac3ad05914c..5684889e85376 100644
--- a/Framework/Core/include/Framework/ServiceSpec.h
+++ b/Framework/Core/include/Framework/ServiceSpec.h
@@ -322,11 +322,6 @@ struct ServicePlugin {
   virtual ServiceSpec* create() = 0;
 };
 
-struct LoadableService {
-  std::string name;
-  std::string library;
-};
-
 } // namespace o2::framework
 
 #endif // O2_FRAMEWORK_SERVICESPEC_H_
diff --git a/Framework/Core/include/Framework/runDataProcessing.h b/Framework/Core/include/Framework/runDataProcessing.h
index 77a4c9f5ba8d6..9ab221af90267 100644
--- a/Framework/Core/include/Framework/runDataProcessing.h
+++ b/Framework/Core/include/Framework/runDataProcessing.h
@@ -29,6 +29,7 @@
 #include "Framework/Logger.h"
 #include "Framework/CheckTypes.h"
 #include "Framework/StructToTuple.h"
+#include "Framework/ConfigParamDiscovery.h"
 
 #include <vector>
 
@@ -192,6 +193,12 @@ int mainNoCatch(int argc, char** argv)
   workflowOptionsStore->preload();
   workflowOptionsStore->activate();
   ConfigParamRegistry workflowOptionsRegistry(std::move(workflowOptionsStore));
+  auto extraOptions = o2::framework::ConfigParamDiscovery::discover(workflowOptionsRegistry, argc, argv);
+  for (auto& extra : extraOptions) {
+    workflowOptions.push_back(extra);
+  }
+  workflowOptionsRegistry.loadExtra(extraOptions);
+
   ConfigContext configContext(workflowOptionsRegistry, argc, argv);
   o2::framework::WorkflowSpec specs = defineDataProcessing(configContext);
   overrideCloning(configContext, specs);
diff --git a/Framework/Core/src/ConfigParamDiscovery.cxx b/Framework/Core/src/ConfigParamDiscovery.cxx
new file mode 100644
index 0000000000000..aef1329417f18
--- /dev/null
+++ b/Framework/Core/src/ConfigParamDiscovery.cxx
@@ -0,0 +1,81 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/PluginManager.h"
+#include "Framework/Signpost.h"
+#include "Framework/Capability.h"
+#include <TFile.h>
+#include <TMap.h>
+#include <TObjString.h>
+
+O2_DECLARE_DYNAMIC_LOG(config_parameter_discovery);
+
+namespace o2::framework
+{
+
+std::vector<ConfigParamSpec> ConfigParamDiscovery::discover(ConfigParamRegistry& registry, int argc, char** argv)
+{
+  std::vector<char const*> capabilitiesSpecs = {
+    "O2FrameworkAnalysisSupport:DiscoverMetadataInAODCapability",
+    "O2FrameworkAnalysisSupport:DiscoverMetadataInCommandLineCapability",
+  };
+
+  // Load all the requested plugins and discover what we can do.
+  O2_SIGNPOST_ID_GENERATE(sid, config_parameter_discovery);
+  std::vector<LoadablePlugin> plugins;
+  for (auto spec : capabilitiesSpecs) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "Parsing Capability plugin: %{public}s", spec);
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+  }
+
+  std::vector<Capability> availableCapabilities;
+  std::vector<char const*> configDiscoverySpec = {};
+  PluginManager::loadFromPlugin<Capability, CapabilityPlugin>(plugins, availableCapabilities);
+  for (auto& capability : availableCapabilities) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- Using Capability of : %{public}s",
+                           capability.name.c_str());
+
+    if (capability.checkIfNeeded(registry, argc, argv)) {
+      O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- Required: %{}s",
+                             capability.requiredPlugin);
+      configDiscoverySpec.push_back(capability.requiredPlugin);
+    } else {
+      O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- No requirements added");
+    }
+  }
+
+  std::vector<LoadablePlugin> configDiscoveryPlugins;
+  for (auto spec : configDiscoverySpec) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "discovery", "Parsing ConfigDiscovery plugin: %{public}s", spec);
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      configDiscoveryPlugins.push_back(extra);
+    }
+  }
+
+  std::vector<ConfigDiscovery> availableConfigDiscovery;
+  PluginManager::loadFromPlugin<ConfigDiscovery, ConfigDiscoveryPlugin>(configDiscoveryPlugins, availableConfigDiscovery);
+  std::vector<ConfigParamSpec> result;
+  for (auto& configDiscovery : availableConfigDiscovery) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "discovery", "Parsing extra configuration as needed");
+    auto extras = configDiscovery.discover(registry, argc, argv);
+    for (auto& extra : extras) {
+      result.push_back(extra);
+    }
+  }
+  return result;
+}
+
+} // namespace o2::framework
diff --git a/Framework/Core/src/ConfigParamStore.cxx b/Framework/Core/src/ConfigParamStore.cxx
index 249a8b0052a67..19e123a23e5d0 100644
--- a/Framework/Core/src/ConfigParamStore.cxx
+++ b/Framework/Core/src/ConfigParamStore.cxx
@@ -41,6 +41,17 @@ void ConfigParamStore::preload()
   }
 }
 
+void ConfigParamStore::load(std::vector<ConfigParamSpec>& specs)
+{
+  mNextStore->clear();
+  mNextProvenance->clear();
+  // By default we populate with code.
+  PropertyTreeHelpers::populateDefaults(specs, *mNextStore, *mNextProvenance);
+  for (auto& retriever : mRetrievers) {
+    retriever->update(specs, *mStore, *mProvenance);
+  }
+}
+
 /// Activate the next store
 void ConfigParamStore::activate()
 {
diff --git a/Framework/Core/src/WorkflowCustomizationHelpers.cxx b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
index ffad8a8720019..05abb5dab98cd 100644
--- a/Framework/Core/src/WorkflowCustomizationHelpers.cxx
+++ b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
@@ -54,6 +54,10 @@ std::vector<ConfigParamSpec> WorkflowCustomizationHelpers::requiredWorkflowOptio
            // options for AOD rate limiting
            {"aod-memory-rate-limit", VariantType::Int64, 0LL, {"Rate limit AOD processing based on memory"}},
 
+           // options for the AOD reader
+           // aod-file needs to be available as workflow option, because we
+           // can configure the workflow based on the contents of the first file.
+           {"aod-file", VariantType::String, "", {"Input AOD file"}},
            // options for AOD writer
            {"aod-writer-json", VariantType::String, "", {"Name of the json configuration file"}},
            {"aod-writer-resdir", VariantType::String, "", {"Name of the output directory"}},
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index bd875ad421f6e..da60bdbc6961e 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -292,7 +292,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
                          static_cast<DataAllocator::SubSpecificationType>(compile_time_hash("internal-dpl-aod-reader")),
                          aodLifetime}},
     .algorithm = AlgorithmSpec::dummyAlgorithm(),
-    .options = {ConfigParamSpec{"aod-file", VariantType::String, {"Input AOD file"}},
+    .options = {ConfigParamSpec{"aod-file-private", VariantType::String, ctx.options().get<std::string>("aod-file"), {"AOD file"}},
                 ConfigParamSpec{"aod-reader-json", VariantType::String, {"json configuration file"}},
                 ConfigParamSpec{"aod-parent-access-level", VariantType::String, {"Allow parent file access up to specified level. Default: no (0)"}},
                 ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}},
diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index bfbdcfc6f169b..21af45422e0b1 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -33,6 +33,10 @@ o2_add_dpl_workflow(diamond-workflow
                   SOURCES src/o2DiamondWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
 
+o2_add_dpl_workflow(analysis-histograms
+                  SOURCES src/o2TestHistograms.cxx
+                  COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(two-timers
                   SOURCES src/o2TwoTimers.cxx
                   COMPONENT_NAME TestWorkflows)
diff --git a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
index 05bf3aabaceab..43d71a9b40528 100644
--- a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
+++ b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
@@ -49,7 +49,14 @@ struct EtaAndClsHistograms {
 
 WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
 {
-  return WorkflowSpec{
-    adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
-  };
+  // For the sake of running without an option, we do not throw an exception
+  // in case the option is not present.
+  if (cfgc.options().hasOption("aod-metadata-Run") == false ||
+      cfgc.options().get<std::string>("aod-metadata-Run") == "2") {
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
+    };
+  } else {
+    throw std::runtime_error("Unsupported run type");
+  }
 }
diff --git a/Framework/TestWorkflows/src/o2TestHistograms.cxx b/Framework/TestWorkflows/src/o2TestHistograms.cxx
new file mode 100644
index 0000000000000..331ee828e1c7b
--- /dev/null
+++ b/Framework/TestWorkflows/src/o2TestHistograms.cxx
@@ -0,0 +1,62 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+///
+/// \brief FullTracks is a join of Tracks, TracksCov, and TracksExtra.
+/// \author
+/// \since
+
+#include "Framework/runDataProcessing.h"
+#include "Framework/AnalysisTask.h"
+#include <TH2F.h>
+#include <cmath>
+
+using namespace o2;
+using namespace o2::framework;
+using namespace o2::framework::expressions;
+
+struct EtaAndClsHistograms {
+  OutputObj<TH3F> etaClsH{TH3F("eta_vs_cls_vs_sigmapT", "#eta vs N_{cls} vs sigma_{1/pT}", 102, -2.01, 2.01, 160, -0.5, 159.5, 100, 0, 10)};
+
+  void process(soa::Join<aod::FullTracks, aod::TracksCov> const& tracks)
+  {
+    for (auto& track : tracks) {
+      etaClsH->Fill(track.eta(), track.tpcNClsFindable(), track.sigma1Pt());
+    }
+  }
+};
+
+struct EtaAndClsHistogramsSimple {
+  OutputObj<TH2F> etaClsH{TH2F("eta_vs_pt", "#eta vs pT", 102, -2.01, 2.01, 100, 0, 10)};
+
+  void process(aod::Tracks const& tracks)
+  {
+    for (auto& track : tracks) {
+      etaClsH->Fill(track.eta(), track.pt(), 0);
+    }
+  }
+};
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  std::string runType = "3";
+  if (cfgc.options().hasOption("aod-metadata-Run")) {
+    runType = cfgc.options().get<std::string>("aod-metadata-Run");
+  }
+  if (runType == "2") {
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistogramsSimple>(cfgc),
+    };
+  } else {
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
+    };
+  }
+}

From 1e30e80fb9bdf0248a981e2d28a61baefcb49c29 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 13 May 2024 16:54:37 +0200
Subject: [PATCH 502/726] DPL: use idiomatic C++ to get rid of whitespaces

---
 .../include/Framework/DataOutputDirector.h    |  2 --
 Framework/Core/src/DataOutputDirector.cxx     | 24 ++++---------------
 2 files changed, 5 insertions(+), 21 deletions(-)

diff --git a/Framework/Core/include/Framework/DataOutputDirector.h b/Framework/Core/include/Framework/DataOutputDirector.h
index 8661dc3eb87e5..bdcb8faf976c0 100644
--- a/Framework/Core/include/Framework/DataOutputDirector.h
+++ b/Framework/Core/include/Framework/DataOutputDirector.h
@@ -51,8 +51,6 @@ struct DataOutputDescriptor {
  private:
   std::string mfilenameBase;
   std::string* mfilenameBasePtr = nullptr;
-
-  std::string remove_ws(const std::string& s);
 };
 
 struct DataOutputDirector {
diff --git a/Framework/Core/src/DataOutputDirector.cxx b/Framework/Core/src/DataOutputDirector.cxx
index 80a31586ef7c1..4b803e1050817 100644
--- a/Framework/Core/src/DataOutputDirector.cxx
+++ b/Framework/Core/src/DataOutputDirector.cxx
@@ -26,9 +26,7 @@
 
 namespace fs = std::filesystem;
 
-namespace o2
-{
-namespace framework
+namespace o2::framework
 {
 using namespace rapidjson;
 
@@ -39,7 +37,7 @@ DataOutputDescriptor::DataOutputDescriptor(std::string inString)
   // the 1st part is used to create a DataDescriptorMatcher
   // the other parts are used to fill treename, colnames, and filename
   // remove all spaces
-  auto cleanString = remove_ws(inString);
+  inString.erase(std::remove_if(inString.begin(), inString.end(), isspace), inString.end());
 
   // reset
   treename = "";
@@ -49,8 +47,8 @@ DataOutputDescriptor::DataOutputDescriptor(std::string inString)
   // analyze the  parts of the input string
   static const std::regex delim1(":");
   std::sregex_token_iterator end;
-  std::sregex_token_iterator iter1(cleanString.begin(),
-                                   cleanString.end(),
+  std::sregex_token_iterator iter1(inString.begin(),
+                                   inString.end(),
                                    delim1,
                                    -1);
 
@@ -134,17 +132,6 @@ void DataOutputDescriptor::printOut()
   }
 }
 
-std::string DataOutputDescriptor::remove_ws(const std::string& s)
-{
-  std::string s_wns;
-  for (auto c : s) {
-    if (!std::isspace(c)) {
-      s_wns += c;
-    }
-  }
-  return s_wns;
-}
-
 DataOutputDirector::DataOutputDirector()
 {
   mfilenameBase = std::string("");
@@ -637,5 +624,4 @@ void DataOutputDirector::setMaximumFileSize(float maxfs)
 {
   mmaxfilesize = maxfs;
 }
-} // namespace framework
-} // namespace o2
+} // namespace o2::framework

From d500f5bd66d96bfa692838ff4baa8ab355a250b3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 14 May 2024 09:18:44 +0200
Subject: [PATCH 503/726] DPL Analysis: provide metadata to workflow
 construction

---
 .../src/AODJAlienReaderHelpers.cxx            |   6 +-
 Framework/AnalysisSupport/src/Plugin.cxx      |  67 ++++++++++++
 Framework/Core/CMakeLists.txt                 |   2 +
 Framework/Core/include/Framework/Capability.h |  44 ++++++++
 .../include/Framework/ConfigParamDiscovery.h  |  37 +++++++
 .../include/Framework/ConfigParamRegistry.h   |   6 ++
 .../Core/include/Framework/ConfigParamStore.h |   4 +-
 .../Core/include/Framework/PluginManager.h    |   2 +-
 Framework/Core/include/Framework/Plugins.h    |   9 ++
 .../Core/include/Framework/ServiceSpec.h      |   5 -
 .../include/Framework/runDataProcessing.h     |   7 ++
 Framework/Core/src/ConfigParamDiscovery.cxx   |  81 ++++++++++++++
 Framework/Core/src/ConfigParamStore.cxx       |  11 ++
 Framework/Core/src/Plugin.cxx                 | 101 ++++++++++++++++++
 .../Core/src/WorkflowCustomizationHelpers.cxx |   4 +
 Framework/Core/src/WorkflowHelpers.cxx        |   2 +-
 Framework/TestWorkflows/CMakeLists.txt        |   4 +
 .../TestWorkflows/src/o2AnalysisWorkflow.cxx  |  13 ++-
 .../TestWorkflows/src/o2TestHistograms.cxx    |  62 +++++++++++
 19 files changed, 453 insertions(+), 14 deletions(-)
 create mode 100644 Framework/Core/include/Framework/Capability.h
 create mode 100644 Framework/Core/include/Framework/ConfigParamDiscovery.h
 create mode 100644 Framework/Core/src/ConfigParamDiscovery.cxx
 create mode 100644 Framework/Core/src/Plugin.cxx
 create mode 100644 Framework/TestWorkflows/src/o2TestHistograms.cxx

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 092857d585a02..5ce2eca49c07b 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -133,12 +133,12 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_MESSAGES_DESTROYED), DataProcessingStats::Op::Set, 0});
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_BYTES_EXPIRED), DataProcessingStats::Op::Set, 0});
 
-    if (!options.isSet("aod-file")) {
+    if (!options.isSet("aod-file-private")) {
       LOGP(fatal, "No input file defined!");
       throw std::runtime_error("Processing is stopped!");
     }
 
-    auto filename = options.get<std::string>("aod-file");
+    auto filename = options.get<std::string>("aod-file-private");
 
     std::string parentFileReplacement;
     if (options.isSet("aod-parent-base-path-replacement")) {
@@ -306,7 +306,7 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
           control.readyToQuit(QuitRequest::Me);
           return;
         }
-      } 
+      }
     });
   })};
 
diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index 8f2e489086afb..2ee8d316f3675 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -12,7 +12,17 @@
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/ServiceSpec.h"
 #include "Framework/ServiceMetricsInfo.h"
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/Capability.h"
+#include "Framework/Signpost.h"
 #include "AODJAlienReaderHelpers.h"
+#include <TFile.h>
+#include <TMap.h>
+#include <TObjString.h>
+#include <TString.h>
+#include <fmt/format.h>
+
+O2_DECLARE_DYNAMIC_LOG(analysis_support);
 
 struct ROOTFileReader : o2::framework::AlgorithmPlugin {
   o2::framework::AlgorithmSpec create() override
@@ -54,7 +64,64 @@ struct RunSummary : o2::framework::ServicePlugin {
   }
 };
 
+struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
+  ConfigDiscovery* create() override
+  {
+    return new ConfigDiscovery{
+      .init = []() {},
+      .discover = [](ConfigParamRegistry& registry, int argc, char** argv) -> std::vector<ConfigParamSpec> {
+        auto filename = registry.get<std::string>("aod-file");
+        if (filename.empty()) {
+          return {};
+        }
+        LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
+        std::vector<ConfigParamSpec> results;
+        TFile* currentFile = nullptr;
+        if (filename.at(0) == '@') {
+          filename.erase(0, 1);
+          // read the text file and set filename to the contents of the first line
+          std::ifstream file(filename);
+          if (!file.is_open()) {
+            LOGP(fatal, "Couldn't open file \"{}\"!", filename);
+          }
+          std::getline(file, filename);
+          file.close();
+          currentFile = TFile::Open(filename.c_str());
+        } else {
+          currentFile = TFile::Open(filename.c_str());
+        }
+        if (!currentFile) {
+          LOGP(fatal, "Couldn't open file \"{}\"!", filename);
+        }
+
+        // Get the metadata, if any
+        auto m = (TMap*)currentFile->Get("metaData");
+        if (!m) {
+          LOGP(warning, "No metadata found in file \"{}\"", filename);
+          return results;
+        }
+        auto it = m->MakeIterator();
+
+        // Serialise metadata into a ; separated string with : separating key and value
+        bool first = true;
+        while (auto obj = it->Next()) {
+          if (first) {
+            LOGP(info, "Metadata for file \"{}\":", filename);
+            first = false;
+          }
+          auto objString = (TObjString*)m->GetValue(obj);
+          LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
+          std::string key = "aod-metadata-" + std::string(obj->GetName());
+          char const* value = strdup(objString->String());
+          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
+        }
+        return results;
+      }};
+  }
+};
+
 DEFINE_DPL_PLUGINS_BEGIN
 DEFINE_DPL_PLUGIN_INSTANCE(ROOTFileReader, CustomAlgorithm);
 DEFINE_DPL_PLUGIN_INSTANCE(RunSummary, CustomService);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAOD, ConfigDiscovery);
 DEFINE_DPL_PLUGINS_END
diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index d392ba42fe8ac..4381b6c4fa29d 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -47,6 +47,7 @@ o2_add_library(Framework
                        src/DefaultsHelpers.cxx
                        src/DomainInfoHeader.cxx
                        src/ProcessingPoliciesHelpers.cxx
+                       src/ConfigParamDiscovery.cxx
                        src/ConfigParamStore.cxx
                        src/ConfigParamsHelper.cxx
                        src/ChannelParamSpec.cxx
@@ -109,6 +110,7 @@ o2_add_library(Framework
                        src/OptionsHelpers.cxx
                        src/PropertyTreeHelpers.cxx
                        src/ProcessingContext.cxx
+                       src/Plugin.cxx
                        src/PluginManager.cxx
                        src/RateLimiter.cxx
                        src/ResourcesMonitoringHelper.cxx
diff --git a/Framework/Core/include/Framework/Capability.h b/Framework/Core/include/Framework/Capability.h
new file mode 100644
index 0000000000000..23ef7b7d9bf17
--- /dev/null
+++ b/Framework/Core/include/Framework/Capability.h
@@ -0,0 +1,44 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_CAPABILITY_H_
+#define O2_FRAMEWORK_CAPABILITY_H_
+
+#include <string_view>
+#include <functional>
+
+namespace o2::framework
+{
+
+struct ConfigParamRegistry;
+
+/// A plugin which is able to discover more option then the ones
+/// provided on the command line. The actual loading is in two parts,
+/// the first one decides if the options are enough to actually perform
+/// the discovery, the second part will do the discovery itself.
+///
+/// Its a good idea to have the Loader part in a standalone library to
+/// minimize dependency on unneed thir party code, e.g. ROOT.
+struct Capability {
+  // Wether or not this capability is required.
+  // FIXME: for now let's pass the arguments to get the metadata to work
+  std::string name;
+  std::function<bool(ConfigParamRegistry&, int argc, char** argv)> checkIfNeeded;
+  char const* requiredPlugin;
+};
+
+struct CapabilityPlugin {
+  virtual Capability* create() = 0;
+};
+
+} // namespace o2::framework
+
+#endif // O2_FRAMEWORK_CAPABILITY_H_
diff --git a/Framework/Core/include/Framework/ConfigParamDiscovery.h b/Framework/Core/include/Framework/ConfigParamDiscovery.h
new file mode 100644
index 0000000000000..3d13d3662ceb1
--- /dev/null
+++ b/Framework/Core/include/Framework/ConfigParamDiscovery.h
@@ -0,0 +1,37 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
+#define O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
+
+#include "Framework/ConfigParamRegistry.h"
+
+namespace o2::framework
+{
+// A plugin which can be used to inject extra configuration
+// options.
+struct ConfigParamDiscovery {
+  static std::vector<ConfigParamSpec> discover(ConfigParamRegistry&, int, char**);
+};
+
+struct ConfigDiscovery {
+  std::function<void()> init;
+  /// A function which uses the arguments available so far to discover more
+  /// @return the extra ConfigParamSpecs it derives.
+  std::function<std::vector<ConfigParamSpec>(ConfigParamRegistry&, int, char**)> discover;
+};
+
+struct ConfigDiscoveryPlugin {
+  virtual ConfigDiscovery* create() = 0;
+};
+
+} // namespace o2::framework
+#endif // O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
diff --git a/Framework/Core/include/Framework/ConfigParamRegistry.h b/Framework/Core/include/Framework/ConfigParamRegistry.h
index 8f2d80cea0948..397a491be9f9c 100644
--- a/Framework/Core/include/Framework/ConfigParamRegistry.h
+++ b/Framework/Core/include/Framework/ConfigParamRegistry.h
@@ -125,6 +125,12 @@ class ConfigParamRegistry
     }
   }
 
+  // Load extra parameters discovered while we process data
+  void loadExtra(std::vector<ConfigParamSpec>& extras)
+  {
+    mStore->load(extras);
+  }
+
  private:
   std::unique_ptr<ConfigParamStore> mStore;
 };
diff --git a/Framework/Core/include/Framework/ConfigParamStore.h b/Framework/Core/include/Framework/ConfigParamStore.h
index 37442a6cf7f41..b3ae76aa47ad1 100644
--- a/Framework/Core/include/Framework/ConfigParamStore.h
+++ b/Framework/Core/include/Framework/ConfigParamStore.h
@@ -34,12 +34,14 @@ class ConfigParamStore
   /// the configuration.
   void preload();
 
+  void load(std::vector<ConfigParamSpec>& specs);
+
   /// Get the store
   boost::property_tree::ptree& store() { return *mStore; };
   boost::property_tree::ptree& provenanceTree() { return *mProvenance; };
 
   /// Get the specs
-  std::vector<ConfigParamSpec> const& specs() const
+  [[nodiscard]] std::vector<ConfigParamSpec> const& specs() const
   {
     return mSpecs;
   }
diff --git a/Framework/Core/include/Framework/PluginManager.h b/Framework/Core/include/Framework/PluginManager.h
index 38966c5a969a9..4c6e965502500 100644
--- a/Framework/Core/include/Framework/PluginManager.h
+++ b/Framework/Core/include/Framework/PluginManager.h
@@ -70,7 +70,7 @@ struct PluginManager {
 
     struct LoadedPlugin {
       std::string name;
-      ServicePlugin* factory;
+      PLUGIN* factory;
     };
     std::vector<LoadedDSO> loadedDSOs;
     std::vector<LoadedPlugin> loadedPlugins;
diff --git a/Framework/Core/include/Framework/Plugins.h b/Framework/Core/include/Framework/Plugins.h
index c29d96f7e0781..23d55a512e1fa 100644
--- a/Framework/Core/include/Framework/Plugins.h
+++ b/Framework/Core/include/Framework/Plugins.h
@@ -27,6 +27,15 @@ enum struct DplPluginKind : int {
   // A plugin which implements a custom Services. Needs to return
   // an object of the kind o2::framework::ServiceSpec
   CustomService,
+  // A plugin which implements a new way to discover extra configuration.
+  // parameters. E.g. it can be used to read metadata from a file or a service
+  // if a certain parameter is available.
+  ConfigDiscovery,
+  // A capability plugin is a plugin used to discover other viable plugins.
+  // For example, if you find out that you have the --aod-file option
+  // set, you might want to load metadata from it and attach it to the
+  // configuration.
+  Capability,
   // A plugin which was not initialised properly.
   Unknown
 };
diff --git a/Framework/Core/include/Framework/ServiceSpec.h b/Framework/Core/include/Framework/ServiceSpec.h
index abac3ad05914c..5684889e85376 100644
--- a/Framework/Core/include/Framework/ServiceSpec.h
+++ b/Framework/Core/include/Framework/ServiceSpec.h
@@ -322,11 +322,6 @@ struct ServicePlugin {
   virtual ServiceSpec* create() = 0;
 };
 
-struct LoadableService {
-  std::string name;
-  std::string library;
-};
-
 } // namespace o2::framework
 
 #endif // O2_FRAMEWORK_SERVICESPEC_H_
diff --git a/Framework/Core/include/Framework/runDataProcessing.h b/Framework/Core/include/Framework/runDataProcessing.h
index 77a4c9f5ba8d6..9ab221af90267 100644
--- a/Framework/Core/include/Framework/runDataProcessing.h
+++ b/Framework/Core/include/Framework/runDataProcessing.h
@@ -29,6 +29,7 @@
 #include "Framework/Logger.h"
 #include "Framework/CheckTypes.h"
 #include "Framework/StructToTuple.h"
+#include "Framework/ConfigParamDiscovery.h"
 
 #include <vector>
 
@@ -192,6 +193,12 @@ int mainNoCatch(int argc, char** argv)
   workflowOptionsStore->preload();
   workflowOptionsStore->activate();
   ConfigParamRegistry workflowOptionsRegistry(std::move(workflowOptionsStore));
+  auto extraOptions = o2::framework::ConfigParamDiscovery::discover(workflowOptionsRegistry, argc, argv);
+  for (auto& extra : extraOptions) {
+    workflowOptions.push_back(extra);
+  }
+  workflowOptionsRegistry.loadExtra(extraOptions);
+
   ConfigContext configContext(workflowOptionsRegistry, argc, argv);
   o2::framework::WorkflowSpec specs = defineDataProcessing(configContext);
   overrideCloning(configContext, specs);
diff --git a/Framework/Core/src/ConfigParamDiscovery.cxx b/Framework/Core/src/ConfigParamDiscovery.cxx
new file mode 100644
index 0000000000000..6c24a44bac566
--- /dev/null
+++ b/Framework/Core/src/ConfigParamDiscovery.cxx
@@ -0,0 +1,81 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/PluginManager.h"
+#include "Framework/Signpost.h"
+#include "Framework/Capability.h"
+#include <TFile.h>
+#include <TMap.h>
+#include <TObjString.h>
+
+O2_DECLARE_DYNAMIC_LOG(config_parameter_discovery);
+
+namespace o2::framework
+{
+
+std::vector<ConfigParamSpec> ConfigParamDiscovery::discover(ConfigParamRegistry& registry, int argc, char** argv)
+{
+  std::vector<char const*> capabilitiesSpecs = {
+    "O2Framework:DiscoverMetadataInAODCapability",
+    "O2Framework:DiscoverMetadataInCommandLineCapability",
+  };
+
+  // Load all the requested plugins and discover what we can do.
+  O2_SIGNPOST_ID_GENERATE(sid, config_parameter_discovery);
+  std::vector<LoadablePlugin> plugins;
+  for (auto spec : capabilitiesSpecs) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "Parsing Capability plugin: %{public}s", spec);
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+  }
+
+  std::vector<Capability> availableCapabilities;
+  std::vector<char const*> configDiscoverySpec = {};
+  PluginManager::loadFromPlugin<Capability, CapabilityPlugin>(plugins, availableCapabilities);
+  for (auto& capability : availableCapabilities) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- Using Capability of : %{public}s",
+                           capability.name.c_str());
+
+    if (capability.checkIfNeeded(registry, argc, argv)) {
+      O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- Required: %{}s",
+                             capability.requiredPlugin);
+      configDiscoverySpec.push_back(capability.requiredPlugin);
+    } else {
+      O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- No requirements added");
+    }
+  }
+
+  std::vector<LoadablePlugin> configDiscoveryPlugins;
+  for (auto spec : configDiscoverySpec) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "discovery", "Parsing ConfigDiscovery plugin: %{public}s", spec);
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      configDiscoveryPlugins.push_back(extra);
+    }
+  }
+
+  std::vector<ConfigDiscovery> availableConfigDiscovery;
+  PluginManager::loadFromPlugin<ConfigDiscovery, ConfigDiscoveryPlugin>(configDiscoveryPlugins, availableConfigDiscovery);
+  std::vector<ConfigParamSpec> result;
+  for (auto& configDiscovery : availableConfigDiscovery) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "discovery", "Parsing extra configuration as needed");
+    auto extras = configDiscovery.discover(registry, argc, argv);
+    for (auto& extra : extras) {
+      result.push_back(extra);
+    }
+  }
+  return result;
+}
+
+} // namespace o2::framework
diff --git a/Framework/Core/src/ConfigParamStore.cxx b/Framework/Core/src/ConfigParamStore.cxx
index 249a8b0052a67..19e123a23e5d0 100644
--- a/Framework/Core/src/ConfigParamStore.cxx
+++ b/Framework/Core/src/ConfigParamStore.cxx
@@ -41,6 +41,17 @@ void ConfigParamStore::preload()
   }
 }
 
+void ConfigParamStore::load(std::vector<ConfigParamSpec>& specs)
+{
+  mNextStore->clear();
+  mNextProvenance->clear();
+  // By default we populate with code.
+  PropertyTreeHelpers::populateDefaults(specs, *mNextStore, *mNextProvenance);
+  for (auto& retriever : mRetrievers) {
+    retriever->update(specs, *mStore, *mProvenance);
+  }
+}
+
 /// Activate the next store
 void ConfigParamStore::activate()
 {
diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx
new file mode 100644
index 0000000000000..e5634a62b3240
--- /dev/null
+++ b/Framework/Core/src/Plugin.cxx
@@ -0,0 +1,101 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/Plugins.h"
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/ConfigParamRegistry.h"
+#include "Framework/Logger.h"
+#include "Framework/Capability.h"
+#include "Framework/Signpost.h"
+#include <string_view>
+
+O2_DECLARE_DYNAMIC_LOG(capabilities);
+namespace o2::framework
+{
+auto lookForAodFile = [](ConfigParamRegistry& registry, int argc, char** argv) -> bool {
+  O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+  if (registry.hasOption("aod-file") && registry.isSet("aod-file")) {
+    for (size_t i = 0; i < argc; i++) {
+      std::string_view arg = argv[i];
+      if (arg.starts_with("--aod-metadata-")) {
+        return false;
+      }
+    }
+    O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInAODCapability", "Metadata found in arguments. Populating from them.");
+    LOGP(info, "Will read metadata from AOD file.");
+    return true;
+  }
+  return false;
+};
+
+auto lookForCommandLineOptions = [](ConfigParamRegistry& registry, int argc, char** argv) -> bool {
+  O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+  for (size_t i = 0; i < argc; i++) {
+    std::string_view arg = argv[i];
+    if (arg.starts_with("--aod-metadata-")) {
+      O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLineCapability", "Metadata found in arguments. Populating from them.");
+      return true;
+    }
+  }
+  return false;
+};
+
+struct DiscoverMetadataInAODCapability : o2::framework::CapabilityPlugin {
+  Capability* create() override
+  {
+    return new Capability{
+      .name = "DiscoverMetadataInAODCapability",
+      .checkIfNeeded = lookForAodFile,
+      .requiredPlugin = "O2FrameworkAnalysisSupport:DiscoverMetadataInAOD"};
+  }
+};
+
+// Trigger discovery of metadata from command line, if needed.
+struct DiscoverMetadataInCommandLineCapability : o2::framework::CapabilityPlugin {
+  Capability* create() override
+  {
+    return new Capability{
+      .name = "DiscoverMetadataInCommandLineCapability",
+      .checkIfNeeded = lookForCommandLineOptions,
+      .requiredPlugin = "O2Framework:DiscoverMetadataInCommandLine"};
+  }
+};
+
+struct DiscoverMetadataInCommandLine : o2::framework::ConfigDiscoveryPlugin {
+  ConfigDiscovery* create() override
+  {
+    return new ConfigDiscovery{
+      .init = []() {},
+      .discover = [](ConfigParamRegistry& registry, int argc, char** argv) -> std::vector<ConfigParamSpec> {
+        O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+        O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLine",
+                               "Discovering metadata for analysis from well known environment variables.");
+        std::vector<ConfigParamSpec> results;
+        for (size_t i = 0; i < argc; i++) {
+          std::string_view arg = argv[i];
+          if (!arg.starts_with("--aod-metadata")) {
+            continue;
+          }
+          std::string key = arg.data() + 2;
+          std::string value = argv[i + 1];
+          O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLine",
+                                 "Found %{public}s with value %{public}s.", key.c_str(), value.c_str());
+          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in command line"}});
+        }
+        return results;
+      }};
+  }
+};
+DEFINE_DPL_PLUGINS_BEGIN
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAODCapability, Capability);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLineCapability, Capability);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLine, ConfigDiscovery);
+DEFINE_DPL_PLUGINS_END
+} // namespace o2::framework
diff --git a/Framework/Core/src/WorkflowCustomizationHelpers.cxx b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
index ffad8a8720019..05abb5dab98cd 100644
--- a/Framework/Core/src/WorkflowCustomizationHelpers.cxx
+++ b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
@@ -54,6 +54,10 @@ std::vector<ConfigParamSpec> WorkflowCustomizationHelpers::requiredWorkflowOptio
            // options for AOD rate limiting
            {"aod-memory-rate-limit", VariantType::Int64, 0LL, {"Rate limit AOD processing based on memory"}},
 
+           // options for the AOD reader
+           // aod-file needs to be available as workflow option, because we
+           // can configure the workflow based on the contents of the first file.
+           {"aod-file", VariantType::String, "", {"Input AOD file"}},
            // options for AOD writer
            {"aod-writer-json", VariantType::String, "", {"Name of the json configuration file"}},
            {"aod-writer-resdir", VariantType::String, "", {"Name of the output directory"}},
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index bd875ad421f6e..da60bdbc6961e 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -292,7 +292,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
                          static_cast<DataAllocator::SubSpecificationType>(compile_time_hash("internal-dpl-aod-reader")),
                          aodLifetime}},
     .algorithm = AlgorithmSpec::dummyAlgorithm(),
-    .options = {ConfigParamSpec{"aod-file", VariantType::String, {"Input AOD file"}},
+    .options = {ConfigParamSpec{"aod-file-private", VariantType::String, ctx.options().get<std::string>("aod-file"), {"AOD file"}},
                 ConfigParamSpec{"aod-reader-json", VariantType::String, {"json configuration file"}},
                 ConfigParamSpec{"aod-parent-access-level", VariantType::String, {"Allow parent file access up to specified level. Default: no (0)"}},
                 ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}},
diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index bfbdcfc6f169b..21af45422e0b1 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -33,6 +33,10 @@ o2_add_dpl_workflow(diamond-workflow
                   SOURCES src/o2DiamondWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
 
+o2_add_dpl_workflow(analysis-histograms
+                  SOURCES src/o2TestHistograms.cxx
+                  COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(two-timers
                   SOURCES src/o2TwoTimers.cxx
                   COMPONENT_NAME TestWorkflows)
diff --git a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
index 05bf3aabaceab..43d71a9b40528 100644
--- a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
+++ b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
@@ -49,7 +49,14 @@ struct EtaAndClsHistograms {
 
 WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
 {
-  return WorkflowSpec{
-    adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
-  };
+  // For the sake of running without an option, we do not throw an exception
+  // in case the option is not present.
+  if (cfgc.options().hasOption("aod-metadata-Run") == false ||
+      cfgc.options().get<std::string>("aod-metadata-Run") == "2") {
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
+    };
+  } else {
+    throw std::runtime_error("Unsupported run type");
+  }
 }
diff --git a/Framework/TestWorkflows/src/o2TestHistograms.cxx b/Framework/TestWorkflows/src/o2TestHistograms.cxx
new file mode 100644
index 0000000000000..331ee828e1c7b
--- /dev/null
+++ b/Framework/TestWorkflows/src/o2TestHistograms.cxx
@@ -0,0 +1,62 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+///
+/// \brief FullTracks is a join of Tracks, TracksCov, and TracksExtra.
+/// \author
+/// \since
+
+#include "Framework/runDataProcessing.h"
+#include "Framework/AnalysisTask.h"
+#include <TH2F.h>
+#include <cmath>
+
+using namespace o2;
+using namespace o2::framework;
+using namespace o2::framework::expressions;
+
+struct EtaAndClsHistograms {
+  OutputObj<TH3F> etaClsH{TH3F("eta_vs_cls_vs_sigmapT", "#eta vs N_{cls} vs sigma_{1/pT}", 102, -2.01, 2.01, 160, -0.5, 159.5, 100, 0, 10)};
+
+  void process(soa::Join<aod::FullTracks, aod::TracksCov> const& tracks)
+  {
+    for (auto& track : tracks) {
+      etaClsH->Fill(track.eta(), track.tpcNClsFindable(), track.sigma1Pt());
+    }
+  }
+};
+
+struct EtaAndClsHistogramsSimple {
+  OutputObj<TH2F> etaClsH{TH2F("eta_vs_pt", "#eta vs pT", 102, -2.01, 2.01, 100, 0, 10)};
+
+  void process(aod::Tracks const& tracks)
+  {
+    for (auto& track : tracks) {
+      etaClsH->Fill(track.eta(), track.pt(), 0);
+    }
+  }
+};
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  std::string runType = "3";
+  if (cfgc.options().hasOption("aod-metadata-Run")) {
+    runType = cfgc.options().get<std::string>("aod-metadata-Run");
+  }
+  if (runType == "2") {
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistogramsSimple>(cfgc),
+    };
+  } else {
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
+    };
+  }
+}

From 38f76e0d206865acb2b550794dd1f21f01bbda5a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 14 May 2024 09:18:44 +0200
Subject: [PATCH 504/726] DPL Analysis: provide metadata to workflow
 construction

---
 .../src/AODJAlienReaderHelpers.cxx            |   6 +-
 Framework/AnalysisSupport/src/Plugin.cxx      |  67 ++++++++++++
 Framework/Core/CMakeLists.txt                 |   2 +
 Framework/Core/include/Framework/Capability.h |  44 ++++++++
 .../include/Framework/ConfigParamDiscovery.h  |  37 +++++++
 .../include/Framework/ConfigParamRegistry.h   |   6 ++
 .../Core/include/Framework/ConfigParamStore.h |   4 +-
 .../Core/include/Framework/PluginManager.h    |   2 +-
 Framework/Core/include/Framework/Plugins.h    |   9 ++
 .../Core/include/Framework/ServiceSpec.h      |   5 -
 .../include/Framework/runDataProcessing.h     |   7 ++
 Framework/Core/src/ConfigParamDiscovery.cxx   |  81 ++++++++++++++
 Framework/Core/src/ConfigParamStore.cxx       |  11 ++
 Framework/Core/src/Plugin.cxx                 | 100 ++++++++++++++++++
 .../Core/src/WorkflowCustomizationHelpers.cxx |   4 +
 Framework/Core/src/WorkflowHelpers.cxx        |   2 +-
 Framework/TestWorkflows/CMakeLists.txt        |   4 +
 .../TestWorkflows/src/o2AnalysisWorkflow.cxx  |  13 ++-
 .../TestWorkflows/src/o2TestHistograms.cxx    |  62 +++++++++++
 19 files changed, 452 insertions(+), 14 deletions(-)
 create mode 100644 Framework/Core/include/Framework/Capability.h
 create mode 100644 Framework/Core/include/Framework/ConfigParamDiscovery.h
 create mode 100644 Framework/Core/src/ConfigParamDiscovery.cxx
 create mode 100644 Framework/Core/src/Plugin.cxx
 create mode 100644 Framework/TestWorkflows/src/o2TestHistograms.cxx

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 092857d585a02..5ce2eca49c07b 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -133,12 +133,12 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_MESSAGES_DESTROYED), DataProcessingStats::Op::Set, 0});
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_BYTES_EXPIRED), DataProcessingStats::Op::Set, 0});
 
-    if (!options.isSet("aod-file")) {
+    if (!options.isSet("aod-file-private")) {
       LOGP(fatal, "No input file defined!");
       throw std::runtime_error("Processing is stopped!");
     }
 
-    auto filename = options.get<std::string>("aod-file");
+    auto filename = options.get<std::string>("aod-file-private");
 
     std::string parentFileReplacement;
     if (options.isSet("aod-parent-base-path-replacement")) {
@@ -306,7 +306,7 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
           control.readyToQuit(QuitRequest::Me);
           return;
         }
-      } 
+      }
     });
   })};
 
diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index 8f2e489086afb..2ee8d316f3675 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -12,7 +12,17 @@
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/ServiceSpec.h"
 #include "Framework/ServiceMetricsInfo.h"
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/Capability.h"
+#include "Framework/Signpost.h"
 #include "AODJAlienReaderHelpers.h"
+#include <TFile.h>
+#include <TMap.h>
+#include <TObjString.h>
+#include <TString.h>
+#include <fmt/format.h>
+
+O2_DECLARE_DYNAMIC_LOG(analysis_support);
 
 struct ROOTFileReader : o2::framework::AlgorithmPlugin {
   o2::framework::AlgorithmSpec create() override
@@ -54,7 +64,64 @@ struct RunSummary : o2::framework::ServicePlugin {
   }
 };
 
+struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
+  ConfigDiscovery* create() override
+  {
+    return new ConfigDiscovery{
+      .init = []() {},
+      .discover = [](ConfigParamRegistry& registry, int argc, char** argv) -> std::vector<ConfigParamSpec> {
+        auto filename = registry.get<std::string>("aod-file");
+        if (filename.empty()) {
+          return {};
+        }
+        LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
+        std::vector<ConfigParamSpec> results;
+        TFile* currentFile = nullptr;
+        if (filename.at(0) == '@') {
+          filename.erase(0, 1);
+          // read the text file and set filename to the contents of the first line
+          std::ifstream file(filename);
+          if (!file.is_open()) {
+            LOGP(fatal, "Couldn't open file \"{}\"!", filename);
+          }
+          std::getline(file, filename);
+          file.close();
+          currentFile = TFile::Open(filename.c_str());
+        } else {
+          currentFile = TFile::Open(filename.c_str());
+        }
+        if (!currentFile) {
+          LOGP(fatal, "Couldn't open file \"{}\"!", filename);
+        }
+
+        // Get the metadata, if any
+        auto m = (TMap*)currentFile->Get("metaData");
+        if (!m) {
+          LOGP(warning, "No metadata found in file \"{}\"", filename);
+          return results;
+        }
+        auto it = m->MakeIterator();
+
+        // Serialise metadata into a ; separated string with : separating key and value
+        bool first = true;
+        while (auto obj = it->Next()) {
+          if (first) {
+            LOGP(info, "Metadata for file \"{}\":", filename);
+            first = false;
+          }
+          auto objString = (TObjString*)m->GetValue(obj);
+          LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
+          std::string key = "aod-metadata-" + std::string(obj->GetName());
+          char const* value = strdup(objString->String());
+          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
+        }
+        return results;
+      }};
+  }
+};
+
 DEFINE_DPL_PLUGINS_BEGIN
 DEFINE_DPL_PLUGIN_INSTANCE(ROOTFileReader, CustomAlgorithm);
 DEFINE_DPL_PLUGIN_INSTANCE(RunSummary, CustomService);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAOD, ConfigDiscovery);
 DEFINE_DPL_PLUGINS_END
diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index d392ba42fe8ac..4381b6c4fa29d 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -47,6 +47,7 @@ o2_add_library(Framework
                        src/DefaultsHelpers.cxx
                        src/DomainInfoHeader.cxx
                        src/ProcessingPoliciesHelpers.cxx
+                       src/ConfigParamDiscovery.cxx
                        src/ConfigParamStore.cxx
                        src/ConfigParamsHelper.cxx
                        src/ChannelParamSpec.cxx
@@ -109,6 +110,7 @@ o2_add_library(Framework
                        src/OptionsHelpers.cxx
                        src/PropertyTreeHelpers.cxx
                        src/ProcessingContext.cxx
+                       src/Plugin.cxx
                        src/PluginManager.cxx
                        src/RateLimiter.cxx
                        src/ResourcesMonitoringHelper.cxx
diff --git a/Framework/Core/include/Framework/Capability.h b/Framework/Core/include/Framework/Capability.h
new file mode 100644
index 0000000000000..23ef7b7d9bf17
--- /dev/null
+++ b/Framework/Core/include/Framework/Capability.h
@@ -0,0 +1,44 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_CAPABILITY_H_
+#define O2_FRAMEWORK_CAPABILITY_H_
+
+#include <string_view>
+#include <functional>
+
+namespace o2::framework
+{
+
+struct ConfigParamRegistry;
+
+/// A plugin which is able to discover more option then the ones
+/// provided on the command line. The actual loading is in two parts,
+/// the first one decides if the options are enough to actually perform
+/// the discovery, the second part will do the discovery itself.
+///
+/// Its a good idea to have the Loader part in a standalone library to
+/// minimize dependency on unneed thir party code, e.g. ROOT.
+struct Capability {
+  // Wether or not this capability is required.
+  // FIXME: for now let's pass the arguments to get the metadata to work
+  std::string name;
+  std::function<bool(ConfigParamRegistry&, int argc, char** argv)> checkIfNeeded;
+  char const* requiredPlugin;
+};
+
+struct CapabilityPlugin {
+  virtual Capability* create() = 0;
+};
+
+} // namespace o2::framework
+
+#endif // O2_FRAMEWORK_CAPABILITY_H_
diff --git a/Framework/Core/include/Framework/ConfigParamDiscovery.h b/Framework/Core/include/Framework/ConfigParamDiscovery.h
new file mode 100644
index 0000000000000..3d13d3662ceb1
--- /dev/null
+++ b/Framework/Core/include/Framework/ConfigParamDiscovery.h
@@ -0,0 +1,37 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
+#define O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
+
+#include "Framework/ConfigParamRegistry.h"
+
+namespace o2::framework
+{
+// A plugin which can be used to inject extra configuration
+// options.
+struct ConfigParamDiscovery {
+  static std::vector<ConfigParamSpec> discover(ConfigParamRegistry&, int, char**);
+};
+
+struct ConfigDiscovery {
+  std::function<void()> init;
+  /// A function which uses the arguments available so far to discover more
+  /// @return the extra ConfigParamSpecs it derives.
+  std::function<std::vector<ConfigParamSpec>(ConfigParamRegistry&, int, char**)> discover;
+};
+
+struct ConfigDiscoveryPlugin {
+  virtual ConfigDiscovery* create() = 0;
+};
+
+} // namespace o2::framework
+#endif // O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
diff --git a/Framework/Core/include/Framework/ConfigParamRegistry.h b/Framework/Core/include/Framework/ConfigParamRegistry.h
index 8f2d80cea0948..397a491be9f9c 100644
--- a/Framework/Core/include/Framework/ConfigParamRegistry.h
+++ b/Framework/Core/include/Framework/ConfigParamRegistry.h
@@ -125,6 +125,12 @@ class ConfigParamRegistry
     }
   }
 
+  // Load extra parameters discovered while we process data
+  void loadExtra(std::vector<ConfigParamSpec>& extras)
+  {
+    mStore->load(extras);
+  }
+
  private:
   std::unique_ptr<ConfigParamStore> mStore;
 };
diff --git a/Framework/Core/include/Framework/ConfigParamStore.h b/Framework/Core/include/Framework/ConfigParamStore.h
index 37442a6cf7f41..b3ae76aa47ad1 100644
--- a/Framework/Core/include/Framework/ConfigParamStore.h
+++ b/Framework/Core/include/Framework/ConfigParamStore.h
@@ -34,12 +34,14 @@ class ConfigParamStore
   /// the configuration.
   void preload();
 
+  void load(std::vector<ConfigParamSpec>& specs);
+
   /// Get the store
   boost::property_tree::ptree& store() { return *mStore; };
   boost::property_tree::ptree& provenanceTree() { return *mProvenance; };
 
   /// Get the specs
-  std::vector<ConfigParamSpec> const& specs() const
+  [[nodiscard]] std::vector<ConfigParamSpec> const& specs() const
   {
     return mSpecs;
   }
diff --git a/Framework/Core/include/Framework/PluginManager.h b/Framework/Core/include/Framework/PluginManager.h
index 38966c5a969a9..4c6e965502500 100644
--- a/Framework/Core/include/Framework/PluginManager.h
+++ b/Framework/Core/include/Framework/PluginManager.h
@@ -70,7 +70,7 @@ struct PluginManager {
 
     struct LoadedPlugin {
       std::string name;
-      ServicePlugin* factory;
+      PLUGIN* factory;
     };
     std::vector<LoadedDSO> loadedDSOs;
     std::vector<LoadedPlugin> loadedPlugins;
diff --git a/Framework/Core/include/Framework/Plugins.h b/Framework/Core/include/Framework/Plugins.h
index c29d96f7e0781..23d55a512e1fa 100644
--- a/Framework/Core/include/Framework/Plugins.h
+++ b/Framework/Core/include/Framework/Plugins.h
@@ -27,6 +27,15 @@ enum struct DplPluginKind : int {
   // A plugin which implements a custom Services. Needs to return
   // an object of the kind o2::framework::ServiceSpec
   CustomService,
+  // A plugin which implements a new way to discover extra configuration.
+  // parameters. E.g. it can be used to read metadata from a file or a service
+  // if a certain parameter is available.
+  ConfigDiscovery,
+  // A capability plugin is a plugin used to discover other viable plugins.
+  // For example, if you find out that you have the --aod-file option
+  // set, you might want to load metadata from it and attach it to the
+  // configuration.
+  Capability,
   // A plugin which was not initialised properly.
   Unknown
 };
diff --git a/Framework/Core/include/Framework/ServiceSpec.h b/Framework/Core/include/Framework/ServiceSpec.h
index abac3ad05914c..5684889e85376 100644
--- a/Framework/Core/include/Framework/ServiceSpec.h
+++ b/Framework/Core/include/Framework/ServiceSpec.h
@@ -322,11 +322,6 @@ struct ServicePlugin {
   virtual ServiceSpec* create() = 0;
 };
 
-struct LoadableService {
-  std::string name;
-  std::string library;
-};
-
 } // namespace o2::framework
 
 #endif // O2_FRAMEWORK_SERVICESPEC_H_
diff --git a/Framework/Core/include/Framework/runDataProcessing.h b/Framework/Core/include/Framework/runDataProcessing.h
index 77a4c9f5ba8d6..9ab221af90267 100644
--- a/Framework/Core/include/Framework/runDataProcessing.h
+++ b/Framework/Core/include/Framework/runDataProcessing.h
@@ -29,6 +29,7 @@
 #include "Framework/Logger.h"
 #include "Framework/CheckTypes.h"
 #include "Framework/StructToTuple.h"
+#include "Framework/ConfigParamDiscovery.h"
 
 #include <vector>
 
@@ -192,6 +193,12 @@ int mainNoCatch(int argc, char** argv)
   workflowOptionsStore->preload();
   workflowOptionsStore->activate();
   ConfigParamRegistry workflowOptionsRegistry(std::move(workflowOptionsStore));
+  auto extraOptions = o2::framework::ConfigParamDiscovery::discover(workflowOptionsRegistry, argc, argv);
+  for (auto& extra : extraOptions) {
+    workflowOptions.push_back(extra);
+  }
+  workflowOptionsRegistry.loadExtra(extraOptions);
+
   ConfigContext configContext(workflowOptionsRegistry, argc, argv);
   o2::framework::WorkflowSpec specs = defineDataProcessing(configContext);
   overrideCloning(configContext, specs);
diff --git a/Framework/Core/src/ConfigParamDiscovery.cxx b/Framework/Core/src/ConfigParamDiscovery.cxx
new file mode 100644
index 0000000000000..6c24a44bac566
--- /dev/null
+++ b/Framework/Core/src/ConfigParamDiscovery.cxx
@@ -0,0 +1,81 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/PluginManager.h"
+#include "Framework/Signpost.h"
+#include "Framework/Capability.h"
+#include <TFile.h>
+#include <TMap.h>
+#include <TObjString.h>
+
+O2_DECLARE_DYNAMIC_LOG(config_parameter_discovery);
+
+namespace o2::framework
+{
+
+std::vector<ConfigParamSpec> ConfigParamDiscovery::discover(ConfigParamRegistry& registry, int argc, char** argv)
+{
+  std::vector<char const*> capabilitiesSpecs = {
+    "O2Framework:DiscoverMetadataInAODCapability",
+    "O2Framework:DiscoverMetadataInCommandLineCapability",
+  };
+
+  // Load all the requested plugins and discover what we can do.
+  O2_SIGNPOST_ID_GENERATE(sid, config_parameter_discovery);
+  std::vector<LoadablePlugin> plugins;
+  for (auto spec : capabilitiesSpecs) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "Parsing Capability plugin: %{public}s", spec);
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+  }
+
+  std::vector<Capability> availableCapabilities;
+  std::vector<char const*> configDiscoverySpec = {};
+  PluginManager::loadFromPlugin<Capability, CapabilityPlugin>(plugins, availableCapabilities);
+  for (auto& capability : availableCapabilities) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- Using Capability of : %{public}s",
+                           capability.name.c_str());
+
+    if (capability.checkIfNeeded(registry, argc, argv)) {
+      O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- Required: %{}s",
+                             capability.requiredPlugin);
+      configDiscoverySpec.push_back(capability.requiredPlugin);
+    } else {
+      O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- No requirements added");
+    }
+  }
+
+  std::vector<LoadablePlugin> configDiscoveryPlugins;
+  for (auto spec : configDiscoverySpec) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "discovery", "Parsing ConfigDiscovery plugin: %{public}s", spec);
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      configDiscoveryPlugins.push_back(extra);
+    }
+  }
+
+  std::vector<ConfigDiscovery> availableConfigDiscovery;
+  PluginManager::loadFromPlugin<ConfigDiscovery, ConfigDiscoveryPlugin>(configDiscoveryPlugins, availableConfigDiscovery);
+  std::vector<ConfigParamSpec> result;
+  for (auto& configDiscovery : availableConfigDiscovery) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "discovery", "Parsing extra configuration as needed");
+    auto extras = configDiscovery.discover(registry, argc, argv);
+    for (auto& extra : extras) {
+      result.push_back(extra);
+    }
+  }
+  return result;
+}
+
+} // namespace o2::framework
diff --git a/Framework/Core/src/ConfigParamStore.cxx b/Framework/Core/src/ConfigParamStore.cxx
index 249a8b0052a67..19e123a23e5d0 100644
--- a/Framework/Core/src/ConfigParamStore.cxx
+++ b/Framework/Core/src/ConfigParamStore.cxx
@@ -41,6 +41,17 @@ void ConfigParamStore::preload()
   }
 }
 
+void ConfigParamStore::load(std::vector<ConfigParamSpec>& specs)
+{
+  mNextStore->clear();
+  mNextProvenance->clear();
+  // By default we populate with code.
+  PropertyTreeHelpers::populateDefaults(specs, *mNextStore, *mNextProvenance);
+  for (auto& retriever : mRetrievers) {
+    retriever->update(specs, *mStore, *mProvenance);
+  }
+}
+
 /// Activate the next store
 void ConfigParamStore::activate()
 {
diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx
new file mode 100644
index 0000000000000..2298896821dab
--- /dev/null
+++ b/Framework/Core/src/Plugin.cxx
@@ -0,0 +1,100 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/Plugins.h"
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/ConfigParamRegistry.h"
+#include "Framework/Logger.h"
+#include "Framework/Capability.h"
+#include "Framework/Signpost.h"
+#include <string_view>
+
+O2_DECLARE_DYNAMIC_LOG(capabilities);
+namespace o2::framework
+{
+auto lookForAodFile = [](ConfigParamRegistry& registry, int argc, char** argv) -> bool {
+  O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+  if (registry.hasOption("aod-file") && registry.isSet("aod-file")) {
+    for (size_t i = 0; i < argc; i++) {
+      std::string_view arg = argv[i];
+      if (arg.starts_with("--aod-metadata-")) {
+        return false;
+      }
+    }
+    O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInAODCapability", "Metadata not found in arguments. Checking in AOD file.");
+    return true;
+  }
+  return false;
+};
+
+auto lookForCommandLineOptions = [](ConfigParamRegistry& registry, int argc, char** argv) -> bool {
+  O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+  for (size_t i = 0; i < argc; i++) {
+    std::string_view arg = argv[i];
+    if (arg.starts_with("--aod-metadata-")) {
+      O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLineCapability", "Metadata found in arguments. Populating from them.");
+      return true;
+    }
+  }
+  return false;
+};
+
+struct DiscoverMetadataInAODCapability : o2::framework::CapabilityPlugin {
+  Capability* create() override
+  {
+    return new Capability{
+      .name = "DiscoverMetadataInAODCapability",
+      .checkIfNeeded = lookForAodFile,
+      .requiredPlugin = "O2FrameworkAnalysisSupport:DiscoverMetadataInAOD"};
+  }
+};
+
+// Trigger discovery of metadata from command line, if needed.
+struct DiscoverMetadataInCommandLineCapability : o2::framework::CapabilityPlugin {
+  Capability* create() override
+  {
+    return new Capability{
+      .name = "DiscoverMetadataInCommandLineCapability",
+      .checkIfNeeded = lookForCommandLineOptions,
+      .requiredPlugin = "O2Framework:DiscoverMetadataInCommandLine"};
+  }
+};
+
+struct DiscoverMetadataInCommandLine : o2::framework::ConfigDiscoveryPlugin {
+  ConfigDiscovery* create() override
+  {
+    return new ConfigDiscovery{
+      .init = []() {},
+      .discover = [](ConfigParamRegistry& registry, int argc, char** argv) -> std::vector<ConfigParamSpec> {
+        O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+        O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLine",
+                               "Discovering metadata for analysis from well known environment variables.");
+        std::vector<ConfigParamSpec> results;
+        for (size_t i = 0; i < argc; i++) {
+          std::string_view arg = argv[i];
+          if (!arg.starts_with("--aod-metadata")) {
+            continue;
+          }
+          std::string key = arg.data() + 2;
+          std::string value = argv[i + 1];
+          O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLine",
+                                 "Found %{public}s with value %{public}s.", key.c_str(), value.c_str());
+          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in command line"}});
+        }
+        return results;
+      }};
+  }
+};
+DEFINE_DPL_PLUGINS_BEGIN
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAODCapability, Capability);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLineCapability, Capability);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLine, ConfigDiscovery);
+DEFINE_DPL_PLUGINS_END
+} // namespace o2::framework
diff --git a/Framework/Core/src/WorkflowCustomizationHelpers.cxx b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
index ffad8a8720019..05abb5dab98cd 100644
--- a/Framework/Core/src/WorkflowCustomizationHelpers.cxx
+++ b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
@@ -54,6 +54,10 @@ std::vector<ConfigParamSpec> WorkflowCustomizationHelpers::requiredWorkflowOptio
            // options for AOD rate limiting
            {"aod-memory-rate-limit", VariantType::Int64, 0LL, {"Rate limit AOD processing based on memory"}},
 
+           // options for the AOD reader
+           // aod-file needs to be available as workflow option, because we
+           // can configure the workflow based on the contents of the first file.
+           {"aod-file", VariantType::String, "", {"Input AOD file"}},
            // options for AOD writer
            {"aod-writer-json", VariantType::String, "", {"Name of the json configuration file"}},
            {"aod-writer-resdir", VariantType::String, "", {"Name of the output directory"}},
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index bd875ad421f6e..da60bdbc6961e 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -292,7 +292,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
                          static_cast<DataAllocator::SubSpecificationType>(compile_time_hash("internal-dpl-aod-reader")),
                          aodLifetime}},
     .algorithm = AlgorithmSpec::dummyAlgorithm(),
-    .options = {ConfigParamSpec{"aod-file", VariantType::String, {"Input AOD file"}},
+    .options = {ConfigParamSpec{"aod-file-private", VariantType::String, ctx.options().get<std::string>("aod-file"), {"AOD file"}},
                 ConfigParamSpec{"aod-reader-json", VariantType::String, {"json configuration file"}},
                 ConfigParamSpec{"aod-parent-access-level", VariantType::String, {"Allow parent file access up to specified level. Default: no (0)"}},
                 ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}},
diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index bfbdcfc6f169b..21af45422e0b1 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -33,6 +33,10 @@ o2_add_dpl_workflow(diamond-workflow
                   SOURCES src/o2DiamondWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
 
+o2_add_dpl_workflow(analysis-histograms
+                  SOURCES src/o2TestHistograms.cxx
+                  COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(two-timers
                   SOURCES src/o2TwoTimers.cxx
                   COMPONENT_NAME TestWorkflows)
diff --git a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
index 05bf3aabaceab..43d71a9b40528 100644
--- a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
+++ b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
@@ -49,7 +49,14 @@ struct EtaAndClsHistograms {
 
 WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
 {
-  return WorkflowSpec{
-    adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
-  };
+  // For the sake of running without an option, we do not throw an exception
+  // in case the option is not present.
+  if (cfgc.options().hasOption("aod-metadata-Run") == false ||
+      cfgc.options().get<std::string>("aod-metadata-Run") == "2") {
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
+    };
+  } else {
+    throw std::runtime_error("Unsupported run type");
+  }
 }
diff --git a/Framework/TestWorkflows/src/o2TestHistograms.cxx b/Framework/TestWorkflows/src/o2TestHistograms.cxx
new file mode 100644
index 0000000000000..331ee828e1c7b
--- /dev/null
+++ b/Framework/TestWorkflows/src/o2TestHistograms.cxx
@@ -0,0 +1,62 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+///
+/// \brief FullTracks is a join of Tracks, TracksCov, and TracksExtra.
+/// \author
+/// \since
+
+#include "Framework/runDataProcessing.h"
+#include "Framework/AnalysisTask.h"
+#include <TH2F.h>
+#include <cmath>
+
+using namespace o2;
+using namespace o2::framework;
+using namespace o2::framework::expressions;
+
+struct EtaAndClsHistograms {
+  OutputObj<TH3F> etaClsH{TH3F("eta_vs_cls_vs_sigmapT", "#eta vs N_{cls} vs sigma_{1/pT}", 102, -2.01, 2.01, 160, -0.5, 159.5, 100, 0, 10)};
+
+  void process(soa::Join<aod::FullTracks, aod::TracksCov> const& tracks)
+  {
+    for (auto& track : tracks) {
+      etaClsH->Fill(track.eta(), track.tpcNClsFindable(), track.sigma1Pt());
+    }
+  }
+};
+
+struct EtaAndClsHistogramsSimple {
+  OutputObj<TH2F> etaClsH{TH2F("eta_vs_pt", "#eta vs pT", 102, -2.01, 2.01, 100, 0, 10)};
+
+  void process(aod::Tracks const& tracks)
+  {
+    for (auto& track : tracks) {
+      etaClsH->Fill(track.eta(), track.pt(), 0);
+    }
+  }
+};
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  std::string runType = "3";
+  if (cfgc.options().hasOption("aod-metadata-Run")) {
+    runType = cfgc.options().get<std::string>("aod-metadata-Run");
+  }
+  if (runType == "2") {
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistogramsSimple>(cfgc),
+    };
+  } else {
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
+    };
+  }
+}

From af9c4fb95795dfd9071c87a12100b5acfd96c1f1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 16 May 2024 12:19:30 +0200
Subject: [PATCH 505/726] DPL: allow short formatting for data header

---
 .../include/Headers/DataHeaderHelpers.h       | 34 ++++++++++++-------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/DataFormats/Headers/include/Headers/DataHeaderHelpers.h b/DataFormats/Headers/include/Headers/DataHeaderHelpers.h
index 0fa179c6e83aa..f264460890494 100644
--- a/DataFormats/Headers/include/Headers/DataHeaderHelpers.h
+++ b/DataFormats/Headers/include/Headers/DataHeaderHelpers.h
@@ -46,15 +46,15 @@ struct fmt::formatter<T, std::enable_if_t<o2::header::is_descriptor<T>::value, c
 
 template <>
 struct fmt::formatter<o2::header::DataHeader> {
-  // Presentation format: 'f' - fixed, 'e' - exponential.
   char presentation = 's';
 
-  // Parses format specifications of the form ['f' | 'e'].
   constexpr auto parse(format_parse_context& ctx)
   {
     auto it = ctx.begin(), end = ctx.end();
     if (it != end && (*it == 's')) {
       presentation = *it++;
+    } else if (it != end && (*it == 'x')) {
+      presentation = *it++;
     }
 
     // Check if reached the end of the range:
@@ -69,17 +69,25 @@ struct fmt::formatter<o2::header::DataHeader> {
   template <typename FormatContext>
   auto format(const o2::header::DataHeader& h, FormatContext& ctx)
   {
-    auto res = fmt::format("Data header version {}, flags: {}\n", h.headerVersion, h.flags) +
-               fmt::format("  origin       : {}\n", h.dataOrigin.str) +
-               fmt::format("  serialization: {}\n", h.payloadSerializationMethod.str) +
-               fmt::format("  description  : {}\n", h.dataDescription.str) +
-               fmt::format("  sub spec.    : {}\n", (long long unsigned int)h.subSpecification) +
-               fmt::format("  header size  : {}\n", h.headerSize) +
-               fmt::format("  payloadSize  : {}\n", (long long unsigned int)h.payloadSize) +
-               fmt::format("  firstTForbit : {}\n", h.firstTForbit) +
-               fmt::format("  tfCounter    : {}\n", h.tfCounter) +
-               fmt::format("  runNumber    : {}\n", h.runNumber);
-    return fmt::format_to(ctx.out(), "{}", res);
+    if (presentation == 's') {
+      auto res = fmt::format("Data header version {}, flags: {}\n", h.headerVersion, h.flags) +
+                 fmt::format("  origin       : {}\n", h.dataOrigin.str) +
+                 fmt::format("  serialization: {}\n", h.payloadSerializationMethod.str) +
+                 fmt::format("  description  : {}\n", h.dataDescription.str) +
+                 fmt::format("  sub spec.    : {}\n", (long long unsigned int)h.subSpecification) +
+                 fmt::format("  header size  : {}\n", h.headerSize) +
+                 fmt::format("  payloadSize  : {}\n", (long long unsigned int)h.payloadSize) +
+                 fmt::format("  firstTForbit : {}\n", h.firstTForbit) +
+                 fmt::format("  tfCounter    : {}\n", h.tfCounter) +
+                 fmt::format("  runNumber    : {}\n", h.runNumber);
+      return fmt::format_to(ctx.out(), "{}", res);
+    } else {
+      auto res = fmt::format("{}/{}/{}",
+                             h.dataOrigin.str,
+                             h.dataDescription.str,
+                             (long long unsigned int)h.subSpecification);
+      return fmt::format_to(ctx.out(), "{}", res);
+    }
   }
 };
 

From 193c7edb278b9e69632e04f71546a1c6b6157076 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 16 May 2024 12:19:30 +0200
Subject: [PATCH 506/726] DPL: improve save in slot message

---
 Framework/Core/src/DataRelayer.cxx | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index bcb5f2a0d9d74..943cd0404f0b3 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -445,7 +445,9 @@ DataRelayer::RelayChoice
                      &cache = mCache,
                      numInputTypes = mDistinctRoutesIndex.size()](TimesliceId timeslice, int input, TimesliceSlot slot) {
     O2_SIGNPOST_ID_GENERATE(aid, data_relayer);
-    O2_SIGNPOST_EVENT_EMIT(data_relayer, aid, "saveInSlot", "saving %zu in slot %zu", timeslice.value, slot.index);
+    O2_SIGNPOST_EVENT_EMIT(data_relayer, aid, "saveInSlot", "saving %{public}s@%zu in slot %zu",
+                           fmt::format("{:x}", *o2::header::get<DataHeader*>(messages[0]->GetData())).c_str(),
+                           timeslice.value, slot.index);
     auto cacheIdx = numInputTypes * slot.index + input;
     MessageSet& target = cache[cacheIdx];
     cachedStateMetrics[cacheIdx] = CacheEntryStatus::PENDING;

From 26ad8f3693f4dbafd316c74f48b8f7dd3d83a060 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 16 May 2024 14:53:53 +0200
Subject: [PATCH 507/726] DPL Analysis: provide metadata to workflow
 construction

---
 .../src/AODJAlienReaderHelpers.cxx            |   6 +-
 Framework/AnalysisSupport/src/Plugin.cxx      |  67 ++++++++++++
 Framework/Core/CMakeLists.txt                 |   2 +
 Framework/Core/include/Framework/Capability.h |  44 ++++++++
 .../include/Framework/ConfigParamDiscovery.h  |  37 +++++++
 .../include/Framework/ConfigParamRegistry.h   |   6 ++
 .../Core/include/Framework/ConfigParamStore.h |   4 +-
 .../Core/include/Framework/PluginManager.h    |   2 +-
 Framework/Core/include/Framework/Plugins.h    |   9 ++
 .../Core/include/Framework/ServiceSpec.h      |   5 -
 .../include/Framework/runDataProcessing.h     |  12 ++-
 Framework/Core/src/ConfigParamDiscovery.cxx   |  81 ++++++++++++++
 Framework/Core/src/ConfigParamStore.cxx       |  11 ++
 Framework/Core/src/DeviceSpecHelpers.cxx      |  35 ++++--
 Framework/Core/src/DeviceSpecHelpers.h        |   1 +
 Framework/Core/src/Plugin.cxx                 | 100 ++++++++++++++++++
 .../Core/src/WorkflowCustomizationHelpers.cxx |   4 +
 Framework/Core/src/WorkflowHelpers.cxx        |   2 +-
 Framework/Core/src/runDataProcessing.cxx      |  11 ++
 .../Core/test/test_DeviceSpecHelpers.cxx      |   2 +-
 .../Core/test/test_FrameworkDataFlowToDDS.cxx |   4 +-
 .../test_FrameworkDataFlowToO2Control.cxx     |   2 +-
 Framework/TestWorkflows/CMakeLists.txt        |   4 +
 .../TestWorkflows/src/o2AnalysisWorkflow.cxx  |  13 ++-
 .../TestWorkflows/src/o2TestHistograms.cxx    |  65 ++++++++++++
 25 files changed, 502 insertions(+), 27 deletions(-)
 create mode 100644 Framework/Core/include/Framework/Capability.h
 create mode 100644 Framework/Core/include/Framework/ConfigParamDiscovery.h
 create mode 100644 Framework/Core/src/ConfigParamDiscovery.cxx
 create mode 100644 Framework/Core/src/Plugin.cxx
 create mode 100644 Framework/TestWorkflows/src/o2TestHistograms.cxx

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 092857d585a02..5ce2eca49c07b 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -133,12 +133,12 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_MESSAGES_DESTROYED), DataProcessingStats::Op::Set, 0});
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_BYTES_EXPIRED), DataProcessingStats::Op::Set, 0});
 
-    if (!options.isSet("aod-file")) {
+    if (!options.isSet("aod-file-private")) {
       LOGP(fatal, "No input file defined!");
       throw std::runtime_error("Processing is stopped!");
     }
 
-    auto filename = options.get<std::string>("aod-file");
+    auto filename = options.get<std::string>("aod-file-private");
 
     std::string parentFileReplacement;
     if (options.isSet("aod-parent-base-path-replacement")) {
@@ -306,7 +306,7 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
           control.readyToQuit(QuitRequest::Me);
           return;
         }
-      } 
+      }
     });
   })};
 
diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index 8f2e489086afb..2ee8d316f3675 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -12,7 +12,17 @@
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/ServiceSpec.h"
 #include "Framework/ServiceMetricsInfo.h"
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/Capability.h"
+#include "Framework/Signpost.h"
 #include "AODJAlienReaderHelpers.h"
+#include <TFile.h>
+#include <TMap.h>
+#include <TObjString.h>
+#include <TString.h>
+#include <fmt/format.h>
+
+O2_DECLARE_DYNAMIC_LOG(analysis_support);
 
 struct ROOTFileReader : o2::framework::AlgorithmPlugin {
   o2::framework::AlgorithmSpec create() override
@@ -54,7 +64,64 @@ struct RunSummary : o2::framework::ServicePlugin {
   }
 };
 
+struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
+  ConfigDiscovery* create() override
+  {
+    return new ConfigDiscovery{
+      .init = []() {},
+      .discover = [](ConfigParamRegistry& registry, int argc, char** argv) -> std::vector<ConfigParamSpec> {
+        auto filename = registry.get<std::string>("aod-file");
+        if (filename.empty()) {
+          return {};
+        }
+        LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
+        std::vector<ConfigParamSpec> results;
+        TFile* currentFile = nullptr;
+        if (filename.at(0) == '@') {
+          filename.erase(0, 1);
+          // read the text file and set filename to the contents of the first line
+          std::ifstream file(filename);
+          if (!file.is_open()) {
+            LOGP(fatal, "Couldn't open file \"{}\"!", filename);
+          }
+          std::getline(file, filename);
+          file.close();
+          currentFile = TFile::Open(filename.c_str());
+        } else {
+          currentFile = TFile::Open(filename.c_str());
+        }
+        if (!currentFile) {
+          LOGP(fatal, "Couldn't open file \"{}\"!", filename);
+        }
+
+        // Get the metadata, if any
+        auto m = (TMap*)currentFile->Get("metaData");
+        if (!m) {
+          LOGP(warning, "No metadata found in file \"{}\"", filename);
+          return results;
+        }
+        auto it = m->MakeIterator();
+
+        // Serialise metadata into a ; separated string with : separating key and value
+        bool first = true;
+        while (auto obj = it->Next()) {
+          if (first) {
+            LOGP(info, "Metadata for file \"{}\":", filename);
+            first = false;
+          }
+          auto objString = (TObjString*)m->GetValue(obj);
+          LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
+          std::string key = "aod-metadata-" + std::string(obj->GetName());
+          char const* value = strdup(objString->String());
+          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
+        }
+        return results;
+      }};
+  }
+};
+
 DEFINE_DPL_PLUGINS_BEGIN
 DEFINE_DPL_PLUGIN_INSTANCE(ROOTFileReader, CustomAlgorithm);
 DEFINE_DPL_PLUGIN_INSTANCE(RunSummary, CustomService);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAOD, ConfigDiscovery);
 DEFINE_DPL_PLUGINS_END
diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index d392ba42fe8ac..4381b6c4fa29d 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -47,6 +47,7 @@ o2_add_library(Framework
                        src/DefaultsHelpers.cxx
                        src/DomainInfoHeader.cxx
                        src/ProcessingPoliciesHelpers.cxx
+                       src/ConfigParamDiscovery.cxx
                        src/ConfigParamStore.cxx
                        src/ConfigParamsHelper.cxx
                        src/ChannelParamSpec.cxx
@@ -109,6 +110,7 @@ o2_add_library(Framework
                        src/OptionsHelpers.cxx
                        src/PropertyTreeHelpers.cxx
                        src/ProcessingContext.cxx
+                       src/Plugin.cxx
                        src/PluginManager.cxx
                        src/RateLimiter.cxx
                        src/ResourcesMonitoringHelper.cxx
diff --git a/Framework/Core/include/Framework/Capability.h b/Framework/Core/include/Framework/Capability.h
new file mode 100644
index 0000000000000..23ef7b7d9bf17
--- /dev/null
+++ b/Framework/Core/include/Framework/Capability.h
@@ -0,0 +1,44 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_CAPABILITY_H_
+#define O2_FRAMEWORK_CAPABILITY_H_
+
+#include <string_view>
+#include <functional>
+
+namespace o2::framework
+{
+
+struct ConfigParamRegistry;
+
+/// A plugin which is able to discover more option then the ones
+/// provided on the command line. The actual loading is in two parts,
+/// the first one decides if the options are enough to actually perform
+/// the discovery, the second part will do the discovery itself.
+///
+/// Its a good idea to have the Loader part in a standalone library to
+/// minimize dependency on unneed thir party code, e.g. ROOT.
+struct Capability {
+  // Wether or not this capability is required.
+  // FIXME: for now let's pass the arguments to get the metadata to work
+  std::string name;
+  std::function<bool(ConfigParamRegistry&, int argc, char** argv)> checkIfNeeded;
+  char const* requiredPlugin;
+};
+
+struct CapabilityPlugin {
+  virtual Capability* create() = 0;
+};
+
+} // namespace o2::framework
+
+#endif // O2_FRAMEWORK_CAPABILITY_H_
diff --git a/Framework/Core/include/Framework/ConfigParamDiscovery.h b/Framework/Core/include/Framework/ConfigParamDiscovery.h
new file mode 100644
index 0000000000000..3d13d3662ceb1
--- /dev/null
+++ b/Framework/Core/include/Framework/ConfigParamDiscovery.h
@@ -0,0 +1,37 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
+#define O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
+
+#include "Framework/ConfigParamRegistry.h"
+
+namespace o2::framework
+{
+// A plugin which can be used to inject extra configuration
+// options.
+struct ConfigParamDiscovery {
+  static std::vector<ConfigParamSpec> discover(ConfigParamRegistry&, int, char**);
+};
+
+struct ConfigDiscovery {
+  std::function<void()> init;
+  /// A function which uses the arguments available so far to discover more
+  /// @return the extra ConfigParamSpecs it derives.
+  std::function<std::vector<ConfigParamSpec>(ConfigParamRegistry&, int, char**)> discover;
+};
+
+struct ConfigDiscoveryPlugin {
+  virtual ConfigDiscovery* create() = 0;
+};
+
+} // namespace o2::framework
+#endif // O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
diff --git a/Framework/Core/include/Framework/ConfigParamRegistry.h b/Framework/Core/include/Framework/ConfigParamRegistry.h
index 8f2d80cea0948..397a491be9f9c 100644
--- a/Framework/Core/include/Framework/ConfigParamRegistry.h
+++ b/Framework/Core/include/Framework/ConfigParamRegistry.h
@@ -125,6 +125,12 @@ class ConfigParamRegistry
     }
   }
 
+  // Load extra parameters discovered while we process data
+  void loadExtra(std::vector<ConfigParamSpec>& extras)
+  {
+    mStore->load(extras);
+  }
+
  private:
   std::unique_ptr<ConfigParamStore> mStore;
 };
diff --git a/Framework/Core/include/Framework/ConfigParamStore.h b/Framework/Core/include/Framework/ConfigParamStore.h
index 37442a6cf7f41..b3ae76aa47ad1 100644
--- a/Framework/Core/include/Framework/ConfigParamStore.h
+++ b/Framework/Core/include/Framework/ConfigParamStore.h
@@ -34,12 +34,14 @@ class ConfigParamStore
   /// the configuration.
   void preload();
 
+  void load(std::vector<ConfigParamSpec>& specs);
+
   /// Get the store
   boost::property_tree::ptree& store() { return *mStore; };
   boost::property_tree::ptree& provenanceTree() { return *mProvenance; };
 
   /// Get the specs
-  std::vector<ConfigParamSpec> const& specs() const
+  [[nodiscard]] std::vector<ConfigParamSpec> const& specs() const
   {
     return mSpecs;
   }
diff --git a/Framework/Core/include/Framework/PluginManager.h b/Framework/Core/include/Framework/PluginManager.h
index 38966c5a969a9..4c6e965502500 100644
--- a/Framework/Core/include/Framework/PluginManager.h
+++ b/Framework/Core/include/Framework/PluginManager.h
@@ -70,7 +70,7 @@ struct PluginManager {
 
     struct LoadedPlugin {
       std::string name;
-      ServicePlugin* factory;
+      PLUGIN* factory;
     };
     std::vector<LoadedDSO> loadedDSOs;
     std::vector<LoadedPlugin> loadedPlugins;
diff --git a/Framework/Core/include/Framework/Plugins.h b/Framework/Core/include/Framework/Plugins.h
index c29d96f7e0781..23d55a512e1fa 100644
--- a/Framework/Core/include/Framework/Plugins.h
+++ b/Framework/Core/include/Framework/Plugins.h
@@ -27,6 +27,15 @@ enum struct DplPluginKind : int {
   // A plugin which implements a custom Services. Needs to return
   // an object of the kind o2::framework::ServiceSpec
   CustomService,
+  // A plugin which implements a new way to discover extra configuration.
+  // parameters. E.g. it can be used to read metadata from a file or a service
+  // if a certain parameter is available.
+  ConfigDiscovery,
+  // A capability plugin is a plugin used to discover other viable plugins.
+  // For example, if you find out that you have the --aod-file option
+  // set, you might want to load metadata from it and attach it to the
+  // configuration.
+  Capability,
   // A plugin which was not initialised properly.
   Unknown
 };
diff --git a/Framework/Core/include/Framework/ServiceSpec.h b/Framework/Core/include/Framework/ServiceSpec.h
index abac3ad05914c..5684889e85376 100644
--- a/Framework/Core/include/Framework/ServiceSpec.h
+++ b/Framework/Core/include/Framework/ServiceSpec.h
@@ -322,11 +322,6 @@ struct ServicePlugin {
   virtual ServiceSpec* create() = 0;
 };
 
-struct LoadableService {
-  std::string name;
-  std::string library;
-};
-
 } // namespace o2::framework
 
 #endif // O2_FRAMEWORK_SERVICESPEC_H_
diff --git a/Framework/Core/include/Framework/runDataProcessing.h b/Framework/Core/include/Framework/runDataProcessing.h
index 77a4c9f5ba8d6..c0d5b2211e23d 100644
--- a/Framework/Core/include/Framework/runDataProcessing.h
+++ b/Framework/Core/include/Framework/runDataProcessing.h
@@ -29,7 +29,8 @@
 #include "Framework/Logger.h"
 #include "Framework/CheckTypes.h"
 #include "Framework/StructToTuple.h"
-
+#include "Framework/ConfigParamDiscovery.h"
+#include <iostream>
 #include <vector>
 
 namespace o2::framework
@@ -137,6 +138,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& specs,
            std::vector<o2::framework::CallbacksPolicy> const& callbacksPolicies,
            std::vector<o2::framework::SendingPolicy> const& sendingPolicies,
            std::vector<o2::framework::ConfigParamSpec> const& workflowOptions,
+           std::vector<o2::framework::ConfigParamSpec> const& detectedOptions,
            o2::framework::ConfigContext& configContext);
 
 void doDefaultWorkflowTerminationHook();
@@ -192,6 +194,12 @@ int mainNoCatch(int argc, char** argv)
   workflowOptionsStore->preload();
   workflowOptionsStore->activate();
   ConfigParamRegistry workflowOptionsRegistry(std::move(workflowOptionsStore));
+  auto extraOptions = o2::framework::ConfigParamDiscovery::discover(workflowOptionsRegistry, argc, argv);
+  for (auto& extra : extraOptions) {
+    workflowOptions.push_back(extra);
+  }
+  workflowOptionsRegistry.loadExtra(extraOptions);
+
   ConfigContext configContext(workflowOptionsRegistry, argc, argv);
   o2::framework::WorkflowSpec specs = defineDataProcessing(configContext);
   overrideCloning(configContext, specs);
@@ -206,7 +214,7 @@ int mainNoCatch(int argc, char** argv)
   channelPolicies.insert(std::end(channelPolicies), std::begin(defaultChannelPolicies), std::end(defaultChannelPolicies));
   return doMain(argc, argv, specs,
                 channelPolicies, completionPolicies, dispatchPolicies,
-                resourcePolicies, callbacksPolicies, sendingPolicies, workflowOptions, configContext);
+                resourcePolicies, callbacksPolicies, sendingPolicies, workflowOptions, extraOptions, configContext);
 }
 
 int callMain(int argc, char** argv, int (*)(int, char**));
diff --git a/Framework/Core/src/ConfigParamDiscovery.cxx b/Framework/Core/src/ConfigParamDiscovery.cxx
new file mode 100644
index 0000000000000..6c24a44bac566
--- /dev/null
+++ b/Framework/Core/src/ConfigParamDiscovery.cxx
@@ -0,0 +1,81 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/PluginManager.h"
+#include "Framework/Signpost.h"
+#include "Framework/Capability.h"
+#include <TFile.h>
+#include <TMap.h>
+#include <TObjString.h>
+
+O2_DECLARE_DYNAMIC_LOG(config_parameter_discovery);
+
+namespace o2::framework
+{
+
+std::vector<ConfigParamSpec> ConfigParamDiscovery::discover(ConfigParamRegistry& registry, int argc, char** argv)
+{
+  std::vector<char const*> capabilitiesSpecs = {
+    "O2Framework:DiscoverMetadataInAODCapability",
+    "O2Framework:DiscoverMetadataInCommandLineCapability",
+  };
+
+  // Load all the requested plugins and discover what we can do.
+  O2_SIGNPOST_ID_GENERATE(sid, config_parameter_discovery);
+  std::vector<LoadablePlugin> plugins;
+  for (auto spec : capabilitiesSpecs) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "Parsing Capability plugin: %{public}s", spec);
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+  }
+
+  std::vector<Capability> availableCapabilities;
+  std::vector<char const*> configDiscoverySpec = {};
+  PluginManager::loadFromPlugin<Capability, CapabilityPlugin>(plugins, availableCapabilities);
+  for (auto& capability : availableCapabilities) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- Using Capability of : %{public}s",
+                           capability.name.c_str());
+
+    if (capability.checkIfNeeded(registry, argc, argv)) {
+      O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- Required: %{}s",
+                             capability.requiredPlugin);
+      configDiscoverySpec.push_back(capability.requiredPlugin);
+    } else {
+      O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- No requirements added");
+    }
+  }
+
+  std::vector<LoadablePlugin> configDiscoveryPlugins;
+  for (auto spec : configDiscoverySpec) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "discovery", "Parsing ConfigDiscovery plugin: %{public}s", spec);
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      configDiscoveryPlugins.push_back(extra);
+    }
+  }
+
+  std::vector<ConfigDiscovery> availableConfigDiscovery;
+  PluginManager::loadFromPlugin<ConfigDiscovery, ConfigDiscoveryPlugin>(configDiscoveryPlugins, availableConfigDiscovery);
+  std::vector<ConfigParamSpec> result;
+  for (auto& configDiscovery : availableConfigDiscovery) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "discovery", "Parsing extra configuration as needed");
+    auto extras = configDiscovery.discover(registry, argc, argv);
+    for (auto& extra : extras) {
+      result.push_back(extra);
+    }
+  }
+  return result;
+}
+
+} // namespace o2::framework
diff --git a/Framework/Core/src/ConfigParamStore.cxx b/Framework/Core/src/ConfigParamStore.cxx
index 249a8b0052a67..19e123a23e5d0 100644
--- a/Framework/Core/src/ConfigParamStore.cxx
+++ b/Framework/Core/src/ConfigParamStore.cxx
@@ -41,6 +41,17 @@ void ConfigParamStore::preload()
   }
 }
 
+void ConfigParamStore::load(std::vector<ConfigParamSpec>& specs)
+{
+  mNextStore->clear();
+  mNextProvenance->clear();
+  // By default we populate with code.
+  PropertyTreeHelpers::populateDefaults(specs, *mNextStore, *mNextProvenance);
+  for (auto& retriever : mRetrievers) {
+    retriever->update(specs, *mStore, *mProvenance);
+  }
+}
+
 /// Activate the next store
 void ConfigParamStore::activate()
 {
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 30197538f2dda..b9552dc093115 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -32,7 +32,7 @@
 #include "Framework/OutputRoute.h"
 #include "Framework/WorkflowSpec.h"
 #include "Framework/ComputingResource.h"
-#include "Framework/Logger.h"
+#include "Framework/Signpost.h"
 #include "Framework/RuntimeError.h"
 #include "Framework/RawDeviceService.h"
 #include "ProcessingPoliciesHelpers.h"
@@ -50,8 +50,7 @@
 
 #include <regex>
 
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wpedantic"
+O2_DECLARE_DYNAMIC_LOG(device_spec_helpers);
 
 namespace bpo = boost::program_options;
 
@@ -125,7 +124,9 @@ struct ExpirationHandlerHelpers {
       if (periods.empty()) {
         std::string defaultRateName = std::string{"period-"} + matcher.binding;
         auto defaultRate = std::chrono::milliseconds(options.get<int>(defaultRateName.c_str()));
-        LOGP(detail, "Using default rate of {} ms as specified by option period-{}", defaultRate.count(), matcher.binding);
+        O2_SIGNPOST_ID_GENERATE(tid, device_spec_helpers);
+        O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, tid, "timeDrivenCreation", "Using default rate of %" PRIi64 " ms as specified by option period-%{public}s", defaultRate.count(),
+                               matcher.binding.c_str());
         periods.emplace_back(defaultRate.count());
         durations.emplace_back(std::chrono::seconds((std::size_t)-1));
       } else {
@@ -1359,12 +1360,15 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
                                          std::vector<DeviceSpec> const& deviceSpecs,
                                          std::vector<DeviceExecution>& deviceExecutions,
                                          std::vector<DeviceControl>& deviceControls,
+                                         std::vector<ConfigParamSpec> const& detectedOptions,
                                          std::string const& uniqueWorkflowId)
 {
   assert(deviceSpecs.size() == deviceExecutions.size());
   assert(deviceControls.size() == deviceExecutions.size());
   for (size_t si = 0; si < deviceSpecs.size(); ++si) {
     auto& spec = deviceSpecs[si];
+    O2_SIGNPOST_ID_GENERATE(poid, device_spec_helpers);
+    O2_SIGNPOST_START(device_spec_helpers, poid, "prepareArguments", "Preparing options for %{public}s", spec.id.c_str());
     auto& control = deviceControls[si];
     auto& execution = deviceExecutions[si];
 
@@ -1373,7 +1377,14 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
 
     int argc;
     char** argv;
-    std::vector<ConfigParamSpec> workflowOptions;
+    // We need to start with the detected options, so that they are not lost.
+    // Notice how detected options can be detected at any moment in the chain,
+    // so it's important that if you rely on them, they get passed on
+    // always.
+    std::vector<ConfigParamSpec> workflowOptions = detectedOptions;
+    for (auto& opt : detectedOptions) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, poid, "prepareArguments", "Processor option %{public}s passed as previously detected", opt.name.c_str());
+    }
     /// Lookup the executable name in the metadata associated with the workflow.
     /// If we find it, we rewrite the command line arguments to be processed
     /// so that they look like the ones passed to the merged workflow.
@@ -1386,8 +1397,17 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
       argv[ai + 1] = strdup(arg.data());
     }
     argv[argc] = nullptr;
-    workflowOptions = pi->workflowOptions;
+    for (auto& opt : pi->workflowOptions) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, poid, "prepareArguments", "Processor option %{public}s found in process description", opt.name.c_str());
+      workflowOptions.push_back(opt);
+    }
+    std::sort(workflowOptions.begin(), workflowOptions.end(), [](ConfigParamSpec const& a, ConfigParamSpec const& b) { return a.name < b.name; });
+    auto last = std::unique(workflowOptions.begin(), workflowOptions.end());
+    workflowOptions.erase(last, workflowOptions.end());
 
+    for (auto& opt : workflowOptions) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, poid, "prepareArguments", "Final unique option %{public}s added to list of workflowOptions", opt.name.c_str());
+    }
     // We duplicate the list of options, filtering only those
     // which are actually relevant for the given device. The additional
     // four are to add
@@ -1662,7 +1682,8 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
       assert(execution.args[ai]);
       str << " " << execution.args[ai];
     }
-    LOG(debug) << "The following options are being forwarded to " << spec.id << ":" << str.str();
+    O2_SIGNPOST_END(device_spec_helpers, poid, "prepareArguments", "The following options are being forwarded to %{public}s: %{public}s",
+                    spec.id.c_str(), str.str().c_str());
   }
 }
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.h b/Framework/Core/src/DeviceSpecHelpers.h
index 17601c81342c1..5b54bc84cf6c8 100644
--- a/Framework/Core/src/DeviceSpecHelpers.h
+++ b/Framework/Core/src/DeviceSpecHelpers.h
@@ -129,6 +129,7 @@ struct DeviceSpecHelpers {
     std::vector<DeviceSpec> const& deviceSpecs,
     std::vector<DeviceExecution>& deviceExecutions,
     std::vector<DeviceControl>& deviceControls,
+    std::vector<ConfigParamSpec> const& detectedOptions,
     std::string const& uniqueWorkflowId);
 
   /// Rework the environment string
diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx
new file mode 100644
index 0000000000000..2298896821dab
--- /dev/null
+++ b/Framework/Core/src/Plugin.cxx
@@ -0,0 +1,100 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/Plugins.h"
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/ConfigParamRegistry.h"
+#include "Framework/Logger.h"
+#include "Framework/Capability.h"
+#include "Framework/Signpost.h"
+#include <string_view>
+
+O2_DECLARE_DYNAMIC_LOG(capabilities);
+namespace o2::framework
+{
+auto lookForAodFile = [](ConfigParamRegistry& registry, int argc, char** argv) -> bool {
+  O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+  if (registry.hasOption("aod-file") && registry.isSet("aod-file")) {
+    for (size_t i = 0; i < argc; i++) {
+      std::string_view arg = argv[i];
+      if (arg.starts_with("--aod-metadata-")) {
+        return false;
+      }
+    }
+    O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInAODCapability", "Metadata not found in arguments. Checking in AOD file.");
+    return true;
+  }
+  return false;
+};
+
+auto lookForCommandLineOptions = [](ConfigParamRegistry& registry, int argc, char** argv) -> bool {
+  O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+  for (size_t i = 0; i < argc; i++) {
+    std::string_view arg = argv[i];
+    if (arg.starts_with("--aod-metadata-")) {
+      O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLineCapability", "Metadata found in arguments. Populating from them.");
+      return true;
+    }
+  }
+  return false;
+};
+
+struct DiscoverMetadataInAODCapability : o2::framework::CapabilityPlugin {
+  Capability* create() override
+  {
+    return new Capability{
+      .name = "DiscoverMetadataInAODCapability",
+      .checkIfNeeded = lookForAodFile,
+      .requiredPlugin = "O2FrameworkAnalysisSupport:DiscoverMetadataInAOD"};
+  }
+};
+
+// Trigger discovery of metadata from command line, if needed.
+struct DiscoverMetadataInCommandLineCapability : o2::framework::CapabilityPlugin {
+  Capability* create() override
+  {
+    return new Capability{
+      .name = "DiscoverMetadataInCommandLineCapability",
+      .checkIfNeeded = lookForCommandLineOptions,
+      .requiredPlugin = "O2Framework:DiscoverMetadataInCommandLine"};
+  }
+};
+
+struct DiscoverMetadataInCommandLine : o2::framework::ConfigDiscoveryPlugin {
+  ConfigDiscovery* create() override
+  {
+    return new ConfigDiscovery{
+      .init = []() {},
+      .discover = [](ConfigParamRegistry& registry, int argc, char** argv) -> std::vector<ConfigParamSpec> {
+        O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+        O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLine",
+                               "Discovering metadata for analysis from well known environment variables.");
+        std::vector<ConfigParamSpec> results;
+        for (size_t i = 0; i < argc; i++) {
+          std::string_view arg = argv[i];
+          if (!arg.starts_with("--aod-metadata")) {
+            continue;
+          }
+          std::string key = arg.data() + 2;
+          std::string value = argv[i + 1];
+          O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLine",
+                                 "Found %{public}s with value %{public}s.", key.c_str(), value.c_str());
+          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in command line"}});
+        }
+        return results;
+      }};
+  }
+};
+DEFINE_DPL_PLUGINS_BEGIN
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAODCapability, Capability);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLineCapability, Capability);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLine, ConfigDiscovery);
+DEFINE_DPL_PLUGINS_END
+} // namespace o2::framework
diff --git a/Framework/Core/src/WorkflowCustomizationHelpers.cxx b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
index ffad8a8720019..05abb5dab98cd 100644
--- a/Framework/Core/src/WorkflowCustomizationHelpers.cxx
+++ b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
@@ -54,6 +54,10 @@ std::vector<ConfigParamSpec> WorkflowCustomizationHelpers::requiredWorkflowOptio
            // options for AOD rate limiting
            {"aod-memory-rate-limit", VariantType::Int64, 0LL, {"Rate limit AOD processing based on memory"}},
 
+           // options for the AOD reader
+           // aod-file needs to be available as workflow option, because we
+           // can configure the workflow based on the contents of the first file.
+           {"aod-file", VariantType::String, "", {"Input AOD file"}},
            // options for AOD writer
            {"aod-writer-json", VariantType::String, "", {"Name of the json configuration file"}},
            {"aod-writer-resdir", VariantType::String, "", {"Name of the output directory"}},
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index bd875ad421f6e..da60bdbc6961e 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -292,7 +292,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
                          static_cast<DataAllocator::SubSpecificationType>(compile_time_hash("internal-dpl-aod-reader")),
                          aodLifetime}},
     .algorithm = AlgorithmSpec::dummyAlgorithm(),
-    .options = {ConfigParamSpec{"aod-file", VariantType::String, {"Input AOD file"}},
+    .options = {ConfigParamSpec{"aod-file-private", VariantType::String, ctx.options().get<std::string>("aod-file"), {"AOD file"}},
                 ConfigParamSpec{"aod-reader-json", VariantType::String, {"json configuration file"}},
                 ConfigParamSpec{"aod-parent-access-level", VariantType::String, {"Allow parent file access up to specified level. Default: no (0)"}},
                 ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}},
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index f91716d28cfd7..a0ef33f812e89 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -194,6 +194,10 @@ char* getIdString(int argc, char** argv)
 
 int callMain(int argc, char** argv, int (*mainNoCatch)(int, char**))
 {
+  std::cout << "This is " << getpid() << " " << argv[0] << std::endl;
+  auto spec1 = fair::VerbositySpec::Make(fair::VerbositySpec::Info::process_name);
+  fair::Logger::DefineVerbosity(fair::Verbosity::user1, spec1);
+  fair::Logger::SetVerbosity(fair::Verbosity::user1);
   static bool noCatch = getenv("O2_NO_CATCHALL_EXCEPTIONS") && strcmp(getenv("O2_NO_CATCHALL_EXCEPTIONS"), "0");
   int result = 1;
   if (noCatch) {
@@ -748,6 +752,9 @@ void spawnDevice(uv_loop_t* loop,
       putenv(strdup(DeviceSpecHelpers::reworkTimeslicePlaceholder(env, spec).data()));
     }
     execvp(execution.args[0], execution.args.data());
+  } else {
+    O2_SIGNPOST_ID_GENERATE(sid, driver);
+    O2_SIGNPOST_EVENT_EMIT(driver, sid, "spawnDevice", "New child at %{pid}d", id);
   }
   close(childFds[ref.index].childstdin[0]);
   close(childFds[ref.index].childstdout[1]);
@@ -1359,6 +1366,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
                     DriverInfo& driverInfo,
                     DriverConfig& driverConfig,
                     std::vector<DeviceMetricsInfo>& metricsInfos,
+                    std::vector<ConfigParamSpec> const& detectedParams,
                     boost::program_options::variables_map& varmap,
                     std::vector<ServiceSpec>& driverServices,
                     std::string frameworkId)
@@ -2048,6 +2056,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
                                               runningWorkflow.devices,
                                               deviceExecutions,
                                               controls,
+                                              detectedParams,
                                               driverInfo.uniqueWorkflowId);
         } catch (o2::framework::RuntimeErrorRef& ref) {
           auto& err = o2::framework::error_from_ref(ref);
@@ -2742,6 +2751,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
            std::vector<CallbacksPolicy> const& callbacksPolicies,
            std::vector<SendingPolicy> const& sendingPolicies,
            std::vector<ConfigParamSpec> const& currentWorkflowOptions,
+           std::vector<ConfigParamSpec> const& detectedParams,
            o2::framework::ConfigContext& configContext)
 {
   std::vector<std::string> currentArgs;
@@ -3173,6 +3183,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
                          driverInfo,
                          driverConfig,
                          gDeviceMetricsInfos,
+                         detectedParams,
                          varmap,
                          driverServices,
                          frameworkId);
diff --git a/Framework/Core/test/test_DeviceSpecHelpers.cxx b/Framework/Core/test/test_DeviceSpecHelpers.cxx
index 762c2df6f82aa..6240e784d09d3 100644
--- a/Framework/Core/test/test_DeviceSpecHelpers.cxx
+++ b/Framework/Core/test/test_DeviceSpecHelpers.cxx
@@ -79,7 +79,7 @@ void check(const std::vector<std::string>& arguments,
                                       dataProcessorInfos,
                                       deviceSpecs,
                                       deviceExecutions,
-                                      deviceControls,
+                                      deviceControls, {},
                                       "workflow-id");
 
   for (size_t index = 0; index < deviceSpecs.size(); index++) {
diff --git a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
index 3c08bbae74475..593728696e77a 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
@@ -153,7 +153,7 @@ TEST_CASE("TestDDS")
   DeviceSpecHelpers::prepareArguments(false, false, false, 8080,
                                       driverConfig,
                                       dataProcessorInfos,
-                                      devices, executions, controls,
+                                      devices, executions, controls, {},
                                       "workflow-id");
   CommandInfo command{"foo"};
   DDSConfigHelpers::dumpDeviceSpec2DDS(ss, DriverMode::STANDALONE, "", workflow, dataProcessorInfos, devices, executions, command);
@@ -417,7 +417,7 @@ TEST_CASE("TestDDSExpendable")
   DeviceSpecHelpers::prepareArguments(false, false, false, 8080,
                                       driverConfig,
                                       dataProcessorInfos,
-                                      devices, executions, controls,
+                                      devices, executions, controls, {},
                                       "workflow-id");
   CommandInfo command{"foo"};
   DDSConfigHelpers::dumpDeviceSpec2DDS(ss, DriverMode::STANDALONE, "", workflow, dataProcessorInfos, devices, executions, command);
diff --git a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
index bbd5b78fbbb68..4aca1bd9680a1 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
@@ -560,7 +560,7 @@ TEST_CASE("TestO2ControlDump")
   DeviceSpecHelpers::prepareArguments(false, false, false, 8080,
                                       driverConfig,
                                       dataProcessorInfos,
-                                      devices, executions, controls,
+                                      devices, executions, controls, {},
                                       "workflow-id");
 
   dumpWorkflow(ss, devices, executions, commandInfo, "testwf", "");
diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index bfbdcfc6f169b..21af45422e0b1 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -33,6 +33,10 @@ o2_add_dpl_workflow(diamond-workflow
                   SOURCES src/o2DiamondWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
 
+o2_add_dpl_workflow(analysis-histograms
+                  SOURCES src/o2TestHistograms.cxx
+                  COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(two-timers
                   SOURCES src/o2TwoTimers.cxx
                   COMPONENT_NAME TestWorkflows)
diff --git a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
index 05bf3aabaceab..43d71a9b40528 100644
--- a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
+++ b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
@@ -49,7 +49,14 @@ struct EtaAndClsHistograms {
 
 WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
 {
-  return WorkflowSpec{
-    adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
-  };
+  // For the sake of running without an option, we do not throw an exception
+  // in case the option is not present.
+  if (cfgc.options().hasOption("aod-metadata-Run") == false ||
+      cfgc.options().get<std::string>("aod-metadata-Run") == "2") {
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
+    };
+  } else {
+    throw std::runtime_error("Unsupported run type");
+  }
 }
diff --git a/Framework/TestWorkflows/src/o2TestHistograms.cxx b/Framework/TestWorkflows/src/o2TestHistograms.cxx
new file mode 100644
index 0000000000000..fd6ad1c261032
--- /dev/null
+++ b/Framework/TestWorkflows/src/o2TestHistograms.cxx
@@ -0,0 +1,65 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+///
+/// \brief FullTracks is a join of Tracks, TracksCov, and TracksExtra.
+/// \author
+/// \since
+
+#include "Framework/runDataProcessing.h"
+#include "Framework/AnalysisTask.h"
+#include <TH2F.h>
+#include <cmath>
+
+using namespace o2;
+using namespace o2::framework;
+using namespace o2::framework::expressions;
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  std::string runType = "3";
+  if (cfgc.options().hasOption("aod-metadata-Run")) {
+    runType = cfgc.options().get<std::string>("aod-metadata-Run");
+  }
+  LOGP(info, "Runtype is {}", runType);
+  if (runType == "2") {
+    struct EtaAndClsHistograms {
+      OutputObj<TH2F> etaClsH{TH2F("eta_vs_pt", "#eta vs pT", 102, -2.01, 2.01, 100, 0, 10)};
+
+      void process(aod::Tracks const& tracks)
+      {
+        LOGP(info, "Invoking the simple one");
+        for (auto& track : tracks) {
+          etaClsH->Fill(track.eta(), track.pt(), 0);
+        }
+      }
+    };
+    LOGP(info, "Using only tracks {}", runType);
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
+    };
+  } else {
+    LOGP(info, "Using tracks extra {}", runType);
+    struct EtaAndClsHistograms {
+      OutputObj<TH3F> etaClsH{TH3F("eta_vs_cls_vs_sigmapT", "#eta vs N_{cls} vs sigma_{1/pT}", 102, -2.01, 2.01, 160, -0.5, 159.5, 100, 0, 10)};
+
+      void process(soa::Join<aod::FullTracks, aod::TracksCov> const& tracks)
+      {
+        LOGP(info, "Invoking the run 3 one");
+        for (auto& track : tracks) {
+          etaClsH->Fill(track.eta(), track.tpcNClsFindable(), track.sigma1Pt());
+        }
+      }
+    };
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
+    };
+  }
+}

From 1bb68dd732e3c0941cf5e8df36dca202df032fcd Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 16 May 2024 15:19:07 +0200
Subject: [PATCH 508/726] DPL Analysis: provide metadata to workflow
 construction

---
 .../src/AODJAlienReaderHelpers.cxx            |   6 +-
 Framework/AnalysisSupport/src/Plugin.cxx      |  67 ++++++++++++
 Framework/Core/CMakeLists.txt                 |   2 +
 Framework/Core/include/Framework/Capability.h |  44 ++++++++
 .../include/Framework/ConfigParamDiscovery.h  |  37 +++++++
 .../include/Framework/ConfigParamRegistry.h   |   6 ++
 .../Core/include/Framework/ConfigParamStore.h |   4 +-
 .../Core/include/Framework/PluginManager.h    |   2 +-
 Framework/Core/include/Framework/Plugins.h    |   9 ++
 .../Core/include/Framework/ServiceSpec.h      |   5 -
 .../include/Framework/runDataProcessing.h     |  12 ++-
 Framework/Core/src/ConfigParamDiscovery.cxx   |  81 ++++++++++++++
 Framework/Core/src/ConfigParamStore.cxx       |  11 ++
 Framework/Core/src/DeviceSpecHelpers.cxx      |  35 ++++--
 Framework/Core/src/DeviceSpecHelpers.h        |   1 +
 Framework/Core/src/Plugin.cxx                 | 100 ++++++++++++++++++
 .../Core/src/WorkflowCustomizationHelpers.cxx |   4 +
 Framework/Core/src/WorkflowHelpers.cxx        |   2 +-
 Framework/Core/src/runDataProcessing.cxx      |  11 ++
 .../Core/test/test_DeviceSpecHelpers.cxx      |   2 +-
 .../Core/test/test_FrameworkDataFlowToDDS.cxx |   4 +-
 .../test_FrameworkDataFlowToO2Control.cxx     |   2 +-
 Framework/TestWorkflows/CMakeLists.txt        |   4 +
 .../TestWorkflows/src/o2AnalysisWorkflow.cxx  |  13 ++-
 .../TestWorkflows/src/o2TestHistograms.cxx    |  68 ++++++++++++
 25 files changed, 505 insertions(+), 27 deletions(-)
 create mode 100644 Framework/Core/include/Framework/Capability.h
 create mode 100644 Framework/Core/include/Framework/ConfigParamDiscovery.h
 create mode 100644 Framework/Core/src/ConfigParamDiscovery.cxx
 create mode 100644 Framework/Core/src/Plugin.cxx
 create mode 100644 Framework/TestWorkflows/src/o2TestHistograms.cxx

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 092857d585a02..5ce2eca49c07b 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -133,12 +133,12 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_MESSAGES_DESTROYED), DataProcessingStats::Op::Set, 0});
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_BYTES_EXPIRED), DataProcessingStats::Op::Set, 0});
 
-    if (!options.isSet("aod-file")) {
+    if (!options.isSet("aod-file-private")) {
       LOGP(fatal, "No input file defined!");
       throw std::runtime_error("Processing is stopped!");
     }
 
-    auto filename = options.get<std::string>("aod-file");
+    auto filename = options.get<std::string>("aod-file-private");
 
     std::string parentFileReplacement;
     if (options.isSet("aod-parent-base-path-replacement")) {
@@ -306,7 +306,7 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
           control.readyToQuit(QuitRequest::Me);
           return;
         }
-      } 
+      }
     });
   })};
 
diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index 8f2e489086afb..2ee8d316f3675 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -12,7 +12,17 @@
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/ServiceSpec.h"
 #include "Framework/ServiceMetricsInfo.h"
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/Capability.h"
+#include "Framework/Signpost.h"
 #include "AODJAlienReaderHelpers.h"
+#include <TFile.h>
+#include <TMap.h>
+#include <TObjString.h>
+#include <TString.h>
+#include <fmt/format.h>
+
+O2_DECLARE_DYNAMIC_LOG(analysis_support);
 
 struct ROOTFileReader : o2::framework::AlgorithmPlugin {
   o2::framework::AlgorithmSpec create() override
@@ -54,7 +64,64 @@ struct RunSummary : o2::framework::ServicePlugin {
   }
 };
 
+struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
+  ConfigDiscovery* create() override
+  {
+    return new ConfigDiscovery{
+      .init = []() {},
+      .discover = [](ConfigParamRegistry& registry, int argc, char** argv) -> std::vector<ConfigParamSpec> {
+        auto filename = registry.get<std::string>("aod-file");
+        if (filename.empty()) {
+          return {};
+        }
+        LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
+        std::vector<ConfigParamSpec> results;
+        TFile* currentFile = nullptr;
+        if (filename.at(0) == '@') {
+          filename.erase(0, 1);
+          // read the text file and set filename to the contents of the first line
+          std::ifstream file(filename);
+          if (!file.is_open()) {
+            LOGP(fatal, "Couldn't open file \"{}\"!", filename);
+          }
+          std::getline(file, filename);
+          file.close();
+          currentFile = TFile::Open(filename.c_str());
+        } else {
+          currentFile = TFile::Open(filename.c_str());
+        }
+        if (!currentFile) {
+          LOGP(fatal, "Couldn't open file \"{}\"!", filename);
+        }
+
+        // Get the metadata, if any
+        auto m = (TMap*)currentFile->Get("metaData");
+        if (!m) {
+          LOGP(warning, "No metadata found in file \"{}\"", filename);
+          return results;
+        }
+        auto it = m->MakeIterator();
+
+        // Serialise metadata into a ; separated string with : separating key and value
+        bool first = true;
+        while (auto obj = it->Next()) {
+          if (first) {
+            LOGP(info, "Metadata for file \"{}\":", filename);
+            first = false;
+          }
+          auto objString = (TObjString*)m->GetValue(obj);
+          LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
+          std::string key = "aod-metadata-" + std::string(obj->GetName());
+          char const* value = strdup(objString->String());
+          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
+        }
+        return results;
+      }};
+  }
+};
+
 DEFINE_DPL_PLUGINS_BEGIN
 DEFINE_DPL_PLUGIN_INSTANCE(ROOTFileReader, CustomAlgorithm);
 DEFINE_DPL_PLUGIN_INSTANCE(RunSummary, CustomService);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAOD, ConfigDiscovery);
 DEFINE_DPL_PLUGINS_END
diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index d392ba42fe8ac..4381b6c4fa29d 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -47,6 +47,7 @@ o2_add_library(Framework
                        src/DefaultsHelpers.cxx
                        src/DomainInfoHeader.cxx
                        src/ProcessingPoliciesHelpers.cxx
+                       src/ConfigParamDiscovery.cxx
                        src/ConfigParamStore.cxx
                        src/ConfigParamsHelper.cxx
                        src/ChannelParamSpec.cxx
@@ -109,6 +110,7 @@ o2_add_library(Framework
                        src/OptionsHelpers.cxx
                        src/PropertyTreeHelpers.cxx
                        src/ProcessingContext.cxx
+                       src/Plugin.cxx
                        src/PluginManager.cxx
                        src/RateLimiter.cxx
                        src/ResourcesMonitoringHelper.cxx
diff --git a/Framework/Core/include/Framework/Capability.h b/Framework/Core/include/Framework/Capability.h
new file mode 100644
index 0000000000000..23ef7b7d9bf17
--- /dev/null
+++ b/Framework/Core/include/Framework/Capability.h
@@ -0,0 +1,44 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_CAPABILITY_H_
+#define O2_FRAMEWORK_CAPABILITY_H_
+
+#include <string_view>
+#include <functional>
+
+namespace o2::framework
+{
+
+struct ConfigParamRegistry;
+
+/// A plugin which is able to discover more option then the ones
+/// provided on the command line. The actual loading is in two parts,
+/// the first one decides if the options are enough to actually perform
+/// the discovery, the second part will do the discovery itself.
+///
+/// Its a good idea to have the Loader part in a standalone library to
+/// minimize dependency on unneed thir party code, e.g. ROOT.
+struct Capability {
+  // Wether or not this capability is required.
+  // FIXME: for now let's pass the arguments to get the metadata to work
+  std::string name;
+  std::function<bool(ConfigParamRegistry&, int argc, char** argv)> checkIfNeeded;
+  char const* requiredPlugin;
+};
+
+struct CapabilityPlugin {
+  virtual Capability* create() = 0;
+};
+
+} // namespace o2::framework
+
+#endif // O2_FRAMEWORK_CAPABILITY_H_
diff --git a/Framework/Core/include/Framework/ConfigParamDiscovery.h b/Framework/Core/include/Framework/ConfigParamDiscovery.h
new file mode 100644
index 0000000000000..3d13d3662ceb1
--- /dev/null
+++ b/Framework/Core/include/Framework/ConfigParamDiscovery.h
@@ -0,0 +1,37 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
+#define O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
+
+#include "Framework/ConfigParamRegistry.h"
+
+namespace o2::framework
+{
+// A plugin which can be used to inject extra configuration
+// options.
+struct ConfigParamDiscovery {
+  static std::vector<ConfigParamSpec> discover(ConfigParamRegistry&, int, char**);
+};
+
+struct ConfigDiscovery {
+  std::function<void()> init;
+  /// A function which uses the arguments available so far to discover more
+  /// @return the extra ConfigParamSpecs it derives.
+  std::function<std::vector<ConfigParamSpec>(ConfigParamRegistry&, int, char**)> discover;
+};
+
+struct ConfigDiscoveryPlugin {
+  virtual ConfigDiscovery* create() = 0;
+};
+
+} // namespace o2::framework
+#endif // O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
diff --git a/Framework/Core/include/Framework/ConfigParamRegistry.h b/Framework/Core/include/Framework/ConfigParamRegistry.h
index 8f2d80cea0948..397a491be9f9c 100644
--- a/Framework/Core/include/Framework/ConfigParamRegistry.h
+++ b/Framework/Core/include/Framework/ConfigParamRegistry.h
@@ -125,6 +125,12 @@ class ConfigParamRegistry
     }
   }
 
+  // Load extra parameters discovered while we process data
+  void loadExtra(std::vector<ConfigParamSpec>& extras)
+  {
+    mStore->load(extras);
+  }
+
  private:
   std::unique_ptr<ConfigParamStore> mStore;
 };
diff --git a/Framework/Core/include/Framework/ConfigParamStore.h b/Framework/Core/include/Framework/ConfigParamStore.h
index 37442a6cf7f41..b3ae76aa47ad1 100644
--- a/Framework/Core/include/Framework/ConfigParamStore.h
+++ b/Framework/Core/include/Framework/ConfigParamStore.h
@@ -34,12 +34,14 @@ class ConfigParamStore
   /// the configuration.
   void preload();
 
+  void load(std::vector<ConfigParamSpec>& specs);
+
   /// Get the store
   boost::property_tree::ptree& store() { return *mStore; };
   boost::property_tree::ptree& provenanceTree() { return *mProvenance; };
 
   /// Get the specs
-  std::vector<ConfigParamSpec> const& specs() const
+  [[nodiscard]] std::vector<ConfigParamSpec> const& specs() const
   {
     return mSpecs;
   }
diff --git a/Framework/Core/include/Framework/PluginManager.h b/Framework/Core/include/Framework/PluginManager.h
index 38966c5a969a9..4c6e965502500 100644
--- a/Framework/Core/include/Framework/PluginManager.h
+++ b/Framework/Core/include/Framework/PluginManager.h
@@ -70,7 +70,7 @@ struct PluginManager {
 
     struct LoadedPlugin {
       std::string name;
-      ServicePlugin* factory;
+      PLUGIN* factory;
     };
     std::vector<LoadedDSO> loadedDSOs;
     std::vector<LoadedPlugin> loadedPlugins;
diff --git a/Framework/Core/include/Framework/Plugins.h b/Framework/Core/include/Framework/Plugins.h
index c29d96f7e0781..23d55a512e1fa 100644
--- a/Framework/Core/include/Framework/Plugins.h
+++ b/Framework/Core/include/Framework/Plugins.h
@@ -27,6 +27,15 @@ enum struct DplPluginKind : int {
   // A plugin which implements a custom Services. Needs to return
   // an object of the kind o2::framework::ServiceSpec
   CustomService,
+  // A plugin which implements a new way to discover extra configuration.
+  // parameters. E.g. it can be used to read metadata from a file or a service
+  // if a certain parameter is available.
+  ConfigDiscovery,
+  // A capability plugin is a plugin used to discover other viable plugins.
+  // For example, if you find out that you have the --aod-file option
+  // set, you might want to load metadata from it and attach it to the
+  // configuration.
+  Capability,
   // A plugin which was not initialised properly.
   Unknown
 };
diff --git a/Framework/Core/include/Framework/ServiceSpec.h b/Framework/Core/include/Framework/ServiceSpec.h
index abac3ad05914c..5684889e85376 100644
--- a/Framework/Core/include/Framework/ServiceSpec.h
+++ b/Framework/Core/include/Framework/ServiceSpec.h
@@ -322,11 +322,6 @@ struct ServicePlugin {
   virtual ServiceSpec* create() = 0;
 };
 
-struct LoadableService {
-  std::string name;
-  std::string library;
-};
-
 } // namespace o2::framework
 
 #endif // O2_FRAMEWORK_SERVICESPEC_H_
diff --git a/Framework/Core/include/Framework/runDataProcessing.h b/Framework/Core/include/Framework/runDataProcessing.h
index 77a4c9f5ba8d6..c0d5b2211e23d 100644
--- a/Framework/Core/include/Framework/runDataProcessing.h
+++ b/Framework/Core/include/Framework/runDataProcessing.h
@@ -29,7 +29,8 @@
 #include "Framework/Logger.h"
 #include "Framework/CheckTypes.h"
 #include "Framework/StructToTuple.h"
-
+#include "Framework/ConfigParamDiscovery.h"
+#include <iostream>
 #include <vector>
 
 namespace o2::framework
@@ -137,6 +138,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& specs,
            std::vector<o2::framework::CallbacksPolicy> const& callbacksPolicies,
            std::vector<o2::framework::SendingPolicy> const& sendingPolicies,
            std::vector<o2::framework::ConfigParamSpec> const& workflowOptions,
+           std::vector<o2::framework::ConfigParamSpec> const& detectedOptions,
            o2::framework::ConfigContext& configContext);
 
 void doDefaultWorkflowTerminationHook();
@@ -192,6 +194,12 @@ int mainNoCatch(int argc, char** argv)
   workflowOptionsStore->preload();
   workflowOptionsStore->activate();
   ConfigParamRegistry workflowOptionsRegistry(std::move(workflowOptionsStore));
+  auto extraOptions = o2::framework::ConfigParamDiscovery::discover(workflowOptionsRegistry, argc, argv);
+  for (auto& extra : extraOptions) {
+    workflowOptions.push_back(extra);
+  }
+  workflowOptionsRegistry.loadExtra(extraOptions);
+
   ConfigContext configContext(workflowOptionsRegistry, argc, argv);
   o2::framework::WorkflowSpec specs = defineDataProcessing(configContext);
   overrideCloning(configContext, specs);
@@ -206,7 +214,7 @@ int mainNoCatch(int argc, char** argv)
   channelPolicies.insert(std::end(channelPolicies), std::begin(defaultChannelPolicies), std::end(defaultChannelPolicies));
   return doMain(argc, argv, specs,
                 channelPolicies, completionPolicies, dispatchPolicies,
-                resourcePolicies, callbacksPolicies, sendingPolicies, workflowOptions, configContext);
+                resourcePolicies, callbacksPolicies, sendingPolicies, workflowOptions, extraOptions, configContext);
 }
 
 int callMain(int argc, char** argv, int (*)(int, char**));
diff --git a/Framework/Core/src/ConfigParamDiscovery.cxx b/Framework/Core/src/ConfigParamDiscovery.cxx
new file mode 100644
index 0000000000000..6c24a44bac566
--- /dev/null
+++ b/Framework/Core/src/ConfigParamDiscovery.cxx
@@ -0,0 +1,81 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/PluginManager.h"
+#include "Framework/Signpost.h"
+#include "Framework/Capability.h"
+#include <TFile.h>
+#include <TMap.h>
+#include <TObjString.h>
+
+O2_DECLARE_DYNAMIC_LOG(config_parameter_discovery);
+
+namespace o2::framework
+{
+
+std::vector<ConfigParamSpec> ConfigParamDiscovery::discover(ConfigParamRegistry& registry, int argc, char** argv)
+{
+  std::vector<char const*> capabilitiesSpecs = {
+    "O2Framework:DiscoverMetadataInAODCapability",
+    "O2Framework:DiscoverMetadataInCommandLineCapability",
+  };
+
+  // Load all the requested plugins and discover what we can do.
+  O2_SIGNPOST_ID_GENERATE(sid, config_parameter_discovery);
+  std::vector<LoadablePlugin> plugins;
+  for (auto spec : capabilitiesSpecs) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "Parsing Capability plugin: %{public}s", spec);
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+  }
+
+  std::vector<Capability> availableCapabilities;
+  std::vector<char const*> configDiscoverySpec = {};
+  PluginManager::loadFromPlugin<Capability, CapabilityPlugin>(plugins, availableCapabilities);
+  for (auto& capability : availableCapabilities) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- Using Capability of : %{public}s",
+                           capability.name.c_str());
+
+    if (capability.checkIfNeeded(registry, argc, argv)) {
+      O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- Required: %{}s",
+                             capability.requiredPlugin);
+      configDiscoverySpec.push_back(capability.requiredPlugin);
+    } else {
+      O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- No requirements added");
+    }
+  }
+
+  std::vector<LoadablePlugin> configDiscoveryPlugins;
+  for (auto spec : configDiscoverySpec) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "discovery", "Parsing ConfigDiscovery plugin: %{public}s", spec);
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      configDiscoveryPlugins.push_back(extra);
+    }
+  }
+
+  std::vector<ConfigDiscovery> availableConfigDiscovery;
+  PluginManager::loadFromPlugin<ConfigDiscovery, ConfigDiscoveryPlugin>(configDiscoveryPlugins, availableConfigDiscovery);
+  std::vector<ConfigParamSpec> result;
+  for (auto& configDiscovery : availableConfigDiscovery) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "discovery", "Parsing extra configuration as needed");
+    auto extras = configDiscovery.discover(registry, argc, argv);
+    for (auto& extra : extras) {
+      result.push_back(extra);
+    }
+  }
+  return result;
+}
+
+} // namespace o2::framework
diff --git a/Framework/Core/src/ConfigParamStore.cxx b/Framework/Core/src/ConfigParamStore.cxx
index 249a8b0052a67..19e123a23e5d0 100644
--- a/Framework/Core/src/ConfigParamStore.cxx
+++ b/Framework/Core/src/ConfigParamStore.cxx
@@ -41,6 +41,17 @@ void ConfigParamStore::preload()
   }
 }
 
+void ConfigParamStore::load(std::vector<ConfigParamSpec>& specs)
+{
+  mNextStore->clear();
+  mNextProvenance->clear();
+  // By default we populate with code.
+  PropertyTreeHelpers::populateDefaults(specs, *mNextStore, *mNextProvenance);
+  for (auto& retriever : mRetrievers) {
+    retriever->update(specs, *mStore, *mProvenance);
+  }
+}
+
 /// Activate the next store
 void ConfigParamStore::activate()
 {
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 30197538f2dda..b9552dc093115 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -32,7 +32,7 @@
 #include "Framework/OutputRoute.h"
 #include "Framework/WorkflowSpec.h"
 #include "Framework/ComputingResource.h"
-#include "Framework/Logger.h"
+#include "Framework/Signpost.h"
 #include "Framework/RuntimeError.h"
 #include "Framework/RawDeviceService.h"
 #include "ProcessingPoliciesHelpers.h"
@@ -50,8 +50,7 @@
 
 #include <regex>
 
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wpedantic"
+O2_DECLARE_DYNAMIC_LOG(device_spec_helpers);
 
 namespace bpo = boost::program_options;
 
@@ -125,7 +124,9 @@ struct ExpirationHandlerHelpers {
       if (periods.empty()) {
         std::string defaultRateName = std::string{"period-"} + matcher.binding;
         auto defaultRate = std::chrono::milliseconds(options.get<int>(defaultRateName.c_str()));
-        LOGP(detail, "Using default rate of {} ms as specified by option period-{}", defaultRate.count(), matcher.binding);
+        O2_SIGNPOST_ID_GENERATE(tid, device_spec_helpers);
+        O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, tid, "timeDrivenCreation", "Using default rate of %" PRIi64 " ms as specified by option period-%{public}s", defaultRate.count(),
+                               matcher.binding.c_str());
         periods.emplace_back(defaultRate.count());
         durations.emplace_back(std::chrono::seconds((std::size_t)-1));
       } else {
@@ -1359,12 +1360,15 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
                                          std::vector<DeviceSpec> const& deviceSpecs,
                                          std::vector<DeviceExecution>& deviceExecutions,
                                          std::vector<DeviceControl>& deviceControls,
+                                         std::vector<ConfigParamSpec> const& detectedOptions,
                                          std::string const& uniqueWorkflowId)
 {
   assert(deviceSpecs.size() == deviceExecutions.size());
   assert(deviceControls.size() == deviceExecutions.size());
   for (size_t si = 0; si < deviceSpecs.size(); ++si) {
     auto& spec = deviceSpecs[si];
+    O2_SIGNPOST_ID_GENERATE(poid, device_spec_helpers);
+    O2_SIGNPOST_START(device_spec_helpers, poid, "prepareArguments", "Preparing options for %{public}s", spec.id.c_str());
     auto& control = deviceControls[si];
     auto& execution = deviceExecutions[si];
 
@@ -1373,7 +1377,14 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
 
     int argc;
     char** argv;
-    std::vector<ConfigParamSpec> workflowOptions;
+    // We need to start with the detected options, so that they are not lost.
+    // Notice how detected options can be detected at any moment in the chain,
+    // so it's important that if you rely on them, they get passed on
+    // always.
+    std::vector<ConfigParamSpec> workflowOptions = detectedOptions;
+    for (auto& opt : detectedOptions) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, poid, "prepareArguments", "Processor option %{public}s passed as previously detected", opt.name.c_str());
+    }
     /// Lookup the executable name in the metadata associated with the workflow.
     /// If we find it, we rewrite the command line arguments to be processed
     /// so that they look like the ones passed to the merged workflow.
@@ -1386,8 +1397,17 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
       argv[ai + 1] = strdup(arg.data());
     }
     argv[argc] = nullptr;
-    workflowOptions = pi->workflowOptions;
+    for (auto& opt : pi->workflowOptions) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, poid, "prepareArguments", "Processor option %{public}s found in process description", opt.name.c_str());
+      workflowOptions.push_back(opt);
+    }
+    std::sort(workflowOptions.begin(), workflowOptions.end(), [](ConfigParamSpec const& a, ConfigParamSpec const& b) { return a.name < b.name; });
+    auto last = std::unique(workflowOptions.begin(), workflowOptions.end());
+    workflowOptions.erase(last, workflowOptions.end());
 
+    for (auto& opt : workflowOptions) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, poid, "prepareArguments", "Final unique option %{public}s added to list of workflowOptions", opt.name.c_str());
+    }
     // We duplicate the list of options, filtering only those
     // which are actually relevant for the given device. The additional
     // four are to add
@@ -1662,7 +1682,8 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
       assert(execution.args[ai]);
       str << " " << execution.args[ai];
     }
-    LOG(debug) << "The following options are being forwarded to " << spec.id << ":" << str.str();
+    O2_SIGNPOST_END(device_spec_helpers, poid, "prepareArguments", "The following options are being forwarded to %{public}s: %{public}s",
+                    spec.id.c_str(), str.str().c_str());
   }
 }
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.h b/Framework/Core/src/DeviceSpecHelpers.h
index 17601c81342c1..5b54bc84cf6c8 100644
--- a/Framework/Core/src/DeviceSpecHelpers.h
+++ b/Framework/Core/src/DeviceSpecHelpers.h
@@ -129,6 +129,7 @@ struct DeviceSpecHelpers {
     std::vector<DeviceSpec> const& deviceSpecs,
     std::vector<DeviceExecution>& deviceExecutions,
     std::vector<DeviceControl>& deviceControls,
+    std::vector<ConfigParamSpec> const& detectedOptions,
     std::string const& uniqueWorkflowId);
 
   /// Rework the environment string
diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx
new file mode 100644
index 0000000000000..2298896821dab
--- /dev/null
+++ b/Framework/Core/src/Plugin.cxx
@@ -0,0 +1,100 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/Plugins.h"
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/ConfigParamRegistry.h"
+#include "Framework/Logger.h"
+#include "Framework/Capability.h"
+#include "Framework/Signpost.h"
+#include <string_view>
+
+O2_DECLARE_DYNAMIC_LOG(capabilities);
+namespace o2::framework
+{
+auto lookForAodFile = [](ConfigParamRegistry& registry, int argc, char** argv) -> bool {
+  O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+  if (registry.hasOption("aod-file") && registry.isSet("aod-file")) {
+    for (size_t i = 0; i < argc; i++) {
+      std::string_view arg = argv[i];
+      if (arg.starts_with("--aod-metadata-")) {
+        return false;
+      }
+    }
+    O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInAODCapability", "Metadata not found in arguments. Checking in AOD file.");
+    return true;
+  }
+  return false;
+};
+
+auto lookForCommandLineOptions = [](ConfigParamRegistry& registry, int argc, char** argv) -> bool {
+  O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+  for (size_t i = 0; i < argc; i++) {
+    std::string_view arg = argv[i];
+    if (arg.starts_with("--aod-metadata-")) {
+      O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLineCapability", "Metadata found in arguments. Populating from them.");
+      return true;
+    }
+  }
+  return false;
+};
+
+struct DiscoverMetadataInAODCapability : o2::framework::CapabilityPlugin {
+  Capability* create() override
+  {
+    return new Capability{
+      .name = "DiscoverMetadataInAODCapability",
+      .checkIfNeeded = lookForAodFile,
+      .requiredPlugin = "O2FrameworkAnalysisSupport:DiscoverMetadataInAOD"};
+  }
+};
+
+// Trigger discovery of metadata from command line, if needed.
+struct DiscoverMetadataInCommandLineCapability : o2::framework::CapabilityPlugin {
+  Capability* create() override
+  {
+    return new Capability{
+      .name = "DiscoverMetadataInCommandLineCapability",
+      .checkIfNeeded = lookForCommandLineOptions,
+      .requiredPlugin = "O2Framework:DiscoverMetadataInCommandLine"};
+  }
+};
+
+struct DiscoverMetadataInCommandLine : o2::framework::ConfigDiscoveryPlugin {
+  ConfigDiscovery* create() override
+  {
+    return new ConfigDiscovery{
+      .init = []() {},
+      .discover = [](ConfigParamRegistry& registry, int argc, char** argv) -> std::vector<ConfigParamSpec> {
+        O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+        O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLine",
+                               "Discovering metadata for analysis from well known environment variables.");
+        std::vector<ConfigParamSpec> results;
+        for (size_t i = 0; i < argc; i++) {
+          std::string_view arg = argv[i];
+          if (!arg.starts_with("--aod-metadata")) {
+            continue;
+          }
+          std::string key = arg.data() + 2;
+          std::string value = argv[i + 1];
+          O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLine",
+                                 "Found %{public}s with value %{public}s.", key.c_str(), value.c_str());
+          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in command line"}});
+        }
+        return results;
+      }};
+  }
+};
+DEFINE_DPL_PLUGINS_BEGIN
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAODCapability, Capability);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLineCapability, Capability);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLine, ConfigDiscovery);
+DEFINE_DPL_PLUGINS_END
+} // namespace o2::framework
diff --git a/Framework/Core/src/WorkflowCustomizationHelpers.cxx b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
index ffad8a8720019..05abb5dab98cd 100644
--- a/Framework/Core/src/WorkflowCustomizationHelpers.cxx
+++ b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
@@ -54,6 +54,10 @@ std::vector<ConfigParamSpec> WorkflowCustomizationHelpers::requiredWorkflowOptio
            // options for AOD rate limiting
            {"aod-memory-rate-limit", VariantType::Int64, 0LL, {"Rate limit AOD processing based on memory"}},
 
+           // options for the AOD reader
+           // aod-file needs to be available as workflow option, because we
+           // can configure the workflow based on the contents of the first file.
+           {"aod-file", VariantType::String, "", {"Input AOD file"}},
            // options for AOD writer
            {"aod-writer-json", VariantType::String, "", {"Name of the json configuration file"}},
            {"aod-writer-resdir", VariantType::String, "", {"Name of the output directory"}},
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index bd875ad421f6e..da60bdbc6961e 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -292,7 +292,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
                          static_cast<DataAllocator::SubSpecificationType>(compile_time_hash("internal-dpl-aod-reader")),
                          aodLifetime}},
     .algorithm = AlgorithmSpec::dummyAlgorithm(),
-    .options = {ConfigParamSpec{"aod-file", VariantType::String, {"Input AOD file"}},
+    .options = {ConfigParamSpec{"aod-file-private", VariantType::String, ctx.options().get<std::string>("aod-file"), {"AOD file"}},
                 ConfigParamSpec{"aod-reader-json", VariantType::String, {"json configuration file"}},
                 ConfigParamSpec{"aod-parent-access-level", VariantType::String, {"Allow parent file access up to specified level. Default: no (0)"}},
                 ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}},
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index f91716d28cfd7..a0ef33f812e89 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -194,6 +194,10 @@ char* getIdString(int argc, char** argv)
 
 int callMain(int argc, char** argv, int (*mainNoCatch)(int, char**))
 {
+  std::cout << "This is " << getpid() << " " << argv[0] << std::endl;
+  auto spec1 = fair::VerbositySpec::Make(fair::VerbositySpec::Info::process_name);
+  fair::Logger::DefineVerbosity(fair::Verbosity::user1, spec1);
+  fair::Logger::SetVerbosity(fair::Verbosity::user1);
   static bool noCatch = getenv("O2_NO_CATCHALL_EXCEPTIONS") && strcmp(getenv("O2_NO_CATCHALL_EXCEPTIONS"), "0");
   int result = 1;
   if (noCatch) {
@@ -748,6 +752,9 @@ void spawnDevice(uv_loop_t* loop,
       putenv(strdup(DeviceSpecHelpers::reworkTimeslicePlaceholder(env, spec).data()));
     }
     execvp(execution.args[0], execution.args.data());
+  } else {
+    O2_SIGNPOST_ID_GENERATE(sid, driver);
+    O2_SIGNPOST_EVENT_EMIT(driver, sid, "spawnDevice", "New child at %{pid}d", id);
   }
   close(childFds[ref.index].childstdin[0]);
   close(childFds[ref.index].childstdout[1]);
@@ -1359,6 +1366,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
                     DriverInfo& driverInfo,
                     DriverConfig& driverConfig,
                     std::vector<DeviceMetricsInfo>& metricsInfos,
+                    std::vector<ConfigParamSpec> const& detectedParams,
                     boost::program_options::variables_map& varmap,
                     std::vector<ServiceSpec>& driverServices,
                     std::string frameworkId)
@@ -2048,6 +2056,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
                                               runningWorkflow.devices,
                                               deviceExecutions,
                                               controls,
+                                              detectedParams,
                                               driverInfo.uniqueWorkflowId);
         } catch (o2::framework::RuntimeErrorRef& ref) {
           auto& err = o2::framework::error_from_ref(ref);
@@ -2742,6 +2751,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
            std::vector<CallbacksPolicy> const& callbacksPolicies,
            std::vector<SendingPolicy> const& sendingPolicies,
            std::vector<ConfigParamSpec> const& currentWorkflowOptions,
+           std::vector<ConfigParamSpec> const& detectedParams,
            o2::framework::ConfigContext& configContext)
 {
   std::vector<std::string> currentArgs;
@@ -3173,6 +3183,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
                          driverInfo,
                          driverConfig,
                          gDeviceMetricsInfos,
+                         detectedParams,
                          varmap,
                          driverServices,
                          frameworkId);
diff --git a/Framework/Core/test/test_DeviceSpecHelpers.cxx b/Framework/Core/test/test_DeviceSpecHelpers.cxx
index 762c2df6f82aa..6240e784d09d3 100644
--- a/Framework/Core/test/test_DeviceSpecHelpers.cxx
+++ b/Framework/Core/test/test_DeviceSpecHelpers.cxx
@@ -79,7 +79,7 @@ void check(const std::vector<std::string>& arguments,
                                       dataProcessorInfos,
                                       deviceSpecs,
                                       deviceExecutions,
-                                      deviceControls,
+                                      deviceControls, {},
                                       "workflow-id");
 
   for (size_t index = 0; index < deviceSpecs.size(); index++) {
diff --git a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
index 3c08bbae74475..593728696e77a 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
@@ -153,7 +153,7 @@ TEST_CASE("TestDDS")
   DeviceSpecHelpers::prepareArguments(false, false, false, 8080,
                                       driverConfig,
                                       dataProcessorInfos,
-                                      devices, executions, controls,
+                                      devices, executions, controls, {},
                                       "workflow-id");
   CommandInfo command{"foo"};
   DDSConfigHelpers::dumpDeviceSpec2DDS(ss, DriverMode::STANDALONE, "", workflow, dataProcessorInfos, devices, executions, command);
@@ -417,7 +417,7 @@ TEST_CASE("TestDDSExpendable")
   DeviceSpecHelpers::prepareArguments(false, false, false, 8080,
                                       driverConfig,
                                       dataProcessorInfos,
-                                      devices, executions, controls,
+                                      devices, executions, controls, {},
                                       "workflow-id");
   CommandInfo command{"foo"};
   DDSConfigHelpers::dumpDeviceSpec2DDS(ss, DriverMode::STANDALONE, "", workflow, dataProcessorInfos, devices, executions, command);
diff --git a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
index bbd5b78fbbb68..4aca1bd9680a1 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
@@ -560,7 +560,7 @@ TEST_CASE("TestO2ControlDump")
   DeviceSpecHelpers::prepareArguments(false, false, false, 8080,
                                       driverConfig,
                                       dataProcessorInfos,
-                                      devices, executions, controls,
+                                      devices, executions, controls, {},
                                       "workflow-id");
 
   dumpWorkflow(ss, devices, executions, commandInfo, "testwf", "");
diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index bfbdcfc6f169b..21af45422e0b1 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -33,6 +33,10 @@ o2_add_dpl_workflow(diamond-workflow
                   SOURCES src/o2DiamondWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
 
+o2_add_dpl_workflow(analysis-histograms
+                  SOURCES src/o2TestHistograms.cxx
+                  COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(two-timers
                   SOURCES src/o2TwoTimers.cxx
                   COMPONENT_NAME TestWorkflows)
diff --git a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
index 05bf3aabaceab..43d71a9b40528 100644
--- a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
+++ b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
@@ -49,7 +49,14 @@ struct EtaAndClsHistograms {
 
 WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
 {
-  return WorkflowSpec{
-    adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
-  };
+  // For the sake of running without an option, we do not throw an exception
+  // in case the option is not present.
+  if (cfgc.options().hasOption("aod-metadata-Run") == false ||
+      cfgc.options().get<std::string>("aod-metadata-Run") == "2") {
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
+    };
+  } else {
+    throw std::runtime_error("Unsupported run type");
+  }
 }
diff --git a/Framework/TestWorkflows/src/o2TestHistograms.cxx b/Framework/TestWorkflows/src/o2TestHistograms.cxx
new file mode 100644
index 0000000000000..4dbea06d03679
--- /dev/null
+++ b/Framework/TestWorkflows/src/o2TestHistograms.cxx
@@ -0,0 +1,68 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+///
+/// \brief FullTracks is a join of Tracks, TracksCov, and TracksExtra.
+/// \author
+/// \since
+
+#include "Framework/runDataProcessing.h"
+#include "Framework/AnalysisTask.h"
+#include <TH2F.h>
+#include <cmath>
+
+using namespace o2;
+using namespace o2::framework;
+using namespace o2::framework::expressions;
+
+struct EtaAndClsHistogramsSimple {
+  OutputObj<TH2F> etaClsH{TH2F("eta_vs_pt", "#eta vs pT", 102, -2.01, 2.01, 100, 0, 10)};
+
+  void process(aod::Tracks const& tracks)
+  {
+    LOGP(info, "Invoking the simple one");
+    for (auto& track : tracks) {
+      etaClsH->Fill(track.eta(), track.pt(), 0);
+    }
+  }
+};
+
+struct EtaAndClsHistogramsFull {
+  OutputObj<TH3F> etaClsH{TH3F("eta_vs_cls_vs_sigmapT", "#eta vs N_{cls} vs sigma_{1/pT}", 102, -2.01, 2.01, 160, -0.5, 159.5, 100, 0, 10)};
+
+  void process(soa::Join<aod::FullTracks, aod::TracksCov> const& tracks)
+  {
+    LOGP(info, "Invoking the run 3 one");
+    for (auto& track : tracks) {
+      etaClsH->Fill(track.eta(), track.tpcNClsFindable(), track.sigma1Pt());
+    }
+  }
+};
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  std::string runType = "3";
+  if (cfgc.options().hasOption("aod-metadata-Run")) {
+    runType = cfgc.options().get<std::string>("aod-metadata-Run");
+  }
+  LOGP(info, "Runtype is {}", runType);
+  // Notice it's important for the tasks to use the same name, otherwise topology generation will be confused.
+  if (runType == "2") {
+    LOGP(info, "Using only tracks {}", runType);
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistogramsSimple>(cfgc, TaskName{"simple-histos"}),
+    };
+  } else {
+    LOGP(info, "Using tracks extra {}", runType);
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistogramsFull>(cfgc, TaskName{"simple-histos"}),
+    };
+  }
+}

From bb4d9423b4c4a6e5ca2791cd78bbb590c46092e3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 16 May 2024 16:50:32 +0200
Subject: [PATCH 509/726] DPL Analysis: provide metadata to workflow
 construction

---
 .../src/AODJAlienReaderHelpers.cxx            |   6 +-
 Framework/AnalysisSupport/src/Plugin.cxx      |  67 ++++++++++++
 Framework/Core/CMakeLists.txt                 |   2 +
 Framework/Core/include/Framework/Capability.h |  44 ++++++++
 .../include/Framework/ConfigParamDiscovery.h  |  37 +++++++
 .../include/Framework/ConfigParamRegistry.h   |   6 ++
 .../Core/include/Framework/ConfigParamStore.h |   4 +-
 .../Core/include/Framework/PluginManager.h    |   2 +-
 Framework/Core/include/Framework/Plugins.h    |   9 ++
 .../Core/include/Framework/ServiceSpec.h      |   5 -
 .../include/Framework/runDataProcessing.h     |  11 +-
 Framework/Core/src/ConfigParamDiscovery.cxx   |  81 ++++++++++++++
 Framework/Core/src/ConfigParamStore.cxx       |  11 ++
 Framework/Core/src/DeviceSpecHelpers.cxx      |  35 ++++--
 Framework/Core/src/DeviceSpecHelpers.h        |   1 +
 Framework/Core/src/Plugin.cxx                 | 100 ++++++++++++++++++
 .../Core/src/WorkflowCustomizationHelpers.cxx |   4 +
 Framework/Core/src/WorkflowHelpers.cxx        |   2 +-
 Framework/Core/src/runDataProcessing.cxx      |  11 ++
 .../Core/test/test_DeviceSpecHelpers.cxx      |   2 +-
 .../Core/test/test_FrameworkDataFlowToDDS.cxx |   4 +-
 .../test_FrameworkDataFlowToO2Control.cxx     |   2 +-
 Framework/TestWorkflows/CMakeLists.txt        |   4 +
 .../TestWorkflows/src/o2AnalysisWorkflow.cxx  |  13 ++-
 .../TestWorkflows/src/o2TestHistograms.cxx    |  68 ++++++++++++
 25 files changed, 504 insertions(+), 27 deletions(-)
 create mode 100644 Framework/Core/include/Framework/Capability.h
 create mode 100644 Framework/Core/include/Framework/ConfigParamDiscovery.h
 create mode 100644 Framework/Core/src/ConfigParamDiscovery.cxx
 create mode 100644 Framework/Core/src/Plugin.cxx
 create mode 100644 Framework/TestWorkflows/src/o2TestHistograms.cxx

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 092857d585a02..5ce2eca49c07b 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -133,12 +133,12 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_MESSAGES_DESTROYED), DataProcessingStats::Op::Set, 0});
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_BYTES_EXPIRED), DataProcessingStats::Op::Set, 0});
 
-    if (!options.isSet("aod-file")) {
+    if (!options.isSet("aod-file-private")) {
       LOGP(fatal, "No input file defined!");
       throw std::runtime_error("Processing is stopped!");
     }
 
-    auto filename = options.get<std::string>("aod-file");
+    auto filename = options.get<std::string>("aod-file-private");
 
     std::string parentFileReplacement;
     if (options.isSet("aod-parent-base-path-replacement")) {
@@ -306,7 +306,7 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
           control.readyToQuit(QuitRequest::Me);
           return;
         }
-      } 
+      }
     });
   })};
 
diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index 8f2e489086afb..2ee8d316f3675 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -12,7 +12,17 @@
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/ServiceSpec.h"
 #include "Framework/ServiceMetricsInfo.h"
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/Capability.h"
+#include "Framework/Signpost.h"
 #include "AODJAlienReaderHelpers.h"
+#include <TFile.h>
+#include <TMap.h>
+#include <TObjString.h>
+#include <TString.h>
+#include <fmt/format.h>
+
+O2_DECLARE_DYNAMIC_LOG(analysis_support);
 
 struct ROOTFileReader : o2::framework::AlgorithmPlugin {
   o2::framework::AlgorithmSpec create() override
@@ -54,7 +64,64 @@ struct RunSummary : o2::framework::ServicePlugin {
   }
 };
 
+struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
+  ConfigDiscovery* create() override
+  {
+    return new ConfigDiscovery{
+      .init = []() {},
+      .discover = [](ConfigParamRegistry& registry, int argc, char** argv) -> std::vector<ConfigParamSpec> {
+        auto filename = registry.get<std::string>("aod-file");
+        if (filename.empty()) {
+          return {};
+        }
+        LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
+        std::vector<ConfigParamSpec> results;
+        TFile* currentFile = nullptr;
+        if (filename.at(0) == '@') {
+          filename.erase(0, 1);
+          // read the text file and set filename to the contents of the first line
+          std::ifstream file(filename);
+          if (!file.is_open()) {
+            LOGP(fatal, "Couldn't open file \"{}\"!", filename);
+          }
+          std::getline(file, filename);
+          file.close();
+          currentFile = TFile::Open(filename.c_str());
+        } else {
+          currentFile = TFile::Open(filename.c_str());
+        }
+        if (!currentFile) {
+          LOGP(fatal, "Couldn't open file \"{}\"!", filename);
+        }
+
+        // Get the metadata, if any
+        auto m = (TMap*)currentFile->Get("metaData");
+        if (!m) {
+          LOGP(warning, "No metadata found in file \"{}\"", filename);
+          return results;
+        }
+        auto it = m->MakeIterator();
+
+        // Serialise metadata into a ; separated string with : separating key and value
+        bool first = true;
+        while (auto obj = it->Next()) {
+          if (first) {
+            LOGP(info, "Metadata for file \"{}\":", filename);
+            first = false;
+          }
+          auto objString = (TObjString*)m->GetValue(obj);
+          LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
+          std::string key = "aod-metadata-" + std::string(obj->GetName());
+          char const* value = strdup(objString->String());
+          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
+        }
+        return results;
+      }};
+  }
+};
+
 DEFINE_DPL_PLUGINS_BEGIN
 DEFINE_DPL_PLUGIN_INSTANCE(ROOTFileReader, CustomAlgorithm);
 DEFINE_DPL_PLUGIN_INSTANCE(RunSummary, CustomService);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAOD, ConfigDiscovery);
 DEFINE_DPL_PLUGINS_END
diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index d392ba42fe8ac..4381b6c4fa29d 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -47,6 +47,7 @@ o2_add_library(Framework
                        src/DefaultsHelpers.cxx
                        src/DomainInfoHeader.cxx
                        src/ProcessingPoliciesHelpers.cxx
+                       src/ConfigParamDiscovery.cxx
                        src/ConfigParamStore.cxx
                        src/ConfigParamsHelper.cxx
                        src/ChannelParamSpec.cxx
@@ -109,6 +110,7 @@ o2_add_library(Framework
                        src/OptionsHelpers.cxx
                        src/PropertyTreeHelpers.cxx
                        src/ProcessingContext.cxx
+                       src/Plugin.cxx
                        src/PluginManager.cxx
                        src/RateLimiter.cxx
                        src/ResourcesMonitoringHelper.cxx
diff --git a/Framework/Core/include/Framework/Capability.h b/Framework/Core/include/Framework/Capability.h
new file mode 100644
index 0000000000000..23ef7b7d9bf17
--- /dev/null
+++ b/Framework/Core/include/Framework/Capability.h
@@ -0,0 +1,44 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_CAPABILITY_H_
+#define O2_FRAMEWORK_CAPABILITY_H_
+
+#include <string_view>
+#include <functional>
+
+namespace o2::framework
+{
+
+struct ConfigParamRegistry;
+
+/// A plugin which is able to discover more option then the ones
+/// provided on the command line. The actual loading is in two parts,
+/// the first one decides if the options are enough to actually perform
+/// the discovery, the second part will do the discovery itself.
+///
+/// Its a good idea to have the Loader part in a standalone library to
+/// minimize dependency on unneed thir party code, e.g. ROOT.
+struct Capability {
+  // Wether or not this capability is required.
+  // FIXME: for now let's pass the arguments to get the metadata to work
+  std::string name;
+  std::function<bool(ConfigParamRegistry&, int argc, char** argv)> checkIfNeeded;
+  char const* requiredPlugin;
+};
+
+struct CapabilityPlugin {
+  virtual Capability* create() = 0;
+};
+
+} // namespace o2::framework
+
+#endif // O2_FRAMEWORK_CAPABILITY_H_
diff --git a/Framework/Core/include/Framework/ConfigParamDiscovery.h b/Framework/Core/include/Framework/ConfigParamDiscovery.h
new file mode 100644
index 0000000000000..3d13d3662ceb1
--- /dev/null
+++ b/Framework/Core/include/Framework/ConfigParamDiscovery.h
@@ -0,0 +1,37 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
+#define O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
+
+#include "Framework/ConfigParamRegistry.h"
+
+namespace o2::framework
+{
+// A plugin which can be used to inject extra configuration
+// options.
+struct ConfigParamDiscovery {
+  static std::vector<ConfigParamSpec> discover(ConfigParamRegistry&, int, char**);
+};
+
+struct ConfigDiscovery {
+  std::function<void()> init;
+  /// A function which uses the arguments available so far to discover more
+  /// @return the extra ConfigParamSpecs it derives.
+  std::function<std::vector<ConfigParamSpec>(ConfigParamRegistry&, int, char**)> discover;
+};
+
+struct ConfigDiscoveryPlugin {
+  virtual ConfigDiscovery* create() = 0;
+};
+
+} // namespace o2::framework
+#endif // O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
diff --git a/Framework/Core/include/Framework/ConfigParamRegistry.h b/Framework/Core/include/Framework/ConfigParamRegistry.h
index 8f2d80cea0948..397a491be9f9c 100644
--- a/Framework/Core/include/Framework/ConfigParamRegistry.h
+++ b/Framework/Core/include/Framework/ConfigParamRegistry.h
@@ -125,6 +125,12 @@ class ConfigParamRegistry
     }
   }
 
+  // Load extra parameters discovered while we process data
+  void loadExtra(std::vector<ConfigParamSpec>& extras)
+  {
+    mStore->load(extras);
+  }
+
  private:
   std::unique_ptr<ConfigParamStore> mStore;
 };
diff --git a/Framework/Core/include/Framework/ConfigParamStore.h b/Framework/Core/include/Framework/ConfigParamStore.h
index 37442a6cf7f41..b3ae76aa47ad1 100644
--- a/Framework/Core/include/Framework/ConfigParamStore.h
+++ b/Framework/Core/include/Framework/ConfigParamStore.h
@@ -34,12 +34,14 @@ class ConfigParamStore
   /// the configuration.
   void preload();
 
+  void load(std::vector<ConfigParamSpec>& specs);
+
   /// Get the store
   boost::property_tree::ptree& store() { return *mStore; };
   boost::property_tree::ptree& provenanceTree() { return *mProvenance; };
 
   /// Get the specs
-  std::vector<ConfigParamSpec> const& specs() const
+  [[nodiscard]] std::vector<ConfigParamSpec> const& specs() const
   {
     return mSpecs;
   }
diff --git a/Framework/Core/include/Framework/PluginManager.h b/Framework/Core/include/Framework/PluginManager.h
index 38966c5a969a9..4c6e965502500 100644
--- a/Framework/Core/include/Framework/PluginManager.h
+++ b/Framework/Core/include/Framework/PluginManager.h
@@ -70,7 +70,7 @@ struct PluginManager {
 
     struct LoadedPlugin {
       std::string name;
-      ServicePlugin* factory;
+      PLUGIN* factory;
     };
     std::vector<LoadedDSO> loadedDSOs;
     std::vector<LoadedPlugin> loadedPlugins;
diff --git a/Framework/Core/include/Framework/Plugins.h b/Framework/Core/include/Framework/Plugins.h
index c29d96f7e0781..23d55a512e1fa 100644
--- a/Framework/Core/include/Framework/Plugins.h
+++ b/Framework/Core/include/Framework/Plugins.h
@@ -27,6 +27,15 @@ enum struct DplPluginKind : int {
   // A plugin which implements a custom Services. Needs to return
   // an object of the kind o2::framework::ServiceSpec
   CustomService,
+  // A plugin which implements a new way to discover extra configuration.
+  // parameters. E.g. it can be used to read metadata from a file or a service
+  // if a certain parameter is available.
+  ConfigDiscovery,
+  // A capability plugin is a plugin used to discover other viable plugins.
+  // For example, if you find out that you have the --aod-file option
+  // set, you might want to load metadata from it and attach it to the
+  // configuration.
+  Capability,
   // A plugin which was not initialised properly.
   Unknown
 };
diff --git a/Framework/Core/include/Framework/ServiceSpec.h b/Framework/Core/include/Framework/ServiceSpec.h
index abac3ad05914c..5684889e85376 100644
--- a/Framework/Core/include/Framework/ServiceSpec.h
+++ b/Framework/Core/include/Framework/ServiceSpec.h
@@ -322,11 +322,6 @@ struct ServicePlugin {
   virtual ServiceSpec* create() = 0;
 };
 
-struct LoadableService {
-  std::string name;
-  std::string library;
-};
-
 } // namespace o2::framework
 
 #endif // O2_FRAMEWORK_SERVICESPEC_H_
diff --git a/Framework/Core/include/Framework/runDataProcessing.h b/Framework/Core/include/Framework/runDataProcessing.h
index 77a4c9f5ba8d6..fbf2843d4db01 100644
--- a/Framework/Core/include/Framework/runDataProcessing.h
+++ b/Framework/Core/include/Framework/runDataProcessing.h
@@ -29,7 +29,7 @@
 #include "Framework/Logger.h"
 #include "Framework/CheckTypes.h"
 #include "Framework/StructToTuple.h"
-
+#include "Framework/ConfigParamDiscovery.h"
 #include <vector>
 
 namespace o2::framework
@@ -137,6 +137,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& specs,
            std::vector<o2::framework::CallbacksPolicy> const& callbacksPolicies,
            std::vector<o2::framework::SendingPolicy> const& sendingPolicies,
            std::vector<o2::framework::ConfigParamSpec> const& workflowOptions,
+           std::vector<o2::framework::ConfigParamSpec> const& detectedOptions,
            o2::framework::ConfigContext& configContext);
 
 void doDefaultWorkflowTerminationHook();
@@ -192,6 +193,12 @@ int mainNoCatch(int argc, char** argv)
   workflowOptionsStore->preload();
   workflowOptionsStore->activate();
   ConfigParamRegistry workflowOptionsRegistry(std::move(workflowOptionsStore));
+  auto extraOptions = o2::framework::ConfigParamDiscovery::discover(workflowOptionsRegistry, argc, argv);
+  for (auto& extra : extraOptions) {
+    workflowOptions.push_back(extra);
+  }
+  workflowOptionsRegistry.loadExtra(extraOptions);
+
   ConfigContext configContext(workflowOptionsRegistry, argc, argv);
   o2::framework::WorkflowSpec specs = defineDataProcessing(configContext);
   overrideCloning(configContext, specs);
@@ -206,7 +213,7 @@ int mainNoCatch(int argc, char** argv)
   channelPolicies.insert(std::end(channelPolicies), std::begin(defaultChannelPolicies), std::end(defaultChannelPolicies));
   return doMain(argc, argv, specs,
                 channelPolicies, completionPolicies, dispatchPolicies,
-                resourcePolicies, callbacksPolicies, sendingPolicies, workflowOptions, configContext);
+                resourcePolicies, callbacksPolicies, sendingPolicies, workflowOptions, extraOptions, configContext);
 }
 
 int callMain(int argc, char** argv, int (*)(int, char**));
diff --git a/Framework/Core/src/ConfigParamDiscovery.cxx b/Framework/Core/src/ConfigParamDiscovery.cxx
new file mode 100644
index 0000000000000..6c24a44bac566
--- /dev/null
+++ b/Framework/Core/src/ConfigParamDiscovery.cxx
@@ -0,0 +1,81 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/PluginManager.h"
+#include "Framework/Signpost.h"
+#include "Framework/Capability.h"
+#include <TFile.h>
+#include <TMap.h>
+#include <TObjString.h>
+
+O2_DECLARE_DYNAMIC_LOG(config_parameter_discovery);
+
+namespace o2::framework
+{
+
+std::vector<ConfigParamSpec> ConfigParamDiscovery::discover(ConfigParamRegistry& registry, int argc, char** argv)
+{
+  std::vector<char const*> capabilitiesSpecs = {
+    "O2Framework:DiscoverMetadataInAODCapability",
+    "O2Framework:DiscoverMetadataInCommandLineCapability",
+  };
+
+  // Load all the requested plugins and discover what we can do.
+  O2_SIGNPOST_ID_GENERATE(sid, config_parameter_discovery);
+  std::vector<LoadablePlugin> plugins;
+  for (auto spec : capabilitiesSpecs) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "Parsing Capability plugin: %{public}s", spec);
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+  }
+
+  std::vector<Capability> availableCapabilities;
+  std::vector<char const*> configDiscoverySpec = {};
+  PluginManager::loadFromPlugin<Capability, CapabilityPlugin>(plugins, availableCapabilities);
+  for (auto& capability : availableCapabilities) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- Using Capability of : %{public}s",
+                           capability.name.c_str());
+
+    if (capability.checkIfNeeded(registry, argc, argv)) {
+      O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- Required: %{}s",
+                             capability.requiredPlugin);
+      configDiscoverySpec.push_back(capability.requiredPlugin);
+    } else {
+      O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- No requirements added");
+    }
+  }
+
+  std::vector<LoadablePlugin> configDiscoveryPlugins;
+  for (auto spec : configDiscoverySpec) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "discovery", "Parsing ConfigDiscovery plugin: %{public}s", spec);
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      configDiscoveryPlugins.push_back(extra);
+    }
+  }
+
+  std::vector<ConfigDiscovery> availableConfigDiscovery;
+  PluginManager::loadFromPlugin<ConfigDiscovery, ConfigDiscoveryPlugin>(configDiscoveryPlugins, availableConfigDiscovery);
+  std::vector<ConfigParamSpec> result;
+  for (auto& configDiscovery : availableConfigDiscovery) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "discovery", "Parsing extra configuration as needed");
+    auto extras = configDiscovery.discover(registry, argc, argv);
+    for (auto& extra : extras) {
+      result.push_back(extra);
+    }
+  }
+  return result;
+}
+
+} // namespace o2::framework
diff --git a/Framework/Core/src/ConfigParamStore.cxx b/Framework/Core/src/ConfigParamStore.cxx
index 249a8b0052a67..19e123a23e5d0 100644
--- a/Framework/Core/src/ConfigParamStore.cxx
+++ b/Framework/Core/src/ConfigParamStore.cxx
@@ -41,6 +41,17 @@ void ConfigParamStore::preload()
   }
 }
 
+void ConfigParamStore::load(std::vector<ConfigParamSpec>& specs)
+{
+  mNextStore->clear();
+  mNextProvenance->clear();
+  // By default we populate with code.
+  PropertyTreeHelpers::populateDefaults(specs, *mNextStore, *mNextProvenance);
+  for (auto& retriever : mRetrievers) {
+    retriever->update(specs, *mStore, *mProvenance);
+  }
+}
+
 /// Activate the next store
 void ConfigParamStore::activate()
 {
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 30197538f2dda..b9552dc093115 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -32,7 +32,7 @@
 #include "Framework/OutputRoute.h"
 #include "Framework/WorkflowSpec.h"
 #include "Framework/ComputingResource.h"
-#include "Framework/Logger.h"
+#include "Framework/Signpost.h"
 #include "Framework/RuntimeError.h"
 #include "Framework/RawDeviceService.h"
 #include "ProcessingPoliciesHelpers.h"
@@ -50,8 +50,7 @@
 
 #include <regex>
 
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wpedantic"
+O2_DECLARE_DYNAMIC_LOG(device_spec_helpers);
 
 namespace bpo = boost::program_options;
 
@@ -125,7 +124,9 @@ struct ExpirationHandlerHelpers {
       if (periods.empty()) {
         std::string defaultRateName = std::string{"period-"} + matcher.binding;
         auto defaultRate = std::chrono::milliseconds(options.get<int>(defaultRateName.c_str()));
-        LOGP(detail, "Using default rate of {} ms as specified by option period-{}", defaultRate.count(), matcher.binding);
+        O2_SIGNPOST_ID_GENERATE(tid, device_spec_helpers);
+        O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, tid, "timeDrivenCreation", "Using default rate of %" PRIi64 " ms as specified by option period-%{public}s", defaultRate.count(),
+                               matcher.binding.c_str());
         periods.emplace_back(defaultRate.count());
         durations.emplace_back(std::chrono::seconds((std::size_t)-1));
       } else {
@@ -1359,12 +1360,15 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
                                          std::vector<DeviceSpec> const& deviceSpecs,
                                          std::vector<DeviceExecution>& deviceExecutions,
                                          std::vector<DeviceControl>& deviceControls,
+                                         std::vector<ConfigParamSpec> const& detectedOptions,
                                          std::string const& uniqueWorkflowId)
 {
   assert(deviceSpecs.size() == deviceExecutions.size());
   assert(deviceControls.size() == deviceExecutions.size());
   for (size_t si = 0; si < deviceSpecs.size(); ++si) {
     auto& spec = deviceSpecs[si];
+    O2_SIGNPOST_ID_GENERATE(poid, device_spec_helpers);
+    O2_SIGNPOST_START(device_spec_helpers, poid, "prepareArguments", "Preparing options for %{public}s", spec.id.c_str());
     auto& control = deviceControls[si];
     auto& execution = deviceExecutions[si];
 
@@ -1373,7 +1377,14 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
 
     int argc;
     char** argv;
-    std::vector<ConfigParamSpec> workflowOptions;
+    // We need to start with the detected options, so that they are not lost.
+    // Notice how detected options can be detected at any moment in the chain,
+    // so it's important that if you rely on them, they get passed on
+    // always.
+    std::vector<ConfigParamSpec> workflowOptions = detectedOptions;
+    for (auto& opt : detectedOptions) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, poid, "prepareArguments", "Processor option %{public}s passed as previously detected", opt.name.c_str());
+    }
     /// Lookup the executable name in the metadata associated with the workflow.
     /// If we find it, we rewrite the command line arguments to be processed
     /// so that they look like the ones passed to the merged workflow.
@@ -1386,8 +1397,17 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
       argv[ai + 1] = strdup(arg.data());
     }
     argv[argc] = nullptr;
-    workflowOptions = pi->workflowOptions;
+    for (auto& opt : pi->workflowOptions) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, poid, "prepareArguments", "Processor option %{public}s found in process description", opt.name.c_str());
+      workflowOptions.push_back(opt);
+    }
+    std::sort(workflowOptions.begin(), workflowOptions.end(), [](ConfigParamSpec const& a, ConfigParamSpec const& b) { return a.name < b.name; });
+    auto last = std::unique(workflowOptions.begin(), workflowOptions.end());
+    workflowOptions.erase(last, workflowOptions.end());
 
+    for (auto& opt : workflowOptions) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, poid, "prepareArguments", "Final unique option %{public}s added to list of workflowOptions", opt.name.c_str());
+    }
     // We duplicate the list of options, filtering only those
     // which are actually relevant for the given device. The additional
     // four are to add
@@ -1662,7 +1682,8 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
       assert(execution.args[ai]);
       str << " " << execution.args[ai];
     }
-    LOG(debug) << "The following options are being forwarded to " << spec.id << ":" << str.str();
+    O2_SIGNPOST_END(device_spec_helpers, poid, "prepareArguments", "The following options are being forwarded to %{public}s: %{public}s",
+                    spec.id.c_str(), str.str().c_str());
   }
 }
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.h b/Framework/Core/src/DeviceSpecHelpers.h
index 17601c81342c1..5b54bc84cf6c8 100644
--- a/Framework/Core/src/DeviceSpecHelpers.h
+++ b/Framework/Core/src/DeviceSpecHelpers.h
@@ -129,6 +129,7 @@ struct DeviceSpecHelpers {
     std::vector<DeviceSpec> const& deviceSpecs,
     std::vector<DeviceExecution>& deviceExecutions,
     std::vector<DeviceControl>& deviceControls,
+    std::vector<ConfigParamSpec> const& detectedOptions,
     std::string const& uniqueWorkflowId);
 
   /// Rework the environment string
diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx
new file mode 100644
index 0000000000000..2298896821dab
--- /dev/null
+++ b/Framework/Core/src/Plugin.cxx
@@ -0,0 +1,100 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/Plugins.h"
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/ConfigParamRegistry.h"
+#include "Framework/Logger.h"
+#include "Framework/Capability.h"
+#include "Framework/Signpost.h"
+#include <string_view>
+
+O2_DECLARE_DYNAMIC_LOG(capabilities);
+namespace o2::framework
+{
+auto lookForAodFile = [](ConfigParamRegistry& registry, int argc, char** argv) -> bool {
+  O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+  if (registry.hasOption("aod-file") && registry.isSet("aod-file")) {
+    for (size_t i = 0; i < argc; i++) {
+      std::string_view arg = argv[i];
+      if (arg.starts_with("--aod-metadata-")) {
+        return false;
+      }
+    }
+    O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInAODCapability", "Metadata not found in arguments. Checking in AOD file.");
+    return true;
+  }
+  return false;
+};
+
+auto lookForCommandLineOptions = [](ConfigParamRegistry& registry, int argc, char** argv) -> bool {
+  O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+  for (size_t i = 0; i < argc; i++) {
+    std::string_view arg = argv[i];
+    if (arg.starts_with("--aod-metadata-")) {
+      O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLineCapability", "Metadata found in arguments. Populating from them.");
+      return true;
+    }
+  }
+  return false;
+};
+
+struct DiscoverMetadataInAODCapability : o2::framework::CapabilityPlugin {
+  Capability* create() override
+  {
+    return new Capability{
+      .name = "DiscoverMetadataInAODCapability",
+      .checkIfNeeded = lookForAodFile,
+      .requiredPlugin = "O2FrameworkAnalysisSupport:DiscoverMetadataInAOD"};
+  }
+};
+
+// Trigger discovery of metadata from command line, if needed.
+struct DiscoverMetadataInCommandLineCapability : o2::framework::CapabilityPlugin {
+  Capability* create() override
+  {
+    return new Capability{
+      .name = "DiscoverMetadataInCommandLineCapability",
+      .checkIfNeeded = lookForCommandLineOptions,
+      .requiredPlugin = "O2Framework:DiscoverMetadataInCommandLine"};
+  }
+};
+
+struct DiscoverMetadataInCommandLine : o2::framework::ConfigDiscoveryPlugin {
+  ConfigDiscovery* create() override
+  {
+    return new ConfigDiscovery{
+      .init = []() {},
+      .discover = [](ConfigParamRegistry& registry, int argc, char** argv) -> std::vector<ConfigParamSpec> {
+        O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+        O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLine",
+                               "Discovering metadata for analysis from well known environment variables.");
+        std::vector<ConfigParamSpec> results;
+        for (size_t i = 0; i < argc; i++) {
+          std::string_view arg = argv[i];
+          if (!arg.starts_with("--aod-metadata")) {
+            continue;
+          }
+          std::string key = arg.data() + 2;
+          std::string value = argv[i + 1];
+          O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLine",
+                                 "Found %{public}s with value %{public}s.", key.c_str(), value.c_str());
+          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in command line"}});
+        }
+        return results;
+      }};
+  }
+};
+DEFINE_DPL_PLUGINS_BEGIN
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAODCapability, Capability);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLineCapability, Capability);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLine, ConfigDiscovery);
+DEFINE_DPL_PLUGINS_END
+} // namespace o2::framework
diff --git a/Framework/Core/src/WorkflowCustomizationHelpers.cxx b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
index ffad8a8720019..05abb5dab98cd 100644
--- a/Framework/Core/src/WorkflowCustomizationHelpers.cxx
+++ b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
@@ -54,6 +54,10 @@ std::vector<ConfigParamSpec> WorkflowCustomizationHelpers::requiredWorkflowOptio
            // options for AOD rate limiting
            {"aod-memory-rate-limit", VariantType::Int64, 0LL, {"Rate limit AOD processing based on memory"}},
 
+           // options for the AOD reader
+           // aod-file needs to be available as workflow option, because we
+           // can configure the workflow based on the contents of the first file.
+           {"aod-file", VariantType::String, "", {"Input AOD file"}},
            // options for AOD writer
            {"aod-writer-json", VariantType::String, "", {"Name of the json configuration file"}},
            {"aod-writer-resdir", VariantType::String, "", {"Name of the output directory"}},
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index bd875ad421f6e..da60bdbc6961e 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -292,7 +292,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
                          static_cast<DataAllocator::SubSpecificationType>(compile_time_hash("internal-dpl-aod-reader")),
                          aodLifetime}},
     .algorithm = AlgorithmSpec::dummyAlgorithm(),
-    .options = {ConfigParamSpec{"aod-file", VariantType::String, {"Input AOD file"}},
+    .options = {ConfigParamSpec{"aod-file-private", VariantType::String, ctx.options().get<std::string>("aod-file"), {"AOD file"}},
                 ConfigParamSpec{"aod-reader-json", VariantType::String, {"json configuration file"}},
                 ConfigParamSpec{"aod-parent-access-level", VariantType::String, {"Allow parent file access up to specified level. Default: no (0)"}},
                 ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}},
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index f91716d28cfd7..a0ef33f812e89 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -194,6 +194,10 @@ char* getIdString(int argc, char** argv)
 
 int callMain(int argc, char** argv, int (*mainNoCatch)(int, char**))
 {
+  std::cout << "This is " << getpid() << " " << argv[0] << std::endl;
+  auto spec1 = fair::VerbositySpec::Make(fair::VerbositySpec::Info::process_name);
+  fair::Logger::DefineVerbosity(fair::Verbosity::user1, spec1);
+  fair::Logger::SetVerbosity(fair::Verbosity::user1);
   static bool noCatch = getenv("O2_NO_CATCHALL_EXCEPTIONS") && strcmp(getenv("O2_NO_CATCHALL_EXCEPTIONS"), "0");
   int result = 1;
   if (noCatch) {
@@ -748,6 +752,9 @@ void spawnDevice(uv_loop_t* loop,
       putenv(strdup(DeviceSpecHelpers::reworkTimeslicePlaceholder(env, spec).data()));
     }
     execvp(execution.args[0], execution.args.data());
+  } else {
+    O2_SIGNPOST_ID_GENERATE(sid, driver);
+    O2_SIGNPOST_EVENT_EMIT(driver, sid, "spawnDevice", "New child at %{pid}d", id);
   }
   close(childFds[ref.index].childstdin[0]);
   close(childFds[ref.index].childstdout[1]);
@@ -1359,6 +1366,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
                     DriverInfo& driverInfo,
                     DriverConfig& driverConfig,
                     std::vector<DeviceMetricsInfo>& metricsInfos,
+                    std::vector<ConfigParamSpec> const& detectedParams,
                     boost::program_options::variables_map& varmap,
                     std::vector<ServiceSpec>& driverServices,
                     std::string frameworkId)
@@ -2048,6 +2056,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
                                               runningWorkflow.devices,
                                               deviceExecutions,
                                               controls,
+                                              detectedParams,
                                               driverInfo.uniqueWorkflowId);
         } catch (o2::framework::RuntimeErrorRef& ref) {
           auto& err = o2::framework::error_from_ref(ref);
@@ -2742,6 +2751,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
            std::vector<CallbacksPolicy> const& callbacksPolicies,
            std::vector<SendingPolicy> const& sendingPolicies,
            std::vector<ConfigParamSpec> const& currentWorkflowOptions,
+           std::vector<ConfigParamSpec> const& detectedParams,
            o2::framework::ConfigContext& configContext)
 {
   std::vector<std::string> currentArgs;
@@ -3173,6 +3183,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
                          driverInfo,
                          driverConfig,
                          gDeviceMetricsInfos,
+                         detectedParams,
                          varmap,
                          driverServices,
                          frameworkId);
diff --git a/Framework/Core/test/test_DeviceSpecHelpers.cxx b/Framework/Core/test/test_DeviceSpecHelpers.cxx
index 762c2df6f82aa..6240e784d09d3 100644
--- a/Framework/Core/test/test_DeviceSpecHelpers.cxx
+++ b/Framework/Core/test/test_DeviceSpecHelpers.cxx
@@ -79,7 +79,7 @@ void check(const std::vector<std::string>& arguments,
                                       dataProcessorInfos,
                                       deviceSpecs,
                                       deviceExecutions,
-                                      deviceControls,
+                                      deviceControls, {},
                                       "workflow-id");
 
   for (size_t index = 0; index < deviceSpecs.size(); index++) {
diff --git a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
index 3c08bbae74475..593728696e77a 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
@@ -153,7 +153,7 @@ TEST_CASE("TestDDS")
   DeviceSpecHelpers::prepareArguments(false, false, false, 8080,
                                       driverConfig,
                                       dataProcessorInfos,
-                                      devices, executions, controls,
+                                      devices, executions, controls, {},
                                       "workflow-id");
   CommandInfo command{"foo"};
   DDSConfigHelpers::dumpDeviceSpec2DDS(ss, DriverMode::STANDALONE, "", workflow, dataProcessorInfos, devices, executions, command);
@@ -417,7 +417,7 @@ TEST_CASE("TestDDSExpendable")
   DeviceSpecHelpers::prepareArguments(false, false, false, 8080,
                                       driverConfig,
                                       dataProcessorInfos,
-                                      devices, executions, controls,
+                                      devices, executions, controls, {},
                                       "workflow-id");
   CommandInfo command{"foo"};
   DDSConfigHelpers::dumpDeviceSpec2DDS(ss, DriverMode::STANDALONE, "", workflow, dataProcessorInfos, devices, executions, command);
diff --git a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
index bbd5b78fbbb68..4aca1bd9680a1 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
@@ -560,7 +560,7 @@ TEST_CASE("TestO2ControlDump")
   DeviceSpecHelpers::prepareArguments(false, false, false, 8080,
                                       driverConfig,
                                       dataProcessorInfos,
-                                      devices, executions, controls,
+                                      devices, executions, controls, {},
                                       "workflow-id");
 
   dumpWorkflow(ss, devices, executions, commandInfo, "testwf", "");
diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index bfbdcfc6f169b..21af45422e0b1 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -33,6 +33,10 @@ o2_add_dpl_workflow(diamond-workflow
                   SOURCES src/o2DiamondWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
 
+o2_add_dpl_workflow(analysis-histograms
+                  SOURCES src/o2TestHistograms.cxx
+                  COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(two-timers
                   SOURCES src/o2TwoTimers.cxx
                   COMPONENT_NAME TestWorkflows)
diff --git a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
index 05bf3aabaceab..43d71a9b40528 100644
--- a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
+++ b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
@@ -49,7 +49,14 @@ struct EtaAndClsHistograms {
 
 WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
 {
-  return WorkflowSpec{
-    adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
-  };
+  // For the sake of running without an option, we do not throw an exception
+  // in case the option is not present.
+  if (cfgc.options().hasOption("aod-metadata-Run") == false ||
+      cfgc.options().get<std::string>("aod-metadata-Run") == "2") {
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
+    };
+  } else {
+    throw std::runtime_error("Unsupported run type");
+  }
 }
diff --git a/Framework/TestWorkflows/src/o2TestHistograms.cxx b/Framework/TestWorkflows/src/o2TestHistograms.cxx
new file mode 100644
index 0000000000000..4dbea06d03679
--- /dev/null
+++ b/Framework/TestWorkflows/src/o2TestHistograms.cxx
@@ -0,0 +1,68 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+///
+/// \brief FullTracks is a join of Tracks, TracksCov, and TracksExtra.
+/// \author
+/// \since
+
+#include "Framework/runDataProcessing.h"
+#include "Framework/AnalysisTask.h"
+#include <TH2F.h>
+#include <cmath>
+
+using namespace o2;
+using namespace o2::framework;
+using namespace o2::framework::expressions;
+
+struct EtaAndClsHistogramsSimple {
+  OutputObj<TH2F> etaClsH{TH2F("eta_vs_pt", "#eta vs pT", 102, -2.01, 2.01, 100, 0, 10)};
+
+  void process(aod::Tracks const& tracks)
+  {
+    LOGP(info, "Invoking the simple one");
+    for (auto& track : tracks) {
+      etaClsH->Fill(track.eta(), track.pt(), 0);
+    }
+  }
+};
+
+struct EtaAndClsHistogramsFull {
+  OutputObj<TH3F> etaClsH{TH3F("eta_vs_cls_vs_sigmapT", "#eta vs N_{cls} vs sigma_{1/pT}", 102, -2.01, 2.01, 160, -0.5, 159.5, 100, 0, 10)};
+
+  void process(soa::Join<aod::FullTracks, aod::TracksCov> const& tracks)
+  {
+    LOGP(info, "Invoking the run 3 one");
+    for (auto& track : tracks) {
+      etaClsH->Fill(track.eta(), track.tpcNClsFindable(), track.sigma1Pt());
+    }
+  }
+};
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  std::string runType = "3";
+  if (cfgc.options().hasOption("aod-metadata-Run")) {
+    runType = cfgc.options().get<std::string>("aod-metadata-Run");
+  }
+  LOGP(info, "Runtype is {}", runType);
+  // Notice it's important for the tasks to use the same name, otherwise topology generation will be confused.
+  if (runType == "2") {
+    LOGP(info, "Using only tracks {}", runType);
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistogramsSimple>(cfgc, TaskName{"simple-histos"}),
+    };
+  } else {
+    LOGP(info, "Using tracks extra {}", runType);
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistogramsFull>(cfgc, TaskName{"simple-histos"}),
+    };
+  }
+}

From 0f407995d3c1460cee2024d1be98f1aa9e03c14d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 16 May 2024 16:53:22 +0200
Subject: [PATCH 510/726] DPL Analysis: provide metadata to workflow
 construction

---
 .../src/AODJAlienReaderHelpers.cxx            |   6 +-
 Framework/AnalysisSupport/src/Plugin.cxx      |  67 ++++++++++++
 Framework/Core/CMakeLists.txt                 |   2 +
 Framework/Core/include/Framework/Capability.h |  44 ++++++++
 .../include/Framework/ConfigParamDiscovery.h  |  37 +++++++
 .../include/Framework/ConfigParamRegistry.h   |   6 ++
 .../Core/include/Framework/ConfigParamStore.h |   4 +-
 .../Core/include/Framework/PluginManager.h    |   2 +-
 Framework/Core/include/Framework/Plugins.h    |   9 ++
 .../Core/include/Framework/ServiceSpec.h      |   5 -
 .../include/Framework/runDataProcessing.h     |  11 +-
 Framework/Core/src/ConfigParamDiscovery.cxx   |  81 ++++++++++++++
 Framework/Core/src/ConfigParamStore.cxx       |  11 ++
 Framework/Core/src/DeviceSpecHelpers.cxx      |  35 ++++--
 Framework/Core/src/DeviceSpecHelpers.h        |   1 +
 Framework/Core/src/Plugin.cxx                 | 100 ++++++++++++++++++
 .../Core/src/WorkflowCustomizationHelpers.cxx |   4 +
 Framework/Core/src/WorkflowHelpers.cxx        |   2 +-
 Framework/Core/src/runDataProcessing.cxx      |   7 ++
 .../Core/test/test_DeviceSpecHelpers.cxx      |   2 +-
 .../Core/test/test_FrameworkDataFlowToDDS.cxx |   4 +-
 .../test_FrameworkDataFlowToO2Control.cxx     |   2 +-
 Framework/TestWorkflows/CMakeLists.txt        |   4 +
 .../TestWorkflows/src/o2AnalysisWorkflow.cxx  |  13 ++-
 .../TestWorkflows/src/o2TestHistograms.cxx    |  68 ++++++++++++
 25 files changed, 500 insertions(+), 27 deletions(-)
 create mode 100644 Framework/Core/include/Framework/Capability.h
 create mode 100644 Framework/Core/include/Framework/ConfigParamDiscovery.h
 create mode 100644 Framework/Core/src/ConfigParamDiscovery.cxx
 create mode 100644 Framework/Core/src/Plugin.cxx
 create mode 100644 Framework/TestWorkflows/src/o2TestHistograms.cxx

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 092857d585a02..5ce2eca49c07b 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -133,12 +133,12 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_MESSAGES_DESTROYED), DataProcessingStats::Op::Set, 0});
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_BYTES_EXPIRED), DataProcessingStats::Op::Set, 0});
 
-    if (!options.isSet("aod-file")) {
+    if (!options.isSet("aod-file-private")) {
       LOGP(fatal, "No input file defined!");
       throw std::runtime_error("Processing is stopped!");
     }
 
-    auto filename = options.get<std::string>("aod-file");
+    auto filename = options.get<std::string>("aod-file-private");
 
     std::string parentFileReplacement;
     if (options.isSet("aod-parent-base-path-replacement")) {
@@ -306,7 +306,7 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
           control.readyToQuit(QuitRequest::Me);
           return;
         }
-      } 
+      }
     });
   })};
 
diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index 8f2e489086afb..2ee8d316f3675 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -12,7 +12,17 @@
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/ServiceSpec.h"
 #include "Framework/ServiceMetricsInfo.h"
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/Capability.h"
+#include "Framework/Signpost.h"
 #include "AODJAlienReaderHelpers.h"
+#include <TFile.h>
+#include <TMap.h>
+#include <TObjString.h>
+#include <TString.h>
+#include <fmt/format.h>
+
+O2_DECLARE_DYNAMIC_LOG(analysis_support);
 
 struct ROOTFileReader : o2::framework::AlgorithmPlugin {
   o2::framework::AlgorithmSpec create() override
@@ -54,7 +64,64 @@ struct RunSummary : o2::framework::ServicePlugin {
   }
 };
 
+struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
+  ConfigDiscovery* create() override
+  {
+    return new ConfigDiscovery{
+      .init = []() {},
+      .discover = [](ConfigParamRegistry& registry, int argc, char** argv) -> std::vector<ConfigParamSpec> {
+        auto filename = registry.get<std::string>("aod-file");
+        if (filename.empty()) {
+          return {};
+        }
+        LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
+        std::vector<ConfigParamSpec> results;
+        TFile* currentFile = nullptr;
+        if (filename.at(0) == '@') {
+          filename.erase(0, 1);
+          // read the text file and set filename to the contents of the first line
+          std::ifstream file(filename);
+          if (!file.is_open()) {
+            LOGP(fatal, "Couldn't open file \"{}\"!", filename);
+          }
+          std::getline(file, filename);
+          file.close();
+          currentFile = TFile::Open(filename.c_str());
+        } else {
+          currentFile = TFile::Open(filename.c_str());
+        }
+        if (!currentFile) {
+          LOGP(fatal, "Couldn't open file \"{}\"!", filename);
+        }
+
+        // Get the metadata, if any
+        auto m = (TMap*)currentFile->Get("metaData");
+        if (!m) {
+          LOGP(warning, "No metadata found in file \"{}\"", filename);
+          return results;
+        }
+        auto it = m->MakeIterator();
+
+        // Serialise metadata into a ; separated string with : separating key and value
+        bool first = true;
+        while (auto obj = it->Next()) {
+          if (first) {
+            LOGP(info, "Metadata for file \"{}\":", filename);
+            first = false;
+          }
+          auto objString = (TObjString*)m->GetValue(obj);
+          LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
+          std::string key = "aod-metadata-" + std::string(obj->GetName());
+          char const* value = strdup(objString->String());
+          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
+        }
+        return results;
+      }};
+  }
+};
+
 DEFINE_DPL_PLUGINS_BEGIN
 DEFINE_DPL_PLUGIN_INSTANCE(ROOTFileReader, CustomAlgorithm);
 DEFINE_DPL_PLUGIN_INSTANCE(RunSummary, CustomService);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAOD, ConfigDiscovery);
 DEFINE_DPL_PLUGINS_END
diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index d392ba42fe8ac..4381b6c4fa29d 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -47,6 +47,7 @@ o2_add_library(Framework
                        src/DefaultsHelpers.cxx
                        src/DomainInfoHeader.cxx
                        src/ProcessingPoliciesHelpers.cxx
+                       src/ConfigParamDiscovery.cxx
                        src/ConfigParamStore.cxx
                        src/ConfigParamsHelper.cxx
                        src/ChannelParamSpec.cxx
@@ -109,6 +110,7 @@ o2_add_library(Framework
                        src/OptionsHelpers.cxx
                        src/PropertyTreeHelpers.cxx
                        src/ProcessingContext.cxx
+                       src/Plugin.cxx
                        src/PluginManager.cxx
                        src/RateLimiter.cxx
                        src/ResourcesMonitoringHelper.cxx
diff --git a/Framework/Core/include/Framework/Capability.h b/Framework/Core/include/Framework/Capability.h
new file mode 100644
index 0000000000000..23ef7b7d9bf17
--- /dev/null
+++ b/Framework/Core/include/Framework/Capability.h
@@ -0,0 +1,44 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_CAPABILITY_H_
+#define O2_FRAMEWORK_CAPABILITY_H_
+
+#include <string_view>
+#include <functional>
+
+namespace o2::framework
+{
+
+struct ConfigParamRegistry;
+
+/// A plugin which is able to discover more option then the ones
+/// provided on the command line. The actual loading is in two parts,
+/// the first one decides if the options are enough to actually perform
+/// the discovery, the second part will do the discovery itself.
+///
+/// Its a good idea to have the Loader part in a standalone library to
+/// minimize dependency on unneed thir party code, e.g. ROOT.
+struct Capability {
+  // Wether or not this capability is required.
+  // FIXME: for now let's pass the arguments to get the metadata to work
+  std::string name;
+  std::function<bool(ConfigParamRegistry&, int argc, char** argv)> checkIfNeeded;
+  char const* requiredPlugin;
+};
+
+struct CapabilityPlugin {
+  virtual Capability* create() = 0;
+};
+
+} // namespace o2::framework
+
+#endif // O2_FRAMEWORK_CAPABILITY_H_
diff --git a/Framework/Core/include/Framework/ConfigParamDiscovery.h b/Framework/Core/include/Framework/ConfigParamDiscovery.h
new file mode 100644
index 0000000000000..3d13d3662ceb1
--- /dev/null
+++ b/Framework/Core/include/Framework/ConfigParamDiscovery.h
@@ -0,0 +1,37 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
+#define O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
+
+#include "Framework/ConfigParamRegistry.h"
+
+namespace o2::framework
+{
+// A plugin which can be used to inject extra configuration
+// options.
+struct ConfigParamDiscovery {
+  static std::vector<ConfigParamSpec> discover(ConfigParamRegistry&, int, char**);
+};
+
+struct ConfigDiscovery {
+  std::function<void()> init;
+  /// A function which uses the arguments available so far to discover more
+  /// @return the extra ConfigParamSpecs it derives.
+  std::function<std::vector<ConfigParamSpec>(ConfigParamRegistry&, int, char**)> discover;
+};
+
+struct ConfigDiscoveryPlugin {
+  virtual ConfigDiscovery* create() = 0;
+};
+
+} // namespace o2::framework
+#endif // O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
diff --git a/Framework/Core/include/Framework/ConfigParamRegistry.h b/Framework/Core/include/Framework/ConfigParamRegistry.h
index 8f2d80cea0948..397a491be9f9c 100644
--- a/Framework/Core/include/Framework/ConfigParamRegistry.h
+++ b/Framework/Core/include/Framework/ConfigParamRegistry.h
@@ -125,6 +125,12 @@ class ConfigParamRegistry
     }
   }
 
+  // Load extra parameters discovered while we process data
+  void loadExtra(std::vector<ConfigParamSpec>& extras)
+  {
+    mStore->load(extras);
+  }
+
  private:
   std::unique_ptr<ConfigParamStore> mStore;
 };
diff --git a/Framework/Core/include/Framework/ConfigParamStore.h b/Framework/Core/include/Framework/ConfigParamStore.h
index 37442a6cf7f41..b3ae76aa47ad1 100644
--- a/Framework/Core/include/Framework/ConfigParamStore.h
+++ b/Framework/Core/include/Framework/ConfigParamStore.h
@@ -34,12 +34,14 @@ class ConfigParamStore
   /// the configuration.
   void preload();
 
+  void load(std::vector<ConfigParamSpec>& specs);
+
   /// Get the store
   boost::property_tree::ptree& store() { return *mStore; };
   boost::property_tree::ptree& provenanceTree() { return *mProvenance; };
 
   /// Get the specs
-  std::vector<ConfigParamSpec> const& specs() const
+  [[nodiscard]] std::vector<ConfigParamSpec> const& specs() const
   {
     return mSpecs;
   }
diff --git a/Framework/Core/include/Framework/PluginManager.h b/Framework/Core/include/Framework/PluginManager.h
index 38966c5a969a9..4c6e965502500 100644
--- a/Framework/Core/include/Framework/PluginManager.h
+++ b/Framework/Core/include/Framework/PluginManager.h
@@ -70,7 +70,7 @@ struct PluginManager {
 
     struct LoadedPlugin {
       std::string name;
-      ServicePlugin* factory;
+      PLUGIN* factory;
     };
     std::vector<LoadedDSO> loadedDSOs;
     std::vector<LoadedPlugin> loadedPlugins;
diff --git a/Framework/Core/include/Framework/Plugins.h b/Framework/Core/include/Framework/Plugins.h
index c29d96f7e0781..23d55a512e1fa 100644
--- a/Framework/Core/include/Framework/Plugins.h
+++ b/Framework/Core/include/Framework/Plugins.h
@@ -27,6 +27,15 @@ enum struct DplPluginKind : int {
   // A plugin which implements a custom Services. Needs to return
   // an object of the kind o2::framework::ServiceSpec
   CustomService,
+  // A plugin which implements a new way to discover extra configuration.
+  // parameters. E.g. it can be used to read metadata from a file or a service
+  // if a certain parameter is available.
+  ConfigDiscovery,
+  // A capability plugin is a plugin used to discover other viable plugins.
+  // For example, if you find out that you have the --aod-file option
+  // set, you might want to load metadata from it and attach it to the
+  // configuration.
+  Capability,
   // A plugin which was not initialised properly.
   Unknown
 };
diff --git a/Framework/Core/include/Framework/ServiceSpec.h b/Framework/Core/include/Framework/ServiceSpec.h
index abac3ad05914c..5684889e85376 100644
--- a/Framework/Core/include/Framework/ServiceSpec.h
+++ b/Framework/Core/include/Framework/ServiceSpec.h
@@ -322,11 +322,6 @@ struct ServicePlugin {
   virtual ServiceSpec* create() = 0;
 };
 
-struct LoadableService {
-  std::string name;
-  std::string library;
-};
-
 } // namespace o2::framework
 
 #endif // O2_FRAMEWORK_SERVICESPEC_H_
diff --git a/Framework/Core/include/Framework/runDataProcessing.h b/Framework/Core/include/Framework/runDataProcessing.h
index 77a4c9f5ba8d6..fbf2843d4db01 100644
--- a/Framework/Core/include/Framework/runDataProcessing.h
+++ b/Framework/Core/include/Framework/runDataProcessing.h
@@ -29,7 +29,7 @@
 #include "Framework/Logger.h"
 #include "Framework/CheckTypes.h"
 #include "Framework/StructToTuple.h"
-
+#include "Framework/ConfigParamDiscovery.h"
 #include <vector>
 
 namespace o2::framework
@@ -137,6 +137,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& specs,
            std::vector<o2::framework::CallbacksPolicy> const& callbacksPolicies,
            std::vector<o2::framework::SendingPolicy> const& sendingPolicies,
            std::vector<o2::framework::ConfigParamSpec> const& workflowOptions,
+           std::vector<o2::framework::ConfigParamSpec> const& detectedOptions,
            o2::framework::ConfigContext& configContext);
 
 void doDefaultWorkflowTerminationHook();
@@ -192,6 +193,12 @@ int mainNoCatch(int argc, char** argv)
   workflowOptionsStore->preload();
   workflowOptionsStore->activate();
   ConfigParamRegistry workflowOptionsRegistry(std::move(workflowOptionsStore));
+  auto extraOptions = o2::framework::ConfigParamDiscovery::discover(workflowOptionsRegistry, argc, argv);
+  for (auto& extra : extraOptions) {
+    workflowOptions.push_back(extra);
+  }
+  workflowOptionsRegistry.loadExtra(extraOptions);
+
   ConfigContext configContext(workflowOptionsRegistry, argc, argv);
   o2::framework::WorkflowSpec specs = defineDataProcessing(configContext);
   overrideCloning(configContext, specs);
@@ -206,7 +213,7 @@ int mainNoCatch(int argc, char** argv)
   channelPolicies.insert(std::end(channelPolicies), std::begin(defaultChannelPolicies), std::end(defaultChannelPolicies));
   return doMain(argc, argv, specs,
                 channelPolicies, completionPolicies, dispatchPolicies,
-                resourcePolicies, callbacksPolicies, sendingPolicies, workflowOptions, configContext);
+                resourcePolicies, callbacksPolicies, sendingPolicies, workflowOptions, extraOptions, configContext);
 }
 
 int callMain(int argc, char** argv, int (*)(int, char**));
diff --git a/Framework/Core/src/ConfigParamDiscovery.cxx b/Framework/Core/src/ConfigParamDiscovery.cxx
new file mode 100644
index 0000000000000..6c24a44bac566
--- /dev/null
+++ b/Framework/Core/src/ConfigParamDiscovery.cxx
@@ -0,0 +1,81 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/PluginManager.h"
+#include "Framework/Signpost.h"
+#include "Framework/Capability.h"
+#include <TFile.h>
+#include <TMap.h>
+#include <TObjString.h>
+
+O2_DECLARE_DYNAMIC_LOG(config_parameter_discovery);
+
+namespace o2::framework
+{
+
+std::vector<ConfigParamSpec> ConfigParamDiscovery::discover(ConfigParamRegistry& registry, int argc, char** argv)
+{
+  std::vector<char const*> capabilitiesSpecs = {
+    "O2Framework:DiscoverMetadataInAODCapability",
+    "O2Framework:DiscoverMetadataInCommandLineCapability",
+  };
+
+  // Load all the requested plugins and discover what we can do.
+  O2_SIGNPOST_ID_GENERATE(sid, config_parameter_discovery);
+  std::vector<LoadablePlugin> plugins;
+  for (auto spec : capabilitiesSpecs) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "Parsing Capability plugin: %{public}s", spec);
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+  }
+
+  std::vector<Capability> availableCapabilities;
+  std::vector<char const*> configDiscoverySpec = {};
+  PluginManager::loadFromPlugin<Capability, CapabilityPlugin>(plugins, availableCapabilities);
+  for (auto& capability : availableCapabilities) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- Using Capability of : %{public}s",
+                           capability.name.c_str());
+
+    if (capability.checkIfNeeded(registry, argc, argv)) {
+      O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- Required: %{}s",
+                             capability.requiredPlugin);
+      configDiscoverySpec.push_back(capability.requiredPlugin);
+    } else {
+      O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "capability", "- No requirements added");
+    }
+  }
+
+  std::vector<LoadablePlugin> configDiscoveryPlugins;
+  for (auto spec : configDiscoverySpec) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "discovery", "Parsing ConfigDiscovery plugin: %{public}s", spec);
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      configDiscoveryPlugins.push_back(extra);
+    }
+  }
+
+  std::vector<ConfigDiscovery> availableConfigDiscovery;
+  PluginManager::loadFromPlugin<ConfigDiscovery, ConfigDiscoveryPlugin>(configDiscoveryPlugins, availableConfigDiscovery);
+  std::vector<ConfigParamSpec> result;
+  for (auto& configDiscovery : availableConfigDiscovery) {
+    O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "discovery", "Parsing extra configuration as needed");
+    auto extras = configDiscovery.discover(registry, argc, argv);
+    for (auto& extra : extras) {
+      result.push_back(extra);
+    }
+  }
+  return result;
+}
+
+} // namespace o2::framework
diff --git a/Framework/Core/src/ConfigParamStore.cxx b/Framework/Core/src/ConfigParamStore.cxx
index 249a8b0052a67..19e123a23e5d0 100644
--- a/Framework/Core/src/ConfigParamStore.cxx
+++ b/Framework/Core/src/ConfigParamStore.cxx
@@ -41,6 +41,17 @@ void ConfigParamStore::preload()
   }
 }
 
+void ConfigParamStore::load(std::vector<ConfigParamSpec>& specs)
+{
+  mNextStore->clear();
+  mNextProvenance->clear();
+  // By default we populate with code.
+  PropertyTreeHelpers::populateDefaults(specs, *mNextStore, *mNextProvenance);
+  for (auto& retriever : mRetrievers) {
+    retriever->update(specs, *mStore, *mProvenance);
+  }
+}
+
 /// Activate the next store
 void ConfigParamStore::activate()
 {
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 30197538f2dda..b9552dc093115 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -32,7 +32,7 @@
 #include "Framework/OutputRoute.h"
 #include "Framework/WorkflowSpec.h"
 #include "Framework/ComputingResource.h"
-#include "Framework/Logger.h"
+#include "Framework/Signpost.h"
 #include "Framework/RuntimeError.h"
 #include "Framework/RawDeviceService.h"
 #include "ProcessingPoliciesHelpers.h"
@@ -50,8 +50,7 @@
 
 #include <regex>
 
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wpedantic"
+O2_DECLARE_DYNAMIC_LOG(device_spec_helpers);
 
 namespace bpo = boost::program_options;
 
@@ -125,7 +124,9 @@ struct ExpirationHandlerHelpers {
       if (periods.empty()) {
         std::string defaultRateName = std::string{"period-"} + matcher.binding;
         auto defaultRate = std::chrono::milliseconds(options.get<int>(defaultRateName.c_str()));
-        LOGP(detail, "Using default rate of {} ms as specified by option period-{}", defaultRate.count(), matcher.binding);
+        O2_SIGNPOST_ID_GENERATE(tid, device_spec_helpers);
+        O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, tid, "timeDrivenCreation", "Using default rate of %" PRIi64 " ms as specified by option period-%{public}s", defaultRate.count(),
+                               matcher.binding.c_str());
         periods.emplace_back(defaultRate.count());
         durations.emplace_back(std::chrono::seconds((std::size_t)-1));
       } else {
@@ -1359,12 +1360,15 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
                                          std::vector<DeviceSpec> const& deviceSpecs,
                                          std::vector<DeviceExecution>& deviceExecutions,
                                          std::vector<DeviceControl>& deviceControls,
+                                         std::vector<ConfigParamSpec> const& detectedOptions,
                                          std::string const& uniqueWorkflowId)
 {
   assert(deviceSpecs.size() == deviceExecutions.size());
   assert(deviceControls.size() == deviceExecutions.size());
   for (size_t si = 0; si < deviceSpecs.size(); ++si) {
     auto& spec = deviceSpecs[si];
+    O2_SIGNPOST_ID_GENERATE(poid, device_spec_helpers);
+    O2_SIGNPOST_START(device_spec_helpers, poid, "prepareArguments", "Preparing options for %{public}s", spec.id.c_str());
     auto& control = deviceControls[si];
     auto& execution = deviceExecutions[si];
 
@@ -1373,7 +1377,14 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
 
     int argc;
     char** argv;
-    std::vector<ConfigParamSpec> workflowOptions;
+    // We need to start with the detected options, so that they are not lost.
+    // Notice how detected options can be detected at any moment in the chain,
+    // so it's important that if you rely on them, they get passed on
+    // always.
+    std::vector<ConfigParamSpec> workflowOptions = detectedOptions;
+    for (auto& opt : detectedOptions) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, poid, "prepareArguments", "Processor option %{public}s passed as previously detected", opt.name.c_str());
+    }
     /// Lookup the executable name in the metadata associated with the workflow.
     /// If we find it, we rewrite the command line arguments to be processed
     /// so that they look like the ones passed to the merged workflow.
@@ -1386,8 +1397,17 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
       argv[ai + 1] = strdup(arg.data());
     }
     argv[argc] = nullptr;
-    workflowOptions = pi->workflowOptions;
+    for (auto& opt : pi->workflowOptions) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, poid, "prepareArguments", "Processor option %{public}s found in process description", opt.name.c_str());
+      workflowOptions.push_back(opt);
+    }
+    std::sort(workflowOptions.begin(), workflowOptions.end(), [](ConfigParamSpec const& a, ConfigParamSpec const& b) { return a.name < b.name; });
+    auto last = std::unique(workflowOptions.begin(), workflowOptions.end());
+    workflowOptions.erase(last, workflowOptions.end());
 
+    for (auto& opt : workflowOptions) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, poid, "prepareArguments", "Final unique option %{public}s added to list of workflowOptions", opt.name.c_str());
+    }
     // We duplicate the list of options, filtering only those
     // which are actually relevant for the given device. The additional
     // four are to add
@@ -1662,7 +1682,8 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
       assert(execution.args[ai]);
       str << " " << execution.args[ai];
     }
-    LOG(debug) << "The following options are being forwarded to " << spec.id << ":" << str.str();
+    O2_SIGNPOST_END(device_spec_helpers, poid, "prepareArguments", "The following options are being forwarded to %{public}s: %{public}s",
+                    spec.id.c_str(), str.str().c_str());
   }
 }
 
diff --git a/Framework/Core/src/DeviceSpecHelpers.h b/Framework/Core/src/DeviceSpecHelpers.h
index 17601c81342c1..5b54bc84cf6c8 100644
--- a/Framework/Core/src/DeviceSpecHelpers.h
+++ b/Framework/Core/src/DeviceSpecHelpers.h
@@ -129,6 +129,7 @@ struct DeviceSpecHelpers {
     std::vector<DeviceSpec> const& deviceSpecs,
     std::vector<DeviceExecution>& deviceExecutions,
     std::vector<DeviceControl>& deviceControls,
+    std::vector<ConfigParamSpec> const& detectedOptions,
     std::string const& uniqueWorkflowId);
 
   /// Rework the environment string
diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx
new file mode 100644
index 0000000000000..2298896821dab
--- /dev/null
+++ b/Framework/Core/src/Plugin.cxx
@@ -0,0 +1,100 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/Plugins.h"
+#include "Framework/ConfigParamDiscovery.h"
+#include "Framework/ConfigParamRegistry.h"
+#include "Framework/Logger.h"
+#include "Framework/Capability.h"
+#include "Framework/Signpost.h"
+#include <string_view>
+
+O2_DECLARE_DYNAMIC_LOG(capabilities);
+namespace o2::framework
+{
+auto lookForAodFile = [](ConfigParamRegistry& registry, int argc, char** argv) -> bool {
+  O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+  if (registry.hasOption("aod-file") && registry.isSet("aod-file")) {
+    for (size_t i = 0; i < argc; i++) {
+      std::string_view arg = argv[i];
+      if (arg.starts_with("--aod-metadata-")) {
+        return false;
+      }
+    }
+    O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInAODCapability", "Metadata not found in arguments. Checking in AOD file.");
+    return true;
+  }
+  return false;
+};
+
+auto lookForCommandLineOptions = [](ConfigParamRegistry& registry, int argc, char** argv) -> bool {
+  O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+  for (size_t i = 0; i < argc; i++) {
+    std::string_view arg = argv[i];
+    if (arg.starts_with("--aod-metadata-")) {
+      O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLineCapability", "Metadata found in arguments. Populating from them.");
+      return true;
+    }
+  }
+  return false;
+};
+
+struct DiscoverMetadataInAODCapability : o2::framework::CapabilityPlugin {
+  Capability* create() override
+  {
+    return new Capability{
+      .name = "DiscoverMetadataInAODCapability",
+      .checkIfNeeded = lookForAodFile,
+      .requiredPlugin = "O2FrameworkAnalysisSupport:DiscoverMetadataInAOD"};
+  }
+};
+
+// Trigger discovery of metadata from command line, if needed.
+struct DiscoverMetadataInCommandLineCapability : o2::framework::CapabilityPlugin {
+  Capability* create() override
+  {
+    return new Capability{
+      .name = "DiscoverMetadataInCommandLineCapability",
+      .checkIfNeeded = lookForCommandLineOptions,
+      .requiredPlugin = "O2Framework:DiscoverMetadataInCommandLine"};
+  }
+};
+
+struct DiscoverMetadataInCommandLine : o2::framework::ConfigDiscoveryPlugin {
+  ConfigDiscovery* create() override
+  {
+    return new ConfigDiscovery{
+      .init = []() {},
+      .discover = [](ConfigParamRegistry& registry, int argc, char** argv) -> std::vector<ConfigParamSpec> {
+        O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+        O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLine",
+                               "Discovering metadata for analysis from well known environment variables.");
+        std::vector<ConfigParamSpec> results;
+        for (size_t i = 0; i < argc; i++) {
+          std::string_view arg = argv[i];
+          if (!arg.starts_with("--aod-metadata")) {
+            continue;
+          }
+          std::string key = arg.data() + 2;
+          std::string value = argv[i + 1];
+          O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLine",
+                                 "Found %{public}s with value %{public}s.", key.c_str(), value.c_str());
+          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in command line"}});
+        }
+        return results;
+      }};
+  }
+};
+DEFINE_DPL_PLUGINS_BEGIN
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAODCapability, Capability);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLineCapability, Capability);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLine, ConfigDiscovery);
+DEFINE_DPL_PLUGINS_END
+} // namespace o2::framework
diff --git a/Framework/Core/src/WorkflowCustomizationHelpers.cxx b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
index ffad8a8720019..05abb5dab98cd 100644
--- a/Framework/Core/src/WorkflowCustomizationHelpers.cxx
+++ b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
@@ -54,6 +54,10 @@ std::vector<ConfigParamSpec> WorkflowCustomizationHelpers::requiredWorkflowOptio
            // options for AOD rate limiting
            {"aod-memory-rate-limit", VariantType::Int64, 0LL, {"Rate limit AOD processing based on memory"}},
 
+           // options for the AOD reader
+           // aod-file needs to be available as workflow option, because we
+           // can configure the workflow based on the contents of the first file.
+           {"aod-file", VariantType::String, "", {"Input AOD file"}},
            // options for AOD writer
            {"aod-writer-json", VariantType::String, "", {"Name of the json configuration file"}},
            {"aod-writer-resdir", VariantType::String, "", {"Name of the output directory"}},
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index bd875ad421f6e..da60bdbc6961e 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -292,7 +292,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
                          static_cast<DataAllocator::SubSpecificationType>(compile_time_hash("internal-dpl-aod-reader")),
                          aodLifetime}},
     .algorithm = AlgorithmSpec::dummyAlgorithm(),
-    .options = {ConfigParamSpec{"aod-file", VariantType::String, {"Input AOD file"}},
+    .options = {ConfigParamSpec{"aod-file-private", VariantType::String, ctx.options().get<std::string>("aod-file"), {"AOD file"}},
                 ConfigParamSpec{"aod-reader-json", VariantType::String, {"json configuration file"}},
                 ConfigParamSpec{"aod-parent-access-level", VariantType::String, {"Allow parent file access up to specified level. Default: no (0)"}},
                 ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}},
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index f91716d28cfd7..930132e972881 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -748,6 +748,9 @@ void spawnDevice(uv_loop_t* loop,
       putenv(strdup(DeviceSpecHelpers::reworkTimeslicePlaceholder(env, spec).data()));
     }
     execvp(execution.args[0], execution.args.data());
+  } else {
+    O2_SIGNPOST_ID_GENERATE(sid, driver);
+    O2_SIGNPOST_EVENT_EMIT(driver, sid, "spawnDevice", "New child at %{pid}d", id);
   }
   close(childFds[ref.index].childstdin[0]);
   close(childFds[ref.index].childstdout[1]);
@@ -1359,6 +1362,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
                     DriverInfo& driverInfo,
                     DriverConfig& driverConfig,
                     std::vector<DeviceMetricsInfo>& metricsInfos,
+                    std::vector<ConfigParamSpec> const& detectedParams,
                     boost::program_options::variables_map& varmap,
                     std::vector<ServiceSpec>& driverServices,
                     std::string frameworkId)
@@ -2048,6 +2052,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
                                               runningWorkflow.devices,
                                               deviceExecutions,
                                               controls,
+                                              detectedParams,
                                               driverInfo.uniqueWorkflowId);
         } catch (o2::framework::RuntimeErrorRef& ref) {
           auto& err = o2::framework::error_from_ref(ref);
@@ -2742,6 +2747,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
            std::vector<CallbacksPolicy> const& callbacksPolicies,
            std::vector<SendingPolicy> const& sendingPolicies,
            std::vector<ConfigParamSpec> const& currentWorkflowOptions,
+           std::vector<ConfigParamSpec> const& detectedParams,
            o2::framework::ConfigContext& configContext)
 {
   std::vector<std::string> currentArgs;
@@ -3173,6 +3179,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
                          driverInfo,
                          driverConfig,
                          gDeviceMetricsInfos,
+                         detectedParams,
                          varmap,
                          driverServices,
                          frameworkId);
diff --git a/Framework/Core/test/test_DeviceSpecHelpers.cxx b/Framework/Core/test/test_DeviceSpecHelpers.cxx
index 762c2df6f82aa..6240e784d09d3 100644
--- a/Framework/Core/test/test_DeviceSpecHelpers.cxx
+++ b/Framework/Core/test/test_DeviceSpecHelpers.cxx
@@ -79,7 +79,7 @@ void check(const std::vector<std::string>& arguments,
                                       dataProcessorInfos,
                                       deviceSpecs,
                                       deviceExecutions,
-                                      deviceControls,
+                                      deviceControls, {},
                                       "workflow-id");
 
   for (size_t index = 0; index < deviceSpecs.size(); index++) {
diff --git a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
index 3c08bbae74475..593728696e77a 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
@@ -153,7 +153,7 @@ TEST_CASE("TestDDS")
   DeviceSpecHelpers::prepareArguments(false, false, false, 8080,
                                       driverConfig,
                                       dataProcessorInfos,
-                                      devices, executions, controls,
+                                      devices, executions, controls, {},
                                       "workflow-id");
   CommandInfo command{"foo"};
   DDSConfigHelpers::dumpDeviceSpec2DDS(ss, DriverMode::STANDALONE, "", workflow, dataProcessorInfos, devices, executions, command);
@@ -417,7 +417,7 @@ TEST_CASE("TestDDSExpendable")
   DeviceSpecHelpers::prepareArguments(false, false, false, 8080,
                                       driverConfig,
                                       dataProcessorInfos,
-                                      devices, executions, controls,
+                                      devices, executions, controls, {},
                                       "workflow-id");
   CommandInfo command{"foo"};
   DDSConfigHelpers::dumpDeviceSpec2DDS(ss, DriverMode::STANDALONE, "", workflow, dataProcessorInfos, devices, executions, command);
diff --git a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
index bbd5b78fbbb68..4aca1bd9680a1 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
@@ -560,7 +560,7 @@ TEST_CASE("TestO2ControlDump")
   DeviceSpecHelpers::prepareArguments(false, false, false, 8080,
                                       driverConfig,
                                       dataProcessorInfos,
-                                      devices, executions, controls,
+                                      devices, executions, controls, {},
                                       "workflow-id");
 
   dumpWorkflow(ss, devices, executions, commandInfo, "testwf", "");
diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index bfbdcfc6f169b..21af45422e0b1 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -33,6 +33,10 @@ o2_add_dpl_workflow(diamond-workflow
                   SOURCES src/o2DiamondWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
 
+o2_add_dpl_workflow(analysis-histograms
+                  SOURCES src/o2TestHistograms.cxx
+                  COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(two-timers
                   SOURCES src/o2TwoTimers.cxx
                   COMPONENT_NAME TestWorkflows)
diff --git a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
index 05bf3aabaceab..43d71a9b40528 100644
--- a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
+++ b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx
@@ -49,7 +49,14 @@ struct EtaAndClsHistograms {
 
 WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
 {
-  return WorkflowSpec{
-    adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
-  };
+  // For the sake of running without an option, we do not throw an exception
+  // in case the option is not present.
+  if (cfgc.options().hasOption("aod-metadata-Run") == false ||
+      cfgc.options().get<std::string>("aod-metadata-Run") == "2") {
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistograms>(cfgc),
+    };
+  } else {
+    throw std::runtime_error("Unsupported run type");
+  }
 }
diff --git a/Framework/TestWorkflows/src/o2TestHistograms.cxx b/Framework/TestWorkflows/src/o2TestHistograms.cxx
new file mode 100644
index 0000000000000..4dbea06d03679
--- /dev/null
+++ b/Framework/TestWorkflows/src/o2TestHistograms.cxx
@@ -0,0 +1,68 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+///
+/// \brief FullTracks is a join of Tracks, TracksCov, and TracksExtra.
+/// \author
+/// \since
+
+#include "Framework/runDataProcessing.h"
+#include "Framework/AnalysisTask.h"
+#include <TH2F.h>
+#include <cmath>
+
+using namespace o2;
+using namespace o2::framework;
+using namespace o2::framework::expressions;
+
+struct EtaAndClsHistogramsSimple {
+  OutputObj<TH2F> etaClsH{TH2F("eta_vs_pt", "#eta vs pT", 102, -2.01, 2.01, 100, 0, 10)};
+
+  void process(aod::Tracks const& tracks)
+  {
+    LOGP(info, "Invoking the simple one");
+    for (auto& track : tracks) {
+      etaClsH->Fill(track.eta(), track.pt(), 0);
+    }
+  }
+};
+
+struct EtaAndClsHistogramsFull {
+  OutputObj<TH3F> etaClsH{TH3F("eta_vs_cls_vs_sigmapT", "#eta vs N_{cls} vs sigma_{1/pT}", 102, -2.01, 2.01, 160, -0.5, 159.5, 100, 0, 10)};
+
+  void process(soa::Join<aod::FullTracks, aod::TracksCov> const& tracks)
+  {
+    LOGP(info, "Invoking the run 3 one");
+    for (auto& track : tracks) {
+      etaClsH->Fill(track.eta(), track.tpcNClsFindable(), track.sigma1Pt());
+    }
+  }
+};
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  std::string runType = "3";
+  if (cfgc.options().hasOption("aod-metadata-Run")) {
+    runType = cfgc.options().get<std::string>("aod-metadata-Run");
+  }
+  LOGP(info, "Runtype is {}", runType);
+  // Notice it's important for the tasks to use the same name, otherwise topology generation will be confused.
+  if (runType == "2") {
+    LOGP(info, "Using only tracks {}", runType);
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistogramsSimple>(cfgc, TaskName{"simple-histos"}),
+    };
+  } else {
+    LOGP(info, "Using tracks extra {}", runType);
+    return WorkflowSpec{
+      adaptAnalysisTask<EtaAndClsHistogramsFull>(cfgc, TaskName{"simple-histos"}),
+    };
+  }
+}

From 6fdba259cd4c649299f7ff863724dcb0456d5fd6 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 17 May 2024 08:44:03 +0200
Subject: [PATCH 511/726] DPL: allow reading metadata from alien files

---
 Framework/AnalysisSupport/src/Plugin.cxx | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index 2ee8d316f3675..2ec12a92857cf 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -18,6 +18,7 @@
 #include "AODJAlienReaderHelpers.h"
 #include <TFile.h>
 #include <TMap.h>
+#include <TGrid.h>
 #include <TObjString.h>
 #include <TString.h>
 #include <fmt/format.h>
@@ -74,6 +75,10 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
         if (filename.empty()) {
           return {};
         }
+        if (filename.rfind("alien://", 0) == 0) {
+          LOGP(debug, "AliEn file requested. Enabling support.");
+          TGrid::Connect("alien://");
+        }
         LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
         std::vector<ConfigParamSpec> results;
         TFile* currentFile = nullptr;

From f807849829f373824edbe35eaf8d8344ab5fe539 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 17 May 2024 08:44:03 +0200
Subject: [PATCH 512/726] DPL: print channel name when tracing messages

---
 .../Core/include/Framework/DataRelayer.h      | 19 ++++++++
 Framework/Core/src/DataProcessingDevice.cxx   | 43 +++++++------------
 Framework/Core/src/DataRelayer.cxx            | 11 ++---
 Framework/Core/test/benchmark_DataRelayer.cxx | 12 +++---
 Framework/Core/test/test_DataRelayer.cxx      | 28 ++++++------
 5 files changed, 60 insertions(+), 53 deletions(-)

diff --git a/Framework/Core/include/Framework/DataRelayer.h b/Framework/Core/include/Framework/DataRelayer.h
index 1ebcf549d6a29..b21af0191caf7 100644
--- a/Framework/Core/include/Framework/DataRelayer.h
+++ b/Framework/Core/include/Framework/DataRelayer.h
@@ -81,6 +81,24 @@ class DataRelayer
     CompletionPolicy::CompletionOp op;
   };
 
+  enum struct InputType : int {
+    Invalid = 0,
+    Data = 1,
+    SourceInfo = 2,
+    DomainInfo = 3
+  };
+
+  struct InputInfo {
+    InputInfo(size_t p, size_t s, InputType t, char const* cn)
+      : position(p), size(s), type(t), channelName(cn)
+    {
+    }
+    size_t position;
+    size_t size;
+    InputType type;
+    char const* channelName = nullptr;
+  };
+
   DataRelayer(CompletionPolicy const&,
               std::vector<InputRoute> const& routes,
               TimesliceIndex&,
@@ -114,6 +132,7 @@ class DataRelayer
   /// Notice that we expect that the header is an O2 Header Stack
   RelayChoice relay(void const* rawHeader,
                     std::unique_ptr<fair::mq::Message>* messages,
+                    char const* channelName,
                     size_t nMessages,
                     size_t nPayloads = 1,
                     OnDropCallback onDrop = nullptr);
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 602fe7cc29548..27aaf5f5f299a 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -1765,29 +1765,15 @@ struct WaitBackpressurePolicy {
 /// boilerplate which the user does not need to care about at top level.
 void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo& info)
 {
+  using InputInfo = DataRelayer::InputInfo;
+  using InputType = DataRelayer::InputType;
+
   auto& context = ref.get<DataProcessorContext>();
   // This is the same id as the upper level function, so we get the events
   // associated with the same interval. We will simply use "handle_data" as
   // the category.
   O2_SIGNPOST_ID_FROM_POINTER(cid, device, &info);
 
-  enum struct InputType : int {
-    Invalid = 0,
-    Data = 1,
-    SourceInfo = 2,
-    DomainInfo = 3
-  };
-
-  struct InputInfo {
-    InputInfo(size_t p, size_t s, InputType t)
-      : position(p), size(s), type(t)
-    {
-    }
-    size_t position;
-    size_t size;
-    InputType type;
-  };
-
   // This is how we validate inputs. I.e. we try to enforce the O2 Data model
   // and we do a few stats. We bind parts as a lambda captured variable, rather
   // than an input, because we do not want the outer loop actually be exposed
@@ -1804,8 +1790,8 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
     results.reserve(parts.Size() / 2);
     size_t nTotalPayloads = 0;
 
-    auto insertInputInfo = [&results, &nTotalPayloads](size_t position, size_t length, InputType type) {
-      results.emplace_back(position, length, type);
+    auto insertInputInfo = [&results, &nTotalPayloads](size_t position, size_t length, InputType type, char const* channelName) {
+      results.emplace_back(position, length, type, channelName);
       if (type != InputType::Invalid && length > 1) {
         nTotalPayloads += length - 1;
       }
@@ -1817,24 +1803,24 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
       if (sih) {
         O2_SIGNPOST_EVENT_EMIT(device, cid, "handle_data", "Got SourceInfoHeader with state %d", (int)sih->state);
         info.state = sih->state;
-        insertInputInfo(pi, 2, InputType::SourceInfo);
+        insertInputInfo(pi, 2, InputType::SourceInfo, info.channel->GetName().c_str());
         *context.wasActive = true;
         continue;
       }
       auto dih = o2::header::get<DomainInfoHeader*>(headerData);
       if (dih) {
-        insertInputInfo(pi, 2, InputType::DomainInfo);
+        insertInputInfo(pi, 2, InputType::DomainInfo, info.channel->GetName().c_str());
         *context.wasActive = true;
         continue;
       }
       auto dh = o2::header::get<DataHeader*>(headerData);
       if (!dh) {
-        insertInputInfo(pi, 0, InputType::Invalid);
+        insertInputInfo(pi, 0, InputType::Invalid, info.channel->GetName().c_str());
         O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "Header is not a DataHeader?");
         continue;
       }
       if (dh->payloadSize > parts.At(pi + 1)->GetSize()) {
-        insertInputInfo(pi, 0, InputType::Invalid);
+        insertInputInfo(pi, 0, InputType::Invalid, info.channel->GetName().c_str());
         O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "DataHeader payloadSize mismatch");
         continue;
       }
@@ -1847,14 +1833,14 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
         O2_SIGNPOST_START(parts, pid, "parts", "Processing DataHeader with splitPayloadParts %d and splitPayloadIndex %d", dh->splitPayloadParts, dh->splitPayloadIndex);
       }
       if (!dph) {
-        insertInputInfo(pi, 2, InputType::Invalid);
+        insertInputInfo(pi, 2, InputType::Invalid, info.channel->GetName().c_str());
         O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "Header stack does not contain DataProcessingHeader");
         continue;
       }
       if (dh->splitPayloadParts > 0 && dh->splitPayloadParts == dh->splitPayloadIndex) {
         // this is indicating a sequence of payloads following the header
         // FIXME: we will probably also set the DataHeader version
-        insertInputInfo(pi, dh->splitPayloadParts + 1, InputType::Data);
+        insertInputInfo(pi, dh->splitPayloadParts + 1, InputType::Data, info.channel->GetName().c_str());
         pi += dh->splitPayloadParts - 1;
       } else {
         // We can set the type for the next splitPayloadParts
@@ -1864,12 +1850,12 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
         size_t finalSplitPayloadIndex = pi + (dh->splitPayloadParts > 0 ? dh->splitPayloadParts : 1) * 2;
         if (finalSplitPayloadIndex > parts.Size()) {
           O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "DataHeader::splitPayloadParts invalid");
-          insertInputInfo(pi, 0, InputType::Invalid);
+          insertInputInfo(pi, 0, InputType::Invalid, info.channel->GetName().c_str());
           continue;
         }
-        insertInputInfo(pi, 2, InputType::Data);
+        insertInputInfo(pi, 2, InputType::Data, info.channel->GetName().c_str());
         for (; pi + 2 < finalSplitPayloadIndex; pi += 2) {
-          insertInputInfo(pi + 2, 2, InputType::Data);
+          insertInputInfo(pi + 2, 2, InputType::Data, info.channel->GetName().c_str());
         }
       }
     }
@@ -1941,6 +1927,7 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
           };
           auto relayed = relayer.relay(parts.At(headerIndex)->GetData(),
                                        &parts.At(headerIndex),
+                                       input.channelName,
                                        nMessages,
                                        nPayloadsPerHeader,
                                        onDrop);
diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index 943cd0404f0b3..3e5db8e4f642d 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -394,6 +394,7 @@ void DataRelayer::pruneCache(TimesliceSlot slot, OnDropCallback onDrop)
 DataRelayer::RelayChoice
   DataRelayer::relay(void const* rawHeader,
                      std::unique_ptr<fair::mq::Message>* messages,
+                     char const* channelName,
                      size_t nMessages,
                      size_t nPayloads,
                      std::function<void(TimesliceSlot, std::vector<MessageSet>&, TimesliceIndex::OldestOutputInfo)> onDrop)
@@ -443,11 +444,11 @@ DataRelayer::RelayChoice
                      &nMessages,
                      &nPayloads,
                      &cache = mCache,
-                     numInputTypes = mDistinctRoutesIndex.size()](TimesliceId timeslice, int input, TimesliceSlot slot) {
+                     numInputTypes = mDistinctRoutesIndex.size()](TimesliceId timeslice, int input, TimesliceSlot slot, char const* channelName) {
     O2_SIGNPOST_ID_GENERATE(aid, data_relayer);
-    O2_SIGNPOST_EVENT_EMIT(data_relayer, aid, "saveInSlot", "saving %{public}s@%zu in slot %zu",
+    O2_SIGNPOST_EVENT_EMIT(data_relayer, aid, "saveInSlot", "saving %{public}s@%zu in slot %zu from %{public}s",
                            fmt::format("{:x}", *o2::header::get<DataHeader*>(messages[0]->GetData())).c_str(),
-                           timeslice.value, slot.index);
+                           timeslice.value, slot.index, channelName);
     auto cacheIdx = numInputTypes * slot.index + input;
     MessageSet& target = cache[cacheIdx];
     cachedStateMetrics[cacheIdx] = CacheEntryStatus::PENDING;
@@ -537,7 +538,7 @@ DataRelayer::RelayChoice
       this->pruneCache(slot, onDrop);
       mPruneOps.erase(std::remove_if(mPruneOps.begin(), mPruneOps.end(), [slot](const auto& x) { return x.slot == slot; }), mPruneOps.end());
     }
-    saveInSlot(timeslice, input, slot);
+    saveInSlot(timeslice, input, slot, channelName);
     index.publishSlot(slot);
     index.markAsDirty(slot, true);
     stats.updateStats({static_cast<short>(ProcessingStatsId::RELAYED_MESSAGES), DataProcessingStats::Op::Add, (int)1});
@@ -619,7 +620,7 @@ DataRelayer::RelayChoice
       // cache still holds the old data, so we prune it.
       this->pruneCache(slot, onDrop);
       mPruneOps.erase(std::remove_if(mPruneOps.begin(), mPruneOps.end(), [slot](const auto& x) { return x.slot == slot; }), mPruneOps.end());
-      saveInSlot(timeslice, input, slot);
+      saveInSlot(timeslice, input, slot, channelName);
       index.publishSlot(slot);
       index.markAsDirty(slot, true);
       return RelayChoice{.type = RelayChoice::Type::WillRelay};
diff --git a/Framework/Core/test/benchmark_DataRelayer.cxx b/Framework/Core/test/benchmark_DataRelayer.cxx
index 3e8302e97b0a7..9e8aaf7a692e4 100644
--- a/Framework/Core/test/benchmark_DataRelayer.cxx
+++ b/Framework/Core/test/benchmark_DataRelayer.cxx
@@ -87,7 +87,7 @@ static void BM_RelaySingleSlot(benchmark::State& state)
   memcpy(inflightMessages[0]->GetData(), stack.data(), stack.size());
 
   for (auto _ : state) {
-    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), inflightMessages.size());
+    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), "test-channel", inflightMessages.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
@@ -144,7 +144,7 @@ static void BM_RelayMultipleSlots(benchmark::State& state)
     Stack stack{dh, DataProcessingHeader{timeslice++, 1}};
     memcpy(inflightMessages[0]->GetData(), stack.data(), stack.size());
 
-    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), inflightMessages.size());
+    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), "test-channel", inflightMessages.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
@@ -211,13 +211,13 @@ static void BM_RelayMultipleRoutes(benchmark::State& state)
   memcpy(inflightMessages[2]->GetData(), stack2.data(), stack2.size());
 
   for (auto _ : state) {
-    relayer.relay(inflightMessages[0]->GetData(), &inflightMessages[0], 2);
+    relayer.relay(inflightMessages[0]->GetData(), &inflightMessages[0], "test-channel", 2);
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
     assert(ready[0].op == CompletionPolicy::CompletionOp::Consume);
 
-    relayer.relay(inflightMessages[2]->GetData(), &inflightMessages[2], 2);
+    relayer.relay(inflightMessages[2]->GetData(), &inflightMessages[2], "test-channel", 2);
     ready.clear();
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
@@ -283,7 +283,7 @@ static void BM_RelaySplitParts(benchmark::State& state)
   }
 
   for (auto _ : state) {
-    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), inflightMessages.size());
+    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), "test-channel", inflightMessages.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
@@ -337,7 +337,7 @@ static void BM_RelayMultiplePayloads(benchmark::State& state)
   }
 
   for (auto _ : state) {
-    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), inflightMessages.size(), nPayloads);
+    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), "test-channel", inflightMessages.size(), nPayloads);
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
diff --git a/Framework/Core/test/test_DataRelayer.cxx b/Framework/Core/test/test_DataRelayer.cxx
index bd1ead40c9773..79238d26f5379 100644
--- a/Framework/Core/test/test_DataRelayer.cxx
+++ b/Framework/Core/test/test_DataRelayer.cxx
@@ -99,7 +99,7 @@ TEST_CASE("DataRelayer")
     messages[1] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header = messages[0];
     fair::mq::MessagePtr& payload = messages[1];
-    relayer.relay(header->GetData(), messages.data(), messages.size());
+    relayer.relay(header->GetData(), messages.data(), "test-channel", messages.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     REQUIRE(ready.size() == 1);
@@ -148,7 +148,7 @@ TEST_CASE("DataRelayer")
     messages[1] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header = messages[0];
     fair::mq::MessagePtr& payload = messages[1];
-    relayer.relay(header->GetData(), messages.data(), messages.size());
+    relayer.relay(header->GetData(), messages.data(), "test-channel", messages.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     REQUIRE(ready.size() == 1);
@@ -201,7 +201,7 @@ TEST_CASE("DataRelayer")
       messages[1] = transport->CreateMessage(1000);
       fair::mq::MessagePtr& header = messages[0];
       fair::mq::MessagePtr& payload = messages[1];
-      relayer.relay(header->GetData(), messages.data(), messages.size());
+      relayer.relay(header->GetData(), messages.data(), "test-channel", messages.size());
       REQUIRE(header.get() == nullptr);
       REQUIRE(payload.get() == nullptr);
     };
@@ -281,7 +281,7 @@ TEST_CASE("DataRelayer")
       messages[1] = transport->CreateMessage(1000);
       fair::mq::MessagePtr& header = messages[0];
       fair::mq::MessagePtr& payload = messages[1];
-      relayer.relay(header->GetData(), messages.data(), messages.size());
+      relayer.relay(header->GetData(), messages.data(), "test-channel", messages.size());
       REQUIRE(header.get() == nullptr);
       REQUIRE(payload.get() == nullptr);
     };
@@ -373,7 +373,7 @@ TEST_CASE("DataRelayer")
       messages[1] = transport->CreateMessage(1000);
       fair::mq::MessagePtr& header = messages[0];
       fair::mq::MessagePtr& payload = messages[1];
-      auto res = relayer.relay(header->GetData(), messages.data(), messages.size());
+      auto res = relayer.relay(header->GetData(), messages.data(), "test-channel", messages.size());
       REQUIRE((res.type != DataRelayer::RelayChoice::Type::WillRelay || header.get() == nullptr));
       REQUIRE((res.type != DataRelayer::RelayChoice::Type::WillRelay || payload.get() == nullptr));
       REQUIRE((res.type != DataRelayer::RelayChoice::Type::Backpressured || header.get() != nullptr));
@@ -455,7 +455,7 @@ TEST_CASE("DataRelayer")
       messages[0] = o2::pmr::getMessage(Stack{channelAlloc, dh, h});
       messages[1] = transport->CreateMessage(1000);
       fair::mq::MessagePtr& header = messages[0];
-      return relayer.relay(header->GetData(), messages.data(), messages.size());
+      return relayer.relay(header->GetData(), messages.data(), "test-channel", messages.size());
     };
 
     // This fills the cache, and then empties it.
@@ -526,7 +526,7 @@ TEST_CASE("DataRelayer")
       messages[0] = o2::pmr::getMessage(Stack{channelAlloc, dh, h});
       messages[1] = transport->CreateMessage(1000);
       fair::mq::MessagePtr& header = messages[0];
-      return relayer.relay(header->GetData(), messages.data(), messages.size());
+      return relayer.relay(header->GetData(), messages.data(), "test-channel", messages.size());
     };
 
     // This fills the cache, and then empties it.
@@ -585,7 +585,7 @@ TEST_CASE("DataRelayer")
     messages[1] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header = messages[0];
     fair::mq::MessagePtr& payload = messages[1];
-    relayer.relay(header->GetData(), &messages[0], 2);
+    relayer.relay(header->GetData(), &messages[0], "test-channel", 2);
     REQUIRE(header.get() == nullptr);
     REQUIRE(payload.get() == nullptr);
     // This fills the cache, and then waits.
@@ -593,7 +593,7 @@ TEST_CASE("DataRelayer")
     messages[3] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header2 = messages[2];
     fair::mq::MessagePtr& payload2 = messages[3];
-    auto action = relayer.relay(header2->GetData(), &messages[2], 2);
+    auto action = relayer.relay(header2->GetData(), &messages[2], "test-channel", 2);
     REQUIRE(action.type == DataRelayer::RelayChoice::Type::Backpressured);
     REQUIRE(header2.get() != nullptr);
     REQUIRE(payload2.get() != nullptr);
@@ -644,7 +644,7 @@ TEST_CASE("DataRelayer")
     messages[1] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header = messages[0];
     fair::mq::MessagePtr& payload = messages[1];
-    relayer.relay(header->GetData(), &messages[0], 2);
+    relayer.relay(header->GetData(), &messages[0], "test-channel", 2);
     REQUIRE(header.get() == nullptr);
     REQUIRE(payload.get() == nullptr);
     // This fills the cache, and then waits.
@@ -652,7 +652,7 @@ TEST_CASE("DataRelayer")
     messages[3] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header2 = messages[2];
     fair::mq::MessagePtr& payload2 = messages[3];
-    auto action = relayer.relay(header2->GetData(), &messages[2], 2);
+    auto action = relayer.relay(header2->GetData(), &messages[2], "test-channel", 2);
     REQUIRE(action.type == DataRelayer::RelayChoice::Type::Backpressured);
     CHECK(action.timeslice.value == 1);
     REQUIRE(header2.get() != nullptr);
@@ -660,7 +660,7 @@ TEST_CASE("DataRelayer")
     // This fills the cache, and then waits.
     messages[4] = o2::pmr::getMessage(Stack{channelAlloc, dh1, DataProcessingHeader{1, 1}});
     messages[5] = transport->CreateMessage(1000);
-    relayer.relay(header2->GetData(), &messages[4], 2);
+    relayer.relay(header2->GetData(), &messages[4], "test-channel", 2);
     REQUIRE(action.type == DataRelayer::RelayChoice::Type::Backpressured);
     CHECK(action.timeslice.value == 1);
     REQUIRE(header2.get() != nullptr);
@@ -707,7 +707,7 @@ TEST_CASE("DataRelayer")
     }
     REQUIRE(splitParts.size() == 2 * nSplitParts);
 
-    relayer.relay(splitParts[0]->GetData(), splitParts.data(), splitParts.size());
+    relayer.relay(splitParts[0]->GetData(), splitParts.data(), "test-channel", splitParts.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     REQUIRE(ready.size() == 1);
@@ -763,7 +763,7 @@ TEST_CASE("DataRelayer")
         ++nTotalPayloads;
       }
       REQUIRE(messages.size() == nPayloads + 1);
-      relayer.relay(messages[0]->GetData(), messages.data(), messages.size(), nPayloads);
+      relayer.relay(messages[0]->GetData(), messages.data(), "test-channel", messages.size(), nPayloads);
       sequenceSize.emplace_back(nPayloads);
     };
     createSequence(100);

From 823dcae7a82eb899675a0fa573d524d3486530f1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 17 May 2024 14:24:06 +0200
Subject: [PATCH 513/726] DPL: print channel name when tracing messages

---
 .../Core/include/Framework/DataRelayer.h      | 19 ++++++++
 Framework/Core/src/DataProcessingDevice.cxx   | 43 +++++++------------
 Framework/Core/src/DataRelayer.cxx            | 13 +++---
 Framework/Core/test/benchmark_DataRelayer.cxx | 18 +++++---
 Framework/Core/test/test_DataRelayer.cxx      | 42 ++++++++++++------
 5 files changed, 82 insertions(+), 53 deletions(-)

diff --git a/Framework/Core/include/Framework/DataRelayer.h b/Framework/Core/include/Framework/DataRelayer.h
index 1ebcf549d6a29..8192db32c4d80 100644
--- a/Framework/Core/include/Framework/DataRelayer.h
+++ b/Framework/Core/include/Framework/DataRelayer.h
@@ -81,6 +81,24 @@ class DataRelayer
     CompletionPolicy::CompletionOp op;
   };
 
+  enum struct InputType : int {
+    Invalid = 0,
+    Data = 1,
+    SourceInfo = 2,
+    DomainInfo = 3
+  };
+
+  struct InputInfo {
+    InputInfo(size_t p, size_t s, InputType t, ChannelIndex i)
+      : position(p), size(s), type(t), index(i)
+    {
+    }
+    size_t position;
+    size_t size;
+    InputType type;
+    ChannelIndex index;
+  };
+
   DataRelayer(CompletionPolicy const&,
               std::vector<InputRoute> const& routes,
               TimesliceIndex&,
@@ -114,6 +132,7 @@ class DataRelayer
   /// Notice that we expect that the header is an O2 Header Stack
   RelayChoice relay(void const* rawHeader,
                     std::unique_ptr<fair::mq::Message>* messages,
+                    InputInfo const&info,
                     size_t nMessages,
                     size_t nPayloads = 1,
                     OnDropCallback onDrop = nullptr);
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 602fe7cc29548..3603d8a6003bb 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -1765,29 +1765,15 @@ struct WaitBackpressurePolicy {
 /// boilerplate which the user does not need to care about at top level.
 void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo& info)
 {
+  using InputInfo = DataRelayer::InputInfo;
+  using InputType = DataRelayer::InputType;
+
   auto& context = ref.get<DataProcessorContext>();
   // This is the same id as the upper level function, so we get the events
   // associated with the same interval. We will simply use "handle_data" as
   // the category.
   O2_SIGNPOST_ID_FROM_POINTER(cid, device, &info);
 
-  enum struct InputType : int {
-    Invalid = 0,
-    Data = 1,
-    SourceInfo = 2,
-    DomainInfo = 3
-  };
-
-  struct InputInfo {
-    InputInfo(size_t p, size_t s, InputType t)
-      : position(p), size(s), type(t)
-    {
-    }
-    size_t position;
-    size_t size;
-    InputType type;
-  };
-
   // This is how we validate inputs. I.e. we try to enforce the O2 Data model
   // and we do a few stats. We bind parts as a lambda captured variable, rather
   // than an input, because we do not want the outer loop actually be exposed
@@ -1804,8 +1790,8 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
     results.reserve(parts.Size() / 2);
     size_t nTotalPayloads = 0;
 
-    auto insertInputInfo = [&results, &nTotalPayloads](size_t position, size_t length, InputType type) {
-      results.emplace_back(position, length, type);
+    auto insertInputInfo = [&results, &nTotalPayloads](size_t position, size_t length, InputType type, ChannelIndex index) {
+      results.emplace_back(position, length, type, index);
       if (type != InputType::Invalid && length > 1) {
         nTotalPayloads += length - 1;
       }
@@ -1817,24 +1803,24 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
       if (sih) {
         O2_SIGNPOST_EVENT_EMIT(device, cid, "handle_data", "Got SourceInfoHeader with state %d", (int)sih->state);
         info.state = sih->state;
-        insertInputInfo(pi, 2, InputType::SourceInfo);
+        insertInputInfo(pi, 2, InputType::SourceInfo, info.id);
         *context.wasActive = true;
         continue;
       }
       auto dih = o2::header::get<DomainInfoHeader*>(headerData);
       if (dih) {
-        insertInputInfo(pi, 2, InputType::DomainInfo);
+        insertInputInfo(pi, 2, InputType::DomainInfo, info.id);
         *context.wasActive = true;
         continue;
       }
       auto dh = o2::header::get<DataHeader*>(headerData);
       if (!dh) {
-        insertInputInfo(pi, 0, InputType::Invalid);
+        insertInputInfo(pi, 0, InputType::Invalid, info.id);
         O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "Header is not a DataHeader?");
         continue;
       }
       if (dh->payloadSize > parts.At(pi + 1)->GetSize()) {
-        insertInputInfo(pi, 0, InputType::Invalid);
+        insertInputInfo(pi, 0, InputType::Invalid, info.id);
         O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "DataHeader payloadSize mismatch");
         continue;
       }
@@ -1847,14 +1833,14 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
         O2_SIGNPOST_START(parts, pid, "parts", "Processing DataHeader with splitPayloadParts %d and splitPayloadIndex %d", dh->splitPayloadParts, dh->splitPayloadIndex);
       }
       if (!dph) {
-        insertInputInfo(pi, 2, InputType::Invalid);
+        insertInputInfo(pi, 2, InputType::Invalid, info.id); 
         O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "Header stack does not contain DataProcessingHeader");
         continue;
       }
       if (dh->splitPayloadParts > 0 && dh->splitPayloadParts == dh->splitPayloadIndex) {
         // this is indicating a sequence of payloads following the header
         // FIXME: we will probably also set the DataHeader version
-        insertInputInfo(pi, dh->splitPayloadParts + 1, InputType::Data);
+        insertInputInfo(pi, dh->splitPayloadParts + 1, InputType::Data, info.id);
         pi += dh->splitPayloadParts - 1;
       } else {
         // We can set the type for the next splitPayloadParts
@@ -1864,12 +1850,12 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
         size_t finalSplitPayloadIndex = pi + (dh->splitPayloadParts > 0 ? dh->splitPayloadParts : 1) * 2;
         if (finalSplitPayloadIndex > parts.Size()) {
           O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "DataHeader::splitPayloadParts invalid");
-          insertInputInfo(pi, 0, InputType::Invalid);
+          insertInputInfo(pi, 0, InputType::Invalid, info.id);
           continue;
         }
-        insertInputInfo(pi, 2, InputType::Data);
+        insertInputInfo(pi, 2, InputType::Data, info.id);
         for (; pi + 2 < finalSplitPayloadIndex; pi += 2) {
-          insertInputInfo(pi + 2, 2, InputType::Data);
+          insertInputInfo(pi + 2, 2, InputType::Data, info.id);
         }
       }
     }
@@ -1941,6 +1927,7 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
           };
           auto relayed = relayer.relay(parts.At(headerIndex)->GetData(),
                                        &parts.At(headerIndex),
+                                       input,
                                        nMessages,
                                        nPayloadsPerHeader,
                                        onDrop);
diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index 943cd0404f0b3..bd0fba7b35e3e 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -394,6 +394,7 @@ void DataRelayer::pruneCache(TimesliceSlot slot, OnDropCallback onDrop)
 DataRelayer::RelayChoice
   DataRelayer::relay(void const* rawHeader,
                      std::unique_ptr<fair::mq::Message>* messages,
+                     InputInfo const& info,
                      size_t nMessages,
                      size_t nPayloads,
                      std::function<void(TimesliceSlot, std::vector<MessageSet>&, TimesliceIndex::OldestOutputInfo)> onDrop)
@@ -443,11 +444,13 @@ DataRelayer::RelayChoice
                      &nMessages,
                      &nPayloads,
                      &cache = mCache,
-                     numInputTypes = mDistinctRoutesIndex.size()](TimesliceId timeslice, int input, TimesliceSlot slot) {
+                     &services = mContext,
+                     numInputTypes = mDistinctRoutesIndex.size()](TimesliceId timeslice, int input, TimesliceSlot slot, InputInfo const& info) {
     O2_SIGNPOST_ID_GENERATE(aid, data_relayer);
-    O2_SIGNPOST_EVENT_EMIT(data_relayer, aid, "saveInSlot", "saving %{public}s@%zu in slot %zu",
+    O2_SIGNPOST_EVENT_EMIT(data_relayer, aid, "saveInSlot", "saving %{public}s@%zu in slot %zu from %{public}s",
                            fmt::format("{:x}", *o2::header::get<DataHeader*>(messages[0]->GetData())).c_str(),
-                           timeslice.value, slot.index);
+                           timeslice.value, slot.index,
+                           info.index.value == ChannelIndex::INVALID ? "invalid" : services.get<FairMQDeviceProxy>().getInputChannel(info.index)->GetName().c_str());
     auto cacheIdx = numInputTypes * slot.index + input;
     MessageSet& target = cache[cacheIdx];
     cachedStateMetrics[cacheIdx] = CacheEntryStatus::PENDING;
@@ -537,7 +540,7 @@ DataRelayer::RelayChoice
       this->pruneCache(slot, onDrop);
       mPruneOps.erase(std::remove_if(mPruneOps.begin(), mPruneOps.end(), [slot](const auto& x) { return x.slot == slot; }), mPruneOps.end());
     }
-    saveInSlot(timeslice, input, slot);
+    saveInSlot(timeslice, input, slot, info);
     index.publishSlot(slot);
     index.markAsDirty(slot, true);
     stats.updateStats({static_cast<short>(ProcessingStatsId::RELAYED_MESSAGES), DataProcessingStats::Op::Add, (int)1});
@@ -619,7 +622,7 @@ DataRelayer::RelayChoice
       // cache still holds the old data, so we prune it.
       this->pruneCache(slot, onDrop);
       mPruneOps.erase(std::remove_if(mPruneOps.begin(), mPruneOps.end(), [slot](const auto& x) { return x.slot == slot; }), mPruneOps.end());
-      saveInSlot(timeslice, input, slot);
+      saveInSlot(timeslice, input, slot, info);
       index.publishSlot(slot);
       index.markAsDirty(slot, true);
       return RelayChoice{.type = RelayChoice::Type::WillRelay};
diff --git a/Framework/Core/test/benchmark_DataRelayer.cxx b/Framework/Core/test/benchmark_DataRelayer.cxx
index 3e8302e97b0a7..dcff3930dbaad 100644
--- a/Framework/Core/test/benchmark_DataRelayer.cxx
+++ b/Framework/Core/test/benchmark_DataRelayer.cxx
@@ -86,8 +86,9 @@ static void BM_RelaySingleSlot(benchmark::State& state)
   inflightMessages.emplace_back(transport->CreateMessage(1000));
   memcpy(inflightMessages[0]->GetData(), stack.data(), stack.size());
 
+  DataRelayer::InputInfo fakeInfo{0, inflightMessages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
   for (auto _ : state) {
-    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), inflightMessages.size());
+    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), fakeInfo, inflightMessages.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
@@ -144,7 +145,8 @@ static void BM_RelayMultipleSlots(benchmark::State& state)
     Stack stack{dh, DataProcessingHeader{timeslice++, 1}};
     memcpy(inflightMessages[0]->GetData(), stack.data(), stack.size());
 
-    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), inflightMessages.size());
+    DataRelayer::InputInfo fakeInfo{0, inflightMessages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), fakeInfo, inflightMessages.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
@@ -211,13 +213,15 @@ static void BM_RelayMultipleRoutes(benchmark::State& state)
   memcpy(inflightMessages[2]->GetData(), stack2.data(), stack2.size());
 
   for (auto _ : state) {
-    relayer.relay(inflightMessages[0]->GetData(), &inflightMessages[0], 2);
+    DataRelayer::InputInfo fakeInfo{0, inflightMessages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(inflightMessages[0]->GetData(), &inflightMessages[0], fakeInfo, 2);
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
     assert(ready[0].op == CompletionPolicy::CompletionOp::Consume);
 
-    relayer.relay(inflightMessages[2]->GetData(), &inflightMessages[2], 2);
+    DataRelayer::InputInfo fakeInfo2{0, inflightMessages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(inflightMessages[2]->GetData(), &inflightMessages[2], fakeInfo2, 2);
     ready.clear();
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
@@ -282,8 +286,9 @@ static void BM_RelaySplitParts(benchmark::State& state)
     inflightMessages.emplace_back(std::move(payload));
   }
 
+  DataRelayer::InputInfo fakeInfo{0, inflightMessages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
   for (auto _ : state) {
-    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), inflightMessages.size());
+    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), fakeInfo, inflightMessages.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
@@ -336,8 +341,9 @@ static void BM_RelayMultiplePayloads(benchmark::State& state)
     inflightMessages.emplace_back(transport->CreateMessage(dh.payloadSize));
   }
 
+  DataRelayer::InputInfo fakeInfo{0, inflightMessages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
   for (auto _ : state) {
-    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), inflightMessages.size(), nPayloads);
+    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), fakeInfo, inflightMessages.size(), nPayloads);
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
diff --git a/Framework/Core/test/test_DataRelayer.cxx b/Framework/Core/test/test_DataRelayer.cxx
index bd1ead40c9773..d112e8736399a 100644
--- a/Framework/Core/test/test_DataRelayer.cxx
+++ b/Framework/Core/test/test_DataRelayer.cxx
@@ -99,7 +99,8 @@ TEST_CASE("DataRelayer")
     messages[1] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header = messages[0];
     fair::mq::MessagePtr& payload = messages[1];
-    relayer.relay(header->GetData(), messages.data(), messages.size());
+    DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(header->GetData(), messages.data(), fakeInfo, messages.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     REQUIRE(ready.size() == 1);
@@ -148,7 +149,8 @@ TEST_CASE("DataRelayer")
     messages[1] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header = messages[0];
     fair::mq::MessagePtr& payload = messages[1];
-    relayer.relay(header->GetData(), messages.data(), messages.size());
+    DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(header->GetData(), messages.data(), fakeInfo, messages.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     REQUIRE(ready.size() == 1);
@@ -201,7 +203,8 @@ TEST_CASE("DataRelayer")
       messages[1] = transport->CreateMessage(1000);
       fair::mq::MessagePtr& header = messages[0];
       fair::mq::MessagePtr& payload = messages[1];
-      relayer.relay(header->GetData(), messages.data(), messages.size());
+      DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+      relayer.relay(header->GetData(), messages.data(), fakeInfo, messages.size());
       REQUIRE(header.get() == nullptr);
       REQUIRE(payload.get() == nullptr);
     };
@@ -281,7 +284,8 @@ TEST_CASE("DataRelayer")
       messages[1] = transport->CreateMessage(1000);
       fair::mq::MessagePtr& header = messages[0];
       fair::mq::MessagePtr& payload = messages[1];
-      relayer.relay(header->GetData(), messages.data(), messages.size());
+      DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+      relayer.relay(header->GetData(), messages.data(), fakeInfo, messages.size());
       REQUIRE(header.get() == nullptr);
       REQUIRE(payload.get() == nullptr);
     };
@@ -373,7 +377,8 @@ TEST_CASE("DataRelayer")
       messages[1] = transport->CreateMessage(1000);
       fair::mq::MessagePtr& header = messages[0];
       fair::mq::MessagePtr& payload = messages[1];
-      auto res = relayer.relay(header->GetData(), messages.data(), messages.size());
+      DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+      auto res = relayer.relay(header->GetData(), messages.data(), fakeInfo, messages.size());
       REQUIRE((res.type != DataRelayer::RelayChoice::Type::WillRelay || header.get() == nullptr));
       REQUIRE((res.type != DataRelayer::RelayChoice::Type::WillRelay || payload.get() == nullptr));
       REQUIRE((res.type != DataRelayer::RelayChoice::Type::Backpressured || header.get() != nullptr));
@@ -455,7 +460,8 @@ TEST_CASE("DataRelayer")
       messages[0] = o2::pmr::getMessage(Stack{channelAlloc, dh, h});
       messages[1] = transport->CreateMessage(1000);
       fair::mq::MessagePtr& header = messages[0];
-      return relayer.relay(header->GetData(), messages.data(), messages.size());
+      DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+      return relayer.relay(header->GetData(), messages.data(), fakeInfo, messages.size());
     };
 
     // This fills the cache, and then empties it.
@@ -526,7 +532,8 @@ TEST_CASE("DataRelayer")
       messages[0] = o2::pmr::getMessage(Stack{channelAlloc, dh, h});
       messages[1] = transport->CreateMessage(1000);
       fair::mq::MessagePtr& header = messages[0];
-      return relayer.relay(header->GetData(), messages.data(), messages.size());
+      DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+      return relayer.relay(header->GetData(), messages.data(), fakeInfo, messages.size());
     };
 
     // This fills the cache, and then empties it.
@@ -585,7 +592,8 @@ TEST_CASE("DataRelayer")
     messages[1] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header = messages[0];
     fair::mq::MessagePtr& payload = messages[1];
-    relayer.relay(header->GetData(), &messages[0], 2);
+    DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(header->GetData(), &messages[0], fakeInfo, 2);
     REQUIRE(header.get() == nullptr);
     REQUIRE(payload.get() == nullptr);
     // This fills the cache, and then waits.
@@ -593,7 +601,8 @@ TEST_CASE("DataRelayer")
     messages[3] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header2 = messages[2];
     fair::mq::MessagePtr& payload2 = messages[3];
-    auto action = relayer.relay(header2->GetData(), &messages[2], 2);
+    DataRelayer::InputInfo fakeInfo2{2, 2, DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    auto action = relayer.relay(header2->GetData(), &messages[2], fakeInfo2, 2);
     REQUIRE(action.type == DataRelayer::RelayChoice::Type::Backpressured);
     REQUIRE(header2.get() != nullptr);
     REQUIRE(payload2.get() != nullptr);
@@ -644,7 +653,8 @@ TEST_CASE("DataRelayer")
     messages[1] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header = messages[0];
     fair::mq::MessagePtr& payload = messages[1];
-    relayer.relay(header->GetData(), &messages[0], 2);
+    DataRelayer::InputInfo fakeInfo{0, 2, DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(header->GetData(), &messages[0], fakeInfo, 2);
     REQUIRE(header.get() == nullptr);
     REQUIRE(payload.get() == nullptr);
     // This fills the cache, and then waits.
@@ -652,7 +662,8 @@ TEST_CASE("DataRelayer")
     messages[3] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header2 = messages[2];
     fair::mq::MessagePtr& payload2 = messages[3];
-    auto action = relayer.relay(header2->GetData(), &messages[2], 2);
+    DataRelayer::InputInfo fakeInfo2{2, 2, DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    auto action = relayer.relay(header2->GetData(), &messages[2], fakeInfo, 2);
     REQUIRE(action.type == DataRelayer::RelayChoice::Type::Backpressured);
     CHECK(action.timeslice.value == 1);
     REQUIRE(header2.get() != nullptr);
@@ -660,7 +671,8 @@ TEST_CASE("DataRelayer")
     // This fills the cache, and then waits.
     messages[4] = o2::pmr::getMessage(Stack{channelAlloc, dh1, DataProcessingHeader{1, 1}});
     messages[5] = transport->CreateMessage(1000);
-    relayer.relay(header2->GetData(), &messages[4], 2);
+    DataRelayer::InputInfo fakeInfo3{4, 2, DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(header2->GetData(), &messages[4], fakeInfo3, 2);
     REQUIRE(action.type == DataRelayer::RelayChoice::Type::Backpressured);
     CHECK(action.timeslice.value == 1);
     REQUIRE(header2.get() != nullptr);
@@ -707,7 +719,8 @@ TEST_CASE("DataRelayer")
     }
     REQUIRE(splitParts.size() == 2 * nSplitParts);
 
-    relayer.relay(splitParts[0]->GetData(), splitParts.data(), splitParts.size());
+    DataRelayer::InputInfo fakeInfo{0, splitParts.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(splitParts[0]->GetData(), splitParts.data(), fakeInfo, splitParts.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     REQUIRE(ready.size() == 1);
@@ -763,7 +776,8 @@ TEST_CASE("DataRelayer")
         ++nTotalPayloads;
       }
       REQUIRE(messages.size() == nPayloads + 1);
-      relayer.relay(messages[0]->GetData(), messages.data(), messages.size(), nPayloads);
+      DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+      relayer.relay(messages[0]->GetData(), messages.data(), fakeInfo, messages.size(), nPayloads);
       sequenceSize.emplace_back(nPayloads);
     };
     createSequence(100);

From 3f5a15cfa4e7e56419615fad8cc679f054d43595 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 17 May 2024 14:27:42 +0200
Subject: [PATCH 514/726] DPL: print channel name when tracing messages

---
 .../Core/include/Framework/DataRelayer.h      | 19 ++++++++
 Framework/Core/src/DataProcessingDevice.cxx   | 43 +++++++------------
 Framework/Core/src/DataRelayer.cxx            | 13 +++---
 Framework/Core/test/benchmark_DataRelayer.cxx | 18 +++++---
 Framework/Core/test/test_DataRelayer.cxx      | 42 ++++++++++++------
 5 files changed, 82 insertions(+), 53 deletions(-)

diff --git a/Framework/Core/include/Framework/DataRelayer.h b/Framework/Core/include/Framework/DataRelayer.h
index 1ebcf549d6a29..012b909096317 100644
--- a/Framework/Core/include/Framework/DataRelayer.h
+++ b/Framework/Core/include/Framework/DataRelayer.h
@@ -81,6 +81,24 @@ class DataRelayer
     CompletionPolicy::CompletionOp op;
   };
 
+  enum struct InputType : int {
+    Invalid = 0,
+    Data = 1,
+    SourceInfo = 2,
+    DomainInfo = 3
+  };
+
+  struct InputInfo {
+    InputInfo(size_t p, size_t s, InputType t, ChannelIndex i)
+      : position(p), size(s), type(t), index(i)
+    {
+    }
+    size_t position;
+    size_t size;
+    InputType type;
+    ChannelIndex index;
+  };
+
   DataRelayer(CompletionPolicy const&,
               std::vector<InputRoute> const& routes,
               TimesliceIndex&,
@@ -114,6 +132,7 @@ class DataRelayer
   /// Notice that we expect that the header is an O2 Header Stack
   RelayChoice relay(void const* rawHeader,
                     std::unique_ptr<fair::mq::Message>* messages,
+                    InputInfo const& info,
                     size_t nMessages,
                     size_t nPayloads = 1,
                     OnDropCallback onDrop = nullptr);
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 602fe7cc29548..6b926d92353be 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -1765,29 +1765,15 @@ struct WaitBackpressurePolicy {
 /// boilerplate which the user does not need to care about at top level.
 void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo& info)
 {
+  using InputInfo = DataRelayer::InputInfo;
+  using InputType = DataRelayer::InputType;
+
   auto& context = ref.get<DataProcessorContext>();
   // This is the same id as the upper level function, so we get the events
   // associated with the same interval. We will simply use "handle_data" as
   // the category.
   O2_SIGNPOST_ID_FROM_POINTER(cid, device, &info);
 
-  enum struct InputType : int {
-    Invalid = 0,
-    Data = 1,
-    SourceInfo = 2,
-    DomainInfo = 3
-  };
-
-  struct InputInfo {
-    InputInfo(size_t p, size_t s, InputType t)
-      : position(p), size(s), type(t)
-    {
-    }
-    size_t position;
-    size_t size;
-    InputType type;
-  };
-
   // This is how we validate inputs. I.e. we try to enforce the O2 Data model
   // and we do a few stats. We bind parts as a lambda captured variable, rather
   // than an input, because we do not want the outer loop actually be exposed
@@ -1804,8 +1790,8 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
     results.reserve(parts.Size() / 2);
     size_t nTotalPayloads = 0;
 
-    auto insertInputInfo = [&results, &nTotalPayloads](size_t position, size_t length, InputType type) {
-      results.emplace_back(position, length, type);
+    auto insertInputInfo = [&results, &nTotalPayloads](size_t position, size_t length, InputType type, ChannelIndex index) {
+      results.emplace_back(position, length, type, index);
       if (type != InputType::Invalid && length > 1) {
         nTotalPayloads += length - 1;
       }
@@ -1817,24 +1803,24 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
       if (sih) {
         O2_SIGNPOST_EVENT_EMIT(device, cid, "handle_data", "Got SourceInfoHeader with state %d", (int)sih->state);
         info.state = sih->state;
-        insertInputInfo(pi, 2, InputType::SourceInfo);
+        insertInputInfo(pi, 2, InputType::SourceInfo, info.id);
         *context.wasActive = true;
         continue;
       }
       auto dih = o2::header::get<DomainInfoHeader*>(headerData);
       if (dih) {
-        insertInputInfo(pi, 2, InputType::DomainInfo);
+        insertInputInfo(pi, 2, InputType::DomainInfo, info.id);
         *context.wasActive = true;
         continue;
       }
       auto dh = o2::header::get<DataHeader*>(headerData);
       if (!dh) {
-        insertInputInfo(pi, 0, InputType::Invalid);
+        insertInputInfo(pi, 0, InputType::Invalid, info.id);
         O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "Header is not a DataHeader?");
         continue;
       }
       if (dh->payloadSize > parts.At(pi + 1)->GetSize()) {
-        insertInputInfo(pi, 0, InputType::Invalid);
+        insertInputInfo(pi, 0, InputType::Invalid, info.id);
         O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "DataHeader payloadSize mismatch");
         continue;
       }
@@ -1847,14 +1833,14 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
         O2_SIGNPOST_START(parts, pid, "parts", "Processing DataHeader with splitPayloadParts %d and splitPayloadIndex %d", dh->splitPayloadParts, dh->splitPayloadIndex);
       }
       if (!dph) {
-        insertInputInfo(pi, 2, InputType::Invalid);
+        insertInputInfo(pi, 2, InputType::Invalid, info.id);
         O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "Header stack does not contain DataProcessingHeader");
         continue;
       }
       if (dh->splitPayloadParts > 0 && dh->splitPayloadParts == dh->splitPayloadIndex) {
         // this is indicating a sequence of payloads following the header
         // FIXME: we will probably also set the DataHeader version
-        insertInputInfo(pi, dh->splitPayloadParts + 1, InputType::Data);
+        insertInputInfo(pi, dh->splitPayloadParts + 1, InputType::Data, info.id);
         pi += dh->splitPayloadParts - 1;
       } else {
         // We can set the type for the next splitPayloadParts
@@ -1864,12 +1850,12 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
         size_t finalSplitPayloadIndex = pi + (dh->splitPayloadParts > 0 ? dh->splitPayloadParts : 1) * 2;
         if (finalSplitPayloadIndex > parts.Size()) {
           O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "DataHeader::splitPayloadParts invalid");
-          insertInputInfo(pi, 0, InputType::Invalid);
+          insertInputInfo(pi, 0, InputType::Invalid, info.id);
           continue;
         }
-        insertInputInfo(pi, 2, InputType::Data);
+        insertInputInfo(pi, 2, InputType::Data, info.id);
         for (; pi + 2 < finalSplitPayloadIndex; pi += 2) {
-          insertInputInfo(pi + 2, 2, InputType::Data);
+          insertInputInfo(pi + 2, 2, InputType::Data, info.id);
         }
       }
     }
@@ -1941,6 +1927,7 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
           };
           auto relayed = relayer.relay(parts.At(headerIndex)->GetData(),
                                        &parts.At(headerIndex),
+                                       input,
                                        nMessages,
                                        nPayloadsPerHeader,
                                        onDrop);
diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index 943cd0404f0b3..bd0fba7b35e3e 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -394,6 +394,7 @@ void DataRelayer::pruneCache(TimesliceSlot slot, OnDropCallback onDrop)
 DataRelayer::RelayChoice
   DataRelayer::relay(void const* rawHeader,
                      std::unique_ptr<fair::mq::Message>* messages,
+                     InputInfo const& info,
                      size_t nMessages,
                      size_t nPayloads,
                      std::function<void(TimesliceSlot, std::vector<MessageSet>&, TimesliceIndex::OldestOutputInfo)> onDrop)
@@ -443,11 +444,13 @@ DataRelayer::RelayChoice
                      &nMessages,
                      &nPayloads,
                      &cache = mCache,
-                     numInputTypes = mDistinctRoutesIndex.size()](TimesliceId timeslice, int input, TimesliceSlot slot) {
+                     &services = mContext,
+                     numInputTypes = mDistinctRoutesIndex.size()](TimesliceId timeslice, int input, TimesliceSlot slot, InputInfo const& info) {
     O2_SIGNPOST_ID_GENERATE(aid, data_relayer);
-    O2_SIGNPOST_EVENT_EMIT(data_relayer, aid, "saveInSlot", "saving %{public}s@%zu in slot %zu",
+    O2_SIGNPOST_EVENT_EMIT(data_relayer, aid, "saveInSlot", "saving %{public}s@%zu in slot %zu from %{public}s",
                            fmt::format("{:x}", *o2::header::get<DataHeader*>(messages[0]->GetData())).c_str(),
-                           timeslice.value, slot.index);
+                           timeslice.value, slot.index,
+                           info.index.value == ChannelIndex::INVALID ? "invalid" : services.get<FairMQDeviceProxy>().getInputChannel(info.index)->GetName().c_str());
     auto cacheIdx = numInputTypes * slot.index + input;
     MessageSet& target = cache[cacheIdx];
     cachedStateMetrics[cacheIdx] = CacheEntryStatus::PENDING;
@@ -537,7 +540,7 @@ DataRelayer::RelayChoice
       this->pruneCache(slot, onDrop);
       mPruneOps.erase(std::remove_if(mPruneOps.begin(), mPruneOps.end(), [slot](const auto& x) { return x.slot == slot; }), mPruneOps.end());
     }
-    saveInSlot(timeslice, input, slot);
+    saveInSlot(timeslice, input, slot, info);
     index.publishSlot(slot);
     index.markAsDirty(slot, true);
     stats.updateStats({static_cast<short>(ProcessingStatsId::RELAYED_MESSAGES), DataProcessingStats::Op::Add, (int)1});
@@ -619,7 +622,7 @@ DataRelayer::RelayChoice
       // cache still holds the old data, so we prune it.
       this->pruneCache(slot, onDrop);
       mPruneOps.erase(std::remove_if(mPruneOps.begin(), mPruneOps.end(), [slot](const auto& x) { return x.slot == slot; }), mPruneOps.end());
-      saveInSlot(timeslice, input, slot);
+      saveInSlot(timeslice, input, slot, info);
       index.publishSlot(slot);
       index.markAsDirty(slot, true);
       return RelayChoice{.type = RelayChoice::Type::WillRelay};
diff --git a/Framework/Core/test/benchmark_DataRelayer.cxx b/Framework/Core/test/benchmark_DataRelayer.cxx
index 3e8302e97b0a7..dcff3930dbaad 100644
--- a/Framework/Core/test/benchmark_DataRelayer.cxx
+++ b/Framework/Core/test/benchmark_DataRelayer.cxx
@@ -86,8 +86,9 @@ static void BM_RelaySingleSlot(benchmark::State& state)
   inflightMessages.emplace_back(transport->CreateMessage(1000));
   memcpy(inflightMessages[0]->GetData(), stack.data(), stack.size());
 
+  DataRelayer::InputInfo fakeInfo{0, inflightMessages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
   for (auto _ : state) {
-    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), inflightMessages.size());
+    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), fakeInfo, inflightMessages.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
@@ -144,7 +145,8 @@ static void BM_RelayMultipleSlots(benchmark::State& state)
     Stack stack{dh, DataProcessingHeader{timeslice++, 1}};
     memcpy(inflightMessages[0]->GetData(), stack.data(), stack.size());
 
-    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), inflightMessages.size());
+    DataRelayer::InputInfo fakeInfo{0, inflightMessages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), fakeInfo, inflightMessages.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
@@ -211,13 +213,15 @@ static void BM_RelayMultipleRoutes(benchmark::State& state)
   memcpy(inflightMessages[2]->GetData(), stack2.data(), stack2.size());
 
   for (auto _ : state) {
-    relayer.relay(inflightMessages[0]->GetData(), &inflightMessages[0], 2);
+    DataRelayer::InputInfo fakeInfo{0, inflightMessages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(inflightMessages[0]->GetData(), &inflightMessages[0], fakeInfo, 2);
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
     assert(ready[0].op == CompletionPolicy::CompletionOp::Consume);
 
-    relayer.relay(inflightMessages[2]->GetData(), &inflightMessages[2], 2);
+    DataRelayer::InputInfo fakeInfo2{0, inflightMessages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(inflightMessages[2]->GetData(), &inflightMessages[2], fakeInfo2, 2);
     ready.clear();
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
@@ -282,8 +286,9 @@ static void BM_RelaySplitParts(benchmark::State& state)
     inflightMessages.emplace_back(std::move(payload));
   }
 
+  DataRelayer::InputInfo fakeInfo{0, inflightMessages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
   for (auto _ : state) {
-    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), inflightMessages.size());
+    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), fakeInfo, inflightMessages.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
@@ -336,8 +341,9 @@ static void BM_RelayMultiplePayloads(benchmark::State& state)
     inflightMessages.emplace_back(transport->CreateMessage(dh.payloadSize));
   }
 
+  DataRelayer::InputInfo fakeInfo{0, inflightMessages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
   for (auto _ : state) {
-    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), inflightMessages.size(), nPayloads);
+    relayer.relay(inflightMessages[0]->GetData(), inflightMessages.data(), fakeInfo, inflightMessages.size(), nPayloads);
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     assert(ready.size() == 1);
diff --git a/Framework/Core/test/test_DataRelayer.cxx b/Framework/Core/test/test_DataRelayer.cxx
index bd1ead40c9773..d112e8736399a 100644
--- a/Framework/Core/test/test_DataRelayer.cxx
+++ b/Framework/Core/test/test_DataRelayer.cxx
@@ -99,7 +99,8 @@ TEST_CASE("DataRelayer")
     messages[1] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header = messages[0];
     fair::mq::MessagePtr& payload = messages[1];
-    relayer.relay(header->GetData(), messages.data(), messages.size());
+    DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(header->GetData(), messages.data(), fakeInfo, messages.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     REQUIRE(ready.size() == 1);
@@ -148,7 +149,8 @@ TEST_CASE("DataRelayer")
     messages[1] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header = messages[0];
     fair::mq::MessagePtr& payload = messages[1];
-    relayer.relay(header->GetData(), messages.data(), messages.size());
+    DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(header->GetData(), messages.data(), fakeInfo, messages.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     REQUIRE(ready.size() == 1);
@@ -201,7 +203,8 @@ TEST_CASE("DataRelayer")
       messages[1] = transport->CreateMessage(1000);
       fair::mq::MessagePtr& header = messages[0];
       fair::mq::MessagePtr& payload = messages[1];
-      relayer.relay(header->GetData(), messages.data(), messages.size());
+      DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+      relayer.relay(header->GetData(), messages.data(), fakeInfo, messages.size());
       REQUIRE(header.get() == nullptr);
       REQUIRE(payload.get() == nullptr);
     };
@@ -281,7 +284,8 @@ TEST_CASE("DataRelayer")
       messages[1] = transport->CreateMessage(1000);
       fair::mq::MessagePtr& header = messages[0];
       fair::mq::MessagePtr& payload = messages[1];
-      relayer.relay(header->GetData(), messages.data(), messages.size());
+      DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+      relayer.relay(header->GetData(), messages.data(), fakeInfo, messages.size());
       REQUIRE(header.get() == nullptr);
       REQUIRE(payload.get() == nullptr);
     };
@@ -373,7 +377,8 @@ TEST_CASE("DataRelayer")
       messages[1] = transport->CreateMessage(1000);
       fair::mq::MessagePtr& header = messages[0];
       fair::mq::MessagePtr& payload = messages[1];
-      auto res = relayer.relay(header->GetData(), messages.data(), messages.size());
+      DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+      auto res = relayer.relay(header->GetData(), messages.data(), fakeInfo, messages.size());
       REQUIRE((res.type != DataRelayer::RelayChoice::Type::WillRelay || header.get() == nullptr));
       REQUIRE((res.type != DataRelayer::RelayChoice::Type::WillRelay || payload.get() == nullptr));
       REQUIRE((res.type != DataRelayer::RelayChoice::Type::Backpressured || header.get() != nullptr));
@@ -455,7 +460,8 @@ TEST_CASE("DataRelayer")
       messages[0] = o2::pmr::getMessage(Stack{channelAlloc, dh, h});
       messages[1] = transport->CreateMessage(1000);
       fair::mq::MessagePtr& header = messages[0];
-      return relayer.relay(header->GetData(), messages.data(), messages.size());
+      DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+      return relayer.relay(header->GetData(), messages.data(), fakeInfo, messages.size());
     };
 
     // This fills the cache, and then empties it.
@@ -526,7 +532,8 @@ TEST_CASE("DataRelayer")
       messages[0] = o2::pmr::getMessage(Stack{channelAlloc, dh, h});
       messages[1] = transport->CreateMessage(1000);
       fair::mq::MessagePtr& header = messages[0];
-      return relayer.relay(header->GetData(), messages.data(), messages.size());
+      DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+      return relayer.relay(header->GetData(), messages.data(), fakeInfo, messages.size());
     };
 
     // This fills the cache, and then empties it.
@@ -585,7 +592,8 @@ TEST_CASE("DataRelayer")
     messages[1] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header = messages[0];
     fair::mq::MessagePtr& payload = messages[1];
-    relayer.relay(header->GetData(), &messages[0], 2);
+    DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(header->GetData(), &messages[0], fakeInfo, 2);
     REQUIRE(header.get() == nullptr);
     REQUIRE(payload.get() == nullptr);
     // This fills the cache, and then waits.
@@ -593,7 +601,8 @@ TEST_CASE("DataRelayer")
     messages[3] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header2 = messages[2];
     fair::mq::MessagePtr& payload2 = messages[3];
-    auto action = relayer.relay(header2->GetData(), &messages[2], 2);
+    DataRelayer::InputInfo fakeInfo2{2, 2, DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    auto action = relayer.relay(header2->GetData(), &messages[2], fakeInfo2, 2);
     REQUIRE(action.type == DataRelayer::RelayChoice::Type::Backpressured);
     REQUIRE(header2.get() != nullptr);
     REQUIRE(payload2.get() != nullptr);
@@ -644,7 +653,8 @@ TEST_CASE("DataRelayer")
     messages[1] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header = messages[0];
     fair::mq::MessagePtr& payload = messages[1];
-    relayer.relay(header->GetData(), &messages[0], 2);
+    DataRelayer::InputInfo fakeInfo{0, 2, DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(header->GetData(), &messages[0], fakeInfo, 2);
     REQUIRE(header.get() == nullptr);
     REQUIRE(payload.get() == nullptr);
     // This fills the cache, and then waits.
@@ -652,7 +662,8 @@ TEST_CASE("DataRelayer")
     messages[3] = transport->CreateMessage(1000);
     fair::mq::MessagePtr& header2 = messages[2];
     fair::mq::MessagePtr& payload2 = messages[3];
-    auto action = relayer.relay(header2->GetData(), &messages[2], 2);
+    DataRelayer::InputInfo fakeInfo2{2, 2, DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    auto action = relayer.relay(header2->GetData(), &messages[2], fakeInfo, 2);
     REQUIRE(action.type == DataRelayer::RelayChoice::Type::Backpressured);
     CHECK(action.timeslice.value == 1);
     REQUIRE(header2.get() != nullptr);
@@ -660,7 +671,8 @@ TEST_CASE("DataRelayer")
     // This fills the cache, and then waits.
     messages[4] = o2::pmr::getMessage(Stack{channelAlloc, dh1, DataProcessingHeader{1, 1}});
     messages[5] = transport->CreateMessage(1000);
-    relayer.relay(header2->GetData(), &messages[4], 2);
+    DataRelayer::InputInfo fakeInfo3{4, 2, DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(header2->GetData(), &messages[4], fakeInfo3, 2);
     REQUIRE(action.type == DataRelayer::RelayChoice::Type::Backpressured);
     CHECK(action.timeslice.value == 1);
     REQUIRE(header2.get() != nullptr);
@@ -707,7 +719,8 @@ TEST_CASE("DataRelayer")
     }
     REQUIRE(splitParts.size() == 2 * nSplitParts);
 
-    relayer.relay(splitParts[0]->GetData(), splitParts.data(), splitParts.size());
+    DataRelayer::InputInfo fakeInfo{0, splitParts.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+    relayer.relay(splitParts[0]->GetData(), splitParts.data(), fakeInfo, splitParts.size());
     std::vector<RecordAction> ready;
     relayer.getReadyToProcess(ready);
     REQUIRE(ready.size() == 1);
@@ -763,7 +776,8 @@ TEST_CASE("DataRelayer")
         ++nTotalPayloads;
       }
       REQUIRE(messages.size() == nPayloads + 1);
-      relayer.relay(messages[0]->GetData(), messages.data(), messages.size(), nPayloads);
+      DataRelayer::InputInfo fakeInfo{0, messages.size(), DataRelayer::InputType::Data, {ChannelIndex::INVALID}};
+      relayer.relay(messages[0]->GetData(), messages.data(), fakeInfo, messages.size(), nPayloads);
       sequenceSize.emplace_back(nPayloads);
     };
     createSequence(100);

From 746672ad06be059ba643c7e031bfdb50e424c596 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 18 May 2024 14:46:14 +0200
Subject: [PATCH 515/726] DPL: fix reading of metadata

Previously, reading metadata from alien:// did not work
when using an @input_list.txt file containting the filename, rather
than specifying alien directly on the command line.
---
 Framework/AnalysisSupport/src/Plugin.cxx | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index 2ec12a92857cf..c69e5c8c56bdd 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -75,11 +75,6 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
         if (filename.empty()) {
           return {};
         }
-        if (filename.rfind("alien://", 0) == 0) {
-          LOGP(debug, "AliEn file requested. Enabling support.");
-          TGrid::Connect("alien://");
-        }
-        LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
         std::vector<ConfigParamSpec> results;
         TFile* currentFile = nullptr;
         if (filename.at(0) == '@') {
@@ -91,10 +86,12 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
           }
           std::getline(file, filename);
           file.close();
-          currentFile = TFile::Open(filename.c_str());
-        } else {
-          currentFile = TFile::Open(filename.c_str());
         }
+        if (filename.rfind("alien://", 0) == 0) {
+          TGrid::Connect("alien://");
+        }
+        LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
+        currentFile = TFile::Open(filename.c_str());
         if (!currentFile) {
           LOGP(fatal, "Couldn't open file \"{}\"!", filename);
         }

From 37a0915399559d2244e9d2d78c60d9817b27627f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 23 May 2024 10:08:46 +0200
Subject: [PATCH 516/726] DPL: avoid memory churn when pushing variable state

---
 Framework/Core/src/DataRelayer.cxx | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index bd0fba7b35e3e..703043477639e 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -260,7 +260,8 @@ void sendVariableContextMetrics(VariableContext& context, TimesliceSlot slot, Da
 
   context.publish([](VariableContext const& variables, TimesliceSlot slot, void* context) {
     auto& states = *static_cast<DataProcessingStates*>(context);
-    std::string state = "";
+    static std::string state = "";
+    state.clear();
     for (size_t i = 0; i < MAX_MATCHING_VARIABLE; ++i) {
       auto var = variables.get(i);
       if (auto pval = std::get_if<uint64_t>(&var)) {

From 089fe67ca3276ab97a89fcb5f0e3373cb7f16eb7 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 24 May 2024 11:34:43 +0200
Subject: [PATCH 517/726] DPL: adapt to support new FairLogger

---
 Framework/DataTakingSupport/src/Plugin.cxx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Framework/DataTakingSupport/src/Plugin.cxx b/Framework/DataTakingSupport/src/Plugin.cxx
index e414b1e49b57f..f719fd5ef7d1f 100644
--- a/Framework/DataTakingSupport/src/Plugin.cxx
+++ b/Framework/DataTakingSupport/src/Plugin.cxx
@@ -117,8 +117,8 @@ auto createInfoLoggerSinkHelper(InfoLogger* logger, InfoLoggerContext* ctx)
       severity,
       level,
       InfoLogger::undefinedMessageOption.errorCode,
-      metadata.file.c_str(),
-      atoi(metadata.line.c_str())};
+      metadata.file.data(),
+      atoi(metadata.line.data())};
 
     if (logger) {
       logger->log(opt, *ctx, "%s", content.c_str());

From eaa94b4bb5db169a31afd2cf8da8b7d410c9bc5e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 24 May 2024 21:55:35 +0200
Subject: [PATCH 518/726] DPL: improve signposts for adding parts

---
 Framework/Core/src/DataAllocator.cxx | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index fe38283d5e2de..0bf0f86f812ec 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -21,6 +21,8 @@
 #include "Framework/DeviceSpec.h"
 #include "Framework/StreamContext.h"
 #include "Framework/Signpost.h"
+#include "Headers/DataHeader.h"
+#include "Headers/DataHeaderHelpers.h"
 #include "Headers/Stack.h"
 
 #include <fairmq/Device.h>
@@ -131,13 +133,13 @@ void DataAllocator::addPartToContext(RouteIndex routeIndex, fair::mq::MessagePtr
 {
   auto headerMessage = headerMessageFromOutput(spec, routeIndex, serializationMethod, 0);
   O2_SIGNPOST_ID_FROM_POINTER(pid, parts, headerMessage->GetData());
-  O2_SIGNPOST_START(parts, pid, "parts", "addPartToContext %p", headerMessage->GetData());
-
   // FIXME: this is kind of ugly, we know that we can change the content of the
   // header message because we have just created it, but the API declares it const
   const DataHeader* cdh = o2::header::get<DataHeader*>(headerMessage->GetData());
   auto* dh = const_cast<DataHeader*>(cdh);
   dh->payloadSize = payloadMessage->GetSize();
+  O2_SIGNPOST_START(parts, pid, "parts", "addPartToContext %{public}s@%p %" PRIu64,
+                    cdh ? fmt::format("{}/{}/{}", cdh->dataOrigin, cdh->dataDescription, cdh->subSpecification).c_str() : "unknown", headerMessage->GetData(), dh->payloadSize);
   auto& context = mRegistry.get<MessageContext>();
   // make_scoped creates the context object inside of a scope handler, since it goes out of
   // scope immediately, the created object is scheduled and can be directly sent if the context
@@ -153,8 +155,10 @@ void DataAllocator::adopt(const Output& spec, std::string* ptr)
   // the correct payload size is set later when sending the
   // StringContext, see DataProcessor::doSend
   auto header = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, 0);
+  const DataHeader* cdh = o2::header::get<DataHeader*>(header->GetData());
   O2_SIGNPOST_ID_FROM_POINTER(pid, parts, header->GetData());
-  O2_SIGNPOST_START(parts, pid, "parts", "addPartToContext %p", header->GetData());
+  O2_SIGNPOST_START(parts, pid, "parts", "addPartToContext %{public}s@%p %" PRIu64,
+                    cdh ? fmt::format("{}/{}/{}", cdh->dataOrigin, cdh->dataDescription, cdh->subSpecification).c_str() : "unknown", header->GetData(), cdh->payloadSize);
   mRegistry.get<StringContext>().addString(std::move(header), std::move(payload), routeIndex);
   assert(payload.get() == nullptr);
 }
@@ -211,8 +215,10 @@ void DataAllocator::adopt(const Output& spec, LifetimeHolder<TableBuilder>& tb)
   auto& timingInfo = mRegistry.get<TimingInfo>();
   RouteIndex routeIndex = matchDataHeader(spec, timingInfo.timeslice);
   auto header = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodArrow, 0);
+  const DataHeader* cdh = o2::header::get<DataHeader*>(header->GetData());
   O2_SIGNPOST_ID_FROM_POINTER(pid, parts, header->GetData());
-  O2_SIGNPOST_START(parts, pid, "parts", "adopt %p", header->GetData());
+  O2_SIGNPOST_START(parts, pid, "parts", "adopt %{public}s@%p %" PRIu64,
+                    cdh ? fmt::format("{}/{}/{}", cdh->dataOrigin, cdh->dataDescription, cdh->subSpecification).c_str() : "unknown", header->GetData(), cdh->payloadSize);
   auto& context = mRegistry.get<ArrowContext>();
 
   auto creator = [transport = context.proxy().getOutputTransport(routeIndex)](size_t s) -> std::unique_ptr<fair::mq::Message> {

From e8ef5f1f7f9ad3855e146f28e36dec51ed893788 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 27 May 2024 10:26:00 +0200
Subject: [PATCH 519/726] Drop support for arrow < 11

---
 dependencies/O2Dependencies.cmake | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/dependencies/O2Dependencies.cmake b/dependencies/O2Dependencies.cmake
index bd0085e8c8cfc..53cd7167e0e41 100644
--- a/dependencies/O2Dependencies.cmake
+++ b/dependencies/O2Dependencies.cmake
@@ -30,11 +30,7 @@ include(FeatureSummary)
 include(FindThreads)
 
 find_package(Arrow CONFIG)
-if(${Arrow_VERSION} VERSION_LESS 11)
-find_package(Gandiva CONFIG PATHS ${Arrow_DIR} QUIET)
-else()
 find_package(Gandiva CONFIG)
-endif()
 set_package_properties(Arrow PROPERTIES TYPE REQUIRED)
 set_package_properties(Gandiva PROPERTIES TYPE REQUIRED)
 

From 62cfbbcaa765f3d0e1a8fcbcbed730df2bf4b688 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 28 May 2024 12:34:49 +0200
Subject: [PATCH 520/726] DPL: drop unneeded signpost protection

---
 Framework/Core/src/DataProcessingDevice.cxx | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 6b926d92353be..f256b49bcc3af 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -1828,10 +1828,8 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
       // We only deal with the tracking of parts if the log is enabled.
       // This is because in principle we should track the size of each of
       // the parts and sum it up. Not for now.
-      if (O2_LOG_ENABLED(parts) == true) {
-        O2_SIGNPOST_ID_FROM_POINTER(pid, parts, headerData);
-        O2_SIGNPOST_START(parts, pid, "parts", "Processing DataHeader with splitPayloadParts %d and splitPayloadIndex %d", dh->splitPayloadParts, dh->splitPayloadIndex);
-      }
+      O2_SIGNPOST_ID_FROM_POINTER(pid, parts, headerData);
+      O2_SIGNPOST_START(parts, pid, "parts", "Processing DataHeader with splitPayloadParts %d and splitPayloadIndex %d", dh->splitPayloadParts, dh->splitPayloadIndex);
       if (!dph) {
         insertInputInfo(pi, 2, InputType::Invalid, info.id);
         O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "Header stack does not contain DataProcessingHeader");

From 7ec4e32ea4d0e145e8caf0f92c05aed36521833d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 28 May 2024 12:37:28 +0200
Subject: [PATCH 521/726] Cleanup headers

---
 Detectors/ZDC/calib/src/InterCalibSpec.cxx | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/Detectors/ZDC/calib/src/InterCalibSpec.cxx b/Detectors/ZDC/calib/src/InterCalibSpec.cxx
index 1a22fc1fe17c1..2c2184b6bca75 100644
--- a/Detectors/ZDC/calib/src/InterCalibSpec.cxx
+++ b/Detectors/ZDC/calib/src/InterCalibSpec.cxx
@@ -17,7 +17,6 @@
 #include <vector>
 #include <string>
 #include <filesystem>
-#include "CCDB/BasicCCDBManager.h"
 #include "CCDB/CCDBTimeStampUtils.h"
 #include "CCDB/CcdbApi.h"
 #include "Framework/Logger.h"
@@ -35,7 +34,6 @@
 #include "ZDCBase/ModuleConfig.h"
 #include "CommonUtils/NameConf.h"
 #include "CommonUtils/MemFileHelper.h"
-#include "CCDB/BasicCCDBManager.h"
 #include "CCDB/CCDBTimeStampUtils.h"
 #include "ZDCReconstruction/RecoConfigZDC.h"
 #include "ZDCReconstruction/ZDCEnergyParam.h"
@@ -45,9 +43,7 @@
 
 using namespace o2::framework;
 
-namespace o2
-{
-namespace zdc
+namespace o2::zdc
 {
 
 InterCalibSpec::InterCalibSpec()
@@ -234,5 +230,4 @@ framework::DataProcessorSpec getInterCalibSpec()
     Options{{"verbosity-level", o2::framework::VariantType::Int, 1, {"Verbosity level"}}}};
 }
 
-} // namespace zdc
 } // namespace o2

From 820d55ef698854d0c7ea365c7e395110210983bc Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 28 May 2024 15:06:23 +0200
Subject: [PATCH 522/726] Attempt at debugging framework:crash error

---
 Framework/Core/test/test_AllCrashTypes.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/test/test_AllCrashTypes.sh b/Framework/Core/test/test_AllCrashTypes.sh
index 54898fd9c4c5d..feebc70f8a1cf 100755
--- a/Framework/Core/test/test_AllCrashTypes.sh
+++ b/Framework/Core/test/test_AllCrashTypes.sh
@@ -7,7 +7,7 @@ o2-framework-crashing-workflow --crash-type=framework-init --completion-policy=q
 printf "ok\nTesting framework-run..."
 o2-framework-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
 printf "ok\nTesting runtime-run..."
-o2-framework-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found" ; exit 1; }
+o2-framework-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found"; o2-framework-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run ; exit 1; }
 printf "ok\n"
 
 export O2_NO_CATCHALL_EXCEPTIONS=1

From 0a326546a153279987993e002524d3696d4b1b17 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 29 May 2024 10:07:12 +0200
Subject: [PATCH 523/726] DPL: run test for the ordered completion policy

---
 Framework/Core/CMakeLists.txt | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index 4381b6c4fa29d..312b821b0f6a1 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -266,9 +266,10 @@ o2_add_test(Timers NAME test_Framework_test_Timers
             LABELS framework
             PUBLIC_LINK_LIBRARIES O2::Framework)
 
-# FIXME: make this a proper test, when it actually does not hang.
-o2_add_executable(test-framework-ConsumeWhenAllOrdered
+o2_add_test(ConsumeWhenAllOrdered NAME test_ConsumeWhenAllOrdered
             SOURCES test/test_ConsumeWhenAllOrdered.cxx
+            COMPONENT_NAME Framework
+            LABELS framework
             PUBLIC_LINK_LIBRARIES O2::Framework)
 
 o2_add_test(SuppressionGenerator NAME test_Framework_test_SuppressionGenerator

From e28bbe37a90afec5a4852e208ee9715a86f9219f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 29 May 2024 10:26:49 +0200
Subject: [PATCH 524/726] DPL: introduce helper functions to detect only timers
 / generate

---
 Framework/Core/src/DataProcessingDevice.cxx | 31 +++++++++++----------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index f256b49bcc3af..2aa239b974136 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -117,6 +117,16 @@ void on_idle_timer(uv_timer_t* handle)
   state->loopReason |= DeviceState::TIMER_EXPIRED;
 }
 
+bool hasOnlyTimers(DeviceSpec const& spec)
+{
+  return std::all_of(spec.inputs.cbegin(), spec.inputs.cend(), [](InputRoute const& route) -> bool { return route.matcher.lifetime == Lifetime::Timer; });
+}
+
+bool hasOnlyGenerated(DeviceSpec const& spec)
+{
+  return (spec.inputChannels.size() == 1) && (spec.inputs[0].matcher.lifetime == Lifetime::Timer || spec.inputs[0].matcher.lifetime == Lifetime::Enumeration);
+}
+
 void on_transition_requested_expired(uv_timer_t* handle)
 {
   auto* state = (DeviceState*)handle->data;
@@ -1281,15 +1291,8 @@ void DataProcessingDevice::Run()
         auto& deviceContext = ref.get<DeviceContext>();
         auto timeout = deviceContext.exitTransitionTimeout;
         // Check if we only have timers
-        bool onlyTimers = true;
         auto& spec = ref.get<DeviceSpec const>();
-        for (auto& route : spec.inputs) {
-          if (route.matcher.lifetime != Lifetime::Timer) {
-            onlyTimers = false;
-            break;
-          }
-        }
-        if (onlyTimers) {
+        if (hasOnlyTimers(spec)) {
           state.streaming = StreamingState::EndOfStreaming;
         }
         if (timeout != 0 && state.streaming != StreamingState::Idle) {
@@ -1689,9 +1692,11 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
     // I guess we will see.
     /// Besides flushing the queues we must make sure we do not have only
     /// timers as they do not need to be further processed.
-    bool hasOnlyGenerated = (spec.inputChannels.size() == 1) && (spec.inputs[0].matcher.lifetime == Lifetime::Timer || spec.inputs[0].matcher.lifetime == Lifetime::Enumeration);
     auto& relayer = ref.get<DataRelayer>();
-    while (DataProcessingDevice::tryDispatchComputation(ref, context.completed) && hasOnlyGenerated == false) {
+
+    bool shouldProcess = hasOnlyGenerated(spec) == false;
+
+    while (DataProcessingDevice::tryDispatchComputation(ref, context.completed) && shouldProcess) {
       relayer.processDanglingInputs(context.expirationHandlers, *context.registry, false);
     }
     EndOfStreamContext eosContext{*context.registry, ref.get<DataAllocator>()};
@@ -1710,11 +1715,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
     // This is needed because the transport is deleted before the device.
     relayer.clear();
     switchState(StreamingState::Idle);
-    if (hasOnlyGenerated) {
-      *context.wasActive = false;
-    } else {
-      *context.wasActive = true;
-    }
+    *context.wasActive = shouldProcess;
     // On end of stream we shut down all output pollers.
     O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "Shutting down output pollers.");
     for (auto& poller : state.activeOutputPollers) {

From a61e5fcbb2d3e967ddd775f46a80f6855fc10de0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 30 May 2024 11:45:36 +0200
Subject: [PATCH 525/726] DPL: add signposts for forwarding

---
 Framework/Core/src/DataProcessingDevice.cxx | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 2aa239b974136..15520206c350f 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -94,6 +94,8 @@ O2_DECLARE_DYNAMIC_LOG(device);
 O2_DECLARE_DYNAMIC_LOG(parts);
 // Special log to track the async queue behavior
 O2_DECLARE_DYNAMIC_LOG(async_queue);
+// Special log to track the forwarding requests
+O2_DECLARE_DYNAMIC_LOG(forwarding);
 
 using namespace o2::framework;
 using ConfigurationInterface = o2::configuration::ConfigurationInterface;
@@ -636,6 +638,9 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
   std::vector<fair::mq::Parts> forwardedParts;
   forwardedParts.resize(proxy.getNumForwards());
   std::vector<ChannelIndex> cachedForwardingChoices{};
+  O2_SIGNPOST_ID_GENERATE(sid, forwarding);
+  O2_SIGNPOST_START(forwarding, sid, "forwardInputs", "Starting forwarding for slot %zu with oldestTimeslice %zu %{public}s%{public}s%{public}s",
+                    slot.index, oldestTimeslice.timeslice.value, copy ? "with copy" : "", copy && consume ? " and " : "", consume ? "with consume" : "");
 
   for (size_t ii = 0, ie = currentSetOfInputs.size(); ii < ie; ++ii) {
     auto& messageSet = currentSetOfInputs[ii];
@@ -663,9 +668,13 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
       if (cachedForwardingChoices.size() > 1) {
         copy = true;
       }
+      auto* dh = o2::header::get<DataHeader*>(header->GetData());
+
       if (copy) {
         for (auto& cachedForwardingChoice : cachedForwardingChoices) {
           auto&& newHeader = header->GetTransport()->CreateMessage();
+          O2_SIGNPOST_EVENT_EMIT(forwarding, sid, "forwardInputs", "Forwarding a copy of %{public}s to route %d.",
+                                 fmt::format(fmt::runtime("{x}"), *dh).c_str(), cachedForwardingChoice.value);
           newHeader->Copy(*header);
           forwardedParts[cachedForwardingChoice.value].AddPart(std::move(newHeader));
 
@@ -676,6 +685,8 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
           }
         }
       } else {
+        O2_SIGNPOST_EVENT_EMIT(forwarding, sid, "forwardInputs", "Forwarding %{public}s to route %d.",
+                               fmt::format(fmt::runtime("{x}"), *dh).c_str(), cachedForwardingChoices.back().value);
         forwardedParts[cachedForwardingChoices.back().value].AddPart(std::move(messageSet.header(pi)));
         for (size_t payloadIndex = 0; payloadIndex < messageSet.getNumberOfPayloads(pi); ++payloadIndex) {
           forwardedParts[cachedForwardingChoices.back().value].AddPart(std::move(messageSet.payload(pi, payloadIndex)));
@@ -683,19 +694,18 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
       }
     }
   }
-  LOG(debug) << "Forwarding " << forwardedParts.size() << " messages";
+  O2_SIGNPOST_EVENT_EMIT(forwarding, sid, "forwardInputs", "Forwarding %zu messages", forwardedParts.size());
   for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
     if (forwardedParts[fi].Size() == 0) {
       continue;
     }
     ForwardChannelInfo info = proxy.getForwardChannelInfo(ChannelIndex{fi});
-    LOG(debug) << "Forwarding to " << info.name << " " << fi;
-    // in DPL we are using subchannel 0 only
     auto& parts = forwardedParts[fi];
     if (info.policy == nullptr) {
-      LOG(error) << "Forwarding to " << info.name << " " << fi << " has no policy";
+      O2_SIGNPOST_EVENT_EMIT_ERROR(forwarding, sid, "forwardInputs", "Forwarding to %{public}s %d has no policy.", info.name.c_str(), fi);
       continue;
     }
+    O2_SIGNPOST_EVENT_EMIT(forwarding, sid, "forwardInputs", "Forwarding to %{public}s %d", info.name.c_str(), fi);
     info.policy->forward(parts, ChannelIndex{fi}, registry);
   }
 
@@ -729,7 +739,7 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
       }
     },
     oldestTimeslice.timeslice, -1);
-  LOG(debug) << "Forwarding done";
+  O2_SIGNPOST_END(forwarding, sid, "forwardInputs", "Forwarding done");
 };
 extern volatile int region_read_global_dummy_variable;
 volatile int region_read_global_dummy_variable;

From b0f1cb9d70b9a5edf64a5803b3e94b0d9cd79d44 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 30 May 2024 12:55:08 +0200
Subject: [PATCH 526/726] DPL: add signposts for forwarding

---
 Framework/Core/src/DataProcessingDevice.cxx | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 2aa239b974136..91b2451b1c21a 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -94,6 +94,8 @@ O2_DECLARE_DYNAMIC_LOG(device);
 O2_DECLARE_DYNAMIC_LOG(parts);
 // Special log to track the async queue behavior
 O2_DECLARE_DYNAMIC_LOG(async_queue);
+// Special log to track the forwarding requests
+O2_DECLARE_DYNAMIC_LOG(forwarding);
 
 using namespace o2::framework;
 using ConfigurationInterface = o2::configuration::ConfigurationInterface;
@@ -636,6 +638,9 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
   std::vector<fair::mq::Parts> forwardedParts;
   forwardedParts.resize(proxy.getNumForwards());
   std::vector<ChannelIndex> cachedForwardingChoices{};
+  O2_SIGNPOST_ID_GENERATE(sid, forwarding);
+  O2_SIGNPOST_START(forwarding, sid, "forwardInputs", "Starting forwarding for slot %zu with oldestTimeslice %zu %{public}s%{public}s%{public}s",
+                    slot.index, oldestTimeslice.timeslice.value, copy ? "with copy" : "", copy && consume ? " and " : "", consume ? "with consume" : "");
 
   for (size_t ii = 0, ie = currentSetOfInputs.size(); ii < ie; ++ii) {
     auto& messageSet = currentSetOfInputs[ii];
@@ -663,9 +668,14 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
       if (cachedForwardingChoices.size() > 1) {
         copy = true;
       }
+      auto* dh = o2::header::get<DataHeader*>(header->GetData());
+      auto* dph = o2::header::get<DataProcessingHeader*>(header->GetData());
+
       if (copy) {
         for (auto& cachedForwardingChoice : cachedForwardingChoices) {
           auto&& newHeader = header->GetTransport()->CreateMessage();
+          O2_SIGNPOST_EVENT_EMIT(forwarding, sid, "forwardInputs", "Forwarding a copy of %{public}s to route %d.",
+                                 fmt::format(fmt::runtime("{x}@timeslice:{} tfCounter:{}"), *dh, dph->startTime, dh->tfCounter).c_str(), cachedForwardingChoice.value);
           newHeader->Copy(*header);
           forwardedParts[cachedForwardingChoice.value].AddPart(std::move(newHeader));
 
@@ -676,6 +686,8 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
           }
         }
       } else {
+        O2_SIGNPOST_EVENT_EMIT(forwarding, sid, "forwardInputs", "Forwarding %{public}s to route %d.",
+                               fmt::format(fmt::runtime("{x}@timeslice:{} tfCounter:{}"), *dh, dph->startTime, dh->tfCounter).c_str(), cachedForwardingChoices.back().value);
         forwardedParts[cachedForwardingChoices.back().value].AddPart(std::move(messageSet.header(pi)));
         for (size_t payloadIndex = 0; payloadIndex < messageSet.getNumberOfPayloads(pi); ++payloadIndex) {
           forwardedParts[cachedForwardingChoices.back().value].AddPart(std::move(messageSet.payload(pi, payloadIndex)));
@@ -683,19 +695,18 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
       }
     }
   }
-  LOG(debug) << "Forwarding " << forwardedParts.size() << " messages";
+  O2_SIGNPOST_EVENT_EMIT(forwarding, sid, "forwardInputs", "Forwarding %zu messages", forwardedParts.size());
   for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
     if (forwardedParts[fi].Size() == 0) {
       continue;
     }
     ForwardChannelInfo info = proxy.getForwardChannelInfo(ChannelIndex{fi});
-    LOG(debug) << "Forwarding to " << info.name << " " << fi;
-    // in DPL we are using subchannel 0 only
     auto& parts = forwardedParts[fi];
     if (info.policy == nullptr) {
-      LOG(error) << "Forwarding to " << info.name << " " << fi << " has no policy";
+      O2_SIGNPOST_EVENT_EMIT_ERROR(forwarding, sid, "forwardInputs", "Forwarding to %{public}s %d has no policy.", info.name.c_str(), fi);
       continue;
     }
+    O2_SIGNPOST_EVENT_EMIT(forwarding, sid, "forwardInputs", "Forwarding to %{public}s %d", info.name.c_str(), fi);
     info.policy->forward(parts, ChannelIndex{fi}, registry);
   }
 
@@ -729,7 +740,7 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
       }
     },
     oldestTimeslice.timeslice, -1);
-  LOG(debug) << "Forwarding done";
+  O2_SIGNPOST_END(forwarding, sid, "forwardInputs", "Forwarding done");
 };
 extern volatile int region_read_global_dummy_variable;
 volatile int region_read_global_dummy_variable;

From 7c45207bbcd3d76f3e84be1ff0360535c7a8ee4f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 30 May 2024 13:09:18 +0200
Subject: [PATCH 527/726] DPL: add signposts for forwarding

---
 Framework/Core/src/DataProcessingDevice.cxx | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 2aa239b974136..93940ef5ec21a 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -94,6 +94,8 @@ O2_DECLARE_DYNAMIC_LOG(device);
 O2_DECLARE_DYNAMIC_LOG(parts);
 // Special log to track the async queue behavior
 O2_DECLARE_DYNAMIC_LOG(async_queue);
+// Special log to track the forwarding requests
+O2_DECLARE_DYNAMIC_LOG(forwarding);
 
 using namespace o2::framework;
 using ConfigurationInterface = o2::configuration::ConfigurationInterface;
@@ -636,6 +638,9 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
   std::vector<fair::mq::Parts> forwardedParts;
   forwardedParts.resize(proxy.getNumForwards());
   std::vector<ChannelIndex> cachedForwardingChoices{};
+  O2_SIGNPOST_ID_GENERATE(sid, forwarding);
+  O2_SIGNPOST_START(forwarding, sid, "forwardInputs", "Starting forwarding for slot %zu with oldestTimeslice %zu %{public}s%{public}s%{public}s",
+                    slot.index, oldestTimeslice.timeslice.value, copy ? "with copy" : "", copy && consume ? " and " : "", consume ? "with consume" : "");
 
   for (size_t ii = 0, ie = currentSetOfInputs.size(); ii < ie; ++ii) {
     auto& messageSet = currentSetOfInputs[ii];
@@ -663,9 +668,14 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
       if (cachedForwardingChoices.size() > 1) {
         copy = true;
       }
+      auto* dh = o2::header::get<DataHeader*>(header->GetData());
+      auto* dph = o2::header::get<DataProcessingHeader*>(header->GetData());
+
       if (copy) {
         for (auto& cachedForwardingChoice : cachedForwardingChoices) {
           auto&& newHeader = header->GetTransport()->CreateMessage();
+          O2_SIGNPOST_EVENT_EMIT(forwarding, sid, "forwardInputs", "Forwarding a copy of %{public}s to route %d.",
+                                 fmt::format("{}/{}/{}@timeslice:{} tfCounter:{}", dh->dataOrigin, dh->dataDescription, dh->subSpecification, dph->startTime, dh->tfCounter).c_str(), cachedForwardingChoice.value);
           newHeader->Copy(*header);
           forwardedParts[cachedForwardingChoice.value].AddPart(std::move(newHeader));
 
@@ -676,6 +686,8 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
           }
         }
       } else {
+        O2_SIGNPOST_EVENT_EMIT(forwarding, sid, "forwardInputs", "Forwarding %{public}s to route %d.",
+                               fmt::format("{}/{}/{}@timeslice:{} tfCounter:{}", dh->dataOrigin, dh->dataDescription, dh->subSpecification, dph->startTime, dh->tfCounter).c_str(), cachedForwardingChoices.back().value);
         forwardedParts[cachedForwardingChoices.back().value].AddPart(std::move(messageSet.header(pi)));
         for (size_t payloadIndex = 0; payloadIndex < messageSet.getNumberOfPayloads(pi); ++payloadIndex) {
           forwardedParts[cachedForwardingChoices.back().value].AddPart(std::move(messageSet.payload(pi, payloadIndex)));
@@ -683,19 +695,18 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
       }
     }
   }
-  LOG(debug) << "Forwarding " << forwardedParts.size() << " messages";
+  O2_SIGNPOST_EVENT_EMIT(forwarding, sid, "forwardInputs", "Forwarding %zu messages", forwardedParts.size());
   for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
     if (forwardedParts[fi].Size() == 0) {
       continue;
     }
     ForwardChannelInfo info = proxy.getForwardChannelInfo(ChannelIndex{fi});
-    LOG(debug) << "Forwarding to " << info.name << " " << fi;
-    // in DPL we are using subchannel 0 only
     auto& parts = forwardedParts[fi];
     if (info.policy == nullptr) {
-      LOG(error) << "Forwarding to " << info.name << " " << fi << " has no policy";
+      O2_SIGNPOST_EVENT_EMIT_ERROR(forwarding, sid, "forwardInputs", "Forwarding to %{public}s %d has no policy.", info.name.c_str(), fi);
       continue;
     }
+    O2_SIGNPOST_EVENT_EMIT(forwarding, sid, "forwardInputs", "Forwarding to %{public}s %d", info.name.c_str(), fi);
     info.policy->forward(parts, ChannelIndex{fi}, registry);
   }
 
@@ -729,7 +740,7 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
       }
     },
     oldestTimeslice.timeslice, -1);
-  LOG(debug) << "Forwarding done";
+  O2_SIGNPOST_END(forwarding, sid, "forwardInputs", "Forwarding done");
 };
 extern volatile int region_read_global_dummy_variable;
 volatile int region_read_global_dummy_variable;

From 2b9ff9ff7456518b687bcdc5c12dec15c01972d0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 1 Jun 2024 09:12:52 +0200
Subject: [PATCH 528/726] DPL: improve message when signpost is enabled

---
 Framework/Core/src/runDataProcessing.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 930132e972881..137f17152cc01 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -3068,7 +3068,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
       auto* selectedName = (char const*)context;
       std::string prefix = "ch.cern.aliceo2.";
       if (strcmp(name, (prefix + selectedName).data()) == 0) {
-        LOGP(info, "Enabling signposts for {}", *selectedName);
+        LOGP(info, "Enabling signposts for stream \"ch.cern.aliceo2.{}\"", selectedName);
         _o2_log_set_stacktrace(log, 1);
         return false;
       } else {

From 11e75e3f3c260e0da6c26240c83c482b863f990b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 3 Jun 2024 10:10:21 +0200
Subject: [PATCH 529/726] DPL Analysis: disable metadata lookup if not found
 immediately

---
 Framework/AnalysisSupport/src/Plugin.cxx | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index c69e5c8c56bdd..e5197e6977835 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -99,7 +99,8 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
         // Get the metadata, if any
         auto m = (TMap*)currentFile->Get("metaData");
         if (!m) {
-          LOGP(warning, "No metadata found in file \"{}\"", filename);
+          LOGP(info, "No metadata found in file \"{}\"", filename);
+          results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
           return results;
         }
         auto it = m->MakeIterator();

From aa9b0fb67edb5cedc47c0f9fecee0105b2d1bfc7 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 5 Jun 2024 22:15:05 +0200
Subject: [PATCH 530/726] DPL: do not print internal error in case we rewinded

When rewinding the timeframe because no data was actually sent,
we expect that the oldest possible timeframe can go back and the
message is therefore wrong.
---
 Framework/Core/include/Framework/TimesliceIndex.h |  3 +--
 Framework/Core/src/CommonServices.cxx             |  4 ++--
 Framework/Core/src/DataRelayer.cxx                |  2 +-
 Framework/Core/src/LifetimeHelpers.cxx            |  9 ++++-----
 Framework/Core/src/TimesliceIndex.cxx             |  4 ++--
 Framework/Core/test/test_TimesliceIndex.cxx       | 10 +++++-----
 6 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/Framework/Core/include/Framework/TimesliceIndex.h b/Framework/Core/include/Framework/TimesliceIndex.h
index c3f3c82bdeb6e..ac3970bec00ee 100644
--- a/Framework/Core/include/Framework/TimesliceIndex.h
+++ b/Framework/Core/include/Framework/TimesliceIndex.h
@@ -26,7 +26,6 @@
 namespace o2::framework
 {
 
-
 /// This class keeps the information relative to a given slot in the cache, in
 /// particular which variables are associated to it (and indirectly which
 /// timeslice which is always mapped to the variable 0) and wether we should
@@ -130,7 +129,7 @@ class TimesliceIndex
   /// and the oldest possible timeslice in-fly which is still dirty.
   [[nodiscard]] OldestInputInfo getOldestPossibleInput() const;
   [[nodiscard]] OldestOutputInfo getOldestPossibleOutput() const;
-  OldestOutputInfo updateOldestPossibleOutput();
+  OldestOutputInfo updateOldestPossibleOutput(bool rewinded);
   [[nodiscard]] InputChannelInfo const& getChannelInfo(ChannelIndex channel) const;
 
   // Reset the TimesliceIndex to its initial state
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 57621ecaa8f33..7b3febca7be09 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -560,7 +560,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
       O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "postForwardingCallbacks", "We are the first one in the topology, we need to update the oldest possible timeslice");
       auto& timesliceIndex = ctx.services().get<TimesliceIndex>();
       auto& relayer = ctx.services().get<DataRelayer>();
-      timesliceIndex.updateOldestPossibleOutput();
+      timesliceIndex.updateOldestPossibleOutput(decongestion->nextEnumerationTimesliceRewinded);
       auto& proxy = ctx.services().get<FairMQDeviceProxy>();
       auto oldestPossibleOutput = relayer.getOldestPossibleOutput();
       if (decongestion->nextEnumerationTimesliceRewinded && decongestion->nextEnumerationTimeslice < oldestPossibleOutput.timeslice.value) {
@@ -632,7 +632,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
       O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Received oldest possible timeframe %" PRIu64 " from channel %d",
                              (uint64_t)oldestPossibleTimeslice, channel.value);
       relayer.setOldestPossibleInput({oldestPossibleTimeslice}, channel);
-      timesliceIndex.updateOldestPossibleOutput();
+      timesliceIndex.updateOldestPossibleOutput(decongestion.nextEnumerationTimesliceRewinded);
       auto oldestPossibleOutput = relayer.getOldestPossibleOutput();
 
       if (oldestPossibleOutput.timeslice.value == decongestion.lastTimeslice) {
diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index e408632723741..b974a51fe82e1 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -759,7 +759,7 @@ void DataRelayer::getReadyToProcess(std::vector<DataRelayer::RecordAction>& comp
         break;
     }
   }
-  mTimesliceIndex.updateOldestPossibleOutput();
+  mTimesliceIndex.updateOldestPossibleOutput(false);
   LOGP(debug, "DataRelayer::getReadyToProcess results notDirty:{}, consume:{}, consumeExisting:{}, process:{}, discard:{}, wait:{}",
        notDirty, countConsume, countConsumeExisting, countProcess,
        countDiscard, countWait);
diff --git a/Framework/Core/src/LifetimeHelpers.cxx b/Framework/Core/src/LifetimeHelpers.cxx
index 8c8605d41849c..657bdcb40326b 100644
--- a/Framework/Core/src/LifetimeHelpers.cxx
+++ b/Framework/Core/src/LifetimeHelpers.cxx
@@ -98,7 +98,7 @@ ExpirationHandler::Creator LifetimeHelpers::enumDrivenCreation(size_t start, siz
         // associated with this.
         LOG(debug) << "Oldest possible input is " << decongestion.nextEnumerationTimeslice;
         [[maybe_unused]] auto newOldest = index.setOldestPossibleInput({decongestion.nextEnumerationTimeslice}, channelIndex);
-        index.updateOldestPossibleOutput();
+        index.updateOldestPossibleOutput(decongestion.nextEnumerationTimesliceRewinded);
         return slot;
       }
     }
@@ -143,7 +143,7 @@ ExpirationHandler::Creator LifetimeHelpers::timeDrivenCreation(std::vector<std::
     // Nothing to do if the time has not expired yet.
     if (timerHasFired == false) {
       [[maybe_unused]] auto newOldest = index.setOldestPossibleInput({decongestion.nextEnumerationTimeslice}, channelIndex);
-      index.updateOldestPossibleOutput();
+      index.updateOldestPossibleOutput(decongestion.nextEnumerationTimesliceRewinded);
       return TimesliceSlot{TimesliceSlot::INVALID};
     }
     // Get the first time we were invoked.
@@ -168,7 +168,7 @@ ExpirationHandler::Creator LifetimeHelpers::timeDrivenCreation(std::vector<std::
       auto& variables = index.getVariablesForSlot(slot);
       if (VariableContextHelpers::getTimeslice(variables).value == current) {
         [[maybe_unused]] auto newOldest = index.setOldestPossibleInput({decongestion.nextEnumerationTimeslice}, channelIndex);
-        index.updateOldestPossibleOutput();
+        index.updateOldestPossibleOutput(decongestion.nextEnumerationTimesliceRewinded);
         return TimesliceSlot{TimesliceSlot::INVALID};
       }
     }
@@ -192,7 +192,7 @@ ExpirationHandler::Creator LifetimeHelpers::timeDrivenCreation(std::vector<std::
     }
 
     auto newOldest = index.setOldestPossibleInput({decongestion.nextEnumerationTimeslice}, channelIndex);
-    index.updateOldestPossibleOutput();
+    index.updateOldestPossibleOutput(decongestion.nextEnumerationTimesliceRewinded);
     return slot;
   };
 }
@@ -442,7 +442,6 @@ ExpirationHandler::Handler LifetimeHelpers::enumerate(ConcreteDataMatcher const&
     *(counter_t*)payload->GetData() = *counter;
     ref.payload = std::move(payload);
     (*counter)++;
-
   };
 }
 
diff --git a/Framework/Core/src/TimesliceIndex.cxx b/Framework/Core/src/TimesliceIndex.cxx
index 5707a66bac4e4..df98eea3cc9d0 100644
--- a/Framework/Core/src/TimesliceIndex.cxx
+++ b/Framework/Core/src/TimesliceIndex.cxx
@@ -192,7 +192,7 @@ bool TimesliceIndex::validateSlot(TimesliceSlot slot, TimesliceId currentOldest)
   return true;
 }
 
-TimesliceIndex::OldestOutputInfo TimesliceIndex::updateOldestPossibleOutput()
+TimesliceIndex::OldestOutputInfo TimesliceIndex::updateOldestPossibleOutput(bool rewinded)
 {
   auto oldestInput = getOldestPossibleInput();
   OldestOutputInfo result{oldestInput.timeslice, oldestInput.channel};
@@ -223,7 +223,7 @@ TimesliceIndex::OldestOutputInfo TimesliceIndex::updateOldestPossibleOutput()
                              mOldestPossibleOutput.timeslice.value, result.timeslice.value);
     }
   }
-  if (mOldestPossibleOutput.timeslice.value > result.timeslice.value) {
+  if (rewinded == false && mOldestPossibleOutput.timeslice.value > result.timeslice.value) {
     LOG(error) << "DPL internal error - oldestPossibleOutput decreased from " << mOldestPossibleOutput.timeslice.value << " to " << result.timeslice.value;
   }
   mOldestPossibleOutput = result;
diff --git a/Framework/Core/test/test_TimesliceIndex.cxx b/Framework/Core/test/test_TimesliceIndex.cxx
index bafba24d17a8c..3a63a9f518e58 100644
--- a/Framework/Core/test/test_TimesliceIndex.cxx
+++ b/Framework/Core/test/test_TimesliceIndex.cxx
@@ -119,7 +119,7 @@ TEST_CASE("TestOldestPossibleTimeslice")
       bool invalidated = index.validateSlot(TimesliceSlot{i}, oldest.timeslice);
       INFO("Slot " << i << " valid: " << invalidated);
     }
-    index.updateOldestPossibleOutput();
+    index.updateOldestPossibleOutput(false);
     REQUIRE(slot.index == 1);
     REQUIRE(action == TimesliceIndex::ActionTaken::ReplaceUnused);
   }
@@ -142,21 +142,21 @@ TEST_CASE("TestOldestPossibleTimeslice")
   for (size_t i = 0; i < 3; ++i) {
     bool invalidated = index.validateSlot(TimesliceSlot{i}, oldest.timeslice);
   }
-  index.updateOldestPossibleOutput();
+  index.updateOldestPossibleOutput(false);
   REQUIRE(index.getOldestPossibleInput().timeslice.value == 10);
   REQUIRE(index.getOldestPossibleOutput().timeslice.value == 9);
   oldest = index.setOldestPossibleInput({11}, {1});
   for (size_t i = 0; i < 3; ++i) {
     bool invalidated = index.validateSlot(TimesliceSlot{i}, oldest.timeslice);
   }
-  index.updateOldestPossibleOutput();
+  index.updateOldestPossibleOutput(false);
   REQUIRE(index.getOldestPossibleInput().timeslice.value == 10);
   REQUIRE(index.getOldestPossibleOutput().timeslice.value == 9);
   // We fake the fact that we have processed the slot 0;
   index.markAsDirty({1}, false);
-  index.updateOldestPossibleOutput();
+  index.updateOldestPossibleOutput(false);
   REQUIRE(index.getOldestPossibleOutput().timeslice.value == 9);
   index.markAsInvalid({1});
-  index.updateOldestPossibleOutput();
+  index.updateOldestPossibleOutput(false);
   REQUIRE(index.getOldestPossibleOutput().timeslice.value == 10);
 }

From 9eda0bb950816876562fde18c83dd34aff372e7b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 10 Jun 2024 22:19:24 +0200
Subject: [PATCH 531/726] DPL Analysis: provide dpl-metadata-tables

Comma separated list of tables in the first AOD to be read.
---
 Framework/AnalysisSupport/src/Plugin.cxx | 31 ++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index e5197e6977835..0f2de76d8b54d 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -65,6 +65,25 @@ struct RunSummary : o2::framework::ServicePlugin {
   }
 };
 
+std::vector<std::string> getListOfTables(TFile* f)
+{
+  std::vector<std::string> r;
+  TList* keyList = f->GetListOfKeys();
+
+  for (auto key : *keyList) {
+    if (!std::string_view(key->GetName()).starts_with("DF_")) {
+      continue;
+    }
+    auto* d = (TDirectory*)f->Get(key->GetName());
+    TList* branchList = d->GetListOfKeys();
+    for (auto b : *branchList) {
+      r.emplace_back(b->GetName());
+    }
+    break;
+  }
+  return r;
+}
+
 struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
   ConfigDiscovery* create() override
   {
@@ -118,6 +137,18 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
           char const* value = strdup(objString->String());
           results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
         }
+
+        auto tables = getListOfTables(currentFile);
+        if (tables.empty() == false) {
+          std::string tableString = "";
+          for (auto& table : tables) {
+            if (!tableString.empty()) {
+              tableString += ",";
+            }
+            tableString += table;
+          }
+          results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::String, tableString, {"Tables in first AOD"}});
+        }
         return results;
       }};
   }

From 3f66661157b704a4c0ab0fdc7bdf1a0805e88f0e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 12 Jun 2024 08:43:39 +0200
Subject: [PATCH 532/726] DPL Analysis: provide dpl-metadata-tables

Comma separated list of tables in the first AOD to be read.
---
 Framework/AnalysisSupport/src/Plugin.cxx      | 24 +++++++++++++++++++
 .../TestWorkflows/src/o2TestHistograms.cxx    | 13 +++++++++-
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index e5197e6977835..baa1b36dd41a5 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -65,6 +65,25 @@ struct RunSummary : o2::framework::ServicePlugin {
   }
 };
 
+std::vector<std::string> getListOfTables(TFile* f)
+{
+  std::vector<std::string> r;
+  TList* keyList = f->GetListOfKeys();
+
+  for (auto key : *keyList) {
+    if (!std::string_view(key->GetName()).starts_with("DF_")) {
+      continue;
+    }
+    auto* d = (TDirectory*)f->Get(key->GetName());
+    TList* branchList = d->GetListOfKeys();
+    for (auto b : *branchList) {
+      r.emplace_back(b->GetName());
+    }
+    break;
+  }
+  return r;
+}
+
 struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
   ConfigDiscovery* create() override
   {
@@ -118,6 +137,11 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
           char const* value = strdup(objString->String());
           results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
         }
+
+        auto tables = getListOfTables(currentFile);
+        if (tables.empty() == false) {
+          results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
+        }
         return results;
       }};
   }
diff --git a/Framework/TestWorkflows/src/o2TestHistograms.cxx b/Framework/TestWorkflows/src/o2TestHistograms.cxx
index 4dbea06d03679..15e7e5235837c 100644
--- a/Framework/TestWorkflows/src/o2TestHistograms.cxx
+++ b/Framework/TestWorkflows/src/o2TestHistograms.cxx
@@ -49,12 +49,23 @@ struct EtaAndClsHistogramsFull {
 WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
 {
   std::string runType = "3";
+  std::vector<std::string> tables;
   if (cfgc.options().hasOption("aod-metadata-Run")) {
     runType = cfgc.options().get<std::string>("aod-metadata-Run");
   }
+  if (cfgc.options().hasOption("aod-metadata-tables")) {
+    tables = cfgc.options().get<std::vector<std::string>>("aod-metadata-tables");
+  }
   LOGP(info, "Runtype is {}", runType);
+  bool hasTrackCov = false;
+  for (auto& table : tables) {
+    if (table.starts_with("O2trackcov")) {
+      hasTrackCov = true;
+    }
+    LOGP(info, "- {} present.", table);
+  }
   // Notice it's important for the tasks to use the same name, otherwise topology generation will be confused.
-  if (runType == "2") {
+  if (runType == "2" || !hasTrackCov) {
     LOGP(info, "Using only tracks {}", runType);
     return WorkflowSpec{
       adaptAnalysisTask<EtaAndClsHistogramsSimple>(cfgc, TaskName{"simple-histos"}),

From 6fae49ac1a6505bfac0410c75109641802dc71b0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 12 Jun 2024 08:44:08 +0200
Subject: [PATCH 533/726] DPL Analysis: provide dpl-metadata-tables

ArrayString of tables in the first AOD to be read.
---
 Framework/AnalysisSupport/src/Plugin.cxx      | 24 +++++++++++++++++++
 .../TestWorkflows/src/o2TestHistograms.cxx    | 13 +++++++++-
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index e5197e6977835..baa1b36dd41a5 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -65,6 +65,25 @@ struct RunSummary : o2::framework::ServicePlugin {
   }
 };
 
+std::vector<std::string> getListOfTables(TFile* f)
+{
+  std::vector<std::string> r;
+  TList* keyList = f->GetListOfKeys();
+
+  for (auto key : *keyList) {
+    if (!std::string_view(key->GetName()).starts_with("DF_")) {
+      continue;
+    }
+    auto* d = (TDirectory*)f->Get(key->GetName());
+    TList* branchList = d->GetListOfKeys();
+    for (auto b : *branchList) {
+      r.emplace_back(b->GetName());
+    }
+    break;
+  }
+  return r;
+}
+
 struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
   ConfigDiscovery* create() override
   {
@@ -118,6 +137,11 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
           char const* value = strdup(objString->String());
           results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
         }
+
+        auto tables = getListOfTables(currentFile);
+        if (tables.empty() == false) {
+          results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
+        }
         return results;
       }};
   }
diff --git a/Framework/TestWorkflows/src/o2TestHistograms.cxx b/Framework/TestWorkflows/src/o2TestHistograms.cxx
index 4dbea06d03679..15e7e5235837c 100644
--- a/Framework/TestWorkflows/src/o2TestHistograms.cxx
+++ b/Framework/TestWorkflows/src/o2TestHistograms.cxx
@@ -49,12 +49,23 @@ struct EtaAndClsHistogramsFull {
 WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
 {
   std::string runType = "3";
+  std::vector<std::string> tables;
   if (cfgc.options().hasOption("aod-metadata-Run")) {
     runType = cfgc.options().get<std::string>("aod-metadata-Run");
   }
+  if (cfgc.options().hasOption("aod-metadata-tables")) {
+    tables = cfgc.options().get<std::vector<std::string>>("aod-metadata-tables");
+  }
   LOGP(info, "Runtype is {}", runType);
+  bool hasTrackCov = false;
+  for (auto& table : tables) {
+    if (table.starts_with("O2trackcov")) {
+      hasTrackCov = true;
+    }
+    LOGP(info, "- {} present.", table);
+  }
   // Notice it's important for the tasks to use the same name, otherwise topology generation will be confused.
-  if (runType == "2") {
+  if (runType == "2" || !hasTrackCov) {
     LOGP(info, "Using only tracks {}", runType);
     return WorkflowSpec{
       adaptAnalysisTask<EtaAndClsHistogramsSimple>(cfgc, TaskName{"simple-histos"}),

From 723c91ebdc323f249de342e8f44c6d2ccfdf4942 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 13 Jun 2024 08:52:50 +0200
Subject: [PATCH 534/726] DPL: correctly handle aod-metadata-table if more than
 one workflow is involved

---
 Framework/Core/src/Plugin.cxx | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx
index 2298896821dab..b58f38a263779 100644
--- a/Framework/Core/src/Plugin.cxx
+++ b/Framework/Core/src/Plugin.cxx
@@ -67,6 +67,25 @@ struct DiscoverMetadataInCommandLineCapability : o2::framework::CapabilityPlugin
   }
 };
 
+std::vector<std::string> getListOfTables(TFile* f)
+{
+  std::vector<std::string> r;
+  TList* keyList = f->GetListOfKeys();
+
+  for (auto key : *keyList) {
+    if (!std::string_view(key->GetName()).starts_with("DF_")) {
+      continue;
+    }
+    auto* d = (TDirectory*)f->Get(key->GetName());
+    TList* branchList = d->GetListOfKeys();
+    for (auto b : *branchList) {
+      r.emplace_back(b->GetName());
+    }
+    break;
+  }
+  return r;
+}
+
 struct DiscoverMetadataInCommandLine : o2::framework::ConfigDiscoveryPlugin {
   ConfigDiscovery* create() override
   {
@@ -86,7 +105,11 @@ struct DiscoverMetadataInCommandLine : o2::framework::ConfigDiscoveryPlugin {
           std::string value = argv[i + 1];
           O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLine",
                                  "Found %{public}s with value %{public}s.", key.c_str(), value.c_str());
-          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in command line"}});
+          if (key == "aod-metadata-tables") {
+            results.push_back(ConfigParamSpec{key, VariantType::ArrayString, getListOfTables(value), {"Metadata in command line"}});
+          } else {
+            results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in command line"}});
+          }
         }
         return results;
       }};

From bf4c9045d87ac712a4500b7a490af5cdb2b3ca1e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 13 Jun 2024 09:27:31 +0200
Subject: [PATCH 535/726] DPL: correctly handle aod-metadata-table if more than
 one workflow is involved

---
 Framework/Core/src/Plugin.cxx | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx
index 2298896821dab..0b8983e050616 100644
--- a/Framework/Core/src/Plugin.cxx
+++ b/Framework/Core/src/Plugin.cxx
@@ -86,7 +86,11 @@ struct DiscoverMetadataInCommandLine : o2::framework::ConfigDiscoveryPlugin {
           std::string value = argv[i + 1];
           O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLine",
                                  "Found %{public}s with value %{public}s.", key.c_str(), value.c_str());
-          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in command line"}});
+          if (key == "aod-metadata-tables") {
+            results.push_back(ConfigParamSpec{key, VariantType::ArrayString, value, {"Metadata in command line"}});
+          } else {
+            results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in command line"}});
+          }
         }
         return results;
       }};

From 0fa3e24b8de3297e2f8e1644d8095725300835e2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 14 Jun 2024 10:56:31 +0200
Subject: [PATCH 536/726] DPL: correctly handle aod-metadata-table if more than
 one workflow is involved

---
 Framework/Core/src/Plugin.cxx | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx
index 2298896821dab..2edf2e62a3633 100644
--- a/Framework/Core/src/Plugin.cxx
+++ b/Framework/Core/src/Plugin.cxx
@@ -14,6 +14,7 @@
 #include "Framework/Logger.h"
 #include "Framework/Capability.h"
 #include "Framework/Signpost.h"
+#include "Framework/VariantJSONHelpers.h"
 #include <string_view>
 
 O2_DECLARE_DYNAMIC_LOG(capabilities);
@@ -86,7 +87,13 @@ struct DiscoverMetadataInCommandLine : o2::framework::ConfigDiscoveryPlugin {
           std::string value = argv[i + 1];
           O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLine",
                                  "Found %{public}s with value %{public}s.", key.c_str(), value.c_str());
-          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in command line"}});
+          if (key == "aod-metadata-tables") {
+            std::stringstream is(value);
+            auto arrayValue = VariantJSONHelpers::read<VariantType::ArrayString>(is);
+            results.push_back(ConfigParamSpec{key, VariantType::ArrayString, arrayValue, {"Metadata in command line"}});
+          } else {
+            results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in command line"}});
+          }
         }
         return results;
       }};

From 6051a92651509d17a439b6de4dc864e6d5298614 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 14 Jun 2024 13:19:16 +0200
Subject: [PATCH 537/726] DPL: improve Graphviz topology visualization

- General refresh of the code
- Add labels to better debug issues with expendable tasks
---
 Framework/Core/src/GraphvizHelpers.cxx   | 22 ++++++++++++++--------
 Framework/Core/src/GraphvizHelpers.h     | 15 ++++++---------
 Framework/Core/src/runDataProcessing.cxx |  6 +-----
 Framework/Core/test/test_Graphviz.cxx    | 14 +++++++-------
 4 files changed, 28 insertions(+), 29 deletions(-)

diff --git a/Framework/Core/src/GraphvizHelpers.cxx b/Framework/Core/src/GraphvizHelpers.cxx
index 47411235faba6..3bae0bdaef598 100644
--- a/Framework/Core/src/GraphvizHelpers.cxx
+++ b/Framework/Core/src/GraphvizHelpers.cxx
@@ -14,9 +14,7 @@
 #include <iostream>
 #include <string>
 
-namespace o2
-{
-namespace framework
+namespace o2::framework
 {
 
 namespace
@@ -25,13 +23,22 @@ std::string quote(std::string const& s) { return R"(")" + s + R"(")"; }
 } // namespace
 
 /// Helper to dump a workflow as a graphviz file
-void GraphvizHelpers::dumpDataProcessorSpec2Graphviz(std::ostream& out, const std::vector<DataProcessorSpec>& specs)
+void GraphvizHelpers::dumpDataProcessorSpec2Graphviz(std::ostream& out, const std::vector<DataProcessorSpec>& specs,
+                                                     std::vector<std::pair<int, int>> const& edges)
 {
   out << "digraph structs {\n";
   out << "  node[shape=record]\n";
   for (auto& spec : specs) {
-    out << R"(  struct [label=")" << spec.name << R"("];)"
-        << "\n";
+    std::string labels = " xlabel=\"";
+    for (auto& label : spec.labels) {
+      labels += labels == " xlabel=\"" ? "" : ",";
+      labels += label.value;
+    }
+    labels += "\"";
+    out << fmt::format("  \"{}\" [label=\"{}\"{}];\n", spec.name, spec.name, labels != " xlabel=\"\"" ? labels : "");
+  }
+  for (auto& e : edges) {
+    out << fmt::format("  \"{}\" -> \"{}\"\n", specs[e.first].name, specs[e.second].name);
   }
   out << "}\n";
 }
@@ -86,5 +93,4 @@ void GraphvizHelpers::dumpDeviceSpec2Graphviz(std::ostream& out, const std::vect
   out << "}\n";
 }
 
-} // namespace framework
-} // namespace o2
+} // namespace o2::framework
diff --git a/Framework/Core/src/GraphvizHelpers.h b/Framework/Core/src/GraphvizHelpers.h
index 1f6fadfac10ba..194cfdaddcd67 100644
--- a/Framework/Core/src/GraphvizHelpers.h
+++ b/Framework/Core/src/GraphvizHelpers.h
@@ -8,25 +8,22 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
-#ifndef FRAMEWORK_GRAPHVIZHELPERS_H
-#define FRAMEWORK_GRAPHVIZHELPERS_H
+#ifndef O2_FRAMEWORK_GRAPHVIZHELPERS_H
+#define O2_FRAMEWORK_GRAPHVIZHELPERS_H
 
 #include "Framework/WorkflowSpec.h"
 #include "Framework/DeviceSpec.h"
 #include <vector>
 #include <iosfwd>
 
-namespace o2
-{
-namespace framework
+namespace o2::framework
 {
 
 struct GraphvizHelpers {
   using Devices = std::vector<DeviceSpec>;
-  static void dumpDataProcessorSpec2Graphviz(std::ostream&, const WorkflowSpec& specs);
+  static void dumpDataProcessorSpec2Graphviz(std::ostream&, const WorkflowSpec& specs, std::vector<std::pair<int, int>> const& edges = {});
   static void dumpDeviceSpec2Graphviz(std::ostream&, const Devices& specs);
 };
 
-} // namespace framework
-} // namespace o2
-#endif // FRAMEWORK_GRAPHVIZHELPERS_H
+} // namespace o2::framework
+#endif // O2_FRAMEWORK_GRAPHVIZHELPERS_H
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 137f17152cc01..89737483fc813 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -2722,11 +2722,7 @@ std::string debugTopoInfo(std::vector<DataProcessorSpec> const& specs,
   for (auto& d : specs) {
     out << "- " << d.name << std::endl;
   }
-  out << "digraph G {\n";
-  for (auto& e : edges) {
-    out << fmt::format("  \"{}\" -> \"{}\"\n", specs[e.first].name, specs[e.second].name);
-  }
-  out << "}\n";
+  GraphvizHelpers::dumpDataProcessorSpec2Graphviz(out, specs, edges);
   return out.str();
 }
 
diff --git a/Framework/Core/test/test_Graphviz.cxx b/Framework/Core/test/test_Graphviz.cxx
index 1d5774630451a..aaebc2b62d1ef 100644
--- a/Framework/Core/test/test_Graphviz.cxx
+++ b/Framework/Core/test/test_Graphviz.cxx
@@ -89,10 +89,10 @@ TEST_CASE("TestGraphviz")
   std::ostringstream str;
   auto expectedResult = R"EXPECTED(digraph structs {
   node[shape=record]
-  struct [label="A"];
-  struct [label="B"];
-  struct [label="C"];
-  struct [label="D"];
+  "A" [label="A"];
+  "B" [label="B"];
+  "C" [label="C"];
+  "D" [label="D"];
 }
 )EXPECTED";
   GraphvizHelpers::dumpDataProcessorSpec2Graphviz(str, workflow);
@@ -130,9 +130,9 @@ TEST_CASE("TestGraphvizWithPipeline")
   std::ostringstream str;
   auto expectedResult = R"EXPECTED(digraph structs {
   node[shape=record]
-  struct [label="A"];
-  struct [label="B"];
-  struct [label="C"];
+  "A" [label="A"];
+  "B" [label="B"];
+  "C" [label="C"];
 }
 )EXPECTED";
   GraphvizHelpers::dumpDataProcessorSpec2Graphviz(str, workflow);

From 48093b7912f876998b1f6f2a8234cf54f9f76837 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 19 Jun 2024 14:24:24 +0200
Subject: [PATCH 538/726] Add module to properly find AliceGridUtils

---
 dependencies/FindAliceGridUtils.cmake | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 dependencies/FindAliceGridUtils.cmake

diff --git a/dependencies/FindAliceGridUtils.cmake b/dependencies/FindAliceGridUtils.cmake
new file mode 100644
index 0000000000000..ca364101432d5
--- /dev/null
+++ b/dependencies/FindAliceGridUtils.cmake
@@ -0,0 +1,7 @@
+include(FindPackageHandleStandardArgs)
+find_path(ALICE_GRID_UTILS_INCLUDE_DIR "TAliceCollection.h"
+        PATH_SUFFIXES "include"
+        HINTS "${ALICE_GRID_UTILS_ROOT}")
+#get_filename_component(ALICE_GRID_UTILS_INCLUDE_DIR ${ALICE_GRID_UTILS_INCLUDE_DIR} DIRECTORY)
+find_package_handle_standard_args(AliceGridUtils DEFAULT_MSG ALICE_GRID_UTILS_INCLUDE_DIR)
+include_directories(${ALICE_GRID_UTILS_INCLUDE_DIR})

From 87a012345dc7b9f094c94c156f9e53393e939086 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 19 Jun 2024 14:26:10 +0200
Subject: [PATCH 539/726] Explicitly find AliceGridUtils

---
 dependencies/O2Dependencies.cmake | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/dependencies/O2Dependencies.cmake b/dependencies/O2Dependencies.cmake
index 53cd7167e0e41..d403002d9cc62 100644
--- a/dependencies/O2Dependencies.cmake
+++ b/dependencies/O2Dependencies.cmake
@@ -111,6 +111,8 @@ set_package_properties(CURL PROPERTIES TYPE REQUIRED)
 find_package(TBB)
 set_package_properties(TBB PROPERTIES TYPE REQUIRED)
 
+find_package(AliceGridUtils MODULE)
+set_package_properties(AliceGridUtils PROPERTIES TYPE RECOMMENDED)
 find_package(JAliEnROOT MODULE)
 set_package_properties(JAliEnROOT PROPERTIES TYPE RECOMMENDED)
 

From f1f32c25d3e36d41f5da7ba05078edbc44284390 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 19 Jun 2024 16:53:17 +0200
Subject: [PATCH 540/726] Remove unneeded pack_to_tuple_t helper

Drop the pack_to_tuple_t helper which is not particularly needed
and adds an implicit dependency of "Framework/Pack.h" on <tuple>.
---
 Framework/Core/include/Framework/GroupedCombinations.h | 3 +++
 Framework/Foundation/include/Framework/Pack.h          | 9 ---------
 Framework/Foundation/test/test_FunctionalHelpers.cxx   | 1 -
 3 files changed, 3 insertions(+), 10 deletions(-)

diff --git a/Framework/Core/include/Framework/GroupedCombinations.h b/Framework/Core/include/Framework/GroupedCombinations.h
index f88a534cc9727..715e2453eafc6 100644
--- a/Framework/Core/include/Framework/GroupedCombinations.h
+++ b/Framework/Core/include/Framework/GroupedCombinations.h
@@ -49,6 +49,9 @@ expressions::BindingNode getMatchingIndexNode()
 
 template <typename T1, typename GroupingPolicy, typename BP, typename G, typename... As>
 struct GroupedCombinationsGenerator {
+  template <typename P>
+  using pack_to_tuple_t = decltype(std::apply([](auto...t) { return std::tuple(t...); }, std::declval<P>()));
+
   using GroupedIteratorType = pack_to_tuple_t<interleaved_pack_t<repeated_type_pack_t<typename G::iterator, sizeof...(As)>, pack<As...>>>;
 
   struct GroupedIterator : public GroupingPolicy {
diff --git a/Framework/Foundation/include/Framework/Pack.h b/Framework/Foundation/include/Framework/Pack.h
index f03476f4fdc0e..a8b6ab5a94e92 100644
--- a/Framework/Foundation/include/Framework/Pack.h
+++ b/Framework/Foundation/include/Framework/Pack.h
@@ -317,15 +317,6 @@ constexpr auto unique_pack(pack<T, Ts...>, PT p2)
 template <typename P>
 using unique_pack_t = decltype(unique_pack(P{}, pack<>{}));
 
-template <typename... Ts>
-inline constexpr std::tuple<Ts...> pack_to_tuple(pack<Ts...>)
-{
-  return std::tuple<Ts...>{};
-}
-
-template <typename P>
-using pack_to_tuple_t = decltype(pack_to_tuple(P{}));
-
 template <typename T, std::size_t... Is>
 inline auto sequence_to_pack(std::integer_sequence<std::size_t, Is...>)
 {
diff --git a/Framework/Foundation/test/test_FunctionalHelpers.cxx b/Framework/Foundation/test/test_FunctionalHelpers.cxx
index 188d91433cdfd..a0e4f9725c597 100644
--- a/Framework/Foundation/test/test_FunctionalHelpers.cxx
+++ b/Framework/Foundation/test/test_FunctionalHelpers.cxx
@@ -58,7 +58,6 @@ TEST_CASE("TestOverride")
 
   static_assert(std::is_same_v<unique_pack_t<pack<int, float, int, float, char, char>>, pack<char, float, int>>, "pack should not have duplicated types");
   static_assert(std::is_same_v<interleaved_pack_t<pack<int, float, int>, pack<char, bool, char>>, pack<int, char, float, bool, int, char>>, "interleaved packs of the same size");
-  static_assert(std::is_same_v<pack_to_tuple_t<pack<int, float, char>>, std::tuple<int, float, char>>, "pack should become a tuple");
   static_assert(std::is_same_v<repeated_type_pack_t<float, 5>, pack<float, float, float, float, float>>, "pack should have float repeated 5 times");
 
   struct ForwardDeclared;

From b967fe9e37934a849b73292f857bb7a1c966e266 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 19 Jun 2024 19:16:13 +0200
Subject: [PATCH 541/726] Remove unneeded pack_to_tuple_t helper

Drop the pack_to_tuple_t helper which is not particularly needed
and adds an implicit dependency of "Framework/Pack.h" on <tuple>.
---
 Framework/Core/include/Framework/GroupedCombinations.h | 3 +++
 Framework/Foundation/include/Framework/Pack.h          | 9 ---------
 Framework/Foundation/test/test_FunctionalHelpers.cxx   | 1 -
 3 files changed, 3 insertions(+), 10 deletions(-)

diff --git a/Framework/Core/include/Framework/GroupedCombinations.h b/Framework/Core/include/Framework/GroupedCombinations.h
index f88a534cc9727..2b0c833e06c25 100644
--- a/Framework/Core/include/Framework/GroupedCombinations.h
+++ b/Framework/Core/include/Framework/GroupedCombinations.h
@@ -49,6 +49,9 @@ expressions::BindingNode getMatchingIndexNode()
 
 template <typename T1, typename GroupingPolicy, typename BP, typename G, typename... As>
 struct GroupedCombinationsGenerator {
+  template <typename P>
+  using pack_to_tuple_t = decltype(std::apply([](auto... t) { return std::tuple(t...); }, P{}));
+
   using GroupedIteratorType = pack_to_tuple_t<interleaved_pack_t<repeated_type_pack_t<typename G::iterator, sizeof...(As)>, pack<As...>>>;
 
   struct GroupedIterator : public GroupingPolicy {
diff --git a/Framework/Foundation/include/Framework/Pack.h b/Framework/Foundation/include/Framework/Pack.h
index f03476f4fdc0e..a8b6ab5a94e92 100644
--- a/Framework/Foundation/include/Framework/Pack.h
+++ b/Framework/Foundation/include/Framework/Pack.h
@@ -317,15 +317,6 @@ constexpr auto unique_pack(pack<T, Ts...>, PT p2)
 template <typename P>
 using unique_pack_t = decltype(unique_pack(P{}, pack<>{}));
 
-template <typename... Ts>
-inline constexpr std::tuple<Ts...> pack_to_tuple(pack<Ts...>)
-{
-  return std::tuple<Ts...>{};
-}
-
-template <typename P>
-using pack_to_tuple_t = decltype(pack_to_tuple(P{}));
-
 template <typename T, std::size_t... Is>
 inline auto sequence_to_pack(std::integer_sequence<std::size_t, Is...>)
 {
diff --git a/Framework/Foundation/test/test_FunctionalHelpers.cxx b/Framework/Foundation/test/test_FunctionalHelpers.cxx
index 188d91433cdfd..a0e4f9725c597 100644
--- a/Framework/Foundation/test/test_FunctionalHelpers.cxx
+++ b/Framework/Foundation/test/test_FunctionalHelpers.cxx
@@ -58,7 +58,6 @@ TEST_CASE("TestOverride")
 
   static_assert(std::is_same_v<unique_pack_t<pack<int, float, int, float, char, char>>, pack<char, float, int>>, "pack should not have duplicated types");
   static_assert(std::is_same_v<interleaved_pack_t<pack<int, float, int>, pack<char, bool, char>>, pack<int, char, float, bool, int, char>>, "interleaved packs of the same size");
-  static_assert(std::is_same_v<pack_to_tuple_t<pack<int, float, char>>, std::tuple<int, float, char>>, "pack should become a tuple");
   static_assert(std::is_same_v<repeated_type_pack_t<float, 5>, pack<float, float, float, float, float>>, "pack should have float repeated 5 times");
 
   struct ForwardDeclared;

From 3523fb5edb53b4e77307bb7f40d2d39945a57b43 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sun, 23 Jun 2024 22:46:45 +0200
Subject: [PATCH 542/726] Use voids to select packs

---
 Framework/Foundation/include/Framework/Pack.h | 32 +++++++++++++++++--
 1 file changed, 29 insertions(+), 3 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Pack.h b/Framework/Foundation/include/Framework/Pack.h
index f03476f4fdc0e..7f5d79d761666 100644
--- a/Framework/Foundation/include/Framework/Pack.h
+++ b/Framework/Foundation/include/Framework/Pack.h
@@ -99,17 +99,43 @@ constexpr auto interleave_pack(pack<Args1...>, pack<Args2...>)
 template <typename P1, typename P2>
 using interleaved_pack_t = decltype(interleave_pack(P1{}, P2{}));
 
+/// Marks as void the types that do not satisfy the condition
+template <template <typename...> typename Condition, typename... Ts>
+using with_condition_pack = pack<std::conditional_t<Condition<Ts>::value, Ts, void>...>;
+
+template <typename... Ts>
+consteval auto count_non_void_pack(pack<Ts...> const&)
+{
+  return ((std::is_void_v<Ts> ? 0 : 1) + ...);
+}
+
+template <typename Result>
+consteval auto prune_voids_pack(Result result, pack<>)
+{
+  return result;
+}
+
+template <typename... Rs, typename T, typename... Ts>
+consteval auto prune_voids_pack(pack<Rs...> result, pack<T, Ts...>)
+{
+  if constexpr (std::is_void_v<T> == false) {
+    return prune_voids_pack(pack<Rs..., T>{}, pack<Ts...>{});
+  } else {
+    return prune_voids_pack(pack<Rs...>{}, pack<Ts...>{});
+  }
+}
+
 /// Selects from the pack types that satisfy the Condition
 /// Multicondition takes the type to check as first template parameter
 /// and any helper types as the following parameters
 template <template <typename...> typename Condition, typename Result, typename... Cs>
-constexpr auto select_pack(Result result, pack<>, pack<Cs...>)
+consteval auto select_pack(Result result, pack<>, pack<Cs...>)
 {
   return result;
 }
 
 template <template <typename...> typename Condition, typename Result, typename T, typename... Cs, typename... Ts>
-constexpr auto select_pack(Result result, pack<T, Ts...>, pack<Cs...> condPack)
+consteval auto select_pack(Result result, pack<T, Ts...>, pack<Cs...> condPack)
 {
   if constexpr (Condition<T, Cs...>()) {
     return select_pack<Condition>(concatenate_pack(result, pack<T>{}), pack<Ts...>{}, condPack);
@@ -119,7 +145,7 @@ constexpr auto select_pack(Result result, pack<T, Ts...>, pack<Cs...> condPack)
 }
 
 template <template <typename...> typename Condition, typename... Types>
-using selected_pack = std::decay_t<decltype(select_pack<Condition>(pack<>{}, pack<Types...>{}, pack<>{}))>;
+using selected_pack = std::decay_t<decltype(prune_voids_pack(pack<>{}, with_condition_pack<Condition, Types...>{}))>;
 template <template <typename...> typename Condition, typename CondPack, typename Pack>
 using selected_pack_multicondition = std::decay_t<decltype(select_pack<Condition>(pack<>{}, Pack{}, CondPack{}))>;
 

From 3f35feafe91a82b0b6daa7edb198020bde1be013 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 25 Jun 2024 13:12:26 +0200
Subject: [PATCH 543/726] DPL Analysis: optimise compilation

Avoids iterating on dynamic and persistent columns separately when
doing the bindings.

As a side effect we can get rid of the dynamic_columns_t type completely.
---
 Framework/Core/include/Framework/ASoA.h | 36 ++++++++++++-------------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h
index 9cc7e0c71f939..465f0fd20642f 100644
--- a/Framework/Core/include/Framework/ASoA.h
+++ b/Framework/Core/include/Framework/ASoA.h
@@ -558,6 +558,9 @@ using is_persistent_t = decltype(persistent_type_helper::test<T>(nullptr));
 template <typename T>
 constexpr auto is_persistent_v = is_persistent_t<T>::value;
 
+template <typename T>
+constexpr auto is_dynamic_v = is_dynamic_t<T>::value;
+
 template <typename T>
 using is_external_index_t = typename std::conditional<is_index_column_v<T>, std::true_type, std::false_type>::type;
 
@@ -784,7 +787,6 @@ struct RowViewCore : public IP, C... {
   using table_t = o2::soa::Table<C...>;
   using all_columns = framework::pack<C...>;
   using persistent_columns_t = framework::selected_pack<is_persistent_t, C...>;
-  using dynamic_columns_t = framework::selected_pack<is_dynamic_t, C...>;
   using index_columns_t = framework::selected_pack<is_index_t, C...>;
   constexpr inline static bool has_index_v = framework::pack_size(index_columns_t{}) > 0;
   using external_index_columns_t = framework::selected_pack<is_external_index_t, C...>;
@@ -794,8 +796,7 @@ struct RowViewCore : public IP, C... {
     : IP{policy},
       C(columnData[framework::has_type_at_v<C>(all_columns{})])...
   {
-    bindIterators(persistent_columns_t{});
-    bindAllDynamicColumns(dynamic_columns_t{});
+    bind();
     // In case we have an index column might need to constrain the actual
     // number of rows in the view to the range provided by the index.
     // FIXME: we should really understand what happens to an index when we
@@ -810,16 +811,14 @@ struct RowViewCore : public IP, C... {
     : IP{static_cast<IP const&>(other)},
       C(static_cast<C const&>(other))...
   {
-    bindIterators(persistent_columns_t{});
-    bindAllDynamicColumns(dynamic_columns_t{});
+    bind();
   }
 
   RowViewCore& operator=(RowViewCore other)
   {
     IP::operator=(static_cast<IP const&>(other));
     (void(static_cast<C&>(*this) = static_cast<C>(other)), ...);
-    bindIterators(persistent_columns_t{});
-    bindAllDynamicColumns(dynamic_columns_t{});
+    bind();
     return *this;
   }
 
@@ -827,8 +826,7 @@ struct RowViewCore : public IP, C... {
     : IP{static_cast<IP const&>(other)},
       C(static_cast<C const&>(other))...
   {
-    bindIterators(persistent_columns_t{});
-    bindAllDynamicColumns(dynamic_columns_t{});
+    bind();
   }
 
   RowViewCore& operator++()
@@ -930,6 +928,9 @@ struct RowViewCore : public IP, C... {
   }
 
  private:
+  template <class... Ts> struct overload : Ts... { using Ts::operator()...; };
+  template <class... Ts> overload(Ts...) -> overload<Ts...>; // clang needs this deduction guide,
+                                                           // even in C++20 for some reasons ...
   /// Helper to move to the correct chunk, if needed.
   /// FIXME: not needed?
   template <typename... PC>
@@ -947,18 +948,15 @@ struct RowViewCore : public IP, C... {
 
   /// Helper which binds all the ColumnIterators to the
   /// index of a the associated RowView
-  template <typename... PC>
-  auto bindIterators(framework::pack<PC...>)
-  {
-    using namespace o2::soa;
-    (void(PC::mColumnIterator.mCurrentPos = &this->mRowIndex), ...);
-  }
-
-  template <typename... DC>
-  auto bindAllDynamicColumns(framework::pack<DC...>)
+  void bind()
   {
     using namespace o2::soa;
-    (bindDynamicColumn<DC>(typename DC::bindings_t{}), ...);
+    auto f = overload  {
+      [this]<typename T>(T&&) -> void requires is_persistent_v<T> { T::mColumnIterator.mCurrentPos = &this->mRowIndex; },
+      [this]<typename T>(T&&) -> void requires is_dynamic_v<T> { bindDynamicColumn<T>(typename T::bindings_t{});},
+      [this]<typename T>(T&&) -> void {},
+    };
+    (f(C{}), ...);
     if constexpr (has_index_v) {
       this->setIndices(this->getIndices());
       this->setOffsets(this->getOffsets());

From 539bb82ed689d78ea9896d4de3ce5cf6fcdee702 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 25 Jun 2024 20:07:22 +0200
Subject: [PATCH 544/726] DPL Analysis: optimise compilation

Avoids iterating on dynamic and persistent columns separately when
doing the bindings.

As a side effect we can get rid of the dynamic_columns_t type completely.
---
 Framework/Core/include/Framework/ASoA.h | 34 +++++++++++--------------
 1 file changed, 15 insertions(+), 19 deletions(-)

diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h
index 9cc7e0c71f939..93960c75b11b7 100644
--- a/Framework/Core/include/Framework/ASoA.h
+++ b/Framework/Core/include/Framework/ASoA.h
@@ -21,6 +21,7 @@
 #include "Framework/ArrowTypes.h"
 #include "Framework/ArrowTableSlicingCache.h"
 #include "Framework/SliceCache.h"
+#include "Framework/VariantHelpers.h"
 #include <arrow/table.h>
 #include <arrow/array.h>
 #include <arrow/util/config.h>
@@ -558,6 +559,9 @@ using is_persistent_t = decltype(persistent_type_helper::test<T>(nullptr));
 template <typename T>
 constexpr auto is_persistent_v = is_persistent_t<T>::value;
 
+template <typename T>
+constexpr auto is_dynamic_v = is_dynamic_t<T>::value;
+
 template <typename T>
 using is_external_index_t = typename std::conditional<is_index_column_v<T>, std::true_type, std::false_type>::type;
 
@@ -784,7 +788,6 @@ struct RowViewCore : public IP, C... {
   using table_t = o2::soa::Table<C...>;
   using all_columns = framework::pack<C...>;
   using persistent_columns_t = framework::selected_pack<is_persistent_t, C...>;
-  using dynamic_columns_t = framework::selected_pack<is_dynamic_t, C...>;
   using index_columns_t = framework::selected_pack<is_index_t, C...>;
   constexpr inline static bool has_index_v = framework::pack_size(index_columns_t{}) > 0;
   using external_index_columns_t = framework::selected_pack<is_external_index_t, C...>;
@@ -794,8 +797,7 @@ struct RowViewCore : public IP, C... {
     : IP{policy},
       C(columnData[framework::has_type_at_v<C>(all_columns{})])...
   {
-    bindIterators(persistent_columns_t{});
-    bindAllDynamicColumns(dynamic_columns_t{});
+    bind();
     // In case we have an index column might need to constrain the actual
     // number of rows in the view to the range provided by the index.
     // FIXME: we should really understand what happens to an index when we
@@ -810,16 +812,14 @@ struct RowViewCore : public IP, C... {
     : IP{static_cast<IP const&>(other)},
       C(static_cast<C const&>(other))...
   {
-    bindIterators(persistent_columns_t{});
-    bindAllDynamicColumns(dynamic_columns_t{});
+    bind();
   }
 
   RowViewCore& operator=(RowViewCore other)
   {
     IP::operator=(static_cast<IP const&>(other));
     (void(static_cast<C&>(*this) = static_cast<C>(other)), ...);
-    bindIterators(persistent_columns_t{});
-    bindAllDynamicColumns(dynamic_columns_t{});
+    bind();
     return *this;
   }
 
@@ -827,8 +827,7 @@ struct RowViewCore : public IP, C... {
     : IP{static_cast<IP const&>(other)},
       C(static_cast<C const&>(other))...
   {
-    bindIterators(persistent_columns_t{});
-    bindAllDynamicColumns(dynamic_columns_t{});
+    bind();
   }
 
   RowViewCore& operator++()
@@ -947,18 +946,15 @@ struct RowViewCore : public IP, C... {
 
   /// Helper which binds all the ColumnIterators to the
   /// index of a the associated RowView
-  template <typename... PC>
-  auto bindIterators(framework::pack<PC...>)
-  {
-    using namespace o2::soa;
-    (void(PC::mColumnIterator.mCurrentPos = &this->mRowIndex), ...);
-  }
-
-  template <typename... DC>
-  auto bindAllDynamicColumns(framework::pack<DC...>)
+  void bind()
   {
     using namespace o2::soa;
-    (bindDynamicColumn<DC>(typename DC::bindings_t{}), ...);
+    auto f = framework::overloaded  {
+      [this]<typename T>(T*) -> void requires is_persistent_v<T> { T::mColumnIterator.mCurrentPos = &this->mRowIndex; },
+      [this]<typename T>(T*) -> void requires is_dynamic_v<T> { bindDynamicColumn<T>(typename T::bindings_t{});},
+      [this]<typename T>(T*) -> void {},
+    };
+    (f(static_cast<C*>(nullptr)), ...);
     if constexpr (has_index_v) {
       this->setIndices(this->getIndices());
       this->setOffsets(this->getOffsets());

From 6be5fc547b456e7f13ec9b9f96ebb1b19f2a7c6a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 26 Jun 2024 08:45:12 +0200
Subject: [PATCH 545/726] Add support for Track_IU

---
 .../TestWorkflows/src/o2TestHistograms.cxx    | 28 ++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/Framework/TestWorkflows/src/o2TestHistograms.cxx b/Framework/TestWorkflows/src/o2TestHistograms.cxx
index 15e7e5235837c..e32149a8bfdc1 100644
--- a/Framework/TestWorkflows/src/o2TestHistograms.cxx
+++ b/Framework/TestWorkflows/src/o2TestHistograms.cxx
@@ -34,6 +34,18 @@ struct EtaAndClsHistogramsSimple {
   }
 };
 
+struct EtaAndClsHistogramsIUSimple {
+  OutputObj<TH2F> etaClsH{TH2F("eta_vs_pt", "#eta vs pT", 102, -2.01, 2.01, 100, 0, 10)};
+
+  void process(aod::TracksIU const& tracks)
+  {
+    LOGP(info, "Invoking the simple one");
+    for (auto& track : tracks) {
+      etaClsH->Fill(track.eta(), track.pt(), 0);
+    }
+  }
+};
+
 struct EtaAndClsHistogramsFull {
   OutputObj<TH3F> etaClsH{TH3F("eta_vs_cls_vs_sigmapT", "#eta vs N_{cls} vs sigma_{1/pT}", 102, -2.01, 2.01, 160, -0.5, 159.5, 100, 0, 10)};
 
@@ -58,20 +70,34 @@ WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
   }
   LOGP(info, "Runtype is {}", runType);
   bool hasTrackCov = false;
+  bool hasTrackIU = false;
   for (auto& table : tables) {
-    if (table.starts_with("O2trackcov")) {
+    if (table == "O2trackcov") {
       hasTrackCov = true;
     }
+    if (table.starts_with("O2track_iu")) {
+      hasTrackIU = true;
+    }
     LOGP(info, "- {} present.", table);
   }
   // Notice it's important for the tasks to use the same name, otherwise topology generation will be confused.
   if (runType == "2" || !hasTrackCov) {
     LOGP(info, "Using only tracks {}", runType);
+    if (hasTrackIU) {
+      return WorkflowSpec{
+        adaptAnalysisTask<EtaAndClsHistogramsIUSimple>(cfgc, TaskName{"simple-histos"}),
+      };
+    }
     return WorkflowSpec{
       adaptAnalysisTask<EtaAndClsHistogramsSimple>(cfgc, TaskName{"simple-histos"}),
     };
   } else {
     LOGP(info, "Using tracks extra {}", runType);
+    if (hasTrackIU) {
+      return WorkflowSpec{
+        adaptAnalysisTask<EtaAndClsHistogramsIUSimple>(cfgc, TaskName{"simple-histos"}),
+      };
+    }
     return WorkflowSpec{
       adaptAnalysisTask<EtaAndClsHistogramsFull>(cfgc, TaskName{"simple-histos"}),
     };

From 6b4722d661b7a6961deef2ce0e6e3afb7874c5e1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 26 Jun 2024 15:19:36 +0200
Subject: [PATCH 546/726] Fix variant for std::vector string

---
 Framework/Core/include/Framework/Variant.h | 93 +++++++++++++---------
 Framework/Core/src/Variant.cxx             | 36 ++++-----
 2 files changed, 74 insertions(+), 55 deletions(-)

diff --git a/Framework/Core/include/Framework/Variant.h b/Framework/Core/include/Framework/Variant.h
index 54e4b54be9606..081fd8db153c4 100644
--- a/Framework/Core/include/Framework/Variant.h
+++ b/Framework/Core/include/Framework/Variant.h
@@ -254,75 +254,80 @@ DECLARE_VARIANT_ARRAY_ELEMENT_TYPE(std::string, LabeledArrayString);
 template <VariantType V>
 using variant_array_element_type_t = typename variant_array_element_type<V>::type;
 
-template <typename S, typename T>
+template <typename T>
 struct variant_helper {
-  static void set(S* store, T value)
+  static void set(void* store, T value)
   {
     new (reinterpret_cast<T*>(store)) T{};
     *(reinterpret_cast<T*>(store)) = value;
   }
-  static void set(S* store, T values, size_t size)
+  static void set(void* store, T values, size_t size)
   {
     *reinterpret_cast<T*>(store) = reinterpret_cast<T>(std::memcpy(std::malloc(size * sizeof(std::remove_pointer_t<T>)), reinterpret_cast<void*>(values), size * sizeof(std::remove_pointer_t<T>)));
   }
 
-  static T get(const S* store) { return *(reinterpret_cast<const T*>(store)); }
+  static T get(const void* store) { return *(reinterpret_cast<const T*>(store)); }
+};
+
+template <>
+struct variant_helper<std::vector<std::string>> {
+  // Allocates a new store and copies into it.
+  static void set(void* store, std::vector<std::string> value)
+  {
+    new (reinterpret_cast<std::vector<std::string>*>(store)) std::vector<std::string>{};
+    *(reinterpret_cast<std::vector<std::string>*>(store)) = value;
+  }
+
+  static std::vector<std::string> const& get(const void* store) { return *(reinterpret_cast<std::vector<std::string> const*>(store)); }
 };
 
-template <typename S>
-struct variant_helper<S, const char*> {
-  static const char* get(const S* store) { return *reinterpret_cast<const char* const*>(store); }
+template <>
+struct variant_helper<const char*> {
+  static const char* get(const void* store) { return *reinterpret_cast<const char* const*>(store); }
 
-  static void set(S* store, const char* value) { *reinterpret_cast<char**>(store) = strdup(value); }
+  static void set(void* store, const char* value) { *reinterpret_cast<char**>(store) = strdup(value); }
 };
 
-template <typename S>
-struct variant_helper<S, std::string_view> {
-  static std::string_view get(const S* store) { return std::string_view(*reinterpret_cast<const char* const*>(store)); }
+template <>
+struct variant_helper<std::string_view> {
+  static std::string_view get(const void* store) { return std::string_view(*reinterpret_cast<const char* const*>(store)); }
 
-  static void set(S* store, std::string_view value) { *reinterpret_cast<char**>(store) = strdup(value.data()); }
+  static void set(void* store, std::string_view value) { *reinterpret_cast<char**>(store) = strdup(value.data()); }
 };
 
-template <typename S>
-struct variant_helper<S, std::string> {
-  static std::string get(const S* store) { return std::string(*reinterpret_cast<const char* const*>(store)); }
+template <>
+struct variant_helper<std::string> {
+  static std::string get(const void* store) { return std::string(*reinterpret_cast<const char* const*>(store)); }
 
-  static void set(S* store, std::string value) { *reinterpret_cast<char**>(store) = strdup(value.data()); }
+  static void set(void* store, std::string value) { *reinterpret_cast<char**>(store) = strdup(value.data()); }
 };
 
 /// Variant for configuration parameter storage. Owns stored data.
 class Variant
 {
-  using storage_t = std::aligned_union<8, int, int8_t, int16_t, int64_t,
-                                       uint8_t, uint16_t, uint32_t, uint64_t,
-                                       const char*, float, double, bool,
-                                       int*, float*, double*, bool*, std::string*,
-                                       Array2D<int>, Array2D<float>, Array2D<double>, Array2D<std::string>,
-                                       LabeledArray<int>, LabeledArray<float>, LabeledArray<double>, LabeledArray<std::string>>::type;
-
  public:
-  Variant(VariantType type = VariantType::Unknown) : mType{type}, mSize{1} {}
+  Variant(VariantType type = VariantType::Unknown) : mType{type} {}
 
   template <typename T>
-  Variant(T value) : mType{variant_trait_v<T>}, mSize{1}
+  Variant(T value) : mType{variant_trait_v<T>}
   {
-    variant_helper<storage_t, decltype(value)>::set(&mStore, value);
+    variant_helper<decltype(value)>::set(&mStore, value);
   }
 
   template <typename T>
   Variant(T values, size_t size) : mType{variant_trait_v<T>}, mSize{size}
   {
-    variant_helper<storage_t, T>::set(&mStore, values, mSize);
+    variant_helper<T>::set(&mStore, values, mSize);
   }
 
   template <typename T>
   Variant(std::vector<T>& values) : mType{variant_trait_v<T*>}, mSize{values.size()}
   {
-    variant_helper<storage_t, T*>::set(&mStore, values.data(), mSize);
+    variant_helper<T*>::set(&mStore, values.data(), mSize);
   }
 
   template <typename T>
-  Variant(std::initializer_list<T>) : mType{VariantType::Unknown}, mSize{1}
+  Variant(std::initializer_list<T>)
   {
     static_assert(sizeof(T) == 0,
                   "brace-enclosed initializer list forbidden for Variant"
@@ -347,36 +352,50 @@ class Variant
     if (mType != variant_trait_v<T>) {
       throw runtime_error("Mismatch between types");
     }
-    return variant_helper<storage_t, T>::get(&mStore);
+    return variant_helper<T>::get(&mStore);
   }
 
   template <typename T>
   void set(T value)
   {
-    return variant_helper<storage_t, T>::set(&mStore, value);
+    return variant_helper<T>::set(&mStore, value);
   }
 
   template <typename T>
   void set(T value, size_t size)
   {
     mSize = size;
-    return variant_helper<storage_t, T>::set(&mStore, value, mSize);
+    return variant_helper<T>::set(&mStore, value, mSize);
+  }
+
+  template <typename T>
+  void set(std::vector<T>& values)
+    requires(std::is_pod_v<T>)
+  {
+    return variant_helper<T*>::set(&mStore, values.data(), values.size());
   }
 
   template <typename T>
   void set(std::vector<T>& values)
+    requires(std::is_same_v<T, std::string>)
   {
-    return variant_helper<storage_t, T*>::set(&mStore, values.data(), values.size());
+    return variant_helper<T*>::set(&mStore, values);
   }
 
-  VariantType type() const { return mType; }
-  size_t size() const { return mSize; }
-  std::string asString() const;
+  [[nodiscard]] VariantType type() const { return mType; }
+  [[nodiscard]] size_t size() const { return mSize; }
+  [[nodiscard]] std::string asString() const;
 
  private:
   friend std::ostream& operator<<(std::ostream& oss, Variant const& val);
+  using storage_t = std::aligned_union<8, int, int8_t, int16_t, int64_t,
+                                       uint8_t, uint16_t, uint32_t, uint64_t,
+                                       const char*, float, double, bool,
+                                       int*, float*, double*, bool*, std::string*,
+                                       Array2D<int>, Array2D<float>, Array2D<double>, Array2D<std::string>,
+                                       LabeledArray<int>, LabeledArray<float>, LabeledArray<double>, LabeledArray<std::string>>::type;
   storage_t mStore;
-  VariantType mType;
+  VariantType mType = VariantType::Unknown;
   size_t mSize = 1;
 };
 
diff --git a/Framework/Core/src/Variant.cxx b/Framework/Core/src/Variant.cxx
index 3f71ec339511e..21eb6103aa56a 100644
--- a/Framework/Core/src/Variant.cxx
+++ b/Framework/Core/src/Variant.cxx
@@ -91,27 +91,27 @@ Variant::Variant(const Variant& other) : mType(other.mType)
   switch (mType) {
     case variant_trait_v<const char*>:
       mSize = other.mSize;
-      variant_helper<storage_t, const char*>::set(&mStore, other.get<const char*>());
+      variant_helper<const char*>::set(&mStore, other.get<const char*>());
       return;
     case variant_trait_v<int*>:
       mSize = other.mSize;
-      variant_helper<storage_t, int*>::set(&mStore, other.get<int*>(), mSize);
+      variant_helper<int*>::set(&mStore, other.get<int*>(), mSize);
       return;
     case variant_trait_v<float*>:
       mSize = other.mSize;
-      variant_helper<storage_t, float*>::set(&mStore, other.get<float*>(), mSize);
+      variant_helper<float*>::set(&mStore, other.get<float*>(), mSize);
       return;
     case variant_trait_v<double*>:
       mSize = other.mSize;
-      variant_helper<storage_t, double*>::set(&mStore, other.get<double*>(), mSize);
+      variant_helper<double*>::set(&mStore, other.get<double*>(), mSize);
       return;
     case variant_trait_v<bool*>:
       mSize = other.mSize;
-      variant_helper<storage_t, bool*>::set(&mStore, other.get<bool*>(), mSize);
+      variant_helper<bool*>::set(&mStore, other.get<bool*>(), mSize);
       return;
     case variant_trait_v<std::string*>:
       mSize = other.mSize;
-      variant_helper<storage_t, std::string*>::set(&mStore, other.get<std::string*>(), mSize);
+      variant_helper<std::string*>::set(&mStore, other.get<std::string*>(), mSize);
       return;
     default:
       mStore = other.mStore;
@@ -172,22 +172,22 @@ Variant& Variant::operator=(const Variant& other)
   mType = other.mType;
   switch (mType) {
     case variant_trait_v<const char*>:
-      variant_helper<storage_t, const char*>::set(&mStore, other.get<const char*>());
+      variant_helper<const char*>::set(&mStore, other.get<const char*>());
       return *this;
     case variant_trait_v<int*>:
-      variant_helper<storage_t, int*>::set(&mStore, other.get<int*>(), mSize);
+      variant_helper<int*>::set(&mStore, other.get<int*>(), mSize);
       return *this;
     case variant_trait_v<float*>:
-      variant_helper<storage_t, float*>::set(&mStore, other.get<float*>(), mSize);
+      variant_helper<float*>::set(&mStore, other.get<float*>(), mSize);
       return *this;
     case variant_trait_v<double*>:
-      variant_helper<storage_t, double*>::set(&mStore, other.get<double*>(), mSize);
+      variant_helper<double*>::set(&mStore, other.get<double*>(), mSize);
       return *this;
     case variant_trait_v<bool*>:
-      variant_helper<storage_t, bool*>::set(&mStore, other.get<bool*>(), mSize);
+      variant_helper<bool*>::set(&mStore, other.get<bool*>(), mSize);
       return *this;
     case variant_trait_v<std::string*>:
-      variant_helper<storage_t, std::string*>::set(&mStore, other.get<std::string*>(), mSize);
+      variant_helper<std::string*>::set(&mStore, other.get<std::string*>(), mSize);
       return *this;
     default:
       mStore = other.mStore;
@@ -201,27 +201,27 @@ Variant& Variant::operator=(Variant&& other) noexcept
   mType = other.mType;
   switch (mType) {
     case variant_trait_v<const char*>:
-      variant_helper<storage_t, const char*>::set(&mStore, other.get<const char*>());
+      variant_helper<const char*>::set(&mStore, other.get<const char*>());
       *reinterpret_cast<char**>(&(other.mStore)) = nullptr;
       return *this;
     case variant_trait_v<int*>:
-      variant_helper<storage_t, int*>::set(&mStore, other.get<int*>(), mSize);
+      variant_helper<int*>::set(&mStore, other.get<int*>(), mSize);
       *reinterpret_cast<int**>(&(other.mStore)) = nullptr;
       return *this;
     case variant_trait_v<float*>:
-      variant_helper<storage_t, float*>::set(&mStore, other.get<float*>(), mSize);
+      variant_helper<float*>::set(&mStore, other.get<float*>(), mSize);
       *reinterpret_cast<float**>(&(other.mStore)) = nullptr;
       return *this;
     case variant_trait_v<double*>:
-      variant_helper<storage_t, double*>::set(&mStore, other.get<double*>(), mSize);
+      variant_helper<double*>::set(&mStore, other.get<double*>(), mSize);
       *reinterpret_cast<double**>(&(other.mStore)) = nullptr;
       return *this;
     case variant_trait_v<bool*>:
-      variant_helper<storage_t, bool*>::set(&mStore, other.get<bool*>(), mSize);
+      variant_helper<bool*>::set(&mStore, other.get<bool*>(), mSize);
       *reinterpret_cast<bool**>(&(other.mStore)) = nullptr;
       return *this;
     case variant_trait_v<std::string*>:
-      variant_helper<storage_t, std::string*>::set(&mStore, other.get<std::string*>(), mSize);
+      variant_helper<std::string*>::set(&mStore, other.get<std::string*>(), mSize);
       *reinterpret_cast<std::string**>(&(other.mStore)) = nullptr;
       return *this;
     default:

From cf0ceefb948a06ce7b1e968bd60025abc725e9a7 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 26 Jun 2024 15:19:36 +0200
Subject: [PATCH 547/726] Fix variant for std::vector string

---
 Framework/Core/include/Framework/Variant.h | 98 ++++++++++++++--------
 Framework/Core/src/Variant.cxx             | 36 ++++----
 2 files changed, 79 insertions(+), 55 deletions(-)

diff --git a/Framework/Core/include/Framework/Variant.h b/Framework/Core/include/Framework/Variant.h
index 54e4b54be9606..54a91e90c3638 100644
--- a/Framework/Core/include/Framework/Variant.h
+++ b/Framework/Core/include/Framework/Variant.h
@@ -254,75 +254,85 @@ DECLARE_VARIANT_ARRAY_ELEMENT_TYPE(std::string, LabeledArrayString);
 template <VariantType V>
 using variant_array_element_type_t = typename variant_array_element_type<V>::type;
 
-template <typename S, typename T>
+template <typename T>
 struct variant_helper {
-  static void set(S* store, T value)
+  static void set(void* store, T value)
   {
     new (reinterpret_cast<T*>(store)) T{};
     *(reinterpret_cast<T*>(store)) = value;
   }
-  static void set(S* store, T values, size_t size)
+  static void set(void* store, T values, size_t size)
   {
     *reinterpret_cast<T*>(store) = reinterpret_cast<T>(std::memcpy(std::malloc(size * sizeof(std::remove_pointer_t<T>)), reinterpret_cast<void*>(values), size * sizeof(std::remove_pointer_t<T>)));
   }
 
-  static T get(const S* store) { return *(reinterpret_cast<const T*>(store)); }
+  static T get(const void* store) { return *(reinterpret_cast<const T*>(store)); }
+};
+
+template <>
+struct variant_helper<std::vector<std::string>> {
+  // Allocates a new store and copies into it.
+  static void set(void* store, std::vector<std::string> value)
+  {
+    new (reinterpret_cast<std::vector<std::string>*>(store)) std::vector<std::string>{};
+    *(reinterpret_cast<std::vector<std::string>*>(store)) = value;
+  }
+
+  static std::vector<std::string> const& get(const void* store) { return *(reinterpret_cast<std::vector<std::string> const*>(store)); }
 };
 
-template <typename S>
-struct variant_helper<S, const char*> {
-  static const char* get(const S* store) { return *reinterpret_cast<const char* const*>(store); }
+template <>
+struct variant_helper<const char*> {
+  static const char* get(const void* store) { return *reinterpret_cast<const char* const*>(store); }
 
-  static void set(S* store, const char* value) { *reinterpret_cast<char**>(store) = strdup(value); }
+  static void set(void* store, const char* value) { *reinterpret_cast<char**>(store) = strdup(value); }
 };
 
-template <typename S>
-struct variant_helper<S, std::string_view> {
-  static std::string_view get(const S* store) { return std::string_view(*reinterpret_cast<const char* const*>(store)); }
+template <>
+struct variant_helper<std::string_view> {
+  static std::string_view get(const void* store) { return std::string_view(*reinterpret_cast<const char* const*>(store)); }
 
-  static void set(S* store, std::string_view value) { *reinterpret_cast<char**>(store) = strdup(value.data()); }
+  static void set(void* store, std::string_view value) { *reinterpret_cast<char**>(store) = strdup(value.data()); }
 };
 
-template <typename S>
-struct variant_helper<S, std::string> {
-  static std::string get(const S* store) { return std::string(*reinterpret_cast<const char* const*>(store)); }
+template <>
+struct variant_helper<std::string> {
+  static std::string get(const void* store) { return std::string(*reinterpret_cast<const char* const*>(store)); }
 
-  static void set(S* store, std::string value) { *reinterpret_cast<char**>(store) = strdup(value.data()); }
+  static void set(void* store, std::string value) { *reinterpret_cast<char**>(store) = strdup(value.data()); }
 };
 
 /// Variant for configuration parameter storage. Owns stored data.
 class Variant
 {
-  using storage_t = std::aligned_union<8, int, int8_t, int16_t, int64_t,
-                                       uint8_t, uint16_t, uint32_t, uint64_t,
-                                       const char*, float, double, bool,
-                                       int*, float*, double*, bool*, std::string*,
-                                       Array2D<int>, Array2D<float>, Array2D<double>, Array2D<std::string>,
-                                       LabeledArray<int>, LabeledArray<float>, LabeledArray<double>, LabeledArray<std::string>>::type;
-
  public:
-  Variant(VariantType type = VariantType::Unknown) : mType{type}, mSize{1} {}
+  Variant(VariantType type = VariantType::Unknown) : mType{type} {}
 
   template <typename T>
-  Variant(T value) : mType{variant_trait_v<T>}, mSize{1}
+  Variant(T value) : mType{variant_trait_v<T>}
   {
-    variant_helper<storage_t, decltype(value)>::set(&mStore, value);
+    variant_helper<decltype(value)>::set(&mStore, value);
   }
 
   template <typename T>
   Variant(T values, size_t size) : mType{variant_trait_v<T>}, mSize{size}
   {
-    variant_helper<storage_t, T>::set(&mStore, values, mSize);
+    variant_helper<T>::set(&mStore, values, mSize);
   }
 
   template <typename T>
   Variant(std::vector<T>& values) : mType{variant_trait_v<T*>}, mSize{values.size()}
   {
-    variant_helper<storage_t, T*>::set(&mStore, values.data(), mSize);
+    variant_helper<T*>::set(&mStore, values.data(), mSize);
+  }
+
+  Variant(std::vector<std::string>& values) : mType{VariantType::ArrayString}, mSize{values.size()}
+  {
+    variant_helper<std::vector<std::string>>::set(&mStore, values);
   }
 
   template <typename T>
-  Variant(std::initializer_list<T>) : mType{VariantType::Unknown}, mSize{1}
+  Variant(std::initializer_list<T>)
   {
     static_assert(sizeof(T) == 0,
                   "brace-enclosed initializer list forbidden for Variant"
@@ -347,36 +357,50 @@ class Variant
     if (mType != variant_trait_v<T>) {
       throw runtime_error("Mismatch between types");
     }
-    return variant_helper<storage_t, T>::get(&mStore);
+    return variant_helper<T>::get(&mStore);
   }
 
   template <typename T>
   void set(T value)
   {
-    return variant_helper<storage_t, T>::set(&mStore, value);
+    return variant_helper<T>::set(&mStore, value);
   }
 
   template <typename T>
   void set(T value, size_t size)
   {
     mSize = size;
-    return variant_helper<storage_t, T>::set(&mStore, value, mSize);
+    return variant_helper<T>::set(&mStore, value, mSize);
   }
 
   template <typename T>
   void set(std::vector<T>& values)
+    requires(std::is_pod_v<T>)
   {
-    return variant_helper<storage_t, T*>::set(&mStore, values.data(), values.size());
+    return variant_helper<T*>::set(&mStore, values.data(), values.size());
   }
 
-  VariantType type() const { return mType; }
-  size_t size() const { return mSize; }
-  std::string asString() const;
+  template <typename T>
+  void set(std::vector<T>& values)
+    requires(std::is_same_v<T, std::string>)
+  {
+    return variant_helper<T*>::set(&mStore, values);
+  }
+
+  [[nodiscard]] VariantType type() const { return mType; }
+  [[nodiscard]] size_t size() const { return mSize; }
+  [[nodiscard]] std::string asString() const;
 
  private:
   friend std::ostream& operator<<(std::ostream& oss, Variant const& val);
+  using storage_t = std::aligned_union<8, int, int8_t, int16_t, int64_t,
+                                       uint8_t, uint16_t, uint32_t, uint64_t,
+                                       const char*, float, double, bool,
+                                       int*, float*, double*, bool*, std::string*,
+                                       Array2D<int>, Array2D<float>, Array2D<double>, Array2D<std::string>,
+                                       LabeledArray<int>, LabeledArray<float>, LabeledArray<double>, LabeledArray<std::string>>::type;
   storage_t mStore;
-  VariantType mType;
+  VariantType mType = VariantType::Unknown;
   size_t mSize = 1;
 };
 
diff --git a/Framework/Core/src/Variant.cxx b/Framework/Core/src/Variant.cxx
index 3f71ec339511e..21eb6103aa56a 100644
--- a/Framework/Core/src/Variant.cxx
+++ b/Framework/Core/src/Variant.cxx
@@ -91,27 +91,27 @@ Variant::Variant(const Variant& other) : mType(other.mType)
   switch (mType) {
     case variant_trait_v<const char*>:
       mSize = other.mSize;
-      variant_helper<storage_t, const char*>::set(&mStore, other.get<const char*>());
+      variant_helper<const char*>::set(&mStore, other.get<const char*>());
       return;
     case variant_trait_v<int*>:
       mSize = other.mSize;
-      variant_helper<storage_t, int*>::set(&mStore, other.get<int*>(), mSize);
+      variant_helper<int*>::set(&mStore, other.get<int*>(), mSize);
       return;
     case variant_trait_v<float*>:
       mSize = other.mSize;
-      variant_helper<storage_t, float*>::set(&mStore, other.get<float*>(), mSize);
+      variant_helper<float*>::set(&mStore, other.get<float*>(), mSize);
       return;
     case variant_trait_v<double*>:
       mSize = other.mSize;
-      variant_helper<storage_t, double*>::set(&mStore, other.get<double*>(), mSize);
+      variant_helper<double*>::set(&mStore, other.get<double*>(), mSize);
       return;
     case variant_trait_v<bool*>:
       mSize = other.mSize;
-      variant_helper<storage_t, bool*>::set(&mStore, other.get<bool*>(), mSize);
+      variant_helper<bool*>::set(&mStore, other.get<bool*>(), mSize);
       return;
     case variant_trait_v<std::string*>:
       mSize = other.mSize;
-      variant_helper<storage_t, std::string*>::set(&mStore, other.get<std::string*>(), mSize);
+      variant_helper<std::string*>::set(&mStore, other.get<std::string*>(), mSize);
       return;
     default:
       mStore = other.mStore;
@@ -172,22 +172,22 @@ Variant& Variant::operator=(const Variant& other)
   mType = other.mType;
   switch (mType) {
     case variant_trait_v<const char*>:
-      variant_helper<storage_t, const char*>::set(&mStore, other.get<const char*>());
+      variant_helper<const char*>::set(&mStore, other.get<const char*>());
       return *this;
     case variant_trait_v<int*>:
-      variant_helper<storage_t, int*>::set(&mStore, other.get<int*>(), mSize);
+      variant_helper<int*>::set(&mStore, other.get<int*>(), mSize);
       return *this;
     case variant_trait_v<float*>:
-      variant_helper<storage_t, float*>::set(&mStore, other.get<float*>(), mSize);
+      variant_helper<float*>::set(&mStore, other.get<float*>(), mSize);
       return *this;
     case variant_trait_v<double*>:
-      variant_helper<storage_t, double*>::set(&mStore, other.get<double*>(), mSize);
+      variant_helper<double*>::set(&mStore, other.get<double*>(), mSize);
       return *this;
     case variant_trait_v<bool*>:
-      variant_helper<storage_t, bool*>::set(&mStore, other.get<bool*>(), mSize);
+      variant_helper<bool*>::set(&mStore, other.get<bool*>(), mSize);
       return *this;
     case variant_trait_v<std::string*>:
-      variant_helper<storage_t, std::string*>::set(&mStore, other.get<std::string*>(), mSize);
+      variant_helper<std::string*>::set(&mStore, other.get<std::string*>(), mSize);
       return *this;
     default:
       mStore = other.mStore;
@@ -201,27 +201,27 @@ Variant& Variant::operator=(Variant&& other) noexcept
   mType = other.mType;
   switch (mType) {
     case variant_trait_v<const char*>:
-      variant_helper<storage_t, const char*>::set(&mStore, other.get<const char*>());
+      variant_helper<const char*>::set(&mStore, other.get<const char*>());
       *reinterpret_cast<char**>(&(other.mStore)) = nullptr;
       return *this;
     case variant_trait_v<int*>:
-      variant_helper<storage_t, int*>::set(&mStore, other.get<int*>(), mSize);
+      variant_helper<int*>::set(&mStore, other.get<int*>(), mSize);
       *reinterpret_cast<int**>(&(other.mStore)) = nullptr;
       return *this;
     case variant_trait_v<float*>:
-      variant_helper<storage_t, float*>::set(&mStore, other.get<float*>(), mSize);
+      variant_helper<float*>::set(&mStore, other.get<float*>(), mSize);
       *reinterpret_cast<float**>(&(other.mStore)) = nullptr;
       return *this;
     case variant_trait_v<double*>:
-      variant_helper<storage_t, double*>::set(&mStore, other.get<double*>(), mSize);
+      variant_helper<double*>::set(&mStore, other.get<double*>(), mSize);
       *reinterpret_cast<double**>(&(other.mStore)) = nullptr;
       return *this;
     case variant_trait_v<bool*>:
-      variant_helper<storage_t, bool*>::set(&mStore, other.get<bool*>(), mSize);
+      variant_helper<bool*>::set(&mStore, other.get<bool*>(), mSize);
       *reinterpret_cast<bool**>(&(other.mStore)) = nullptr;
       return *this;
     case variant_trait_v<std::string*>:
-      variant_helper<storage_t, std::string*>::set(&mStore, other.get<std::string*>(), mSize);
+      variant_helper<std::string*>::set(&mStore, other.get<std::string*>(), mSize);
       *reinterpret_cast<std::string**>(&(other.mStore)) = nullptr;
       return *this;
     default:

From 48133602752d44e9e46f52d2a2f4b0e90559009f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 27 Jun 2024 09:26:52 +0200
Subject: [PATCH 548/726] DPL Analyis: make has_type value based, rather than
 type based.

---
 Framework/Core/include/Framework/ASoA.h       |  6 +++---
 .../Core/include/Framework/AnalysisHelpers.h  |  2 +-
 .../Core/include/Framework/AnalysisManagers.h |  2 +-
 .../Core/include/Framework/BinningPolicy.h    |  2 +-
 Framework/Foundation/include/Framework/Pack.h | 20 +++++++------------
 .../test/test_FunctionalHelpers.cxx           |  6 +++---
 6 files changed, 16 insertions(+), 22 deletions(-)

diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h
index 93960c75b11b7..34fa8ce05dbfb 100644
--- a/Framework/Core/include/Framework/ASoA.h
+++ b/Framework/Core/include/Framework/ASoA.h
@@ -1466,7 +1466,7 @@ class Table
     auto getId() const
     {
       using decayed = std::decay_t<TI>;
-      if constexpr (framework::has_type_v<decayed, bindings_pack_t>) { // index to another table
+      if constexpr (framework::has_type<decayed>(bindings_pack_t{})) { // index to another table
         constexpr auto idx = framework::has_type_at_v<decayed>(bindings_pack_t{});
         return framework::pack_element_t<idx, external_index_columns_t>::getId();
       } else if constexpr (std::is_same_v<decayed, Parent>) { // self index
@@ -2777,12 +2777,12 @@ struct Join : TableWrap<Ts...>::table_t {
   }
 
   template <typename T>
-  static constexpr bool contains()
+  static consteval bool contains()
   {
     if constexpr (is_type_with_originals_v<T>) {
       return contains(typename T::originals{});
     } else {
-      return framework::has_type_v<T, originals>;
+      return framework::has_type<T>(originals{});
     }
   }
 
diff --git a/Framework/Core/include/Framework/AnalysisHelpers.h b/Framework/Core/include/Framework/AnalysisHelpers.h
index 65543bd067f1e..f1c0d2e35a2b2 100644
--- a/Framework/Core/include/Framework/AnalysisHelpers.h
+++ b/Framework/Core/include/Framework/AnalysisHelpers.h
@@ -90,7 +90,7 @@ struct WritingCursor<soa::Table<PC...>> {
     if constexpr (soa::is_soa_iterator_v<T>) {
       return arg.globalIndex();
     } else {
-      static_assert(!framework::has_type_v<T, framework::pack<PC...>>, "Argument type mismatch");
+      static_assert(!framework::has_type<T>(framework::pack<PC...>{}), "Argument type mismatch");
       return arg;
     }
   }
diff --git a/Framework/Core/include/Framework/AnalysisManagers.h b/Framework/Core/include/Framework/AnalysisManagers.h
index f341f60392e9f..3183069326d52 100644
--- a/Framework/Core/include/Framework/AnalysisManagers.h
+++ b/Framework/Core/include/Framework/AnalysisManagers.h
@@ -50,7 +50,7 @@ struct GroupedCombinationManager<GroupedCombinationsGenerator<T1, GroupingPolicy
   {
     static_assert(sizeof...(T2s) > 0, "There must be associated tables in process() for a correct pair");
     if constexpr (std::is_same_v<G, TG>) {
-      static_assert(std::conjunction_v<framework::has_type<As, pack<T2s...>>...>, "You didn't subscribed to all tables requested for mixing");
+      static_assert((framework::has_type<As>(pack<T2s...>{}) && ...), "You didn't subscribed to all tables requested for mixing");
       comb.setTables(grouping, associated);
     }
   }
diff --git a/Framework/Core/include/Framework/BinningPolicy.h b/Framework/Core/include/Framework/BinningPolicy.h
index ddc696fbcc3af..1ac2375ea6d4d 100644
--- a/Framework/Core/include/Framework/BinningPolicy.h
+++ b/Framework/Core/include/Framework/BinningPolicy.h
@@ -240,7 +240,7 @@ struct FlexibleBinningPolicy<std::tuple<Ls...>, Ts...> : BinningPolicyBase<sizeo
   template <typename T, typename T2>
   auto getBinningValue(T& rowIterator, arrow::Table* table, uint64_t ci = -1, uint64_t ai = -1, uint64_t globalIndex = -1) const
   {
-    if constexpr (has_type_v<T2, pack<Ls...>>) {
+    if constexpr (has_type<T2>(pack<Ls...>{})) {
       if (globalIndex != -1) {
         rowIterator.setCursor(globalIndex);
       }
diff --git a/Framework/Foundation/include/Framework/Pack.h b/Framework/Foundation/include/Framework/Pack.h
index 7f5d79d761666..c534738ea3524 100644
--- a/Framework/Foundation/include/Framework/Pack.h
+++ b/Framework/Foundation/include/Framework/Pack.h
@@ -175,16 +175,10 @@ void print_pack()
 template <template <typename> typename Condition, typename... Types>
 using filtered_pack = std::decay_t<decltype(filter_pack<Condition>(pack<>{}, pack<Types...>{}))>;
 
-/// Check if a given pack Pack has a type T inside.
-template <typename T, typename Pack>
-struct has_type;
-
-template <typename T, typename... Us>
-struct has_type<T, pack<Us...>> : std::disjunction<std::is_same<T, Us>...> {
-};
-
-template <typename T, typename... Us>
-inline constexpr bool has_type_v = has_type<T, Us...>::value;
+template <typename T, typename...Us>
+bool consteval has_type(framework::pack<Us...>) {
+  return (std::is_same_v<T, Us> || ...);
+}
 
 template <template <typename, typename> typename Condition, typename T, typename Pack>
 struct has_type_conditional;
@@ -207,7 +201,7 @@ constexpr size_t has_type_at(pack<T1, Ts...> const&)
 {
   if constexpr (std::is_same_v<T, T1>) {
     return 0;
-  } else if constexpr (has_type_v<T, pack<Ts...>>) {
+  } else if constexpr (has_type<T>(pack<Ts...>{})) {
     return 1 + has_type_at<T>(pack<Ts...>{});
   }
   return sizeof...(Ts) + 2;
@@ -274,7 +268,7 @@ struct intersect_pack {
   {
     return filtered_pack<std::is_void,
                          std::conditional_t<
-                           has_type_v<pack_element_t<Indices, S1>, S2>,
+                           has_type<pack_element_t<Indices, S1>>(S2{}),
                            pack_element_t<Indices, S1>, void>...>{};
   }
   using type = decltype(make_intersection(std::make_index_sequence<pack_size(S1{})>{}));
@@ -306,7 +300,7 @@ struct subtract_pack {
   {
     return filtered_pack<std::is_void,
                          std::conditional_t<
-                           !has_type_v<pack_element_t<Indices, S1>, S2>,
+                           !has_type<pack_element_t<Indices, S1>>(S2{}),
                            pack_element_t<Indices, S1>, void>...>{};
   }
   using type = decltype(make_subtraction(std::make_index_sequence<pack_size(S1{})>{}));
diff --git a/Framework/Foundation/test/test_FunctionalHelpers.cxx b/Framework/Foundation/test/test_FunctionalHelpers.cxx
index 188d91433cdfd..bc4f39fbfc0eb 100644
--- a/Framework/Foundation/test/test_FunctionalHelpers.cxx
+++ b/Framework/Foundation/test/test_FunctionalHelpers.cxx
@@ -28,8 +28,8 @@ struct TestStruct {
 TEST_CASE("TestOverride")
 {
   static_assert(pack_size(pack<int, float>{}) == 2, "Bad size for the pack");
-  static_assert(has_type_v<int, pack<int, float>> == true, "int should be in the pack");
-  static_assert(has_type_v<double, pack<int, float>> == false, "double should not be in the pack");
+  static_assert(has_type<int>(pack<int, float>{}) == true, "int should be in the pack");
+  static_assert(has_type<double>(pack<int, float>{}) == false, "double should not be in the pack");
   static_assert(has_type_conditional_v<std::is_same, int, pack<int, float>> == true, "int should be in the pack");
   static_assert(has_type_conditional_v<std::is_same, double, pack<int, float>> == false, "double should not be in the pack");
 
@@ -43,7 +43,7 @@ TEST_CASE("TestOverride")
   static_assert(std::is_same_v<selected_pack_multicondition<is_same_as_second_t, pack<int>, pack<int, float, char>>, pack<int>>, "multiselector should select int");
   static_assert(std::is_same_v<filtered_pack<is_int_t, int, float, char>, pack<float, char>>, "filter should remove int");
   static_assert(std::is_same_v<intersected_pack_t<pack<int, float, char>, pack<float, double>>, pack<float>>, "filter intersect two packs");
-  static_assert(has_type_v<pack_element_t<0, pack<int>>, pack<int>> == true, "foo");
+  static_assert(has_type<pack_element_t<0, pack<int>>>(pack<int>{}) == true, "foo");
   print_pack<intersected_pack_t<pack<int>, pack<int>>>();
   print_pack<intersected_pack_t<pack<TestStruct<0, -1>, int>, pack<TestStruct<0, -1>, float>>>();
   static_assert(std::is_same_v<intersected_pack_t<pack<TestStruct<0, -1>, int>, pack<TestStruct<0, -1>, float>>, pack<TestStruct<0, -1>>>, "filter intersect two packs");

From f949cd79526a819585af61b16fb032634d6f36a5 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 27 Jun 2024 11:43:56 +0200
Subject: [PATCH 549/726] DPL Analyis: introduce value based has_type

Using the std::disjunction is recursive in the pack elements,
therefore slow. This version allow to expand the pack without recursive
type creations and seems to be compiled much faster.

We keep the old version around because it's still needed in some places
in O2Physics and therefore we need to do an incremental migration.
---
 Framework/Core/include/Framework/ASoA.h       |  6 +++---
 .../Core/include/Framework/AnalysisHelpers.h  |  2 +-
 .../Core/include/Framework/AnalysisManagers.h |  2 +-
 .../Core/include/Framework/BinningPolicy.h    |  2 +-
 Framework/Foundation/include/Framework/Pack.h | 19 ++++++++-----------
 .../test/test_FunctionalHelpers.cxx           |  6 +++---
 6 files changed, 17 insertions(+), 20 deletions(-)

diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h
index 93960c75b11b7..34fa8ce05dbfb 100644
--- a/Framework/Core/include/Framework/ASoA.h
+++ b/Framework/Core/include/Framework/ASoA.h
@@ -1466,7 +1466,7 @@ class Table
     auto getId() const
     {
       using decayed = std::decay_t<TI>;
-      if constexpr (framework::has_type_v<decayed, bindings_pack_t>) { // index to another table
+      if constexpr (framework::has_type<decayed>(bindings_pack_t{})) { // index to another table
         constexpr auto idx = framework::has_type_at_v<decayed>(bindings_pack_t{});
         return framework::pack_element_t<idx, external_index_columns_t>::getId();
       } else if constexpr (std::is_same_v<decayed, Parent>) { // self index
@@ -2777,12 +2777,12 @@ struct Join : TableWrap<Ts...>::table_t {
   }
 
   template <typename T>
-  static constexpr bool contains()
+  static consteval bool contains()
   {
     if constexpr (is_type_with_originals_v<T>) {
       return contains(typename T::originals{});
     } else {
-      return framework::has_type_v<T, originals>;
+      return framework::has_type<T>(originals{});
     }
   }
 
diff --git a/Framework/Core/include/Framework/AnalysisHelpers.h b/Framework/Core/include/Framework/AnalysisHelpers.h
index 65543bd067f1e..f1c0d2e35a2b2 100644
--- a/Framework/Core/include/Framework/AnalysisHelpers.h
+++ b/Framework/Core/include/Framework/AnalysisHelpers.h
@@ -90,7 +90,7 @@ struct WritingCursor<soa::Table<PC...>> {
     if constexpr (soa::is_soa_iterator_v<T>) {
       return arg.globalIndex();
     } else {
-      static_assert(!framework::has_type_v<T, framework::pack<PC...>>, "Argument type mismatch");
+      static_assert(!framework::has_type<T>(framework::pack<PC...>{}), "Argument type mismatch");
       return arg;
     }
   }
diff --git a/Framework/Core/include/Framework/AnalysisManagers.h b/Framework/Core/include/Framework/AnalysisManagers.h
index f341f60392e9f..3183069326d52 100644
--- a/Framework/Core/include/Framework/AnalysisManagers.h
+++ b/Framework/Core/include/Framework/AnalysisManagers.h
@@ -50,7 +50,7 @@ struct GroupedCombinationManager<GroupedCombinationsGenerator<T1, GroupingPolicy
   {
     static_assert(sizeof...(T2s) > 0, "There must be associated tables in process() for a correct pair");
     if constexpr (std::is_same_v<G, TG>) {
-      static_assert(std::conjunction_v<framework::has_type<As, pack<T2s...>>...>, "You didn't subscribed to all tables requested for mixing");
+      static_assert((framework::has_type<As>(pack<T2s...>{}) && ...), "You didn't subscribed to all tables requested for mixing");
       comb.setTables(grouping, associated);
     }
   }
diff --git a/Framework/Core/include/Framework/BinningPolicy.h b/Framework/Core/include/Framework/BinningPolicy.h
index ddc696fbcc3af..1ac2375ea6d4d 100644
--- a/Framework/Core/include/Framework/BinningPolicy.h
+++ b/Framework/Core/include/Framework/BinningPolicy.h
@@ -240,7 +240,7 @@ struct FlexibleBinningPolicy<std::tuple<Ls...>, Ts...> : BinningPolicyBase<sizeo
   template <typename T, typename T2>
   auto getBinningValue(T& rowIterator, arrow::Table* table, uint64_t ci = -1, uint64_t ai = -1, uint64_t globalIndex = -1) const
   {
-    if constexpr (has_type_v<T2, pack<Ls...>>) {
+    if constexpr (has_type<T2>(pack<Ls...>{})) {
       if (globalIndex != -1) {
         rowIterator.setCursor(globalIndex);
       }
diff --git a/Framework/Foundation/include/Framework/Pack.h b/Framework/Foundation/include/Framework/Pack.h
index 7f5d79d761666..9cc19064a8a0e 100644
--- a/Framework/Foundation/include/Framework/Pack.h
+++ b/Framework/Foundation/include/Framework/Pack.h
@@ -175,16 +175,13 @@ void print_pack()
 template <template <typename> typename Condition, typename... Types>
 using filtered_pack = std::decay_t<decltype(filter_pack<Condition>(pack<>{}, pack<Types...>{}))>;
 
-/// Check if a given pack Pack has a type T inside.
-template <typename T, typename Pack>
-struct has_type;
-
-template <typename T, typename... Us>
-struct has_type<T, pack<Us...>> : std::disjunction<std::is_same<T, Us>...> {
-};
+template <typename T, typename...Us>
+bool consteval has_type(framework::pack<Us...>) {
+  return (std::is_same_v<T, Us> || ...);
+}
 
 template <typename T, typename... Us>
-inline constexpr bool has_type_v = has_type<T, Us...>::value;
+inline constexpr bool has_type_v = has_type<T>(framework::pack<Us...>{});
 
 template <template <typename, typename> typename Condition, typename T, typename Pack>
 struct has_type_conditional;
@@ -207,7 +204,7 @@ constexpr size_t has_type_at(pack<T1, Ts...> const&)
 {
   if constexpr (std::is_same_v<T, T1>) {
     return 0;
-  } else if constexpr (has_type_v<T, pack<Ts...>>) {
+  } else if constexpr (has_type<T>(pack<Ts...>{})) {
     return 1 + has_type_at<T>(pack<Ts...>{});
   }
   return sizeof...(Ts) + 2;
@@ -274,7 +271,7 @@ struct intersect_pack {
   {
     return filtered_pack<std::is_void,
                          std::conditional_t<
-                           has_type_v<pack_element_t<Indices, S1>, S2>,
+                           has_type<pack_element_t<Indices, S1>>(S2{}),
                            pack_element_t<Indices, S1>, void>...>{};
   }
   using type = decltype(make_intersection(std::make_index_sequence<pack_size(S1{})>{}));
@@ -306,7 +303,7 @@ struct subtract_pack {
   {
     return filtered_pack<std::is_void,
                          std::conditional_t<
-                           !has_type_v<pack_element_t<Indices, S1>, S2>,
+                           !has_type<pack_element_t<Indices, S1>>(S2{}),
                            pack_element_t<Indices, S1>, void>...>{};
   }
   using type = decltype(make_subtraction(std::make_index_sequence<pack_size(S1{})>{}));
diff --git a/Framework/Foundation/test/test_FunctionalHelpers.cxx b/Framework/Foundation/test/test_FunctionalHelpers.cxx
index 188d91433cdfd..bc4f39fbfc0eb 100644
--- a/Framework/Foundation/test/test_FunctionalHelpers.cxx
+++ b/Framework/Foundation/test/test_FunctionalHelpers.cxx
@@ -28,8 +28,8 @@ struct TestStruct {
 TEST_CASE("TestOverride")
 {
   static_assert(pack_size(pack<int, float>{}) == 2, "Bad size for the pack");
-  static_assert(has_type_v<int, pack<int, float>> == true, "int should be in the pack");
-  static_assert(has_type_v<double, pack<int, float>> == false, "double should not be in the pack");
+  static_assert(has_type<int>(pack<int, float>{}) == true, "int should be in the pack");
+  static_assert(has_type<double>(pack<int, float>{}) == false, "double should not be in the pack");
   static_assert(has_type_conditional_v<std::is_same, int, pack<int, float>> == true, "int should be in the pack");
   static_assert(has_type_conditional_v<std::is_same, double, pack<int, float>> == false, "double should not be in the pack");
 
@@ -43,7 +43,7 @@ TEST_CASE("TestOverride")
   static_assert(std::is_same_v<selected_pack_multicondition<is_same_as_second_t, pack<int>, pack<int, float, char>>, pack<int>>, "multiselector should select int");
   static_assert(std::is_same_v<filtered_pack<is_int_t, int, float, char>, pack<float, char>>, "filter should remove int");
   static_assert(std::is_same_v<intersected_pack_t<pack<int, float, char>, pack<float, double>>, pack<float>>, "filter intersect two packs");
-  static_assert(has_type_v<pack_element_t<0, pack<int>>, pack<int>> == true, "foo");
+  static_assert(has_type<pack_element_t<0, pack<int>>>(pack<int>{}) == true, "foo");
   print_pack<intersected_pack_t<pack<int>, pack<int>>>();
   print_pack<intersected_pack_t<pack<TestStruct<0, -1>, int>, pack<TestStruct<0, -1>, float>>>();
   static_assert(std::is_same_v<intersected_pack_t<pack<TestStruct<0, -1>, int>, pack<TestStruct<0, -1>, float>>, pack<TestStruct<0, -1>>>, "filter intersect two packs");

From 261c1506812e990d7d63d8f9a0f3073321ecc190 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 28 Jun 2024 16:37:06 +0200
Subject: [PATCH 550/726] DPL Analyis: introduce value based has_type

Using the std::disjunction is recursive in the pack elements,
therefore slow. This version allow to expand the pack without recursive
type creations and seems to be compiled much faster.

We keep the old version around because it's still needed in some places
in O2Physics and therefore we need to do an incremental migration.
---
 Framework/Core/include/Framework/ASoA.h       |  6 +++---
 .../Core/include/Framework/AnalysisHelpers.h  |  2 +-
 .../Core/include/Framework/AnalysisManagers.h |  2 +-
 .../Core/include/Framework/BinningPolicy.h    |  2 +-
 Framework/Foundation/include/Framework/Pack.h | 20 +++++++++----------
 .../test/test_FunctionalHelpers.cxx           |  4 +++-
 6 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h
index 93960c75b11b7..34fa8ce05dbfb 100644
--- a/Framework/Core/include/Framework/ASoA.h
+++ b/Framework/Core/include/Framework/ASoA.h
@@ -1466,7 +1466,7 @@ class Table
     auto getId() const
     {
       using decayed = std::decay_t<TI>;
-      if constexpr (framework::has_type_v<decayed, bindings_pack_t>) { // index to another table
+      if constexpr (framework::has_type<decayed>(bindings_pack_t{})) { // index to another table
         constexpr auto idx = framework::has_type_at_v<decayed>(bindings_pack_t{});
         return framework::pack_element_t<idx, external_index_columns_t>::getId();
       } else if constexpr (std::is_same_v<decayed, Parent>) { // self index
@@ -2777,12 +2777,12 @@ struct Join : TableWrap<Ts...>::table_t {
   }
 
   template <typename T>
-  static constexpr bool contains()
+  static consteval bool contains()
   {
     if constexpr (is_type_with_originals_v<T>) {
       return contains(typename T::originals{});
     } else {
-      return framework::has_type_v<T, originals>;
+      return framework::has_type<T>(originals{});
     }
   }
 
diff --git a/Framework/Core/include/Framework/AnalysisHelpers.h b/Framework/Core/include/Framework/AnalysisHelpers.h
index 65543bd067f1e..f1c0d2e35a2b2 100644
--- a/Framework/Core/include/Framework/AnalysisHelpers.h
+++ b/Framework/Core/include/Framework/AnalysisHelpers.h
@@ -90,7 +90,7 @@ struct WritingCursor<soa::Table<PC...>> {
     if constexpr (soa::is_soa_iterator_v<T>) {
       return arg.globalIndex();
     } else {
-      static_assert(!framework::has_type_v<T, framework::pack<PC...>>, "Argument type mismatch");
+      static_assert(!framework::has_type<T>(framework::pack<PC...>{}), "Argument type mismatch");
       return arg;
     }
   }
diff --git a/Framework/Core/include/Framework/AnalysisManagers.h b/Framework/Core/include/Framework/AnalysisManagers.h
index f341f60392e9f..3183069326d52 100644
--- a/Framework/Core/include/Framework/AnalysisManagers.h
+++ b/Framework/Core/include/Framework/AnalysisManagers.h
@@ -50,7 +50,7 @@ struct GroupedCombinationManager<GroupedCombinationsGenerator<T1, GroupingPolicy
   {
     static_assert(sizeof...(T2s) > 0, "There must be associated tables in process() for a correct pair");
     if constexpr (std::is_same_v<G, TG>) {
-      static_assert(std::conjunction_v<framework::has_type<As, pack<T2s...>>...>, "You didn't subscribed to all tables requested for mixing");
+      static_assert((framework::has_type<As>(pack<T2s...>{}) && ...), "You didn't subscribed to all tables requested for mixing");
       comb.setTables(grouping, associated);
     }
   }
diff --git a/Framework/Core/include/Framework/BinningPolicy.h b/Framework/Core/include/Framework/BinningPolicy.h
index ddc696fbcc3af..1ac2375ea6d4d 100644
--- a/Framework/Core/include/Framework/BinningPolicy.h
+++ b/Framework/Core/include/Framework/BinningPolicy.h
@@ -240,7 +240,7 @@ struct FlexibleBinningPolicy<std::tuple<Ls...>, Ts...> : BinningPolicyBase<sizeo
   template <typename T, typename T2>
   auto getBinningValue(T& rowIterator, arrow::Table* table, uint64_t ci = -1, uint64_t ai = -1, uint64_t globalIndex = -1) const
   {
-    if constexpr (has_type_v<T2, pack<Ls...>>) {
+    if constexpr (has_type<T2>(pack<Ls...>{})) {
       if (globalIndex != -1) {
         rowIterator.setCursor(globalIndex);
       }
diff --git a/Framework/Foundation/include/Framework/Pack.h b/Framework/Foundation/include/Framework/Pack.h
index 7f5d79d761666..f44f08d2bf2f5 100644
--- a/Framework/Foundation/include/Framework/Pack.h
+++ b/Framework/Foundation/include/Framework/Pack.h
@@ -175,16 +175,14 @@ void print_pack()
 template <template <typename> typename Condition, typename... Types>
 using filtered_pack = std::decay_t<decltype(filter_pack<Condition>(pack<>{}, pack<Types...>{}))>;
 
-/// Check if a given pack Pack has a type T inside.
-template <typename T, typename Pack>
-struct has_type;
-
 template <typename T, typename... Us>
-struct has_type<T, pack<Us...>> : std::disjunction<std::is_same<T, Us>...> {
-};
+bool consteval has_type(framework::pack<Us...>)
+{
+  return (std::is_same_v<T, Us> || ...);
+}
 
-template <typename T, typename... Us>
-inline constexpr bool has_type_v = has_type<T, Us...>::value;
+template <typename T, typename P>
+inline constexpr bool has_type_v = has_type<T>(P{});
 
 template <template <typename, typename> typename Condition, typename T, typename Pack>
 struct has_type_conditional;
@@ -207,7 +205,7 @@ constexpr size_t has_type_at(pack<T1, Ts...> const&)
 {
   if constexpr (std::is_same_v<T, T1>) {
     return 0;
-  } else if constexpr (has_type_v<T, pack<Ts...>>) {
+  } else if constexpr (has_type<T>(pack<Ts...>{})) {
     return 1 + has_type_at<T>(pack<Ts...>{});
   }
   return sizeof...(Ts) + 2;
@@ -274,7 +272,7 @@ struct intersect_pack {
   {
     return filtered_pack<std::is_void,
                          std::conditional_t<
-                           has_type_v<pack_element_t<Indices, S1>, S2>,
+                           has_type<pack_element_t<Indices, S1>>(S2{}),
                            pack_element_t<Indices, S1>, void>...>{};
   }
   using type = decltype(make_intersection(std::make_index_sequence<pack_size(S1{})>{}));
@@ -306,7 +304,7 @@ struct subtract_pack {
   {
     return filtered_pack<std::is_void,
                          std::conditional_t<
-                           !has_type_v<pack_element_t<Indices, S1>, S2>,
+                           !has_type<pack_element_t<Indices, S1>>(S2{}),
                            pack_element_t<Indices, S1>, void>...>{};
   }
   using type = decltype(make_subtraction(std::make_index_sequence<pack_size(S1{})>{}));
diff --git a/Framework/Foundation/test/test_FunctionalHelpers.cxx b/Framework/Foundation/test/test_FunctionalHelpers.cxx
index 188d91433cdfd..334da9ac1d5f5 100644
--- a/Framework/Foundation/test/test_FunctionalHelpers.cxx
+++ b/Framework/Foundation/test/test_FunctionalHelpers.cxx
@@ -28,6 +28,8 @@ struct TestStruct {
 TEST_CASE("TestOverride")
 {
   static_assert(pack_size(pack<int, float>{}) == 2, "Bad size for the pack");
+  static_assert(has_type<int>(pack<int, float>{}) == true, "int should be in the pack");
+  static_assert(has_type<double>(pack<int, float>{}) == false, "double should not be in the pack");
   static_assert(has_type_v<int, pack<int, float>> == true, "int should be in the pack");
   static_assert(has_type_v<double, pack<int, float>> == false, "double should not be in the pack");
   static_assert(has_type_conditional_v<std::is_same, int, pack<int, float>> == true, "int should be in the pack");
@@ -43,7 +45,7 @@ TEST_CASE("TestOverride")
   static_assert(std::is_same_v<selected_pack_multicondition<is_same_as_second_t, pack<int>, pack<int, float, char>>, pack<int>>, "multiselector should select int");
   static_assert(std::is_same_v<filtered_pack<is_int_t, int, float, char>, pack<float, char>>, "filter should remove int");
   static_assert(std::is_same_v<intersected_pack_t<pack<int, float, char>, pack<float, double>>, pack<float>>, "filter intersect two packs");
-  static_assert(has_type_v<pack_element_t<0, pack<int>>, pack<int>> == true, "foo");
+  static_assert(has_type<pack_element_t<0, pack<int>>>(pack<int>{}) == true, "foo");
   print_pack<intersected_pack_t<pack<int>, pack<int>>>();
   print_pack<intersected_pack_t<pack<TestStruct<0, -1>, int>, pack<TestStruct<0, -1>, float>>>();
   static_assert(std::is_same_v<intersected_pack_t<pack<TestStruct<0, -1>, int>, pack<TestStruct<0, -1>, float>>, pack<TestStruct<0, -1>>>, "filter intersect two packs");

From 594c6bab72b70f7428f0e6137cd8f54b8f776466 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 28 Jun 2024 21:34:12 +0200
Subject: [PATCH 551/726] Always test with address sanitizer a few base classes

---
 Framework/Core/CMakeLists.txt | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index 4381b6c4fa29d..423765a88f613 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -178,6 +178,17 @@ o2_target_root_dictionary(Framework
                           include/Framework/StepTHn.h
                           LINKDEF test/FrameworkCoreTestLinkDef.h)
 
+add_executable(o2-test-framework-core-sanitized
+               test/test_Variants.cxx
+               src/Array2D.cxx
+               src/Variant.cxx
+               )
+
+target_link_libraries(o2-test-framework-core-sanitized PRIVATE O2::Catch2 O2::FrameworkFoundation Boost::boost RapidJSON::RapidJSON)
+target_include_directories(o2-test-framework-core-sanitized PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include O2::FrameworkFoundation)
+target_compile_options(o2-test-framework-core-sanitized PRIVATE -fsanitize=address)
+target_link_options(o2-test-framework-core-sanitized PRIVATE -fsanitize=address)
+
 add_executable(o2-test-framework-core
               test/test_AlgorithmSpec.cxx
               test/test_AnalysisTask.cxx
@@ -251,8 +262,10 @@ target_link_libraries(o2-test-framework-core PRIVATE O2::Catch2)
 
 get_filename_component(outdir ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/../tests ABSOLUTE)
 set_property(TARGET o2-test-framework-core PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
+set_property(TARGET o2-test-framework-core-sanitized PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
 
 add_test(NAME framework:core COMMAND o2-test-framework-core --skip-benchmarks)
+add_test(NAME framework:sanitized COMMAND o2-test-framework-core-sanitized --skip-benchmarks)
 
 o2_add_test(AlgorithmWrapper NAME test_Framework_test_AlgorithmWrapper
             SOURCES test/test_AlgorithmWrapper.cxx

From 6fb944192389b1982c6fad42fc5ecad92db477c8 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 28 Jun 2024 22:03:01 +0200
Subject: [PATCH 552/726] Explicitly find AliceGridUtils

---
 dependencies/O2Dependencies.cmake | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/dependencies/O2Dependencies.cmake b/dependencies/O2Dependencies.cmake
index 53cd7167e0e41..9f35a89449601 100644
--- a/dependencies/O2Dependencies.cmake
+++ b/dependencies/O2Dependencies.cmake
@@ -111,6 +111,12 @@ set_package_properties(CURL PROPERTIES TYPE REQUIRED)
 find_package(TBB)
 set_package_properties(TBB PROPERTIES TYPE REQUIRED)
 
+# The Ifdef is to avoid merging at the same time alidist and AliceO2 PRs.
+if (ALICE_GRID_UTILS_INCLUDE_DIR)
+find_package(AliceGridUtils MODULE)
+set_package_properties(AliceGridUtils PROPERTIES TYPE RECOMMENDED)
+endif()
+
 find_package(JAliEnROOT MODULE)
 set_package_properties(JAliEnROOT PROPERTIES TYPE RECOMMENDED)
 

From 765a0bc56c68cafea899f38bb73a1e124170878e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 29 Jun 2024 15:13:22 +0200
Subject: [PATCH 553/726] Get rid of one more disjunction

---
 Framework/Core/include/Framework/ASoA.h       |  2 +-
 Framework/Foundation/include/Framework/Pack.h | 13 ++++++-------
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h
index 34fa8ce05dbfb..41fae3087fbbe 100644
--- a/Framework/Core/include/Framework/ASoA.h
+++ b/Framework/Core/include/Framework/ASoA.h
@@ -1567,7 +1567,7 @@ class Table
   template <typename Key>
   inline arrow::ChunkedArray* getIndexToKey()
   {
-    if constexpr (framework::has_type_conditional_v<is_binding_compatible, Key, external_index_columns_t>) {
+    if constexpr (framework::has_type_conditional<is_binding_compatible, Key>(external_index_columns_t{})) {
       using IC = framework::pack_element_t<framework::has_type_at_conditional<is_binding_compatible, Key>(external_index_columns_t{}), external_index_columns_t>;
       return mColumnChunks[framework::has_type_at<IC>(persistent_columns_t{})];
     } else if constexpr (std::is_same_v<table_t, Key>) {
diff --git a/Framework/Foundation/include/Framework/Pack.h b/Framework/Foundation/include/Framework/Pack.h
index f44f08d2bf2f5..e0593a6175b3a 100644
--- a/Framework/Foundation/include/Framework/Pack.h
+++ b/Framework/Foundation/include/Framework/Pack.h
@@ -184,15 +184,14 @@ bool consteval has_type(framework::pack<Us...>)
 template <typename T, typename P>
 inline constexpr bool has_type_v = has_type<T>(P{});
 
-template <template <typename, typename> typename Condition, typename T, typename Pack>
-struct has_type_conditional;
-
 template <template <typename, typename> typename Condition, typename T, typename... Us>
-struct has_type_conditional<Condition, T, pack<Us...>> : std::disjunction<Condition<T, Us>...> {
-};
+bool consteval has_type_conditional(framework::pack<Us...>)
+{
+  return (Condition<T, Us>::value || ...);
+}
 
-template <template <typename, typename> typename Condition, typename T, typename... Us>
-inline constexpr bool has_type_conditional_v = has_type_conditional<Condition, T, Us...>::value;
+template <template <typename, typename> typename Condition, typename T, typename P>
+inline constexpr bool has_type_conditional_v = has_type_conditional<Condition, T>(P{});
 
 template <typename T>
 constexpr size_t has_type_at(pack<> const&)

From 5d2f524e667a1ac23faab849e07d77366d76ac89 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 29 Jun 2024 15:13:22 +0200
Subject: [PATCH 554/726] DPL Analysis: mark a few functions consteval

---
 Framework/Core/include/Framework/ASoA.h | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h
index 41fae3087fbbe..4273c79a57eed 100644
--- a/Framework/Core/include/Framework/ASoA.h
+++ b/Framework/Core/include/Framework/ASoA.h
@@ -1081,25 +1081,25 @@ static constexpr std::string getLabelFromTypeForKey(std::string const& key)
 }
 
 template <typename B, typename... C>
-constexpr static bool hasIndexTo(framework::pack<C...>&&)
+consteval static bool hasIndexTo(framework::pack<C...>&&)
 {
   return (o2::soa::is_binding_compatible_v<B, typename C::binding_t>() || ...);
 }
 
 template <typename B, typename... C>
-constexpr static bool hasSortedIndexTo(framework::pack<C...>&&)
+consteval static bool hasSortedIndexTo(framework::pack<C...>&&)
 {
   return ((C::sorted && o2::soa::is_binding_compatible_v<B, typename C::binding_t>()) || ...);
 }
 
 template <typename B, typename Z>
-constexpr static bool relatedByIndex()
+consteval static bool relatedByIndex()
 {
   return hasIndexTo<B>(typename Z::table_t::external_index_columns_t{});
 }
 
 template <typename B, typename Z>
-constexpr static bool relatedBySortedIndex()
+consteval static bool relatedBySortedIndex()
 {
   return hasSortedIndexTo<B>(typename Z::table_t::external_index_columns_t{});
 }
@@ -1187,7 +1187,7 @@ template <typename T>
 inline constexpr bool is_soa_iterator_v = framework::is_base_of_template_v<RowViewCore, T> || framework::is_specialization_v<T, RowViewCore>;
 
 template <typename T>
-inline constexpr bool is_soa_filtered_iterator_v()
+inline consteval bool is_soa_filtered_iterator_v()
 {
   if constexpr (!is_soa_iterator_v<T>) {
     return false;
@@ -1216,7 +1216,7 @@ inline constexpr bool is_soa_filtered_v = framework::is_base_of_template_v<soa::
 
 /// Helper function to extract bound indices
 template <typename... Is>
-static constexpr auto extractBindings(framework::pack<Is...>)
+static consteval auto extractBindings(framework::pack<Is...>)
 {
   return framework::pack<typename Is::binding_t...>{};
 }

From c437054dfdbc3ce16bfc5acebc71e250cfdc91c9 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 29 Jun 2024 15:15:15 +0200
Subject: [PATCH 555/726] DPL Analysis: speedup prune_voids_pack

---
 Framework/Foundation/include/Framework/Pack.h | 41 ++++++++++++++++---
 .../test/test_FunctionalHelpers.cxx           |  3 ++
 2 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Pack.h b/Framework/Foundation/include/Framework/Pack.h
index e0593a6175b3a..89555ee45f20c 100644
--- a/Framework/Foundation/include/Framework/Pack.h
+++ b/Framework/Foundation/include/Framework/Pack.h
@@ -115,14 +115,45 @@ consteval auto prune_voids_pack(Result result, pack<>)
   return result;
 }
 
+// The first one is non void, but one of the others is void
 template <typename... Rs, typename T, typename... Ts>
+  requires(!std::is_void_v<T>)
 consteval auto prune_voids_pack(pack<Rs...> result, pack<T, Ts...>)
 {
-  if constexpr (std::is_void_v<T> == false) {
-    return prune_voids_pack(pack<Rs..., T>{}, pack<Ts...>{});
-  } else {
-    return prune_voids_pack(pack<Rs...>{}, pack<Ts...>{});
-  }
+  return prune_voids_pack(pack<Rs..., T>{}, pack<Ts...>{});
+}
+
+// The first one is void
+template <typename... Rs, typename V, typename... Ts>
+  requires(std::is_void_v<V>)
+consteval auto prune_voids_pack(pack<Rs...> result, pack<V, Ts...>)
+{
+  return prune_voids_pack(pack<Rs...>{}, pack<Ts...>{});
+}
+
+// The first one is non void, but one of the others is void
+template <typename... Rs, typename T1, typename T2, typename... Ts>
+  requires(!std::is_void_v<T1> && !std::is_void_v<T2>)
+consteval auto prune_voids_pack(pack<Rs...> result, pack<T1, T2, Ts...>)
+{
+  return prune_voids_pack(pack<Rs..., T1, T2>{}, pack<Ts...>{});
+}
+
+// Eats 4 types at the time
+template <typename... Rs, typename T1, typename T2, typename T3, typename T4, typename... Ts>
+  requires(!std::is_void_v<T1> && !std::is_void_v<T2> && !std::is_void_v<T3> && !std::is_void_v<T4>)
+consteval auto prune_voids_pack(pack<Rs...> result, pack<T1, T2, T3, T4, Ts...>)
+{
+  return prune_voids_pack(pack<Rs..., T1, T2, T3, T4>{}, pack<Ts...>{});
+}
+
+// Eats 8 types at the time
+template <typename... Rs, typename T1, typename T2, typename T3, typename T4,
+          typename T5, typename T6, typename T7, typename T8, typename... Ts>
+  requires(!std::is_void_v<T1> && !std::is_void_v<T2> && !std::is_void_v<T3> && !std::is_void_v<T4> && !std::is_void_v<T5> && !std::is_void_v<T6> && !std::is_void_v<T7> && !std::is_void_v<T8>)
+consteval auto prune_voids_pack(pack<Rs...> result, pack<T1, T2, T3, T4, T5, T6, T7, T8, Ts...>)
+{
+  return prune_voids_pack(pack<Rs..., T1, T2, T3, T4, T5, T6, T7, T8>{}, pack<Ts...>{});
 }
 
 /// Selects from the pack types that satisfy the Condition
diff --git a/Framework/Foundation/test/test_FunctionalHelpers.cxx b/Framework/Foundation/test/test_FunctionalHelpers.cxx
index 334da9ac1d5f5..895133ae6428d 100644
--- a/Framework/Foundation/test/test_FunctionalHelpers.cxx
+++ b/Framework/Foundation/test/test_FunctionalHelpers.cxx
@@ -32,6 +32,7 @@ TEST_CASE("TestOverride")
   static_assert(has_type<double>(pack<int, float>{}) == false, "double should not be in the pack");
   static_assert(has_type_v<int, pack<int, float>> == true, "int should be in the pack");
   static_assert(has_type_v<double, pack<int, float>> == false, "double should not be in the pack");
+  static_assert(std::is_same_v<decltype(prune_voids_pack({}, pack<int, int, int, int, int, int, int, int, void>{})), pack<int, int, int, int, int, int, int, int>>, "prune last void");
   static_assert(has_type_conditional_v<std::is_same, int, pack<int, float>> == true, "int should be in the pack");
   static_assert(has_type_conditional_v<std::is_same, double, pack<int, float>> == false, "double should not be in the pack");
 
@@ -42,6 +43,8 @@ TEST_CASE("TestOverride")
   static_assert(has_type_at_conditional<std::is_same, bool>(pack<int, float, double>()) == 3 + 1, "bool is not in the pack so the function returns size + 1");
 
   static_assert(std::is_same_v<selected_pack<is_int_t, int, float, char>, pack<int>>, "selector should select int");
+  static_assert(std::is_same_v<selected_pack<is_int_t, int, int, float, char>, pack<int, int>>, "selector should select int");
+  static_assert(std::is_same_v<selected_pack<is_int_t, float, int, int, float, char>, pack<int, int>>, "selector should select int");
   static_assert(std::is_same_v<selected_pack_multicondition<is_same_as_second_t, pack<int>, pack<int, float, char>>, pack<int>>, "multiselector should select int");
   static_assert(std::is_same_v<filtered_pack<is_int_t, int, float, char>, pack<float, char>>, "filter should remove int");
   static_assert(std::is_same_v<intersected_pack_t<pack<int, float, char>, pack<float, double>>, pack<float>>, "filter intersect two packs");

From 6c2e2858f8c051b850c5f4747516ae60f7980014 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 1 Jul 2024 09:17:22 +0200
Subject: [PATCH 556/726] DPL: properly handle getting non-existing bindinds

The current code only works because the template specialization happens
late enough.

This might or might not be the case, depending for example if one decides to
force a given table's code out-of-line.
---
 Framework/Core/include/Framework/ASoA.h | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h
index 4273c79a57eed..fc5c9cac4a2e5 100644
--- a/Framework/Core/include/Framework/ASoA.h
+++ b/Framework/Core/include/Framework/ASoA.h
@@ -781,6 +781,11 @@ struct ColumnDataHolder {
   arrow::ChunkedArray* second;
 };
 
+template<typename T, typename B>
+concept HasBinding = requires(T& t) {
+    { t.B::mColumnIterator };
+};
+
 template <typename IP, typename... C>
 struct RowViewCore : public IP, C... {
  public:
@@ -964,7 +969,23 @@ struct RowViewCore : public IP, C... {
   template <typename DC, typename... B>
   auto bindDynamicColumn(framework::pack<B...>)
   {
-    DC::boundIterators = std::make_tuple(&(B::mColumnIterator)...);
+    DC::boundIterators = std::make_tuple(getDynamicBinding<B>()...);
+  }
+
+  // Sometimes dynamic columns are defined for tables in
+  // the hope that it will be joined / extended with another one which provides
+  // the full set of bindings. This is to avoid a compilation
+  // error if constructor for the table or any other thing involving a missing 
+  // binding is preinstanciated.
+  template <typename B>
+    requires (HasBinding<table_t, B>)
+  auto getDynamicBinding() {
+    return &B::mColumnIterator;
+  }
+
+  template <typename B>
+  auto getDynamicBinding() {
+    return nullptr;
   }
 };
 

From 86847032327600759c9ba5ad61dab41ccde63458 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 1 Jul 2024 09:36:02 +0200
Subject: [PATCH 557/726] DPL: properly handle getting non-existing bindinds

The current code only works because the template specialization happens
late enough.

This might or might not be the case, depending for example if one decides to
force a given table's code out-of-line.
---
 Framework/Core/include/Framework/ASoA.h | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h
index 4273c79a57eed..d2de55b0b10bc 100644
--- a/Framework/Core/include/Framework/ASoA.h
+++ b/Framework/Core/include/Framework/ASoA.h
@@ -781,6 +781,11 @@ struct ColumnDataHolder {
   arrow::ChunkedArray* second;
 };
 
+template <typename T, typename B>
+concept HasBinding = requires(T& t) {
+  { t.B::mColumnIterator };
+};
+
 template <typename IP, typename... C>
 struct RowViewCore : public IP, C... {
  public:
@@ -964,7 +969,25 @@ struct RowViewCore : public IP, C... {
   template <typename DC, typename... B>
   auto bindDynamicColumn(framework::pack<B...>)
   {
-    DC::boundIterators = std::make_tuple(&(B::mColumnIterator)...);
+    DC::boundIterators = std::make_tuple(getDynamicBinding<B>()...);
+  }
+
+  // Sometimes dynamic columns are defined for tables in
+  // the hope that it will be joined / extended with another one which provides
+  // the full set of bindings. This is to avoid a compilation
+  // error if constructor for the table or any other thing involving a missing
+  // binding is preinstanciated.
+  template <typename B>
+    requires(HasBinding<table_t, B>)
+  auto getDynamicBinding()
+  {
+    return &B::mColumnIterator;
+  }
+
+  template <typename B>
+  auto getDynamicBinding()
+  {
+    return nullptr;
   }
 };
 

From d72f22afcc6c7ad8c6318701a6d8c5a4e2461557 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 1 Jul 2024 12:45:57 +0200
Subject: [PATCH 558/726] DPL: properly handle getting non-existing bindinds

The current code only works because the template specialization happens
late enough.

This might or might not be the case, depending for example if one decides to
force a given table's code out-of-line.
---
 Framework/Core/include/Framework/ASoA.h | 27 ++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h
index 4273c79a57eed..fa9888a698790 100644
--- a/Framework/Core/include/Framework/ASoA.h
+++ b/Framework/Core/include/Framework/ASoA.h
@@ -781,6 +781,11 @@ struct ColumnDataHolder {
   arrow::ChunkedArray* second;
 };
 
+template <typename T, typename B>
+concept CanBind = requires(T&& t) {
+  { t.B::mColumnIterator };
+};
+
 template <typename IP, typename... C>
 struct RowViewCore : public IP, C... {
  public:
@@ -964,7 +969,27 @@ struct RowViewCore : public IP, C... {
   template <typename DC, typename... B>
   auto bindDynamicColumn(framework::pack<B...>)
   {
-    DC::boundIterators = std::make_tuple(&(B::mColumnIterator)...);
+    DC::boundIterators = std::make_tuple(getDynamicBinding<B>()...);
+  }
+
+  // Sometimes dynamic columns are defined for tables in
+  // the hope that it will be joined / extended with another one which provides
+  // the full set of bindings. This is to avoid a compilation
+  // error if constructor for the table or any other thing involving a missing
+  // binding is preinstanciated.
+  template <typename B>
+    requires(CanBind<typename table_t::iterator, B>)
+  decltype(auto) getDynamicBinding()
+  {
+    static_assert(std::is_same_v<decltype(&(static_cast<B*>(this)->mColumnIterator)), std::decay_t<decltype(B::mColumnIterator)>*>, "foo");
+    return &(static_cast<B*>(this)->mColumnIterator);
+    //return static_cast<std::decay_t<decltype(B::mColumnIterator)>*>(nullptr);
+  }
+
+  template <typename B>
+  decltype(auto) getDynamicBinding()
+  {
+    return static_cast<std::decay_t<decltype(B::mColumnIterator)>*>(nullptr);
   }
 };
 

From 99affa8a22c44bfc9e99bd91c1f39b9d601e2dfc Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 1 Jul 2024 20:44:40 +0200
Subject: [PATCH 559/726] DPL Analysis: add support for ProducesGroup

---
 .../Core/include/Framework/AnalysisHelpers.h  | 11 ++++++++
 .../Core/include/Framework/AnalysisManagers.h | 26 +++++++++++++++----
 2 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/Framework/Core/include/Framework/AnalysisHelpers.h b/Framework/Core/include/Framework/AnalysisHelpers.h
index f1c0d2e35a2b2..d23183a340216 100644
--- a/Framework/Core/include/Framework/AnalysisHelpers.h
+++ b/Framework/Core/include/Framework/AnalysisHelpers.h
@@ -131,6 +131,17 @@ template <template <typename...> class T, typename... C>
 struct Produces<T<C...>> : WritingCursor<typename soa::PackToTable<typename T<C...>::table_t::persistent_columns_t>::table> {
 };
 
+/// Use this to group together produces. Useful to separate them logically
+/// or simply to stay within the 100 elements per Task limit.
+/// Use as:
+///
+/// struct MySetOfProduces : ProducesGroup {
+/// } products;
+///
+/// Notice the label MySetOfProduces is just a mnemonic and can be omitted.
+struct ProducesGroup {
+};
+
 /// Helper template for table transformations
 template <typename METADATA>
 struct TableTransform {
diff --git a/Framework/Core/include/Framework/AnalysisManagers.h b/Framework/Core/include/Framework/AnalysisManagers.h
index 3183069326d52..7b6dcd0f3d5fd 100644
--- a/Framework/Core/include/Framework/AnalysisManagers.h
+++ b/Framework/Core/include/Framework/AnalysisManagers.h
@@ -198,30 +198,46 @@ struct ConditionManager<Condition<OBJ>> {
   }
 };
 
-/// SFINAE placeholder
+/// SFINAE placeholder, also handles recursion in ProcessGroup
 template <typename T>
 struct OutputManager {
   template <typename ANY>
-  static bool appendOutput(std::vector<OutputSpec>&, ANY&, uint32_t)
+  static bool appendOutput(std::vector<OutputSpec>& outputs, ANY& what, uint32_t v)
   {
+    if constexpr (std::is_base_of_v<ProducesGroup, ANY>) {
+      homogeneous_apply_refs<true>([&outputs, v](auto& p) { return OutputManager<std::decay_t<decltype(p)>>::appendOption(outputs, p, v); }, what);
+      return true;
+    }
     return false;
   }
 
   template <typename ANY>
-  static bool prepare(ProcessingContext&, ANY&)
+  static bool prepare(ProcessingContext& context, ANY& what)
   {
+    if constexpr (std::is_base_of_v<ProducesGroup, ANY>) {
+      homogeneous_apply_refs<true>([&context](auto& p) { return OutputManager<std::decay_t<decltype(p)>>::prepare(context, p); }, what);
+      return true;
+    }
     return false;
   }
 
   template <typename ANY>
-  static bool postRun(EndOfStreamContext&, ANY&)
+  static bool postRun(EndOfStreamContext& context, ANY& what)
   {
+    if constexpr (std::is_base_of_v<ProducesGroup, ANY>) {
+      homogeneous_apply_refs<true>([&context](auto& p) { return OutputManager<std::decay_t<decltype(p)>>::postRun(context, p); }, what);
+      return true;
+    }
     return true;
   }
 
   template <typename ANY>
-  static bool finalize(ProcessingContext&, ANY&)
+  static bool finalize(ProcessingContext& context, ANY& what)
   {
+    if constexpr (std::is_base_of_v<ProducesGroup, ANY>) {
+      homogeneous_apply_refs<true>([&context](auto& p) { return OutputManager<std::decay_t<decltype(p)>>::finalize(context, p); }, what);
+      return true;
+    }
     return true;
   }
 };

From 3f18b3ce18e1ae2e5e65354fde71e89ba1930419 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 1 Jul 2024 21:36:58 +0200
Subject: [PATCH 560/726] DPL Analysis: add support for ProducesGroup

---
 .../Core/include/Framework/AnalysisHelpers.h  | 11 ++++++++
 .../Core/include/Framework/AnalysisManagers.h | 26 +++++++++++++++----
 2 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/Framework/Core/include/Framework/AnalysisHelpers.h b/Framework/Core/include/Framework/AnalysisHelpers.h
index f1c0d2e35a2b2..d23183a340216 100644
--- a/Framework/Core/include/Framework/AnalysisHelpers.h
+++ b/Framework/Core/include/Framework/AnalysisHelpers.h
@@ -131,6 +131,17 @@ template <template <typename...> class T, typename... C>
 struct Produces<T<C...>> : WritingCursor<typename soa::PackToTable<typename T<C...>::table_t::persistent_columns_t>::table> {
 };
 
+/// Use this to group together produces. Useful to separate them logically
+/// or simply to stay within the 100 elements per Task limit.
+/// Use as:
+///
+/// struct MySetOfProduces : ProducesGroup {
+/// } products;
+///
+/// Notice the label MySetOfProduces is just a mnemonic and can be omitted.
+struct ProducesGroup {
+};
+
 /// Helper template for table transformations
 template <typename METADATA>
 struct TableTransform {
diff --git a/Framework/Core/include/Framework/AnalysisManagers.h b/Framework/Core/include/Framework/AnalysisManagers.h
index 3183069326d52..e1cdbbeb138d1 100644
--- a/Framework/Core/include/Framework/AnalysisManagers.h
+++ b/Framework/Core/include/Framework/AnalysisManagers.h
@@ -198,30 +198,46 @@ struct ConditionManager<Condition<OBJ>> {
   }
 };
 
-/// SFINAE placeholder
+/// SFINAE placeholder, also handles recursion in ProcessGroup
 template <typename T>
 struct OutputManager {
   template <typename ANY>
-  static bool appendOutput(std::vector<OutputSpec>&, ANY&, uint32_t)
+  static bool appendOutput(std::vector<OutputSpec>& outputs, ANY& what, uint32_t v)
   {
+    if constexpr (std::is_base_of_v<ProducesGroup, ANY>) {
+      homogeneous_apply_refs<true>([&outputs, v](auto& p) { return OutputManager<std::decay_t<decltype(p)>>::appendOutput(outputs, p, v); }, what);
+      return true;
+    }
     return false;
   }
 
   template <typename ANY>
-  static bool prepare(ProcessingContext&, ANY&)
+  static bool prepare(ProcessingContext& context, ANY& what)
   {
+    if constexpr (std::is_base_of_v<ProducesGroup, ANY>) {
+      homogeneous_apply_refs<true>([&context](auto& p) { return OutputManager<std::decay_t<decltype(p)>>::prepare(context, p); }, what);
+      return true;
+    }
     return false;
   }
 
   template <typename ANY>
-  static bool postRun(EndOfStreamContext&, ANY&)
+  static bool postRun(EndOfStreamContext& context, ANY& what)
   {
+    if constexpr (std::is_base_of_v<ProducesGroup, ANY>) {
+      homogeneous_apply_refs<true>([&context](auto& p) { return OutputManager<std::decay_t<decltype(p)>>::postRun(context, p); }, what);
+      return true;
+    }
     return true;
   }
 
   template <typename ANY>
-  static bool finalize(ProcessingContext&, ANY&)
+  static bool finalize(ProcessingContext& context, ANY& what)
   {
+    if constexpr (std::is_base_of_v<ProducesGroup, ANY>) {
+      homogeneous_apply_refs<true>([&context](auto& p) { return OutputManager<std::decay_t<decltype(p)>>::finalize(context, p); }, what);
+      return true;
+    }
     return true;
   }
 };

From 3bc81da6afd26e63946f6789b8d293c6bbff3db3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 2 Jul 2024 16:54:31 +0200
Subject: [PATCH 561/726] DPL: improve detection of first device in chain

Only rely on the stdin state to fetch configuration.
---
 Framework/Core/src/runDataProcessing.cxx | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 89737483fc813..f20b53e547c8d 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -2329,15 +2329,25 @@ bool isOutputToPipe()
 
 bool isInputConfig()
 {
+  O2_SIGNPOST_ID_GENERATE(sid, driver);
+  O2_SIGNPOST_START(driver, sid, "isInputConfig", "Checking if stdin is a configuration file");
   struct stat s;
   int r = fstat(STDIN_FILENO, &s);
   // If stdin cannot be statted, we assume the shell is some sort of
   // non-interactive container thing
   if (r < 0) {
+    O2_SIGNPOST_END(driver, sid, "isInputConfig", "Could not open stdin for reading.");
     return false;
   }
   // If stdin is a pipe or a file, we try to fetch configuration from there
-  return ((s.st_mode & S_IFIFO) != 0 || (s.st_mode & S_IFREG) != 0);
+  bool isPipe = ((s.st_mode & S_IFIFO) != 0);
+  bool isStandardFile = ((s.st_mode & S_IFREG) != 0);
+  if (isPipe || isStandardFile) {
+    O2_SIGNPOST_END(driver, sid, "isInputConfig", "stdin is a %{public}s. Reading config file from it.", isPipe ? "pipe" : "file");
+    return true;
+  }
+  O2_SIGNPOST_END(driver, sid, "isInputConfig", "stdin is neither a pipe nor a file. We are first workflow a chain.");
+  return false;
 }
 
 void overrideCloning(ConfigContext& ctx, WorkflowSpec& workflow)
@@ -2829,7 +2839,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
   std::vector<DataProcessorInfo> dataProcessorInfos;
   CommandInfo commandInfo{};
 
-  if (isatty(STDIN_FILENO) == false && isInputConfig()) {
+  if (isInputConfig()) {
     std::vector<DataProcessorSpec> importedWorkflow;
     bool previousWorked = WorkflowSerializationHelpers::import(std::cin, importedWorkflow, dataProcessorInfos, commandInfo);
     if (previousWorked == false) {

From 5722fdc494178ab0c0f307a46277b4e1fd8380fe Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 4 Jul 2024 09:34:00 +0200
Subject: [PATCH 562/726] ARM Support: use correct _MAX type

---
 Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx b/Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx
index 69d6cc1d53c59..46eb1642c3f7c 100644
--- a/Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx
+++ b/Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx
@@ -50,7 +50,7 @@ BOOST_AUTO_TEST_CASE(TRUDataHandler_test)
   }
 
   // test error handling
-  for (int8_t index = o2::emcal::TriggerMappingV2::PATCHESINTRU; index < CHAR_MAX; index++) {
+  for (int8_t index = o2::emcal::TriggerMappingV2::PATCHESINTRU; index < SCHAR_MAX; index++) {
     BOOST_CHECK_EXCEPTION(testhandler.hasPatch(index), o2::emcal::TRUDataHandler::PatchIndexException, [index](const o2::emcal::TRUDataHandler::PatchIndexException& e) { return e.getIndex() == index; });
     BOOST_CHECK_EXCEPTION(testhandler.setPatch(index, 8), o2::emcal::TRUDataHandler::PatchIndexException, [index](const o2::emcal::TRUDataHandler::PatchIndexException& e) { return e.getIndex() == index; });
   }
@@ -89,4 +89,4 @@ BOOST_AUTO_TEST_CASE(TRUDataHandler_test)
 
 } // namespace emcal
 
-} // namespace o2
\ No newline at end of file
+} // namespace o2

From 0ad5dc925de461bfb4604936f57233f74fcf991b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 4 Jul 2024 09:44:36 +0200
Subject: [PATCH 563/726] ARM Support: use correct _MAX type

---
 Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx b/Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx
index 69d6cc1d53c59..5b51228e2417b 100644
--- a/Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx
+++ b/Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx
@@ -50,7 +50,7 @@ BOOST_AUTO_TEST_CASE(TRUDataHandler_test)
   }
 
   // test error handling
-  for (int8_t index = o2::emcal::TriggerMappingV2::PATCHESINTRU; index < CHAR_MAX; index++) {
+  for (int8_t index = o2::emcal::TriggerMappingV2::PATCHESINTRU; index < INT8_MAX; index++) {
     BOOST_CHECK_EXCEPTION(testhandler.hasPatch(index), o2::emcal::TRUDataHandler::PatchIndexException, [index](const o2::emcal::TRUDataHandler::PatchIndexException& e) { return e.getIndex() == index; });
     BOOST_CHECK_EXCEPTION(testhandler.setPatch(index, 8), o2::emcal::TRUDataHandler::PatchIndexException, [index](const o2::emcal::TRUDataHandler::PatchIndexException& e) { return e.getIndex() == index; });
   }
@@ -89,4 +89,4 @@ BOOST_AUTO_TEST_CASE(TRUDataHandler_test)
 
 } // namespace emcal
 
-} // namespace o2
\ No newline at end of file
+} // namespace o2

From 75eddee342f3240fdac8a6dc3be06c9eddc34da4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 4 Jul 2024 09:45:20 +0200
Subject: [PATCH 564/726] ARM Support: use correct _MAX type

---
 Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx b/Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx
index 69d6cc1d53c59..5bab7bf04cc1f 100644
--- a/Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx
+++ b/Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx
@@ -50,7 +50,7 @@ BOOST_AUTO_TEST_CASE(TRUDataHandler_test)
   }
 
   // test error handling
-  for (int8_t index = o2::emcal::TriggerMappingV2::PATCHESINTRU; index < CHAR_MAX; index++) {
+  for (int8_t index = o2::emcal::TriggerMappingV2::PATCHESINTRU; index < INT8_MAX; index++) {
     BOOST_CHECK_EXCEPTION(testhandler.hasPatch(index), o2::emcal::TRUDataHandler::PatchIndexException, [index](const o2::emcal::TRUDataHandler::PatchIndexException& e) { return e.getIndex() == index; });
     BOOST_CHECK_EXCEPTION(testhandler.setPatch(index, 8), o2::emcal::TRUDataHandler::PatchIndexException, [index](const o2::emcal::TRUDataHandler::PatchIndexException& e) { return e.getIndex() == index; });
   }

From 27b24e0794db9843d31cd6202069f093fd175264 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 5 Jul 2024 12:32:39 +0200
Subject: [PATCH 565/726] Attempt at fixing errors with the new ROOT

---
 Detectors/TPC/monitor/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Detectors/TPC/monitor/CMakeLists.txt b/Detectors/TPC/monitor/CMakeLists.txt
index 152eba83d45cd..a37f6a7dfd6f8 100644
--- a/Detectors/TPC/monitor/CMakeLists.txt
+++ b/Detectors/TPC/monitor/CMakeLists.txt
@@ -28,4 +28,5 @@ o2_add_test_root_macro(macro/RunCompareMode3.C
                        LABELS tpc)
 o2_add_test_root_macro(macro/RunFindAdcError.C
                        PUBLIC_LINK_LIBRARIES O2::TPCReconstruction
+                       COMPILE_ONLY
                        LABELS tpc)

From 13da664cff6a007f22235cc91bea48925ef0e8bf Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 15 Jul 2024 09:47:38 +0200
Subject: [PATCH 566/726] Attempt at fixing errors with the new ROOT

---
 Detectors/TPC/monitor/CMakeLists.txt          | 1 +
 Detectors/TPC/monitor/macro/RunFindAdcError.C | 7 +++----
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Detectors/TPC/monitor/CMakeLists.txt b/Detectors/TPC/monitor/CMakeLists.txt
index 152eba83d45cd..a37f6a7dfd6f8 100644
--- a/Detectors/TPC/monitor/CMakeLists.txt
+++ b/Detectors/TPC/monitor/CMakeLists.txt
@@ -28,4 +28,5 @@ o2_add_test_root_macro(macro/RunCompareMode3.C
                        LABELS tpc)
 o2_add_test_root_macro(macro/RunFindAdcError.C
                        PUBLIC_LINK_LIBRARIES O2::TPCReconstruction
+                       COMPILE_ONLY
                        LABELS tpc)
diff --git a/Detectors/TPC/monitor/macro/RunFindAdcError.C b/Detectors/TPC/monitor/macro/RunFindAdcError.C
index 3975c59280461..5e8f806d43497 100644
--- a/Detectors/TPC/monitor/macro/RunFindAdcError.C
+++ b/Detectors/TPC/monitor/macro/RunFindAdcError.C
@@ -73,9 +73,8 @@ void loopReader(std::shared_ptr<RawReader> reader_ptr, std::vector<Result>& resu
 void RunFindAdcError(int run_min, int run_max)
 {
   std::string DATADIR = "/local/data/tpc-beam-test-2017";
-  FairLogger* logger = FairLogger::GetLogger();
-  logger->SetLogVerbosityLevel("LOW");
-  logger->SetLogScreenLevel("INFO");
+  fair::Logger::SetVerbosity(fair::Verbosity::low);
+  fair::Logger::SetConsoleSeverity(fair::Severity::info);
 
   // ===========================================================================
   // Preparing File Infos
@@ -141,7 +140,7 @@ void RunFindAdcError(int run_min, int run_max)
   // ===========================================================================
   // Analyse outcome
   // ===========================================================================
-  logger->SetLogScreenLevel("DEBUG");
+  fair::Logger::SetConsoleSeverity(fair::Severity::debug);
   hAdcError = new TH2F("hAdcError", "occurrence of ADC errors", 36, 0, 36, 20, 0, 20);
   hAdcError->GetXaxis()->SetTitle("SampaID");
   hAdcError->GetYaxis()->SetTitle("Timebin");

From 5f3709fa8fc0431180c66d76290deb9a60f1b21b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 19 Jul 2024 11:50:04 +0200
Subject: [PATCH 567/726] Trying to get ROOT 6.32.0 to digest the type

---
 .../include/CommonDataFormat/AbstractRef.h     | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/DataFormats/common/include/CommonDataFormat/AbstractRef.h b/DataFormats/common/include/CommonDataFormat/AbstractRef.h
index 8dce94d502ffb..a5ff382d11051 100644
--- a/DataFormats/common/include/CommonDataFormat/AbstractRef.h
+++ b/DataFormats/common/include/CommonDataFormat/AbstractRef.h
@@ -20,9 +20,7 @@
 #include "GPUCommonRtypes.h"
 #include "GPUCommonTypeTraits.h"
 
-namespace o2
-{
-namespace dataformats
+namespace o2::dataformats
 {
 
 template <int NBIdx, int NBSrc, int NBFlg>
@@ -32,8 +30,15 @@ class AbstractRef
   static constexpr auto MVAR()
   {
     static_assert(NBIT <= 64, "> 64 bits not supported");
-    typename std::conditional<(NBIT > 32), uint64_t, typename std::conditional<(NBIT > 16), uint32_t, typename std::conditional<(NBIT > 8), uint16_t, uint8_t>::type>::type>::type tp = 0;
-    return tp;
+    if constexpr (NBIT > 32) {
+      return uint64_t{};
+    } else if constexpr (NBIT > 16) {
+      return uint32_t{};
+    } else if constexpr (NBIT > 8) {
+      return uint16_t{};
+    } else {
+      return uint8_t{};
+    }
   }
 
  public:
@@ -87,7 +92,6 @@ class AbstractRef
   ClassDefNV(AbstractRef, 1);
 };
 
-} // namespace dataformats
-} // namespace o2
+} // namespace o2::dataformats
 
 #endif

From 55a9793ac430228084f4f49ff46f58a78da4ecdd Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 19 Jul 2024 14:18:48 +0200
Subject: [PATCH 568/726] Trying to get ROOT 6.32.0 to digest the type

---
 .../include/CommonDataFormat/AbstractRef.h     | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/DataFormats/common/include/CommonDataFormat/AbstractRef.h b/DataFormats/common/include/CommonDataFormat/AbstractRef.h
index 8dce94d502ffb..b7af3297b1059 100644
--- a/DataFormats/common/include/CommonDataFormat/AbstractRef.h
+++ b/DataFormats/common/include/CommonDataFormat/AbstractRef.h
@@ -20,9 +20,7 @@
 #include "GPUCommonRtypes.h"
 #include "GPUCommonTypeTraits.h"
 
-namespace o2
-{
-namespace dataformats
+namespace o2::dataformats
 {
 
 template <int NBIdx, int NBSrc, int NBFlg>
@@ -32,8 +30,15 @@ class AbstractRef
   static constexpr auto MVAR()
   {
     static_assert(NBIT <= 64, "> 64 bits not supported");
-    typename std::conditional<(NBIT > 32), uint64_t, typename std::conditional<(NBIT > 16), uint32_t, typename std::conditional<(NBIT > 8), uint16_t, uint8_t>::type>::type>::type tp = 0;
-    return tp;
+    if constexpr (NBIT > 32) {
+      return uint64_t();
+    } else if constexpr (NBIT > 16) {
+      return uint32_t();
+    } else if constexpr (NBIT > 8) {
+      return uint16_t();
+    } else {
+      return uint8_t();
+    }
   }
 
  public:
@@ -87,7 +92,6 @@ class AbstractRef
   ClassDefNV(AbstractRef, 1);
 };
 
-} // namespace dataformats
-} // namespace o2
+} // namespace o2::dataformats
 
 #endif

From 1f102e4d9d8fe63455743f7f440550e89e501f12 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 19 Jul 2024 22:44:42 +0200
Subject: [PATCH 569/726] Trying to get ROOT 6.32.0 to digest the type

---
 .../include/CommonDataFormat/AbstractRef.h    | 22 +++++++++++++------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/DataFormats/common/include/CommonDataFormat/AbstractRef.h b/DataFormats/common/include/CommonDataFormat/AbstractRef.h
index 8dce94d502ffb..403bab3cbd62f 100644
--- a/DataFormats/common/include/CommonDataFormat/AbstractRef.h
+++ b/DataFormats/common/include/CommonDataFormat/AbstractRef.h
@@ -20,9 +20,7 @@
 #include "GPUCommonRtypes.h"
 #include "GPUCommonTypeTraits.h"
 
-namespace o2
-{
-namespace dataformats
+namespace o2::dataformats
 {
 
 template <int NBIdx, int NBSrc, int NBFlg>
@@ -32,8 +30,19 @@ class AbstractRef
   static constexpr auto MVAR()
   {
     static_assert(NBIT <= 64, "> 64 bits not supported");
-    typename std::conditional<(NBIT > 32), uint64_t, typename std::conditional<(NBIT > 16), uint32_t, typename std::conditional<(NBIT > 8), uint16_t, uint8_t>::type>::type>::type tp = 0;
-    return tp;
+    if constexpr (NBIT > 32) {
+      uint64_t tp = 0;
+      return tp;
+    } else if constexpr (NBIT > 16) {
+      uint32_t tp = 0;
+      return tp;
+    } else if constexpr (NBIT > 8) {
+      uint16_t tp = 0;
+      return tp;
+    } else {
+      uint8_t tp = 0;
+      return tp;
+    }
   }
 
  public:
@@ -87,7 +96,6 @@ class AbstractRef
   ClassDefNV(AbstractRef, 1);
 };
 
-} // namespace dataformats
-} // namespace o2
+} // namespace o2::dataformats
 
 #endif

From 7b75b3fbf45799af08fad3a788bed9e916a0d5a3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 20 Jul 2024 10:20:03 +0200
Subject: [PATCH 570/726] DPL Data Model: introduce bit to keep track of data
 created after EoS

---
 DataFormats/Headers/include/Headers/DataHeader.h        | 5 +++--
 Framework/Core/include/Framework/DataProcessingHeader.h | 3 +++
 Framework/Core/include/Framework/TimingInfo.h           | 4 +++-
 Framework/Core/src/DataAllocator.cxx                    | 1 +
 Framework/Core/src/DataProcessingDevice.cxx             | 9 ++++++++-
 5 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/DataFormats/Headers/include/Headers/DataHeader.h b/DataFormats/Headers/include/Headers/DataHeader.h
index c37eff9b34f20..e4ddaded20aba 100644
--- a/DataFormats/Headers/include/Headers/DataHeader.h
+++ b/DataFormats/Headers/include/Headers/DataHeader.h
@@ -372,8 +372,9 @@ struct BaseHeader {
   union {
     uint32_t flags;
     struct {
-      uint32_t flagsNextHeader : 1, // do we have a next header after this one?
-        flagsUnused : 31;           // currently unused
+      uint32_t flagsNextHeader : 1,        // do we have a next header after this one?
+               flagsReserved : 15,         // reserved for future use
+               flagsDerivedHeader : 16;    // reserved for usage by the derived header
     };
   };
 
diff --git a/Framework/Core/include/Framework/DataProcessingHeader.h b/Framework/Core/include/Framework/DataProcessingHeader.h
index 5c068b4e4179a..484dbb9d51a8e 100644
--- a/Framework/Core/include/Framework/DataProcessingHeader.h
+++ b/Framework/Core/include/Framework/DataProcessingHeader.h
@@ -42,6 +42,9 @@ namespace o2::framework
 /// @ingroup aliceo2_dataformats_dataheader
 struct DataProcessingHeader : public header::BaseHeader {
   static constexpr uint64_t DUMMY_CREATION_TIME_OFFSET = 0x8000000000000000;
+  // The following flags are used to indicate the behavior of the data processing
+  static constexpr int32_t KEEP_AT_EOS_FLAG = 1;
+
   /// We return some number of milliseconds, offsetting int by 0x8000000000000000
   /// to make sure we can understand when the dummy constructor of DataProcessingHeader was
   /// used without overriding it with an actual real time from epoch.
diff --git a/Framework/Core/include/Framework/TimingInfo.h b/Framework/Core/include/Framework/TimingInfo.h
index 84f3971ad3a4a..1734da8dd3941 100644
--- a/Framework/Core/include/Framework/TimingInfo.h
+++ b/Framework/Core/include/Framework/TimingInfo.h
@@ -36,9 +36,11 @@ struct TimingInfo {
   /// from a new run, as being processed by the current stream.
   /// FIXME: for now this is the same as the above.
   bool streamRunNumberChanged = false;
+  /// Wether this kind of data should be flushed during end of stream.
+  bool keepAtEndOfStream = false;
 
   static bool timesliceIsTimer(size_t timeslice) { return timeslice > 1652945069870351; }
-  bool isTimer() const { return timesliceIsTimer(timeslice); };
+  [[nodiscard]] bool isTimer() const { return timesliceIsTimer(timeslice); };
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index 0bf0f86f812ec..bae40f2b47947 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -121,6 +121,7 @@ fair::mq::MessagePtr DataAllocator::headerMessageFromOutput(Output const& spec,
   dh.runNumber = timingInfo.runNumber;
 
   DataProcessingHeader dph{timingInfo.timeslice, 1, timingInfo.creation};
+  static_cast<o2::header::BaseHeader&>(dph).flagsDerivedHeader |= timingInfo.keepAtEndOfStream ? DataProcessingHeader::KEEP_AT_EOS_FLAG : 0;
   auto& proxy = mRegistry.get<FairMQDeviceProxy>();
   auto* transport = proxy.getOutputTransport(routeIndex);
 
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 93940ef5ec21a..452681a06dfd1 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -1298,6 +1298,7 @@ void DataProcessingDevice::Run()
         state.loopReason |= DeviceState::LoopReason::NEW_STATE_PENDING;
       }
       if (state.transitionHandling == TransitionHandlingState::NoTransition && NewStatePending()) {
+        O2_SIGNPOST_EVENT_EMIT(calibration, lid, "timer_setup", "A new state is pending so we requested a new transition.");
         state.transitionHandling = TransitionHandlingState::Requested;
         auto& deviceContext = ref.get<DeviceContext>();
         auto timeout = deviceContext.exitTransitionTimeout;
@@ -1697,7 +1698,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
   }
 
   if (state.streaming == StreamingState::EndOfStreaming) {
-    O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "We are in EndOfStreaming. Flushing queues.");
+    O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "Flushing queues.");
     // We keep processing data until we are Idle.
     // FIXME: not sure this is the correct way to drain the queues, but
     // I guess we will see.
@@ -1710,6 +1711,12 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
     while (DataProcessingDevice::tryDispatchComputation(ref, context.completed) && shouldProcess) {
       relayer.processDanglingInputs(context.expirationHandlers, *context.registry, false);
     }
+
+    auto& timingInfo = ref.get<TimingInfo>();
+    // We should keep the data generated at end of stream only for those
+    // which are not sources.
+    timingInfo.keepAtEndOfStream = shouldProcess;
+
     EndOfStreamContext eosContext{*context.registry, ref.get<DataAllocator>()};
 
     context.preEOSCallbacks(eosContext);

From 78c94a0ef4f5af5a4fe669cebfdeff2acd74ac79 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 22 Jul 2024 11:42:43 +0200
Subject: [PATCH 571/726] Add dependency to Bookkeeping in the correct place

---
 dependencies/O2Dependencies.cmake | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/dependencies/O2Dependencies.cmake b/dependencies/O2Dependencies.cmake
index 9f35a89449601..48a5d57d17c96 100644
--- a/dependencies/O2Dependencies.cmake
+++ b/dependencies/O2Dependencies.cmake
@@ -99,6 +99,9 @@ set_package_properties(Configuration PROPERTIES TYPE REQUIRED)
 find_package(Monitoring CONFIG)
 set_package_properties(Monitoring PROPERTIES TYPE REQUIRED)
 
+find_package(BookkeepingApi CONFIG)
+set_package_properties(BookeepingApi PROPERTIES TYPE REQUIRED)
+
 find_package(Common CONFIG)
 set_package_properties(Common PROPERTIES TYPE REQUIRED)
 

From 28dbd24f9307b2dc031143fb1c63a74962a9f084 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 22 Jul 2024 12:05:50 +0200
Subject: [PATCH 572/726] DPL Data Model: introduce bit to keep track of data
 created after EoS

---
 DataFormats/Headers/include/Headers/DataHeader.h        | 3 ++-
 Framework/Core/include/Framework/DataProcessingHeader.h | 3 +++
 Framework/Core/include/Framework/TimingInfo.h           | 6 ++++--
 Framework/Core/src/DataAllocator.cxx                    | 1 +
 Framework/Core/src/DataProcessingDevice.cxx             | 9 ++++++++-
 5 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/DataFormats/Headers/include/Headers/DataHeader.h b/DataFormats/Headers/include/Headers/DataHeader.h
index c37eff9b34f20..2dbfbd67d8d6c 100644
--- a/DataFormats/Headers/include/Headers/DataHeader.h
+++ b/DataFormats/Headers/include/Headers/DataHeader.h
@@ -373,7 +373,8 @@ struct BaseHeader {
     uint32_t flags;
     struct {
       uint32_t flagsNextHeader : 1, // do we have a next header after this one?
-        flagsUnused : 31;           // currently unused
+        flagsReserved : 15,         // reserved for future use
+        flagsDerivedHeader : 16;    // reserved for usage by the derived header
     };
   };
 
diff --git a/Framework/Core/include/Framework/DataProcessingHeader.h b/Framework/Core/include/Framework/DataProcessingHeader.h
index 5c068b4e4179a..484dbb9d51a8e 100644
--- a/Framework/Core/include/Framework/DataProcessingHeader.h
+++ b/Framework/Core/include/Framework/DataProcessingHeader.h
@@ -42,6 +42,9 @@ namespace o2::framework
 /// @ingroup aliceo2_dataformats_dataheader
 struct DataProcessingHeader : public header::BaseHeader {
   static constexpr uint64_t DUMMY_CREATION_TIME_OFFSET = 0x8000000000000000;
+  // The following flags are used to indicate the behavior of the data processing
+  static constexpr int32_t KEEP_AT_EOS_FLAG = 1;
+
   /// We return some number of milliseconds, offsetting int by 0x8000000000000000
   /// to make sure we can understand when the dummy constructor of DataProcessingHeader was
   /// used without overriding it with an actual real time from epoch.
diff --git a/Framework/Core/include/Framework/TimingInfo.h b/Framework/Core/include/Framework/TimingInfo.h
index 84f3971ad3a4a..6b61b8e4ef5b5 100644
--- a/Framework/Core/include/Framework/TimingInfo.h
+++ b/Framework/Core/include/Framework/TimingInfo.h
@@ -23,7 +23,7 @@ namespace o2::framework
 
 struct TimingInfo {
   constexpr static ServiceKind service_kind = ServiceKind::Stream;
-  size_t timeslice; /// the timeslice associated to current processing
+  size_t timeslice;           /// the timeslice associated to current processing
   uint32_t firstTForbit = -1; /// the orbit the TF begins
   uint32_t tfCounter = -1;    // the counter associated to a TF
   uint32_t runNumber = -1;
@@ -36,9 +36,11 @@ struct TimingInfo {
   /// from a new run, as being processed by the current stream.
   /// FIXME: for now this is the same as the above.
   bool streamRunNumberChanged = false;
+  /// Wether this kind of data should be flushed during end of stream.
+  bool keepAtEndOfStream = false;
 
   static bool timesliceIsTimer(size_t timeslice) { return timeslice > 1652945069870351; }
-  bool isTimer() const { return timesliceIsTimer(timeslice); };
+  [[nodiscard]] bool isTimer() const { return timesliceIsTimer(timeslice); };
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index 0bf0f86f812ec..bae40f2b47947 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -121,6 +121,7 @@ fair::mq::MessagePtr DataAllocator::headerMessageFromOutput(Output const& spec,
   dh.runNumber = timingInfo.runNumber;
 
   DataProcessingHeader dph{timingInfo.timeslice, 1, timingInfo.creation};
+  static_cast<o2::header::BaseHeader&>(dph).flagsDerivedHeader |= timingInfo.keepAtEndOfStream ? DataProcessingHeader::KEEP_AT_EOS_FLAG : 0;
   auto& proxy = mRegistry.get<FairMQDeviceProxy>();
   auto* transport = proxy.getOutputTransport(routeIndex);
 
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 93940ef5ec21a..452681a06dfd1 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -1298,6 +1298,7 @@ void DataProcessingDevice::Run()
         state.loopReason |= DeviceState::LoopReason::NEW_STATE_PENDING;
       }
       if (state.transitionHandling == TransitionHandlingState::NoTransition && NewStatePending()) {
+        O2_SIGNPOST_EVENT_EMIT(calibration, lid, "timer_setup", "A new state is pending so we requested a new transition.");
         state.transitionHandling = TransitionHandlingState::Requested;
         auto& deviceContext = ref.get<DeviceContext>();
         auto timeout = deviceContext.exitTransitionTimeout;
@@ -1697,7 +1698,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
   }
 
   if (state.streaming == StreamingState::EndOfStreaming) {
-    O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "We are in EndOfStreaming. Flushing queues.");
+    O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "Flushing queues.");
     // We keep processing data until we are Idle.
     // FIXME: not sure this is the correct way to drain the queues, but
     // I guess we will see.
@@ -1710,6 +1711,12 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
     while (DataProcessingDevice::tryDispatchComputation(ref, context.completed) && shouldProcess) {
       relayer.processDanglingInputs(context.expirationHandlers, *context.registry, false);
     }
+
+    auto& timingInfo = ref.get<TimingInfo>();
+    // We should keep the data generated at end of stream only for those
+    // which are not sources.
+    timingInfo.keepAtEndOfStream = shouldProcess;
+
     EndOfStreamContext eosContext{*context.registry, ref.get<DataAllocator>()};
 
     context.preEOSCallbacks(eosContext);

From 10dba70360a868c55d03b5f131713540b9e23a9f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 22 Jul 2024 15:59:30 +0200
Subject: [PATCH 573/726] DPL Data Model: introduce bit to keep track of data
 created after EoS

---
 DataFormats/Headers/include/Headers/DataHeader.h        | 3 ++-
 Framework/Core/include/Framework/DataProcessingHeader.h | 3 +++
 Framework/Core/include/Framework/TimingInfo.h           | 6 ++++--
 Framework/Core/src/DataAllocator.cxx                    | 1 +
 Framework/Core/src/DataProcessingDevice.cxx             | 6 ++++++
 5 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/DataFormats/Headers/include/Headers/DataHeader.h b/DataFormats/Headers/include/Headers/DataHeader.h
index c37eff9b34f20..2dbfbd67d8d6c 100644
--- a/DataFormats/Headers/include/Headers/DataHeader.h
+++ b/DataFormats/Headers/include/Headers/DataHeader.h
@@ -373,7 +373,8 @@ struct BaseHeader {
     uint32_t flags;
     struct {
       uint32_t flagsNextHeader : 1, // do we have a next header after this one?
-        flagsUnused : 31;           // currently unused
+        flagsReserved : 15,         // reserved for future use
+        flagsDerivedHeader : 16;    // reserved for usage by the derived header
     };
   };
 
diff --git a/Framework/Core/include/Framework/DataProcessingHeader.h b/Framework/Core/include/Framework/DataProcessingHeader.h
index 5c068b4e4179a..484dbb9d51a8e 100644
--- a/Framework/Core/include/Framework/DataProcessingHeader.h
+++ b/Framework/Core/include/Framework/DataProcessingHeader.h
@@ -42,6 +42,9 @@ namespace o2::framework
 /// @ingroup aliceo2_dataformats_dataheader
 struct DataProcessingHeader : public header::BaseHeader {
   static constexpr uint64_t DUMMY_CREATION_TIME_OFFSET = 0x8000000000000000;
+  // The following flags are used to indicate the behavior of the data processing
+  static constexpr int32_t KEEP_AT_EOS_FLAG = 1;
+
   /// We return some number of milliseconds, offsetting int by 0x8000000000000000
   /// to make sure we can understand when the dummy constructor of DataProcessingHeader was
   /// used without overriding it with an actual real time from epoch.
diff --git a/Framework/Core/include/Framework/TimingInfo.h b/Framework/Core/include/Framework/TimingInfo.h
index 84f3971ad3a4a..6b61b8e4ef5b5 100644
--- a/Framework/Core/include/Framework/TimingInfo.h
+++ b/Framework/Core/include/Framework/TimingInfo.h
@@ -23,7 +23,7 @@ namespace o2::framework
 
 struct TimingInfo {
   constexpr static ServiceKind service_kind = ServiceKind::Stream;
-  size_t timeslice; /// the timeslice associated to current processing
+  size_t timeslice;           /// the timeslice associated to current processing
   uint32_t firstTForbit = -1; /// the orbit the TF begins
   uint32_t tfCounter = -1;    // the counter associated to a TF
   uint32_t runNumber = -1;
@@ -36,9 +36,11 @@ struct TimingInfo {
   /// from a new run, as being processed by the current stream.
   /// FIXME: for now this is the same as the above.
   bool streamRunNumberChanged = false;
+  /// Wether this kind of data should be flushed during end of stream.
+  bool keepAtEndOfStream = false;
 
   static bool timesliceIsTimer(size_t timeslice) { return timeslice > 1652945069870351; }
-  bool isTimer() const { return timesliceIsTimer(timeslice); };
+  [[nodiscard]] bool isTimer() const { return timesliceIsTimer(timeslice); };
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index 0bf0f86f812ec..bae40f2b47947 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -121,6 +121,7 @@ fair::mq::MessagePtr DataAllocator::headerMessageFromOutput(Output const& spec,
   dh.runNumber = timingInfo.runNumber;
 
   DataProcessingHeader dph{timingInfo.timeslice, 1, timingInfo.creation};
+  static_cast<o2::header::BaseHeader&>(dph).flagsDerivedHeader |= timingInfo.keepAtEndOfStream ? DataProcessingHeader::KEEP_AT_EOS_FLAG : 0;
   auto& proxy = mRegistry.get<FairMQDeviceProxy>();
   auto* transport = proxy.getOutputTransport(routeIndex);
 
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 93940ef5ec21a..dd0528c1b2b42 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -1710,6 +1710,12 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
     while (DataProcessingDevice::tryDispatchComputation(ref, context.completed) && shouldProcess) {
       relayer.processDanglingInputs(context.expirationHandlers, *context.registry, false);
     }
+
+    auto& timingInfo = ref.get<TimingInfo>();
+    // We should keep the data generated at end of stream only for those
+    // which are not sources.
+    timingInfo.keepAtEndOfStream = shouldProcess;
+
     EndOfStreamContext eosContext{*context.registry, ref.get<DataAllocator>()};
 
     context.preEOSCallbacks(eosContext);

From fc458311c1c4feafb02ace8323c5816cb3bad823 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 22 Jul 2024 23:15:11 +0200
Subject: [PATCH 574/726] CCDB Downloader: example of how to use signposts

Signposts, when properly configured, can provide stacktrace for each
logging line.
---
 CCDB/src/CCDBDownloader.cxx | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/CCDB/src/CCDBDownloader.cxx b/CCDB/src/CCDBDownloader.cxx
index eac0c07f03131..222ab15859ede 100644
--- a/CCDB/src/CCDBDownloader.cxx
+++ b/CCDB/src/CCDBDownloader.cxx
@@ -12,6 +12,7 @@
 #include <CCDB/CCDBDownloader.h>
 #include "CommonUtils/StringUtils.h"
 #include "CCDB/CCDBTimeStampUtils.h"
+#include "Framework/Signpost.h"
 
 #include <curl/curl.h>
 #include <unordered_map>
@@ -29,6 +30,8 @@
 #include <fairlogger/Logger.h>
 #include <boost/asio/ip/host_name.hpp>
 
+O2_DECLARE_DYNAMIC_LOG(ccdb);
+
 namespace o2::ccdb
 {
 
@@ -37,21 +40,24 @@ void uvErrorCheck(int code)
   if (code != 0) {
     char buf[1000];
     uv_strerror_r(code, buf, 1000);
-    LOG(error) << "CCDBDownloader: UV error - " << buf;
+    O2_SIGNPOST_ID_GENERATE(sid, ccdb);
+    O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb, sid, "CCDBDownloader", "UV error - %{public}s", buf);
   }
 }
 
 void curlEasyErrorCheck(CURLcode code)
 {
   if (code != CURLE_OK) {
-    LOG(error) << "CCDBDownloader: CURL error - " << curl_easy_strerror(code);
+    O2_SIGNPOST_ID_GENERATE(sid, ccdb);
+    O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb, sid, "CCDBDownloader", "CURL error - %{public}s", curl_easy_strerror(code));
   }
 }
 
 void curlMultiErrorCheck(CURLMcode code)
 {
   if (code != CURLM_OK) {
-    LOG(error) << "CCDBDownloader: CURL error - " << curl_multi_strerror(code);
+    O2_SIGNPOST_ID_GENERATE(sid, ccdb);
+    O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb, sid, "CCDBDownloader", "CURL error - %{public}s", curl_multi_strerror(code));
   }
 }
 namespace
@@ -155,12 +161,14 @@ void CCDBDownloader::closesocketCallback(void* clientp, curl_socket_t item)
       }
       CD->mSocketTimerMap.erase(item);
       if (close(item) == -1) {
-        LOG(error) << "CCDBDownloader: Socket failed to close";
+        O2_SIGNPOST_ID_GENERATE(sid, ccdb);
+        O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb, sid, "CCDBDownloader", "CCDBDownloader: Socket failed to close");
       }
     }
   } else {
     if (close(item) == -1) {
-      LOG(error) << "CCDBDownloader: Socket failed to close";
+      O2_SIGNPOST_ID_GENERATE(sid, ccdb);
+      O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb, sid, "CCDBDownloader", "CCDBDownloader: Socket failed to close");
     }
   }
 }
@@ -170,7 +178,8 @@ curl_socket_t opensocketCallback(void* clientp, curlsocktype purpose, struct cur
   auto CD = (CCDBDownloader*)clientp;
   auto sock = socket(address->family, address->socktype, address->protocol);
   if (sock == -1) {
-    LOG(error) << "CCDBDownloader: Socket failed to open";
+    O2_SIGNPOST_ID_GENERATE(sid, ccdb);
+    O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb, sid, "CCDBDownloader", "CCDBDownloader: Socket failed to open");
   }
 
   if (CD->mExternalLoop) {
@@ -197,7 +206,8 @@ void CCDBDownloader::closeSocketByTimer(uv_timer_t* handle)
     uvErrorCheck(uv_timer_stop(CD->mSocketTimerMap[sock]));
     CD->mSocketTimerMap.erase(sock);
     if (close(sock) == -1) {
-      LOG(error) << "CCDBDownloader: Socket failed to close";
+      O2_SIGNPOST_ID_GENERATE(sid, ccdb);
+      O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb, sid, "CCDBDownloader", "CCDBDownloader: Socket failed to close");
     }
     delete data;
   }

From 7e155f5ea72a7c9e131cb382516db3754ac673de Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 23 Jul 2024 09:39:50 +0200
Subject: [PATCH 575/726] CCDB Downloader: example of how to use signposts

Signposts, when properly configured, can provide stacktrace for each
logging line.
---
 CCDB/src/CCDBDownloader.cxx | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/CCDB/src/CCDBDownloader.cxx b/CCDB/src/CCDBDownloader.cxx
index eac0c07f03131..681f75941eda0 100644
--- a/CCDB/src/CCDBDownloader.cxx
+++ b/CCDB/src/CCDBDownloader.cxx
@@ -12,6 +12,7 @@
 #include <CCDB/CCDBDownloader.h>
 #include "CommonUtils/StringUtils.h"
 #include "CCDB/CCDBTimeStampUtils.h"
+#include "Framework/Signpost.h"
 
 #include <curl/curl.h>
 #include <unordered_map>
@@ -29,6 +30,8 @@
 #include <fairlogger/Logger.h>
 #include <boost/asio/ip/host_name.hpp>
 
+O2_DECLARE_DYNAMIC_STACKTRACE_LOG(ccdb_downloader);
+
 namespace o2::ccdb
 {
 
@@ -37,21 +40,24 @@ void uvErrorCheck(int code)
   if (code != 0) {
     char buf[1000];
     uv_strerror_r(code, buf, 1000);
-    LOG(error) << "CCDBDownloader: UV error - " << buf;
+    O2_SIGNPOST_ID_GENERATE(sid, ccdb_downloader);
+    O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb_downloader, sid, "CCDBDownloader", "UV error - %{public}s", buf);
   }
 }
 
 void curlEasyErrorCheck(CURLcode code)
 {
   if (code != CURLE_OK) {
-    LOG(error) << "CCDBDownloader: CURL error - " << curl_easy_strerror(code);
+    O2_SIGNPOST_ID_GENERATE(sid, ccdb_downloader);
+    O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb_downloader, sid, "CCDBDownloader", "CURL error - %{public}s", curl_easy_strerror(code));
   }
 }
 
 void curlMultiErrorCheck(CURLMcode code)
 {
   if (code != CURLM_OK) {
-    LOG(error) << "CCDBDownloader: CURL error - " << curl_multi_strerror(code);
+    O2_SIGNPOST_ID_GENERATE(sid, ccdb_downloader);
+    O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb_downloader, sid, "CCDBDownloader", "CURL error - %{public}s", curl_multi_strerror(code));
   }
 }
 namespace
@@ -155,12 +161,14 @@ void CCDBDownloader::closesocketCallback(void* clientp, curl_socket_t item)
       }
       CD->mSocketTimerMap.erase(item);
       if (close(item) == -1) {
-        LOG(error) << "CCDBDownloader: Socket failed to close";
+        O2_SIGNPOST_ID_GENERATE(sid, ccdb_downloader);
+        O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb_downloader, sid, "CCDBDownloader", "CCDBDownloader: Socket failed to close");
       }
     }
   } else {
     if (close(item) == -1) {
-      LOG(error) << "CCDBDownloader: Socket failed to close";
+      O2_SIGNPOST_ID_GENERATE(sid, ccdb_downloader);
+      O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb_downloader, sid, "CCDBDownloader", "CCDBDownloader: Socket failed to close");
     }
   }
 }
@@ -170,7 +178,8 @@ curl_socket_t opensocketCallback(void* clientp, curlsocktype purpose, struct cur
   auto CD = (CCDBDownloader*)clientp;
   auto sock = socket(address->family, address->socktype, address->protocol);
   if (sock == -1) {
-    LOG(error) << "CCDBDownloader: Socket failed to open";
+    O2_SIGNPOST_ID_GENERATE(sid, ccdb_downloader);
+    O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb_downloader, sid, "CCDBDownloader", "CCDBDownloader: Socket failed to open");
   }
 
   if (CD->mExternalLoop) {
@@ -197,7 +206,8 @@ void CCDBDownloader::closeSocketByTimer(uv_timer_t* handle)
     uvErrorCheck(uv_timer_stop(CD->mSocketTimerMap[sock]));
     CD->mSocketTimerMap.erase(sock);
     if (close(sock) == -1) {
-      LOG(error) << "CCDBDownloader: Socket failed to close";
+      O2_SIGNPOST_ID_GENERATE(sid, ccdb_downloader);
+      O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb_downloader, sid, "CCDBDownloader", "CCDBDownloader: Socket failed to close");
     }
     delete data;
   }

From 3721636dbbaf8f03e87b6ae4d67dcb92e849656d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 23 Jul 2024 14:47:41 +0200
Subject: [PATCH 576/726] DPL: improve debug message

---
 Framework/Core/src/DataProcessingDevice.cxx | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 93940ef5ec21a..460c39dff741d 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -1841,7 +1841,8 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
       // This is because in principle we should track the size of each of
       // the parts and sum it up. Not for now.
       O2_SIGNPOST_ID_FROM_POINTER(pid, parts, headerData);
-      O2_SIGNPOST_START(parts, pid, "parts", "Processing DataHeader with splitPayloadParts %d and splitPayloadIndex %d", dh->splitPayloadParts, dh->splitPayloadIndex);
+      O2_SIGNPOST_START(parts, pid, "parts", "Processing DataHeader %{public}-4s/%{public}-16s/%d with splitPayloadParts %d and splitPayloadIndex %d", 
+                        dh->dataOrigin.str, dh->dataDescription.str, dh->subSpecification, dh->splitPayloadParts, dh->splitPayloadIndex);
       if (!dph) {
         insertInputInfo(pi, 2, InputType::Invalid, info.id);
         O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "Header stack does not contain DataProcessingHeader");

From 9bd03442f687b224a33e485eda1e27e7b26441e9 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 23 Jul 2024 13:37:57 +0200
Subject: [PATCH 577/726] DPL GUI: add dispatch policy to the inspector

---
 Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
index 3ef8b99ed26bb..0e7d21f5eaf27 100644
--- a/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDeviceInspector.cxx
@@ -365,6 +365,7 @@ void displayDeviceInspector(DeviceSpec const& spec,
   if (ImGui::CollapsingHeader("Policies")) {
     ImGui::Text("Completion: %s", spec.completionPolicy.name.c_str());
     ImGui::Text("Sending: %s", spec.sendingPolicy.name.c_str());
+    ImGui::Text("Dispatching: %s", spec.dispatchPolicy.name.c_str());
   }
 
   if (ImGui::CollapsingHeader("Signals", ImGuiTreeNodeFlags_DefaultOpen)) {

From 697c30a9ba1dc98903d7259db7fee96e0da6c6b4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 23 Jul 2024 15:53:37 +0200
Subject: [PATCH 578/726] DPL: improve debug message

---
 Framework/Core/src/DataProcessingDevice.cxx | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 93940ef5ec21a..3691934959962 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -1841,7 +1841,8 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
       // This is because in principle we should track the size of each of
       // the parts and sum it up. Not for now.
       O2_SIGNPOST_ID_FROM_POINTER(pid, parts, headerData);
-      O2_SIGNPOST_START(parts, pid, "parts", "Processing DataHeader with splitPayloadParts %d and splitPayloadIndex %d", dh->splitPayloadParts, dh->splitPayloadIndex);
+      O2_SIGNPOST_START(parts, pid, "parts", "Processing DataHeader %{public}-4s/%{public}-16s/%d with splitPayloadParts %d and splitPayloadIndex %d",
+                        dh->dataOrigin.str, dh->dataDescription.str, dh->subSpecification, dh->splitPayloadParts, dh->splitPayloadIndex);
       if (!dph) {
         insertInputInfo(pi, 2, InputType::Invalid, info.id);
         O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "handle_data", "Header stack does not contain DataProcessingHeader");

From 228969e1ca4fbad30ab5f754ff85c4514d0014af Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 23 Jul 2024 15:53:37 +0200
Subject: [PATCH 579/726] DPL: provide ability to customise consumption order

---
 Framework/Core/include/Framework/CompletionPolicy.h | 8 ++++++++
 Framework/Core/src/DataProcessingDevice.cxx         | 8 ++++++++
 2 files changed, 16 insertions(+)

diff --git a/Framework/Core/include/Framework/CompletionPolicy.h b/Framework/Core/include/Framework/CompletionPolicy.h
index 55d3014166956..8bce37b3563f7 100644
--- a/Framework/Core/include/Framework/CompletionPolicy.h
+++ b/Framework/Core/include/Framework/CompletionPolicy.h
@@ -62,6 +62,12 @@ struct CompletionPolicy {
     Retry,
   };
 
+  /// Order in which the completed slots must be consumed
+  enum CompletionOrder {
+    Any,
+    Timeslice
+  };
+
   using Matcher = std::function<bool(DeviceSpec const& device)>;
   using InputSetElement = DataRef;
   using CallbackFull = std::function<CompletionOp(InputSpan const&, std::vector<InputSpec> const&, ServiceRegistryRef&)>;
@@ -91,6 +97,8 @@ struct CompletionPolicy {
   /// data.
   bool balanceChannels = true;
 
+  CompletionOrder order = CompletionOrder::Any;
+
   /// Helper to create the default configuration.
   static std::vector<CompletionPolicy> createDefaultPolicies();
 };
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index daae1e25483c9..19780faee41dc 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -2301,6 +2301,14 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   using namespace o2::framework;
   stats.updateStats({(int)ProcessingStatsId::PENDING_INPUTS, DataProcessingStats::Op::Set, static_cast<int64_t>(relayer.getParallelTimeslices() - completed.size())});
   stats.updateStats({(int)ProcessingStatsId::INCOMPLETE_INPUTS, DataProcessingStats::Op::Set, completed.empty() ? 1 : 0});
+  switch (spec.completionPolicy.order) {
+    case CompletionPolicy::Timeslice:
+      std::sort(completed.begin(), completed.end(), [](auto const& a, auto const& b) { return a.timeslice.value < b.timeslice.value; });
+      break;
+    case CompletionPolicy::Any:
+    default:
+      break;
+  }
 
   for (auto action : completed) {
     O2_SIGNPOST_ID_GENERATE(aid, device);

From 13ea9c4870cc4e7688f8f388fe9a3d860989a7c2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 23 Jul 2024 15:53:37 +0200
Subject: [PATCH 580/726] DPL: simplify code

Remove lambdas which were meant to simplify code, and made it more obscure in the end.
---
 Framework/Core/src/DataProcessingDevice.cxx | 34 ++++++---------------
 1 file changed, 10 insertions(+), 24 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 3691934959962..daae1e25483c9 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -2102,28 +2102,6 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   // should work just fine.
   std::vector<MessageSet> currentSetOfInputs;
 
-  // For the moment we have a simple "immediately dispatch" policy for stuff
-  // in the cache. This could be controlled from the outside e.g. by waiting
-  // for a few sets of inputs to arrive before we actually dispatch the
-  // computation, however this can be defined at a later stage.
-  auto canDispatchSomeComputation = [&completed, ref]() -> bool {
-    ref.get<DataRelayer>().getReadyToProcess(completed);
-    return completed.empty() == false;
-  };
-
-  // We use this to get a list with the actual indexes in the cache which
-  // indicate a complete set of inputs. Notice how I fill the completed
-  // vector and return it, so that I can have a nice for loop iteration later
-  // on.
-  auto getReadyActions = [&completed, ref]() -> std::vector<DataRelayer::RecordAction> {
-    auto& stats = ref.get<DataProcessingStats>();
-    auto& relayer = ref.get<DataRelayer>();
-    using namespace o2::framework;
-    stats.updateStats({(int)ProcessingStatsId::PENDING_INPUTS, DataProcessingStats::Op::Set, static_cast<int64_t>(relayer.getParallelTimeslices() - completed.size())});
-    stats.updateStats({(int)ProcessingStatsId::INCOMPLETE_INPUTS, DataProcessingStats::Op::Set, completed.empty() ? 1 : 0});
-    return completed;
-  };
-
   //
   auto getInputSpan = [ref, &currentSetOfInputs](TimesliceSlot slot, bool consume = true) {
     auto& relayer = ref.get<DataRelayer>();
@@ -2258,7 +2236,8 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
     control.notifyStreamingState(state.streaming);
   };
 
-  if (canDispatchSomeComputation() == false) {
+  ref.get<DataRelayer>().getReadyToProcess(completed);
+  if (completed.empty() == true) {
     LOGP(debug, "No computations available for dispatching.");
     return false;
   }
@@ -2316,7 +2295,14 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   auto& streamContext = ref.get<StreamContext>();
   O2_SIGNPOST_ID_GENERATE(sid, device);
   O2_SIGNPOST_START(device, sid, "device", "Start processing ready actions");
-  for (auto action : getReadyActions()) {
+
+  auto& stats = ref.get<DataProcessingStats>();
+  auto& relayer = ref.get<DataRelayer>();
+  using namespace o2::framework;
+  stats.updateStats({(int)ProcessingStatsId::PENDING_INPUTS, DataProcessingStats::Op::Set, static_cast<int64_t>(relayer.getParallelTimeslices() - completed.size())});
+  stats.updateStats({(int)ProcessingStatsId::INCOMPLETE_INPUTS, DataProcessingStats::Op::Set, completed.empty() ? 1 : 0});
+
+  for (auto action : completed) {
     O2_SIGNPOST_ID_GENERATE(aid, device);
     O2_SIGNPOST_START(device, aid, "device", "Processing action on slot %lu for action %{public}s", action.slot.index, fmt::format("{}", action.op).c_str());
     if (action.op == CompletionPolicy::CompletionOp::Wait) {

From f6a89270cf0960453a4bc4d2cbd6546ad3866da1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 23 Jul 2024 17:37:44 +0200
Subject: [PATCH 581/726] DPL: provide ability to customise consumption order

---
 Framework/Core/include/Framework/CompletionPolicy.h |  9 +++++++++
 Framework/Core/src/DataProcessingDevice.cxx         | 11 +++++++++++
 2 files changed, 20 insertions(+)

diff --git a/Framework/Core/include/Framework/CompletionPolicy.h b/Framework/Core/include/Framework/CompletionPolicy.h
index 55d3014166956..987adfac90e7a 100644
--- a/Framework/Core/include/Framework/CompletionPolicy.h
+++ b/Framework/Core/include/Framework/CompletionPolicy.h
@@ -62,6 +62,13 @@ struct CompletionPolicy {
     Retry,
   };
 
+  /// Order in which the completed slots must be consumed
+  enum struct CompletionOrder {
+    Any,
+    Timeslice,
+    Slot
+  };
+
   using Matcher = std::function<bool(DeviceSpec const& device)>;
   using InputSetElement = DataRef;
   using CallbackFull = std::function<CompletionOp(InputSpan const&, std::vector<InputSpec> const&, ServiceRegistryRef&)>;
@@ -91,6 +98,8 @@ struct CompletionPolicy {
   /// data.
   bool balanceChannels = true;
 
+  CompletionOrder order = CompletionOrder::Any;
+
   /// Helper to create the default configuration.
   static std::vector<CompletionPolicy> createDefaultPolicies();
 };
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index daae1e25483c9..49b50e9406e3d 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -2301,6 +2301,17 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   using namespace o2::framework;
   stats.updateStats({(int)ProcessingStatsId::PENDING_INPUTS, DataProcessingStats::Op::Set, static_cast<int64_t>(relayer.getParallelTimeslices() - completed.size())});
   stats.updateStats({(int)ProcessingStatsId::INCOMPLETE_INPUTS, DataProcessingStats::Op::Set, completed.empty() ? 1 : 0});
+  switch (spec.completionPolicy.order) {
+    case CompletionPolicy::Timeslice:
+      std::sort(completed.begin(), completed.end(), [](auto const& a, auto const& b) { return a.timeslice.value < b.timeslice.value; });
+      break;
+    case CompletionPolicy::Slot:
+      std::sort(completed.begin(), completed.end(), [](auto const& a, auto const& b) { return a.slot.index < b.slot.index; });
+      break;
+    case CompletionPolicy::Any:
+    default:
+      break;
+  }
 
   for (auto action : completed) {
     O2_SIGNPOST_ID_GENERATE(aid, device);

From 947c18ce64b2623807f6d13e6dc0e22aca644069 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 23 Jul 2024 21:56:00 +0200
Subject: [PATCH 582/726] DPL: provide ability to customise consumption order

---
 Framework/Core/include/Framework/CompletionPolicy.h |  9 +++++++++
 Framework/Core/src/DataProcessingDevice.cxx         | 11 +++++++++++
 2 files changed, 20 insertions(+)

diff --git a/Framework/Core/include/Framework/CompletionPolicy.h b/Framework/Core/include/Framework/CompletionPolicy.h
index 55d3014166956..987adfac90e7a 100644
--- a/Framework/Core/include/Framework/CompletionPolicy.h
+++ b/Framework/Core/include/Framework/CompletionPolicy.h
@@ -62,6 +62,13 @@ struct CompletionPolicy {
     Retry,
   };
 
+  /// Order in which the completed slots must be consumed
+  enum struct CompletionOrder {
+    Any,
+    Timeslice,
+    Slot
+  };
+
   using Matcher = std::function<bool(DeviceSpec const& device)>;
   using InputSetElement = DataRef;
   using CallbackFull = std::function<CompletionOp(InputSpan const&, std::vector<InputSpec> const&, ServiceRegistryRef&)>;
@@ -91,6 +98,8 @@ struct CompletionPolicy {
   /// data.
   bool balanceChannels = true;
 
+  CompletionOrder order = CompletionOrder::Any;
+
   /// Helper to create the default configuration.
   static std::vector<CompletionPolicy> createDefaultPolicies();
 };
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index daae1e25483c9..590d35b94ac40 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -2301,6 +2301,17 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
   using namespace o2::framework;
   stats.updateStats({(int)ProcessingStatsId::PENDING_INPUTS, DataProcessingStats::Op::Set, static_cast<int64_t>(relayer.getParallelTimeslices() - completed.size())});
   stats.updateStats({(int)ProcessingStatsId::INCOMPLETE_INPUTS, DataProcessingStats::Op::Set, completed.empty() ? 1 : 0});
+  switch (spec.completionPolicy.order) {
+    case CompletionPolicy::CompletionOrder::Timeslice:
+      std::sort(completed.begin(), completed.end(), [](auto const& a, auto const& b) { return a.timeslice.value < b.timeslice.value; });
+      break;
+    case CompletionPolicy::CompletionOrder::Slot:
+      std::sort(completed.begin(), completed.end(), [](auto const& a, auto const& b) { return a.slot.index < b.slot.index; });
+      break;
+    case CompletionPolicy::CompletionOrder::Any:
+    default:
+      break;
+  }
 
   for (auto action : completed) {
     O2_SIGNPOST_ID_GENERATE(aid, device);

From 4f4748e70291665574d61946389dfb4c22b99c86 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 24 Jul 2024 08:56:09 +0200
Subject: [PATCH 583/726] DPL: add dummy workflow mimicking CCDB populator
 behaviour

---
 Framework/TestWorkflows/CMakeLists.txt        |  4 +
 .../src/o2DummyPopulatorWorkflow.cxx          | 83 +++++++++++++++++++
 2 files changed, 87 insertions(+)
 create mode 100644 Framework/TestWorkflows/src/o2DummyPopulatorWorkflow.cxx

diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index 21af45422e0b1..8548d6570e4a4 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -33,6 +33,10 @@ o2_add_dpl_workflow(diamond-workflow
                   SOURCES src/o2DiamondWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
 
+o2_add_dpl_workflow(dummy-populator-workflow
+                  SOURCES src/o2DummyPopulatorWorkflow.cxx
+                  COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(analysis-histograms
                   SOURCES src/o2TestHistograms.cxx
                   COMPONENT_NAME TestWorkflows)
diff --git a/Framework/TestWorkflows/src/o2DummyPopulatorWorkflow.cxx b/Framework/TestWorkflows/src/o2DummyPopulatorWorkflow.cxx
new file mode 100644
index 0000000000000..caf72dd0e12a0
--- /dev/null
+++ b/Framework/TestWorkflows/src/o2DummyPopulatorWorkflow.cxx
@@ -0,0 +1,83 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/DataTakingContext.h"
+#include "Framework/CompletionPolicyHelpers.h"
+#include "Framework/DeviceSpec.h"
+#include "Framework/RawDeviceService.h"
+#include "Framework/ControlService.h"
+#include "Framework/Configurable.h"
+#include "Framework/RunningWorkflowInfo.h"
+#include "Framework/CallbackService.h"
+#include "Framework/RateLimiter.h"
+#include <fairmq/Device.h>
+
+#include <iostream>
+#include <chrono>
+#include <thread>
+#include <vector>
+
+using namespace o2::framework;
+
+struct WorkflowOptions {
+  Configurable<int> anInt{"anInt", 1, ""};
+  Configurable<float> aFloat{"aFloat", 2.0f, {"a float option"}};
+  Configurable<double> aDouble{"aDouble", 3., {"a double option"}};
+  Configurable<std::string> aString{"aString", "foobar", {"a string option"}};
+  Configurable<bool> aBool{"aBool", true, {"a boolean option"}};
+};
+
+void customize(std::vector<CompletionPolicy>& policies)
+{
+  auto a = CompletionPolicyHelpers::consumeWhenAll();
+  a.order = CompletionPolicy::CompletionOrder::Timeslice;
+  policies.clear();
+  policies.push_back(a);
+}
+
+#include "Framework/runDataProcessing.h"
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& specs)
+{
+  DataProcessorSpec a{
+    .name = "A",
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, RawDeviceService& device, DataTakingContext& context, ProcessingContext& pcx) {
+        for (unsigned int i = 0; i < 10; i++) {
+          outputs.snapshot(Output{"TS1", "A1", i}, i);
+          outputs.snapshot(Output{"TS2", "A2", i}, i);
+        }
+      })},
+    .options = {ConfigParamSpec{"some-device-param", VariantType::Int, 1, {"Some device parameter"}},
+                }};
+
+  a.outputs.emplace_back(ConcreteDataTypeMatcher{"TS1", "A1"}, Lifetime::Sporadic);
+  a.outputs.emplace_back(ConcreteDataTypeMatcher{"TS2", "A2"}, Lifetime::Sporadic);
+
+  DataProcessorSpec d{.name = "D",
+                      .inputs = {InputSpec{"a", "TS1", Lifetime::Sporadic}, InputSpec{"b", "TS2", Lifetime::Sporadic}},
+                      .algorithm = AlgorithmSpec{adaptStateless(
+                        [](InputRecord& inputs) {
+                          auto ref = inputs.get("b");
+                          if (!ref.header) {
+                            LOG(info) << "Header is not there";
+                            return;
+                          }
+                          auto dph = o2::header::get<const DataProcessingHeader*>(ref.header);
+                          auto dh = o2::header::get<const o2::header::DataHeader*>(ref.header);
+                          LOG(info) << "Start time: " << dph->startTime;
+                          LOG(info) << "Subspec: " << dh->subSpecification;
+                        })},
+                      };
+
+  return workflow::concat(WorkflowSpec{a}, WorkflowSpec{d});
+}

From 376e6f30fa18e28d7fb04b989e8daff090a802a0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 24 Jul 2024 09:12:17 +0200
Subject: [PATCH 584/726] DPL: add dummy workflow mimicking CCDB populator
 behaviour

---
 Framework/TestWorkflows/CMakeLists.txt        |  4 +
 .../src/o2DummyPopulatorWorkflow.cxx          | 85 +++++++++++++++++++
 2 files changed, 89 insertions(+)
 create mode 100644 Framework/TestWorkflows/src/o2DummyPopulatorWorkflow.cxx

diff --git a/Framework/TestWorkflows/CMakeLists.txt b/Framework/TestWorkflows/CMakeLists.txt
index 21af45422e0b1..8548d6570e4a4 100644
--- a/Framework/TestWorkflows/CMakeLists.txt
+++ b/Framework/TestWorkflows/CMakeLists.txt
@@ -33,6 +33,10 @@ o2_add_dpl_workflow(diamond-workflow
                   SOURCES src/o2DiamondWorkflow.cxx
                   COMPONENT_NAME TestWorkflows)
 
+o2_add_dpl_workflow(dummy-populator-workflow
+                  SOURCES src/o2DummyPopulatorWorkflow.cxx
+                  COMPONENT_NAME TestWorkflows)
+
 o2_add_dpl_workflow(analysis-histograms
                   SOURCES src/o2TestHistograms.cxx
                   COMPONENT_NAME TestWorkflows)
diff --git a/Framework/TestWorkflows/src/o2DummyPopulatorWorkflow.cxx b/Framework/TestWorkflows/src/o2DummyPopulatorWorkflow.cxx
new file mode 100644
index 0000000000000..c7a88dd3c5d32
--- /dev/null
+++ b/Framework/TestWorkflows/src/o2DummyPopulatorWorkflow.cxx
@@ -0,0 +1,85 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/DataTakingContext.h"
+#include "Framework/CompletionPolicyHelpers.h"
+#include "Framework/DeviceSpec.h"
+#include "Framework/RawDeviceService.h"
+#include "Framework/ControlService.h"
+#include "Framework/Configurable.h"
+#include "Framework/RunningWorkflowInfo.h"
+#include "Framework/CallbackService.h"
+#include "Framework/RateLimiter.h"
+#include <fairmq/Device.h>
+
+#include <iostream>
+#include <chrono>
+#include <thread>
+#include <vector>
+
+using namespace o2::framework;
+
+struct WorkflowOptions {
+  Configurable<int> anInt{"anInt", 1, ""};
+  Configurable<float> aFloat{"aFloat", 2.0f, {"a float option"}};
+  Configurable<double> aDouble{"aDouble", 3., {"a double option"}};
+  Configurable<std::string> aString{"aString", "foobar", {"a string option"}};
+  Configurable<bool> aBool{"aBool", true, {"a boolean option"}};
+};
+
+void customize(std::vector<CompletionPolicy>& policies)
+{
+  auto a = CompletionPolicyHelpers::consumeWhenAll();
+  a.order = CompletionPolicy::CompletionOrder::Timeslice;
+  policies.clear();
+  policies.push_back(a);
+}
+
+#include "Framework/runDataProcessing.h"
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& specs)
+{
+  DataProcessorSpec a{
+    .name = "A",
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](DataAllocator& outputs, RawDeviceService& device, DataTakingContext& context, ProcessingContext& pcx) {
+        for (unsigned int i = 0; i < 10; i++) {
+          outputs.snapshot(Output{"TS1", "A1", i}, i);
+          outputs.snapshot(Output{"TS2", "A2", i}, i);
+        }
+      })},
+    .options = {
+      ConfigParamSpec{"some-device-param", VariantType::Int, 1, {"Some device parameter"}},
+    }};
+
+  a.outputs.emplace_back(ConcreteDataTypeMatcher{"TS1", "A1"}, Lifetime::Sporadic);
+  a.outputs.emplace_back(ConcreteDataTypeMatcher{"TS2", "A2"}, Lifetime::Sporadic);
+
+  DataProcessorSpec d{
+    .name = "D",
+    .inputs = {InputSpec{"a", "TS1", Lifetime::Sporadic}, InputSpec{"b", "TS2", Lifetime::Sporadic}},
+    .algorithm = AlgorithmSpec{adaptStateless(
+      [](InputRecord& inputs) {
+        auto ref = inputs.get("b");
+        if (!ref.header) {
+          LOG(info) << "Header is not there";
+          return;
+        }
+        auto dph = o2::header::get<const DataProcessingHeader*>(ref.header);
+        auto dh = o2::header::get<const o2::header::DataHeader*>(ref.header);
+        LOG(info) << "Start time: " << dph->startTime;
+        LOG(info) << "Subspec: " << dh->subSpecification;
+      })},
+  };
+
+  return workflow::concat(WorkflowSpec{a}, WorkflowSpec{d});
+}

From 0f6170d0663779d7611f9d1579d01a65f776c185 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 24 Jul 2024 14:27:27 +0200
Subject: [PATCH 585/726] DPL: have helper which returns true if running online

---
 Framework/Core/include/Framework/DefaultsHelpers.h |  2 ++
 Framework/Core/src/CommonServices.cxx              |  8 +++-----
 Framework/Core/src/DefaultsHelpers.cxx             | 12 ++++++++++++
 3 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/Framework/Core/include/Framework/DefaultsHelpers.h b/Framework/Core/include/Framework/DefaultsHelpers.h
index 9905b2033d62d..16d41d03baa7f 100644
--- a/Framework/Core/include/Framework/DefaultsHelpers.h
+++ b/Framework/Core/include/Framework/DefaultsHelpers.h
@@ -18,6 +18,8 @@ enum struct DeploymentMode;
 
 struct DefaultsHelpers {
   static DeploymentMode deploymentMode();
+  /// @true if running online
+  static bool onlineDeploymentMode();
   /// get max number of timeslices in the queue
   static unsigned int pipelineLength();
 };
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 7b3febca7be09..39c17599abb9d 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -848,8 +848,7 @@ o2::framework::ServiceSpec CommonServices::dataProcessingStats()
       bool enableDebugMetrics = true;
 #endif
       bool arrowAndResourceLimitingMetrics = false;
-      DeploymentMode deploymentMode = DefaultsHelpers::deploymentMode();
-      if (deploymentMode != DeploymentMode::OnlineDDS && deploymentMode != DeploymentMode::OnlineECS && deploymentMode != DeploymentMode::OnlineAUX && deploymentMode != DeploymentMode::FST) {
+      if (!DefaultsHelpers::onlineDeploymentMode() && DefaultsHelpers::deploymentMode() != DeploymentMode::FST) {
         arrowAndResourceLimitingMetrics = true;
       }
       // Input proxies should not report cpu_usage_fraction,
@@ -1243,8 +1242,7 @@ std::vector<ServiceSpec> CommonServices::defaultServices(std::string extraPlugin
     objectCache(),
     ccdbSupportSpec()};
 
-  DeploymentMode deploymentMode = DefaultsHelpers::deploymentMode();
-  if (deploymentMode != DeploymentMode::OnlineDDS && deploymentMode != DeploymentMode::OnlineECS && deploymentMode != DeploymentMode::OnlineAUX && deploymentMode != DeploymentMode::FST) {
+  if (!DefaultsHelpers::onlineDeploymentMode() && DefaultsHelpers::deploymentMode() != DeploymentMode::FST) {
     specs.push_back(ArrowSupport::arrowBackendSpec());
   }
   specs.push_back(CommonMessageBackends::fairMQBackendSpec());
@@ -1253,7 +1251,7 @@ std::vector<ServiceSpec> CommonServices::defaultServices(std::string extraPlugin
 
   std::string loadableServicesStr = extraPlugins;
   // Do not load InfoLogger by default if we are not at P2.
-  if (deploymentMode == DeploymentMode::OnlineDDS || deploymentMode == DeploymentMode::OnlineECS || deploymentMode == DeploymentMode::OnlineAUX) {
+  if (DefaultsHelpers::onlineDeploymentMode()) {
     if (loadableServicesStr.empty() == false) {
       loadableServicesStr += ",";
     }
diff --git a/Framework/Core/src/DefaultsHelpers.cxx b/Framework/Core/src/DefaultsHelpers.cxx
index 88956b93a855d..4dcc734216f0c 100644
--- a/Framework/Core/src/DefaultsHelpers.cxx
+++ b/Framework/Core/src/DefaultsHelpers.cxx
@@ -64,4 +64,16 @@ DeploymentMode DefaultsHelpers::deploymentMode()
   return retVal;
 }
 
+bool DefaultsHelpers::onlineDeploymentMode()
+{
+  switch (DefaultsHelpers::deploymentMode()) {
+    case DeploymentMode::OnlineAUX:
+    case DeploymentMode::OnlineECS:
+    case DeploymentMode::OnlineDDS:
+      return true;
+    default:
+      return false;
+  }
+}
+
 } // namespace o2::framework

From bd3d13201f9b1a6a6276db480d0dc07977b4551d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 24 Jul 2024 14:27:27 +0200
Subject: [PATCH 586/726] DPL: demote error to warning when dropping on a stop
 transition

Data might be missing for real here, and dropping the processing of
those timeslices is the unfortunate but correct thing to do.
---
 Framework/Core/src/DataRelayer.cxx | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index b974a51fe82e1..ad085afac0b03 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -307,7 +307,12 @@ void DataRelayer::setOldestPossibleInput(TimesliceId proposed, ChannelIndex chan
       if (element.size() != 0) {
         if (input.lifetime != Lifetime::Condition && mCompletionPolicy.name != "internal-dpl-injected-dummy-sink") {
           didDrop = true;
-          LOGP(error, "Dropping incomplete {} Lifetime::{} data in slot {} with timestamp {} < {} as it can never be completed.", DataSpecUtils::describe(input), input.lifetime, si, timestamp.value, newOldest.timeslice.value);
+          auto& state = mContext.get<DeviceState>();
+          if (state.transitionHandling != TransitionHandlingState::NoTransition && DefaultsHelpers::onlineDeploymentMode()) {
+            LOGP(warning, "Stop transition requested. Dropping incomplete {} Lifetime::{} data in slot {} with timestamp {} < {} as it will never be completed.", DataSpecUtils::describe(input), input.lifetime, si, timestamp.value, newOldest.timeslice.value);
+          } else {
+            LOGP(error, "Dropping incomplete {} Lifetime::{} data in slot {} with timestamp {} < {} as it can never be completed.", DataSpecUtils::describe(input), input.lifetime, si, timestamp.value, newOldest.timeslice.value);
+          }
         } else {
           LOGP(debug,
                "Silently dropping data {} in pipeline slot {} because it has timeslice {} < {} after receiving data from channel {}."
@@ -326,7 +331,12 @@ void DataRelayer::setOldestPossibleInput(TimesliceId proposed, ChannelIndex chan
         }
         auto& element = mCache[si * mInputs.size() + mi];
         if (element.size() == 0) {
-          LOGP(error, "Missing {} (lifetime:{}) while dropping incomplete data in slot {} with timestamp {} < {}.", DataSpecUtils::describe(input), input.lifetime, si, timestamp.value, newOldest.timeslice.value);
+          auto& state = mContext.get<DeviceState>();
+          if (state.transitionHandling != TransitionHandlingState::NoTransition && DefaultsHelpers::onlineDeploymentMode()) {
+            LOGP(warning, "Missing {} (lifetime:{}) while dropping incomplete data in slot {} with timestamp {} < {}.", DataSpecUtils::describe(input), input.lifetime, si, timestamp.value, newOldest.timeslice.value);
+          } else {
+            LOGP(error, "Missing {} (lifetime:{}) while dropping incomplete data in slot {} with timestamp {} < {}.", DataSpecUtils::describe(input), input.lifetime, si, timestamp.value, newOldest.timeslice.value);
+          }
         }
       }
     }

From 3e53fdeba751677f219f4e1f66b615e44dca1ae9 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 25 Jul 2024 15:05:14 +0200
Subject: [PATCH 587/726] DPL: run data processing on separate thread

---
 .../include/Framework/DataProcessingDevice.h  |  2 +-
 Framework/Core/src/DataProcessingDevice.cxx   | 20 ++++++++++++-------
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/Framework/Core/include/Framework/DataProcessingDevice.h b/Framework/Core/include/Framework/DataProcessingDevice.h
index 779e1aeff5340..d9565ebef84a2 100644
--- a/Framework/Core/include/Framework/DataProcessingDevice.h
+++ b/Framework/Core/include/Framework/DataProcessingDevice.h
@@ -63,7 +63,7 @@ struct TaskStreamInfo {
   /// The registry associated to the task being run
   ServiceRegistry* registry;
   /// The libuv task handle
-  uv_work_t task;
+  uv_work_t* task;
   /// Wether or not this task is running
   bool running = false;
 };
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 36f12156bab85..e0a0d1aad16e2 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -1264,6 +1264,12 @@ void DataProcessingDevice::Run()
   bool firstLoop = true;
   O2_SIGNPOST_ID_FROM_POINTER(lid, device, state.loop);
   O2_SIGNPOST_START(device, lid, "device_state", "First iteration of the device loop");
+
+  bool dplEnableMultithreding = getenv("DPL_THREADPOOL_SIZE") != nullptr;
+  if (dplEnableMultithreding) {
+    setenv("UV_THREADPOOL_SIZE", "1", 1);
+  }
+
   while (state.transitionHandling != TransitionHandlingState::Expired) {
     if (state.nextFairMQState.empty() == false) {
       (void)this->ChangeState(state.nextFairMQState.back());
@@ -1448,13 +1454,13 @@ void DataProcessingDevice::Run()
         stream.id = streamRef;
         stream.running = true;
         stream.registry = &mServiceRegistry;
-#ifdef DPL_ENABLE_THREADING
-        stream.task.data = &handle;
-        uv_queue_work(state.loop, &stream.task, run_callback, run_completion);
-#else
-        run_callback(&handle);
-        run_completion(&handle, 0);
-#endif
+        if (dplEnableMultithreding) [[unlikely]] {
+          stream.task = &handle;
+          uv_queue_work(state.loop, stream.task, run_callback, run_completion);
+        } else {
+          run_callback(&handle);
+          run_completion(&handle, 0);
+        }
       } else {
         auto ref = ServiceRegistryRef{mServiceRegistry};
         ref.get<ComputingQuotaEvaluator>().handleExpired(reportExpiredOffer);

From 742a0e7940e9769c24b2828979ad9685d68b0398 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 25 Jul 2024 22:03:19 +0200
Subject: [PATCH 588/726] DPL: improved calibration mode

---
 .../Core/include/Framework/DeviceContext.h    | 10 +++
 .../Core/include/Framework/DeviceStateEnums.h |  2 +
 Framework/Core/src/DataProcessingContext.cxx  |  4 ++
 Framework/Core/src/DataProcessingDevice.cxx   | 71 +++++++++++++++----
 Framework/Core/src/DeviceSpecHelpers.cxx      |  2 +
 Framework/Core/src/O2ControlHelpers.cxx       |  2 +
 Framework/Core/src/runDataProcessing.cxx      |  2 +
 .../test_FrameworkDataFlowToO2Control.cxx     |  8 +++
 .../src/o2DummyCalibrationWorkflow.cxx        |  6 +-
 9 files changed, 92 insertions(+), 15 deletions(-)

diff --git a/Framework/Core/include/Framework/DeviceContext.h b/Framework/Core/include/Framework/DeviceContext.h
index 3777e7f608b75..f0c3021d86dc8 100644
--- a/Framework/Core/include/Framework/DeviceContext.h
+++ b/Framework/Core/include/Framework/DeviceContext.h
@@ -28,8 +28,18 @@ struct ComputingQuotaStats;
 struct DeviceContext {
   ComputingQuotaStats* quotaStats = nullptr;
   uv_timer_t* gracePeriodTimer = nullptr;
+  uv_timer_t* dataProcessingGracePeriodTimer = nullptr;
   uv_signal_t* sigusr1Handle = nullptr;
   int expectedRegionCallbacks = 0;
+  // The timeout for the data processing to stop on this device.
+  // After this is reached, incoming data not marked to be kept will
+  // be dropped and the data processing will be stopped. However the
+  // calibrations will still be done and objects resulting from calibrations
+  // will be marked to be kept.
+  int dataProcessingTimeout = 0;
+  // The timeout for the whole processing to stop on this device.
+  // This includes the grace period for processing and the time
+  // for the calibrations to be done.
   int exitTransitionTimeout = 0;
 };
 
diff --git a/Framework/Core/include/Framework/DeviceStateEnums.h b/Framework/Core/include/Framework/DeviceStateEnums.h
index 291faac0ac982..a4c02c70c2bf6 100644
--- a/Framework/Core/include/Framework/DeviceStateEnums.h
+++ b/Framework/Core/include/Framework/DeviceStateEnums.h
@@ -30,6 +30,8 @@ enum struct TransitionHandlingState {
   NoTransition,
   /// A transition was notified to be requested
   Requested,
+  /// Only calibrations can be done
+  DataProcessingExpired,
   /// A transition needs to be fullfilled ASAP
   Expired
 };
diff --git a/Framework/Core/src/DataProcessingContext.cxx b/Framework/Core/src/DataProcessingContext.cxx
index 86409bf5434bb..394f50064bb37 100644
--- a/Framework/Core/src/DataProcessingContext.cxx
+++ b/Framework/Core/src/DataProcessingContext.cxx
@@ -11,9 +11,13 @@
 
 #include "Framework/DataProcessingContext.h"
 #include "Framework/DataProcessorSpec.h"
+#include "Framework/EndOfStreamContext.h"
+#include "Framework/TimingInfo.h"
 #include "Framework/Signpost.h"
 
 O2_DECLARE_DYNAMIC_LOG(data_processor_context);
+O2_DECLARE_DYNAMIC_LOG(calibration);
+
 namespace o2::framework
 {
 
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index e0a0d1aad16e2..4180ba9ff4965 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -92,6 +92,8 @@ struct formatter<o2::framework::CompletionPolicy::CompletionOp> : ostream_format
 O2_DECLARE_DYNAMIC_LOG(device);
 // Special log to keep track of the lifetime of the parts
 O2_DECLARE_DYNAMIC_LOG(parts);
+// Stream which keeps track of the calibration lifetime logic
+O2_DECLARE_DYNAMIC_LOG(calibration);
 // Special log to track the async queue behavior
 O2_DECLARE_DYNAMIC_LOG(async_queue);
 // Special log to track the forwarding requests
@@ -131,11 +133,28 @@ bool hasOnlyGenerated(DeviceSpec const& spec)
 
 void on_transition_requested_expired(uv_timer_t* handle)
 {
-  auto* state = (DeviceState*)handle->data;
-  state->loopReason |= DeviceState::TIMER_EXPIRED;
+  auto* ref = (ServiceRegistryRef*)handle->data;
+  auto& state = ref->get<DeviceState>();
+  state.loopReason |= DeviceState::TIMER_EXPIRED;
+  // Check if this is a source device
+  O2_SIGNPOST_ID_FROM_POINTER(cid, device, handle);
+  auto& spec = ref->get<DeviceSpec const>();
+  if (hasOnlyGenerated(spec)) {
+    O2_SIGNPOST_EVENT_EMIT_INFO(calibration, cid, "callback", "Grace period for source expired. Exiting.");
+  } else {
+    O2_SIGNPOST_EVENT_EMIT_ERROR(calibration, cid, "callback", "Grace period for calibration expired. Exiting.");
+  }
+  state.transitionHandling = TransitionHandlingState::Expired;
+}
+
+void on_data_processing_grace_expired(uv_timer_t* handle)
+{
+  auto* ref = (ServiceRegistryRef*)handle->data;
+  auto& state = ref->get<DeviceState>();
+  state.loopReason |= DeviceState::TIMER_EXPIRED;
   O2_SIGNPOST_ID_FROM_POINTER(cid, device, handle);
-  O2_SIGNPOST_EVENT_EMIT_WARN(device, cid, "callback", "Exit transition timer expired. Exiting.");
-  state->transitionHandling = TransitionHandlingState::Expired;
+  O2_SIGNPOST_EVENT_EMIT_INFO(calibration, cid, "callback", "Grace period for Data Processing expired. Waiting for calibration");
+  state.transitionHandling = TransitionHandlingState::DataProcessingExpired;
 }
 
 void on_communication_requested(uv_async_t* s)
@@ -928,8 +947,13 @@ void DataProcessingDevice::startPollers()
   }
 
   deviceContext.gracePeriodTimer = (uv_timer_t*)malloc(sizeof(uv_timer_t));
-  deviceContext.gracePeriodTimer->data = &state;
+  deviceContext.gracePeriodTimer->data = new ServiceRegistryRef(mServiceRegistry);
   uv_timer_init(state.loop, deviceContext.gracePeriodTimer);
+
+  // Prepare the timers if needed
+  deviceContext.dataProcessingGracePeriodTimer = (uv_timer_t*)malloc(sizeof(uv_timer_t));
+  deviceContext.dataProcessingGracePeriodTimer->data = new ServiceRegistryRef(mServiceRegistry);
+  uv_timer_init(state.loop, deviceContext.dataProcessingGracePeriodTimer);
 }
 
 void DataProcessingDevice::stopPollers()
@@ -960,6 +984,10 @@ void DataProcessingDevice::stopPollers()
   uv_timer_stop(deviceContext.gracePeriodTimer);
   free(deviceContext.gracePeriodTimer);
   deviceContext.gracePeriodTimer = nullptr;
+
+  uv_timer_stop(deviceContext.dataProcessingGracePeriodTimer);
+  free(deviceContext.dataProcessingGracePeriodTimer);
+  deviceContext.dataProcessingGracePeriodTimer = nullptr;
 }
 
 void DataProcessingDevice::InitTask()
@@ -994,6 +1022,7 @@ void DataProcessingDevice::InitTask()
   }
 
   deviceContext.expectedRegionCallbacks = std::stoi(fConfig->GetValue<std::string>("expected-region-callbacks"));
+  deviceContext.dataProcessingTimeout = std::stoi(fConfig->GetValue<std::string>("data-processing-timeout"));
   deviceContext.exitTransitionTimeout = std::stoi(fConfig->GetValue<std::string>("exit-transition-timeout"));
 
   for (auto& channel : GetChannels()) {
@@ -1306,17 +1335,28 @@ void DataProcessingDevice::Run()
       if (state.transitionHandling == TransitionHandlingState::NoTransition && NewStatePending()) {
         state.transitionHandling = TransitionHandlingState::Requested;
         auto& deviceContext = ref.get<DeviceContext>();
-        auto timeout = deviceContext.exitTransitionTimeout;
         // Check if we only have timers
         auto& spec = ref.get<DeviceSpec const>();
         if (hasOnlyTimers(spec)) {
           state.streaming = StreamingState::EndOfStreaming;
         }
-        if (timeout != 0 && state.streaming != StreamingState::Idle) {
+
+        if (deviceContext.exitTransitionTimeout != 0 && state.streaming != StreamingState::Idle) {
           state.transitionHandling = TransitionHandlingState::Requested;
           ref.get<CallbackService>().call<CallbackService::Id::ExitRequested>(ServiceRegistryRef{ref});
           uv_update_time(state.loop);
-          uv_timer_start(deviceContext.gracePeriodTimer, on_transition_requested_expired, timeout * 1000, 0);
+          O2_SIGNPOST_EVENT_EMIT(calibration, lid, "timer_setup", "Starting %d s timer for exitTransitionTimeout.", deviceContext.exitTransitionTimeout);
+          uv_timer_start(deviceContext.gracePeriodTimer, on_transition_requested_expired, deviceContext.exitTransitionTimeout * 1000, 0);
+          // In case we have a calibration grace period it will always be longer than the data processing timeout
+          if (hasOnlyGenerated(spec)) {
+            O2_SIGNPOST_EVENT_EMIT(calibration, lid, "timer_setup", "Source devices should never start calibration timer.");
+          } else if (deviceContext.dataProcessingTimeout != deviceContext.exitTransitionTimeout) {
+            O2_SIGNPOST_EVENT_EMIT(calibration, lid, "timer_setup", "Starting %d s timer for dataProcessingTimeout.", deviceContext.dataProcessingTimeout);
+            uv_timer_start(deviceContext.dataProcessingGracePeriodTimer, on_data_processing_grace_expired, deviceContext.dataProcessingTimeout * 1000, 0);
+          } else {
+            O2_SIGNPOST_EVENT_EMIT(calibration, lid, "timer_setup", "No special timer requested for calibration.");
+          }
+
           if (mProcessingPolicies.termination == TerminationPolicy::QUIT) {
             O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state requested. Waiting for %d seconds before quitting.", (int)deviceContext.exitTransitionTimeout);
           } else {
@@ -1331,12 +1371,12 @@ void DataProcessingDevice::Run()
           } else if (mProcessingPolicies.termination == TerminationPolicy::QUIT) {
             O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state pending and we are already idle, quitting immediately as per --completion-policy");
           } else {
-            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "runb_loop", "New state pending and we are already idle, switching to READY immediately.");
+            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state pending and we are already idle, switching to READY immediately.");
           }
         }
       }
-      // If we are Idle, we can then consider the transition to be expired.
-      if (state.transitionHandling == TransitionHandlingState::Requested && state.streaming == StreamingState::Idle) {
+      // If we are Idle, we can then consider the transition to be expired when it was requested or when the data processing timeout expired.
+      if ((state.transitionHandling == TransitionHandlingState::Requested || state.transitionHandling == TransitionHandlingState::DataProcessingExpired) && state.streaming == StreamingState::Idle) {
         O2_SIGNPOST_EVENT_EMIT(device, lid, "run_loop", "State transition requested and we are now in Idle. We can consider it to be completed.");
         state.transitionHandling = TransitionHandlingState::Expired;
       }
@@ -1702,8 +1742,13 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
     *context.wasActive = true;
   }
 
+  auto& deviceContext = ref.get<DeviceContext>();
+
   if (state.streaming == StreamingState::EndOfStreaming) {
-    O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "We are in EndOfStreaming. Flushing queues.");
+    O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "Flushing queues.");
+    O2_SIGNPOST_EVENT_EMIT(calibration, dpid, "state", "EndOfStreaming reached. %{public}s %{public}s",
+                           uv_timer_get_due_in(deviceContext.gracePeriodTimer) ? fmt::format("grace timer still running for {}", uv_timer_get_due_in(deviceContext.gracePeriodTimer)).c_str() : "Grace timer expired",
+                           uv_timer_get_due_in(deviceContext.dataProcessingGracePeriodTimer) ? fmt::format("data processing timer still running for {}", uv_timer_get_due_in(deviceContext.dataProcessingGracePeriodTimer)).c_str() : "DataProcessingTimer timer expired");
     // We keep processing data until we are Idle.
     // FIXME: not sure this is the correct way to drain the queues, but
     // I guess we will see.
@@ -1721,6 +1766,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
     // We should keep the data generated at end of stream only for those
     // which are not sources.
     timingInfo.keepAtEndOfStream = shouldProcess;
+    O2_SIGNPOST_EVENT_EMIT(calibration, dpid, "calibration", "TimingInfo.keepAtEndOfStream %d", timingInfo.keepAtEndOfStream);
 
     EndOfStreamContext eosContext{*context.registry, ref.get<DataAllocator>()};
 
@@ -2348,7 +2394,6 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
                        *context.registry};
     ProcessingContext processContext{record, ref, ref.get<DataAllocator>()};
     {
-      O2_SIGNPOST_EVENT_EMIT(device, aid, "device", "Invoking preProcessingCallbacks");
       // Notice this should be thread safe and reentrant
       // as it is called from many threads.
       streamContext.preProcessingCallbacks(processContext);
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index b9552dc093115..5218de0828e89 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1515,6 +1515,7 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
         realOdesc.add_options()("child-driver", bpo::value<std::string>());
         realOdesc.add_options()("rate", bpo::value<std::string>());
         realOdesc.add_options()("exit-transition-timeout", bpo::value<std::string>());
+        realOdesc.add_options()("data-processing-timeout", bpo::value<std::string>());
         realOdesc.add_options()("expected-region-callbacks", bpo::value<std::string>());
         realOdesc.add_options()("timeframes-rate-limit", bpo::value<std::string>());
         realOdesc.add_options()("environment", bpo::value<std::string>());
@@ -1701,6 +1702,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
     ("control-port", bpo::value<std::string>(), "Utility port to be used by O2 Control")                                                                             //
     ("rate", bpo::value<std::string>(), "rate for a data source device (Hz)")                                                                                        //
     ("exit-transition-timeout", bpo::value<std::string>(), "timeout before switching to READY state")                                                                //
+    ("data-processing-timeout", bpo::value<std::string>(), "timeout before switching to calibration processing mode")                                                //
     ("expected-region-callbacks", bpo::value<std::string>(), "region callbacks to expect before starting")                                                           //
     ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframes can be in fly")                                                    //
     ("shm-monitor", bpo::value<std::string>(), "whether to use the shared memory monitor")                                                                           //
diff --git a/Framework/Core/src/O2ControlHelpers.cxx b/Framework/Core/src/O2ControlHelpers.cxx
index ad539dc4af03f..a75799a4d9712 100644
--- a/Framework/Core/src/O2ControlHelpers.cxx
+++ b/Framework/Core/src/O2ControlHelpers.cxx
@@ -262,6 +262,8 @@ void dumpCommand(std::ostream& dumpOut, const DeviceExecution& execution, std::s
   dumpOut << indLevel << indScheme << "- \"-b\"\n";
   dumpOut << indLevel << indScheme << "- \"--exit-transition-timeout\"\n";
   dumpOut << indLevel << indScheme << "- \"'{{ exit_transition_timeout }}'\"\n";
+  dumpOut << indLevel << indScheme << "- \"--data-processing-timeout\"\n";
+  dumpOut << indLevel << indScheme << "- \"'{{ data_processing_timeout }}'\"\n";
   dumpOut << indLevel << indScheme << "- \"--monitoring-backend\"\n";
   dumpOut << indLevel << indScheme << "- \"'{{ monitoring_dpl_url }}'\"\n";
   dumpOut << indLevel << indScheme << "- \"--session\"\n";
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 89737483fc813..a89abfba33204 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1037,6 +1037,7 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
   // declared in the workflow definition are allowed.
   runner.AddHook<fair::mq::hooks::SetCustomCmdLineOptions>([&spec, driverConfig, defaultDriverClient](fair::mq::DeviceRunner& r) {
     std::string defaultExitTransitionTimeout = "0";
+    std::string defaultDataProcessingTimeout = "0";
     std::string defaultInfologgerMode = "";
     o2::framework::DeploymentMode deploymentMode = o2::framework::DefaultsHelpers::deploymentMode();
     if (deploymentMode == o2::framework::DeploymentMode::OnlineDDS) {
@@ -1055,6 +1056,7 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
       ("signposts", bpo::value<std::string>()->default_value(defaultSignposts ? defaultSignposts : ""), "comma separated list of signposts to enable")                                     //
       ("expected-region-callbacks", bpo::value<std::string>()->default_value("0"), "how many region callbacks we are expecting")                                                           //
       ("exit-transition-timeout", bpo::value<std::string>()->default_value(defaultExitTransitionTimeout), "how many second to wait before switching from RUN to READY")                    //
+      ("data-processing-timeout", bpo::value<std::string>()->default_value(defaultDataProcessingTimeout), "how many second to wait before switching from RUN to CALIBRATION")              //
       ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframe can be in fly at the same moment (0 disables)")                                         //
       ("configuration,cfg", bpo::value<std::string>()->default_value("command-line"), "configuration backend")                                                                             //
       ("infologger-mode", bpo::value<std::string>()->default_value(defaultInfologgerMode), "O2_INFOLOGGER_MODE override");
diff --git a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
index 4cda5fd76336a..a05ce5f4c3d1a 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
@@ -173,6 +173,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"
@@ -270,6 +272,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"
@@ -367,6 +371,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"
@@ -461,6 +467,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"
diff --git a/Framework/TestWorkflows/src/o2DummyCalibrationWorkflow.cxx b/Framework/TestWorkflows/src/o2DummyCalibrationWorkflow.cxx
index 94916cfa36984..0ece9da5aa7d0 100644
--- a/Framework/TestWorkflows/src/o2DummyCalibrationWorkflow.cxx
+++ b/Framework/TestWorkflows/src/o2DummyCalibrationWorkflow.cxx
@@ -37,8 +37,9 @@ WorkflowSpec defineDataProcessing(ConfigContext const& specs)
         static int counter = 0;
         auto& aData = outputs.make<int>(OutputRef{"counter"});
         aData = counter++;
-        if (counter == 10) {
-          pcx.services().get<ControlService>().endOfStream();
+        sleep(1);
+        if (counter == 5) {
+          pcx.services().get<ControlService>().readyToQuit(QuitRequest::All);
         }
       })},
   };
@@ -57,6 +58,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& specs)
         callbacks.set<CallbackService::Id::EndOfStream>(eosCallback);
         return adaptStateless([](Input<"x", int> const& x)
           {
+            sleep(1);
             sum += x;
             std::cout << "Sum: " << sum << std::endl;
         }); })};

From 94172272e6263db1d6421c6769dc875f5006dfa5 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 25 Jul 2024 22:27:08 +0200
Subject: [PATCH 589/726] DPL: improve exitTransitionTimeout handling

---
 Framework/Core/src/DataProcessingContext.cxx |  4 +++
 Framework/Core/src/DataProcessingDevice.cxx  | 30 ++++++++++++++------
 2 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/Framework/Core/src/DataProcessingContext.cxx b/Framework/Core/src/DataProcessingContext.cxx
index 86409bf5434bb..394f50064bb37 100644
--- a/Framework/Core/src/DataProcessingContext.cxx
+++ b/Framework/Core/src/DataProcessingContext.cxx
@@ -11,9 +11,13 @@
 
 #include "Framework/DataProcessingContext.h"
 #include "Framework/DataProcessorSpec.h"
+#include "Framework/EndOfStreamContext.h"
+#include "Framework/TimingInfo.h"
 #include "Framework/Signpost.h"
 
 O2_DECLARE_DYNAMIC_LOG(data_processor_context);
+O2_DECLARE_DYNAMIC_LOG(calibration);
+
 namespace o2::framework
 {
 
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index e0a0d1aad16e2..c23f43256ae61 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -92,6 +92,8 @@ struct formatter<o2::framework::CompletionPolicy::CompletionOp> : ostream_format
 O2_DECLARE_DYNAMIC_LOG(device);
 // Special log to keep track of the lifetime of the parts
 O2_DECLARE_DYNAMIC_LOG(parts);
+// Stream which keeps track of the calibration lifetime logic
+O2_DECLARE_DYNAMIC_LOG(calibration);
 // Special log to track the async queue behavior
 O2_DECLARE_DYNAMIC_LOG(async_queue);
 // Special log to track the forwarding requests
@@ -131,8 +133,18 @@ bool hasOnlyGenerated(DeviceSpec const& spec)
 
 void on_transition_requested_expired(uv_timer_t* handle)
 {
-  auto* state = (DeviceState*)handle->data;
-  state->loopReason |= DeviceState::TIMER_EXPIRED;
+  auto* ref = (ServiceRegistryRef*)handle->data;
+  auto& state = ref->get<DeviceState>();
+  state.loopReason |= DeviceState::TIMER_EXPIRED;
+  // Check if this is a source device
+  O2_SIGNPOST_ID_FROM_POINTER(cid, device, handle);
+  auto& spec = ref->get<DeviceSpec const>();
+  if (hasOnlyGenerated(spec)) {
+    O2_SIGNPOST_EVENT_EMIT_INFO(calibration, cid, "callback", "Grace period for source expired. Exiting.");
+  } else {
+    O2_SIGNPOST_EVENT_EMIT_ERROR(calibration, cid, "callback", "Grace period for data / calibration expired. Exiting.");
+  }
+  state.transitionHandling = TransitionHandlingState::Expired;
   O2_SIGNPOST_ID_FROM_POINTER(cid, device, handle);
   O2_SIGNPOST_EVENT_EMIT_WARN(device, cid, "callback", "Exit transition timer expired. Exiting.");
   state->transitionHandling = TransitionHandlingState::Expired;
@@ -928,7 +940,7 @@ void DataProcessingDevice::startPollers()
   }
 
   deviceContext.gracePeriodTimer = (uv_timer_t*)malloc(sizeof(uv_timer_t));
-  deviceContext.gracePeriodTimer->data = &state;
+  deviceContext.gracePeriodTimer->data = new ServiceRegistryRef(mServiceRegistry);
   uv_timer_init(state.loop, deviceContext.gracePeriodTimer);
 }
 
@@ -958,6 +970,7 @@ void DataProcessingDevice::stopPollers()
   }
 
   uv_timer_stop(deviceContext.gracePeriodTimer);
+  delete (ServiceRegistryRef*)deviceContext.gracePeriodTimer->data;
   free(deviceContext.gracePeriodTimer);
   deviceContext.gracePeriodTimer = nullptr;
 }
@@ -1306,17 +1319,18 @@ void DataProcessingDevice::Run()
       if (state.transitionHandling == TransitionHandlingState::NoTransition && NewStatePending()) {
         state.transitionHandling = TransitionHandlingState::Requested;
         auto& deviceContext = ref.get<DeviceContext>();
-        auto timeout = deviceContext.exitTransitionTimeout;
         // Check if we only have timers
         auto& spec = ref.get<DeviceSpec const>();
         if (hasOnlyTimers(spec)) {
           state.streaming = StreamingState::EndOfStreaming;
         }
-        if (timeout != 0 && state.streaming != StreamingState::Idle) {
+
+        if (deviceContext.exitTransitionTimeout != 0 && state.streaming != StreamingState::Idle) {
           state.transitionHandling = TransitionHandlingState::Requested;
           ref.get<CallbackService>().call<CallbackService::Id::ExitRequested>(ServiceRegistryRef{ref});
           uv_update_time(state.loop);
-          uv_timer_start(deviceContext.gracePeriodTimer, on_transition_requested_expired, timeout * 1000, 0);
+          O2_SIGNPOST_EVENT_EMIT(calibration, lid, "timer_setup", "Starting %d s timer for exitTransitionTimeout.", deviceContext.exitTransitionTimeout);
+          uv_timer_start(deviceContext.gracePeriodTimer, on_transition_requested_expired, deviceContext.exitTransitionTimeout * 1000, 0);
           if (mProcessingPolicies.termination == TerminationPolicy::QUIT) {
             O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state requested. Waiting for %d seconds before quitting.", (int)deviceContext.exitTransitionTimeout);
           } else {
@@ -1331,7 +1345,7 @@ void DataProcessingDevice::Run()
           } else if (mProcessingPolicies.termination == TerminationPolicy::QUIT) {
             O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state pending and we are already idle, quitting immediately as per --completion-policy");
           } else {
-            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "runb_loop", "New state pending and we are already idle, switching to READY immediately.");
+            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state pending and we are already idle, switching to READY immediately.");
           }
         }
       }
@@ -1721,6 +1735,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
     // We should keep the data generated at end of stream only for those
     // which are not sources.
     timingInfo.keepAtEndOfStream = shouldProcess;
+    O2_SIGNPOST_EVENT_EMIT(calibration, dpid, "calibration", "TimingInfo.keepAtEndOfStream %d", timingInfo.keepAtEndOfStream);
 
     EndOfStreamContext eosContext{*context.registry, ref.get<DataAllocator>()};
 
@@ -2348,7 +2363,6 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
                        *context.registry};
     ProcessingContext processContext{record, ref, ref.get<DataAllocator>()};
     {
-      O2_SIGNPOST_EVENT_EMIT(device, aid, "device", "Invoking preProcessingCallbacks");
       // Notice this should be thread safe and reentrant
       // as it is called from many threads.
       streamContext.preProcessingCallbacks(processContext);

From 3c9df52c1722c037da77fc79c051e191085f3fde Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 26 Jul 2024 08:36:47 +0200
Subject: [PATCH 590/726] DPL: improve exitTransitionTimeout handling

---
 Framework/Core/src/DataProcessingContext.cxx |  4 +++
 Framework/Core/src/DataProcessingDevice.cxx  | 31 +++++++++++++-------
 2 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/Framework/Core/src/DataProcessingContext.cxx b/Framework/Core/src/DataProcessingContext.cxx
index 86409bf5434bb..394f50064bb37 100644
--- a/Framework/Core/src/DataProcessingContext.cxx
+++ b/Framework/Core/src/DataProcessingContext.cxx
@@ -11,9 +11,13 @@
 
 #include "Framework/DataProcessingContext.h"
 #include "Framework/DataProcessorSpec.h"
+#include "Framework/EndOfStreamContext.h"
+#include "Framework/TimingInfo.h"
 #include "Framework/Signpost.h"
 
 O2_DECLARE_DYNAMIC_LOG(data_processor_context);
+O2_DECLARE_DYNAMIC_LOG(calibration);
+
 namespace o2::framework
 {
 
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index e0a0d1aad16e2..2bbec748d2b11 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -92,6 +92,8 @@ struct formatter<o2::framework::CompletionPolicy::CompletionOp> : ostream_format
 O2_DECLARE_DYNAMIC_LOG(device);
 // Special log to keep track of the lifetime of the parts
 O2_DECLARE_DYNAMIC_LOG(parts);
+// Stream which keeps track of the calibration lifetime logic
+O2_DECLARE_DYNAMIC_LOG(calibration);
 // Special log to track the async queue behavior
 O2_DECLARE_DYNAMIC_LOG(async_queue);
 // Special log to track the forwarding requests
@@ -131,11 +133,18 @@ bool hasOnlyGenerated(DeviceSpec const& spec)
 
 void on_transition_requested_expired(uv_timer_t* handle)
 {
-  auto* state = (DeviceState*)handle->data;
-  state->loopReason |= DeviceState::TIMER_EXPIRED;
+  auto* ref = (ServiceRegistryRef*)handle->data;
+  auto& state = ref->get<DeviceState>();
+  state.loopReason |= DeviceState::TIMER_EXPIRED;
+  // Check if this is a source device
   O2_SIGNPOST_ID_FROM_POINTER(cid, device, handle);
-  O2_SIGNPOST_EVENT_EMIT_WARN(device, cid, "callback", "Exit transition timer expired. Exiting.");
-  state->transitionHandling = TransitionHandlingState::Expired;
+  auto& spec = ref->get<DeviceSpec const>();
+  if (hasOnlyGenerated(spec)) {
+    O2_SIGNPOST_EVENT_EMIT_INFO(calibration, cid, "callback", "Grace period for source expired. Exiting.");
+  } else {
+    O2_SIGNPOST_EVENT_EMIT_ERROR(calibration, cid, "callback", "Grace period for data / calibration expired. Exiting.");
+  }
+  state.transitionHandling = TransitionHandlingState::Expired;
 }
 
 void on_communication_requested(uv_async_t* s)
@@ -928,7 +937,7 @@ void DataProcessingDevice::startPollers()
   }
 
   deviceContext.gracePeriodTimer = (uv_timer_t*)malloc(sizeof(uv_timer_t));
-  deviceContext.gracePeriodTimer->data = &state;
+  deviceContext.gracePeriodTimer->data = new ServiceRegistryRef(mServiceRegistry);
   uv_timer_init(state.loop, deviceContext.gracePeriodTimer);
 }
 
@@ -958,6 +967,7 @@ void DataProcessingDevice::stopPollers()
   }
 
   uv_timer_stop(deviceContext.gracePeriodTimer);
+  delete (ServiceRegistryRef*)deviceContext.gracePeriodTimer->data;
   free(deviceContext.gracePeriodTimer);
   deviceContext.gracePeriodTimer = nullptr;
 }
@@ -1306,17 +1316,18 @@ void DataProcessingDevice::Run()
       if (state.transitionHandling == TransitionHandlingState::NoTransition && NewStatePending()) {
         state.transitionHandling = TransitionHandlingState::Requested;
         auto& deviceContext = ref.get<DeviceContext>();
-        auto timeout = deviceContext.exitTransitionTimeout;
         // Check if we only have timers
         auto& spec = ref.get<DeviceSpec const>();
         if (hasOnlyTimers(spec)) {
           state.streaming = StreamingState::EndOfStreaming;
         }
-        if (timeout != 0 && state.streaming != StreamingState::Idle) {
+
+        if (deviceContext.exitTransitionTimeout != 0 && state.streaming != StreamingState::Idle) {
           state.transitionHandling = TransitionHandlingState::Requested;
           ref.get<CallbackService>().call<CallbackService::Id::ExitRequested>(ServiceRegistryRef{ref});
           uv_update_time(state.loop);
-          uv_timer_start(deviceContext.gracePeriodTimer, on_transition_requested_expired, timeout * 1000, 0);
+          O2_SIGNPOST_EVENT_EMIT(calibration, lid, "timer_setup", "Starting %d s timer for exitTransitionTimeout.", deviceContext.exitTransitionTimeout);
+          uv_timer_start(deviceContext.gracePeriodTimer, on_transition_requested_expired, deviceContext.exitTransitionTimeout * 1000, 0);
           if (mProcessingPolicies.termination == TerminationPolicy::QUIT) {
             O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state requested. Waiting for %d seconds before quitting.", (int)deviceContext.exitTransitionTimeout);
           } else {
@@ -1331,7 +1342,7 @@ void DataProcessingDevice::Run()
           } else if (mProcessingPolicies.termination == TerminationPolicy::QUIT) {
             O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state pending and we are already idle, quitting immediately as per --completion-policy");
           } else {
-            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "runb_loop", "New state pending and we are already idle, switching to READY immediately.");
+            O2_SIGNPOST_EVENT_EMIT_INFO(device, lid, "run_loop", "New state pending and we are already idle, switching to READY immediately.");
           }
         }
       }
@@ -1721,6 +1732,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
     // We should keep the data generated at end of stream only for those
     // which are not sources.
     timingInfo.keepAtEndOfStream = shouldProcess;
+    O2_SIGNPOST_EVENT_EMIT(calibration, dpid, "calibration", "TimingInfo.keepAtEndOfStream %d", timingInfo.keepAtEndOfStream);
 
     EndOfStreamContext eosContext{*context.registry, ref.get<DataAllocator>()};
 
@@ -2348,7 +2360,6 @@ bool DataProcessingDevice::tryDispatchComputation(ServiceRegistryRef ref, std::v
                        *context.registry};
     ProcessingContext processContext{record, ref, ref.get<DataAllocator>()};
     {
-      O2_SIGNPOST_EVENT_EMIT(device, aid, "device", "Invoking preProcessingCallbacks");
       // Notice this should be thread safe and reentrant
       // as it is called from many threads.
       streamContext.preProcessingCallbacks(processContext);

From 90b8fcd5daea4aa09a8e76f6b6c828c616d3f623 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 26 Jul 2024 09:05:46 +0200
Subject: [PATCH 591/726] DPL: factorise out backtrace support

---
 Framework/Core/src/DataProcessingDevice.cxx   |   8 +-
 Framework/Core/src/runDataProcessing.cxx      |   9 +-
 Framework/Foundation/CMakeLists.txt           |   1 +
 .../include/Framework/BacktraceHelpers.h      |  22 +++
 .../include/Framework/RuntimeError.h          |   7 +-
 Framework/Foundation/src/BacktraceHelpers.cxx | 125 ++++++++++++++++++
 Framework/Foundation/src/RuntimeError.cxx     | 110 +--------------
 7 files changed, 163 insertions(+), 119 deletions(-)
 create mode 100644 Framework/Foundation/include/Framework/BacktraceHelpers.h
 create mode 100644 Framework/Foundation/src/BacktraceHelpers.cxx

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index e0a0d1aad16e2..57010c27ffb09 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -431,7 +431,7 @@ void DataProcessingDevice::Init()
       auto& err = error_from_ref(e);
       O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
       O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "Init", "Exception caught while in Init: %{public}s. Invoking errorCallback.", err.what);
-      demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
+      BacktraceHelpers::demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
       auto& stats = ref.get<DataProcessingStats>();
       stats.updateStats({(int)ProcessingStatsId::EXCEPTION_COUNT, DataProcessingStats::Op::Add, 1});
       InitErrorContext errorContext{ref, e};
@@ -446,7 +446,7 @@ void DataProcessingDevice::Init()
       auto& context = ref.get<DataProcessorContext>();
       O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
       O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "Init", "Exception caught while in Init: %{public}s. Exiting with 1.", err.what);
-      demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
+      BacktraceHelpers::demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
       auto& stats = ref.get<DataProcessingStats>();
       stats.updateStats({(int)ProcessingStatsId::EXCEPTION_COUNT, DataProcessingStats::Op::Add, 1});
       exit(1);
@@ -1111,7 +1111,7 @@ void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceCont
       auto& context = ref.get<DataProcessorContext>();
       O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
       O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "Run", "Exception while running: %{public}s. Invoking callback.", err.what);
-      demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
+      BacktraceHelpers::demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
       auto& stats = ref.get<DataProcessingStats>();
       stats.updateStats({(int)ProcessingStatsId::EXCEPTION_COUNT, DataProcessingStats::Op::Add, 1});
       ErrorContext errorContext{record, ref, e};
@@ -1126,7 +1126,7 @@ void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceCont
       ServiceRegistryRef ref{serviceRegistry, ServiceRegistry::globalDeviceSalt()};
       auto& context = ref.get<DataProcessorContext>();
       O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
-      demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
+      BacktraceHelpers::demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
       auto& stats = ref.get<DataProcessingStats>();
       stats.updateStats({(int)ProcessingStatsId::EXCEPTION_COUNT, DataProcessingStats::Op::Add, 1});
       switch (errorPolicy) {
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 89737483fc813..6b95a073ed21d 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -11,6 +11,7 @@
 #define BOOST_BIND_GLOBAL_PLACEHOLDERS
 #include <stdexcept>
 #include "Framework/BoostOptionsRetriever.h"
+#include "Framework/BacktraceHelpers.h"
 #include "Framework/CallbacksPolicy.h"
 #include "Framework/ChannelConfigurationPolicy.h"
 #include "Framework/ChannelMatching.h"
@@ -656,7 +657,7 @@ void handle_crash(int sig)
     auto retVal = write(STDERR_FILENO, buffer, strlen(buffer));
     (void)retVal;
   }
-  demangled_backtrace_symbols(array, size, STDERR_FILENO);
+  BacktraceHelpers::demangled_backtrace_symbols(array, size, STDERR_FILENO);
   {
     char const* msg = "Backtrace complete.\n";
     int len = strlen(msg); /* the byte length of the string */
@@ -982,7 +983,7 @@ void doDPLException(RuntimeErrorRef& e, char const* processName)
          " Reason: {}"
          "\n Backtrace follow: \n",
          processName, err.what);
-    demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
+    BacktraceHelpers::demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
   } else {
     LOGP(fatal,
          "Unhandled o2::framework::runtime_error reached the top of main of {}, device shutting down."
@@ -1928,7 +1929,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
         } catch (o2::framework::RuntimeErrorRef ref) {
           auto& err = o2::framework::error_from_ref(ref);
 #ifdef DPL_ENABLE_BACKTRACE
-          demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
+          BacktraceHelpers::demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
 #endif
           LOGP(error, "invalid workflow in {}: {}", driverInfo.argv[0], err.what);
           return 1;
@@ -2059,7 +2060,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
           LOGP(error, "unable to merge configurations in {}: {}", driverInfo.argv[0], err.what);
 #ifdef DPL_ENABLE_BACKTRACE
           std::cerr << "\nStacktrace follows:\n\n";
-          demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
+          BacktraceHelpers::demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
 #endif
           return 1;
         }
diff --git a/Framework/Foundation/CMakeLists.txt b/Framework/Foundation/CMakeLists.txt
index bcde0f7a69f34..1cffe7f0487ff 100644
--- a/Framework/Foundation/CMakeLists.txt
+++ b/Framework/Foundation/CMakeLists.txt
@@ -16,6 +16,7 @@ install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/include/Framework
 o2_add_library(FrameworkFoundation
                SOURCES src/RuntimeError.cxx
                src/Signpost.cxx
+               src/BacktraceHelpers.cxx
                TARGETVARNAME targetName
                PUBLIC_LINK_LIBRARIES O2::FrameworkFoundation3rdparty
               )
diff --git a/Framework/Foundation/include/Framework/BacktraceHelpers.h b/Framework/Foundation/include/Framework/BacktraceHelpers.h
new file mode 100644
index 0000000000000..1f897d4d6f860
--- /dev/null
+++ b/Framework/Foundation/include/Framework/BacktraceHelpers.h
@@ -0,0 +1,22 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef O2_FRAMEWORK_BACKTRACE_HELPERS_H_
+#define O2_FRAMEWORK_BACKTRACE_HELPERS_H_
+
+namespace o2::framework
+{
+struct BacktraceHelpers {
+  static constexpr unsigned int MAX_BACKTRACE_SIZE = 100;
+  static void demangled_backtrace_symbols(void** backtrace, unsigned int total, int fd);
+};
+} // namespace o2::framework
+
+#endif // O2_FRAMEWORK_BACKTRACE_HELPERS_H_
diff --git a/Framework/Foundation/include/Framework/RuntimeError.h b/Framework/Foundation/include/Framework/RuntimeError.h
index a8335ae77c509..6c0cf601dfe2d 100644
--- a/Framework/Foundation/include/Framework/RuntimeError.h
+++ b/Framework/Foundation/include/Framework/RuntimeError.h
@@ -11,15 +11,16 @@
 #ifndef O2_FRAMEWORK_RUNTIMEERROR_H_
 #define O2_FRAMEWORK_RUNTIMEERROR_H_
 
+#include "BacktraceHelpers.h"
+
 namespace o2::framework
 {
 
 struct RuntimeError {
   static constexpr unsigned int MAX_RUNTIME_ERRORS = 64;
   static constexpr unsigned int MAX_RUNTIME_ERROR_SIZE = 1024;
-  static constexpr unsigned int MAX_BACKTRACE_SIZE = 100;
   char what[MAX_RUNTIME_ERROR_SIZE];
-  void* backtrace[MAX_BACKTRACE_SIZE];
+  void* backtrace[BacktraceHelpers::MAX_BACKTRACE_SIZE];
   int maxBacktrace = 0;
 };
 
@@ -27,8 +28,6 @@ struct RuntimeErrorRef {
   int index = 0;
 };
 
-void demangled_backtrace_symbols(void** backtrace, unsigned int total, int fd);
-
 RuntimeErrorRef runtime_error(const char*);
 RuntimeErrorRef runtime_error_f(const char*, ...);
 RuntimeError& error_from_ref(RuntimeErrorRef);
diff --git a/Framework/Foundation/src/BacktraceHelpers.cxx b/Framework/Foundation/src/BacktraceHelpers.cxx
new file mode 100644
index 0000000000000..cf37464f7e0bd
--- /dev/null
+++ b/Framework/Foundation/src/BacktraceHelpers.cxx
@@ -0,0 +1,125 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/BacktraceHelpers.h"
+#include <climits>
+#include <cstddef>
+#include <cstdio>
+#include <cstdlib>
+#include <unistd.h>
+#include <cxxabi.h>
+#include <execinfo.h>
+
+namespace o2::framework
+{
+
+void BacktraceHelpers::demangled_backtrace_symbols(void** stackTrace, unsigned int stackDepth, int fd)
+{
+  char** stackStrings;
+  stackStrings = backtrace_symbols(stackTrace, stackDepth);
+  char exe[PATH_MAX];
+  bool hasExe = false;
+  int exeSize = 0;
+
+#if __linux__
+  exeSize = readlink("/proc/self/exe", exe, PATH_MAX);
+  if (exeSize == -1) {
+    dprintf(fd, "Unable to detect exectuable name\n");
+    hasExe = false;
+  } else {
+    dprintf(fd, "Executable is %.*s\n", exeSize, exe);
+    hasExe = true;
+  }
+#endif
+
+  for (size_t i = 1; i < stackDepth; i++) {
+
+    size_t sz = 64000; // 64K ought to be enough for our templates...
+    char* function = static_cast<char*>(malloc(sz));
+    char *begin = nullptr, *end = nullptr;
+    // find the last space and address offset surrounding the mangled name
+#if __APPLE__
+    for (char* j = stackStrings[i]; *j; ++j) {
+      if (*j == ' ' && *(j + 1) != '+') {
+        begin = j;
+      } else if (*j == ' ' && *(j + 1) == '+') {
+        end = j;
+        break;
+      }
+    }
+    bool tryAddr2Line = false;
+#else
+    for (char* j = stackStrings[i]; j && *j; ++j) {
+      if (*j == '(') {
+        begin = j;
+      } else if (*j == '+') {
+        end = j;
+        break;
+      }
+    }
+    bool tryAddr2Line = true;
+#endif
+    if (begin && end) {
+      *begin++ = '\0';
+      *end = '\0';
+      // found our mangled name, now in [begin, end)
+
+      int status;
+      char* ret = abi::__cxa_demangle(begin, function, &sz, &status);
+      if (ret) {
+        // return value may be a realloc() of the input
+        function = ret;
+        dprintf(fd, "    %s: %s\n", stackStrings[i], function);
+        tryAddr2Line = false;
+      }
+    }
+    if (tryAddr2Line) {
+      // didn't find the mangled name, just print the whole line
+      dprintf(fd, "    %s: ", stackStrings[i]);
+      if (stackTrace[i] && hasExe) {
+        char syscom[4096 + PATH_MAX];
+
+        // Find c++filt from the environment
+        // This is needed for platforms where we still need c++filt -r
+        char const* cxxfilt = getenv("CXXFILT");
+        if (cxxfilt == nullptr) {
+          cxxfilt = "c++filt";
+        }
+        // Do the same for addr2line, just in case we wanted to pass some options
+        char const* addr2line = getenv("ADDR2LINE");
+        if (addr2line == nullptr) {
+          addr2line = "addr2line";
+        }
+        snprintf(syscom, 4096, "%s %p -p -s -f -e %.*s 2>/dev/null | %s ", addr2line, stackTrace[i], exeSize, exe, cxxfilt); // last parameter is the name of this app
+
+        FILE* fp;
+        char path[1024];
+
+        fp = popen(syscom, "r");
+        if (fp == nullptr) {
+          dprintf(fd, "-- no source could be retrieved --\n");
+          continue;
+        }
+
+        while (fgets(path, sizeof(path) - 1, fp) != nullptr) {
+          dprintf(fd, "    %s", path);
+        }
+
+        pclose(fp);
+      } else {
+        dprintf(fd, "-- no source avaliable --\n");
+      }
+    }
+    free(function);
+  }
+  free(stackStrings); // malloc()ed by backtrace_symbols
+  fsync(fd);
+}
+} // namespace o2::framework
diff --git a/Framework/Foundation/src/RuntimeError.cxx b/Framework/Foundation/src/RuntimeError.cxx
index 747b339cc6dbb..636dff1a2f9e1 100644
--- a/Framework/Foundation/src/RuntimeError.cxx
+++ b/Framework/Foundation/src/RuntimeError.cxx
@@ -15,11 +15,10 @@
 #include <climits>
 #include <atomic>
 #include <cstdarg>
-#include <execinfo.h>
 #include <unistd.h>
 #include <cstdlib>
-#include <cstring>
 #include <cxxabi.h>
+#include <execinfo.h>
 #include <stdexcept>
 
 namespace o2::framework
@@ -71,7 +70,7 @@ RuntimeErrorRef runtime_error_f(const char* format, ...)
   va_start(args, format);
   vsnprintf(gError[i].what, RuntimeError::MAX_RUNTIME_ERROR_SIZE, format, args);
   va_end(args);
-  gError[i].maxBacktrace = canDumpBacktrace() ? backtrace(gError[i].backtrace, RuntimeError::MAX_BACKTRACE_SIZE) : 0;
+  gError[i].maxBacktrace = canDumpBacktrace() ? backtrace(gError[i].backtrace, BacktraceHelpers::MAX_BACKTRACE_SIZE) : 0;
   return RuntimeErrorRef{i};
 }
 
@@ -83,7 +82,7 @@ RuntimeErrorRef runtime_error(const char* s)
     ++i;
   }
   strncpy(gError[i].what, s, RuntimeError::MAX_RUNTIME_ERROR_SIZE);
-  gError[i].maxBacktrace = canDumpBacktrace() ? backtrace(gError[i].backtrace, RuntimeError::MAX_BACKTRACE_SIZE) : 0;
+  gError[i].maxBacktrace = canDumpBacktrace() ? backtrace(gError[i].backtrace, BacktraceHelpers::MAX_BACKTRACE_SIZE) : 0;
   return RuntimeErrorRef{i};
 }
 
@@ -92,107 +91,4 @@ void throw_error(RuntimeErrorRef ref)
   throw ref;
 }
 
-void demangled_backtrace_symbols(void** stackTrace, unsigned int stackDepth, int fd)
-{
-  char** stackStrings;
-  stackStrings = backtrace_symbols(stackTrace, stackDepth);
-  char exe[PATH_MAX];
-  bool hasExe = false;
-  int exeSize = 0;
-
-#if __linux__
-  exeSize = readlink("/proc/self/exe", exe, PATH_MAX);
-  if (exeSize == -1) {
-    dprintf(fd, "Unable to detect exectuable name\n");
-    hasExe = false;
-  } else {
-    dprintf(fd, "Executable is %.*s\n", exeSize, exe);
-    hasExe = true;
-  }
-#endif
-
-  for (size_t i = 1; i < stackDepth; i++) {
-
-    size_t sz = 64000; // 64K ought to be enough for our templates...
-    char* function = static_cast<char*>(malloc(sz));
-    char *begin = nullptr, *end = nullptr;
-    // find the last space and address offset surrounding the mangled name
-#if __APPLE__
-    for (char* j = stackStrings[i]; *j; ++j) {
-      if (*j == ' ' && *(j + 1) != '+') {
-        begin = j;
-      } else if (*j == ' ' && *(j + 1) == '+') {
-        end = j;
-        break;
-      }
-    }
-    bool tryAddr2Line = false;
-#else
-    for (char* j = stackStrings[i]; j && *j; ++j) {
-      if (*j == '(') {
-        begin = j;
-      } else if (*j == '+') {
-        end = j;
-        break;
-      }
-    }
-    bool tryAddr2Line = true;
-#endif
-    if (begin && end) {
-      *begin++ = '\0';
-      *end = '\0';
-      // found our mangled name, now in [begin, end)
-
-      int status;
-      char* ret = abi::__cxa_demangle(begin, function, &sz, &status);
-      if (ret) {
-        // return value may be a realloc() of the input
-        function = ret;
-        dprintf(fd, "    %s: %s\n", stackStrings[i], function);
-        tryAddr2Line = false;
-      }
-    }
-    if (tryAddr2Line) {
-      // didn't find the mangled name, just print the whole line
-      dprintf(fd, "    %s: ", stackStrings[i]);
-      if (stackTrace[i] && hasExe) {
-        char syscom[4096 + PATH_MAX];
-
-        // Find c++filt from the environment
-        // This is needed for platforms where we still need c++filt -r
-        char const* cxxfilt = getenv("CXXFILT");
-        if (cxxfilt == nullptr) {
-          cxxfilt = "c++filt";
-        }
-        // Do the same for addr2line, just in case we wanted to pass some options
-        char const* addr2line = getenv("ADDR2LINE");
-        if (addr2line == nullptr) {
-          addr2line = "addr2line";
-        }
-        snprintf(syscom, 4096, "%s %p -p -s -f -e %.*s 2>/dev/null | %s ", addr2line, stackTrace[i], exeSize, exe, cxxfilt); // last parameter is the name of this app
-
-        FILE* fp;
-        char path[1024];
-
-        fp = popen(syscom, "r");
-        if (fp == nullptr) {
-          dprintf(fd, "-- no source could be retrieved --\n");
-          continue;
-        }
-
-        while (fgets(path, sizeof(path) - 1, fp) != nullptr) {
-          dprintf(fd, "    %s", path);
-        }
-
-        pclose(fp);
-      } else {
-        dprintf(fd, "-- no source avaliable --\n");
-      }
-    }
-    free(function);
-  }
-  free(stackStrings); // malloc()ed by backtrace_symbols
-  fsync(fd);
-}
-
 } // namespace o2::framework

From bc3b40f42fb59e5bf2a57fdae078e772006942c3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 26 Jul 2024 09:05:46 +0200
Subject: [PATCH 592/726] DPL: factorise out backtrace support

---
 Framework/Core/src/DataProcessingDevice.cxx   |   8 +-
 Framework/Core/src/runDataProcessing.cxx      |   9 +-
 Framework/Foundation/CMakeLists.txt           |   1 +
 .../include/Framework/BacktraceHelpers.h      |  22 +++
 .../include/Framework/RuntimeError.h          |   7 +-
 Framework/Foundation/src/BacktraceHelpers.cxx | 125 ++++++++++++++++++
 Framework/Foundation/src/RuntimeError.cxx     | 111 +---------------
 .../Foundation/test/test_RuntimeError.cxx     |   3 +-
 8 files changed, 166 insertions(+), 120 deletions(-)
 create mode 100644 Framework/Foundation/include/Framework/BacktraceHelpers.h
 create mode 100644 Framework/Foundation/src/BacktraceHelpers.cxx

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index e0a0d1aad16e2..57010c27ffb09 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -431,7 +431,7 @@ void DataProcessingDevice::Init()
       auto& err = error_from_ref(e);
       O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
       O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "Init", "Exception caught while in Init: %{public}s. Invoking errorCallback.", err.what);
-      demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
+      BacktraceHelpers::demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
       auto& stats = ref.get<DataProcessingStats>();
       stats.updateStats({(int)ProcessingStatsId::EXCEPTION_COUNT, DataProcessingStats::Op::Add, 1});
       InitErrorContext errorContext{ref, e};
@@ -446,7 +446,7 @@ void DataProcessingDevice::Init()
       auto& context = ref.get<DataProcessorContext>();
       O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
       O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "Init", "Exception caught while in Init: %{public}s. Exiting with 1.", err.what);
-      demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
+      BacktraceHelpers::demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
       auto& stats = ref.get<DataProcessingStats>();
       stats.updateStats({(int)ProcessingStatsId::EXCEPTION_COUNT, DataProcessingStats::Op::Add, 1});
       exit(1);
@@ -1111,7 +1111,7 @@ void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceCont
       auto& context = ref.get<DataProcessorContext>();
       O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
       O2_SIGNPOST_EVENT_EMIT_ERROR(device, cid, "Run", "Exception while running: %{public}s. Invoking callback.", err.what);
-      demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
+      BacktraceHelpers::demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
       auto& stats = ref.get<DataProcessingStats>();
       stats.updateStats({(int)ProcessingStatsId::EXCEPTION_COUNT, DataProcessingStats::Op::Add, 1});
       ErrorContext errorContext{record, ref, e};
@@ -1126,7 +1126,7 @@ void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceCont
       ServiceRegistryRef ref{serviceRegistry, ServiceRegistry::globalDeviceSalt()};
       auto& context = ref.get<DataProcessorContext>();
       O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
-      demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
+      BacktraceHelpers::demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
       auto& stats = ref.get<DataProcessingStats>();
       stats.updateStats({(int)ProcessingStatsId::EXCEPTION_COUNT, DataProcessingStats::Op::Add, 1});
       switch (errorPolicy) {
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 89737483fc813..6b95a073ed21d 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -11,6 +11,7 @@
 #define BOOST_BIND_GLOBAL_PLACEHOLDERS
 #include <stdexcept>
 #include "Framework/BoostOptionsRetriever.h"
+#include "Framework/BacktraceHelpers.h"
 #include "Framework/CallbacksPolicy.h"
 #include "Framework/ChannelConfigurationPolicy.h"
 #include "Framework/ChannelMatching.h"
@@ -656,7 +657,7 @@ void handle_crash(int sig)
     auto retVal = write(STDERR_FILENO, buffer, strlen(buffer));
     (void)retVal;
   }
-  demangled_backtrace_symbols(array, size, STDERR_FILENO);
+  BacktraceHelpers::demangled_backtrace_symbols(array, size, STDERR_FILENO);
   {
     char const* msg = "Backtrace complete.\n";
     int len = strlen(msg); /* the byte length of the string */
@@ -982,7 +983,7 @@ void doDPLException(RuntimeErrorRef& e, char const* processName)
          " Reason: {}"
          "\n Backtrace follow: \n",
          processName, err.what);
-    demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
+    BacktraceHelpers::demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
   } else {
     LOGP(fatal,
          "Unhandled o2::framework::runtime_error reached the top of main of {}, device shutting down."
@@ -1928,7 +1929,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
         } catch (o2::framework::RuntimeErrorRef ref) {
           auto& err = o2::framework::error_from_ref(ref);
 #ifdef DPL_ENABLE_BACKTRACE
-          demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
+          BacktraceHelpers::demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
 #endif
           LOGP(error, "invalid workflow in {}: {}", driverInfo.argv[0], err.what);
           return 1;
@@ -2059,7 +2060,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
           LOGP(error, "unable to merge configurations in {}: {}", driverInfo.argv[0], err.what);
 #ifdef DPL_ENABLE_BACKTRACE
           std::cerr << "\nStacktrace follows:\n\n";
-          demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
+          BacktraceHelpers::demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
 #endif
           return 1;
         }
diff --git a/Framework/Foundation/CMakeLists.txt b/Framework/Foundation/CMakeLists.txt
index bcde0f7a69f34..1cffe7f0487ff 100644
--- a/Framework/Foundation/CMakeLists.txt
+++ b/Framework/Foundation/CMakeLists.txt
@@ -16,6 +16,7 @@ install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/include/Framework
 o2_add_library(FrameworkFoundation
                SOURCES src/RuntimeError.cxx
                src/Signpost.cxx
+               src/BacktraceHelpers.cxx
                TARGETVARNAME targetName
                PUBLIC_LINK_LIBRARIES O2::FrameworkFoundation3rdparty
               )
diff --git a/Framework/Foundation/include/Framework/BacktraceHelpers.h b/Framework/Foundation/include/Framework/BacktraceHelpers.h
new file mode 100644
index 0000000000000..1f897d4d6f860
--- /dev/null
+++ b/Framework/Foundation/include/Framework/BacktraceHelpers.h
@@ -0,0 +1,22 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef O2_FRAMEWORK_BACKTRACE_HELPERS_H_
+#define O2_FRAMEWORK_BACKTRACE_HELPERS_H_
+
+namespace o2::framework
+{
+struct BacktraceHelpers {
+  static constexpr unsigned int MAX_BACKTRACE_SIZE = 100;
+  static void demangled_backtrace_symbols(void** backtrace, unsigned int total, int fd);
+};
+} // namespace o2::framework
+
+#endif // O2_FRAMEWORK_BACKTRACE_HELPERS_H_
diff --git a/Framework/Foundation/include/Framework/RuntimeError.h b/Framework/Foundation/include/Framework/RuntimeError.h
index a8335ae77c509..6c0cf601dfe2d 100644
--- a/Framework/Foundation/include/Framework/RuntimeError.h
+++ b/Framework/Foundation/include/Framework/RuntimeError.h
@@ -11,15 +11,16 @@
 #ifndef O2_FRAMEWORK_RUNTIMEERROR_H_
 #define O2_FRAMEWORK_RUNTIMEERROR_H_
 
+#include "BacktraceHelpers.h"
+
 namespace o2::framework
 {
 
 struct RuntimeError {
   static constexpr unsigned int MAX_RUNTIME_ERRORS = 64;
   static constexpr unsigned int MAX_RUNTIME_ERROR_SIZE = 1024;
-  static constexpr unsigned int MAX_BACKTRACE_SIZE = 100;
   char what[MAX_RUNTIME_ERROR_SIZE];
-  void* backtrace[MAX_BACKTRACE_SIZE];
+  void* backtrace[BacktraceHelpers::MAX_BACKTRACE_SIZE];
   int maxBacktrace = 0;
 };
 
@@ -27,8 +28,6 @@ struct RuntimeErrorRef {
   int index = 0;
 };
 
-void demangled_backtrace_symbols(void** backtrace, unsigned int total, int fd);
-
 RuntimeErrorRef runtime_error(const char*);
 RuntimeErrorRef runtime_error_f(const char*, ...);
 RuntimeError& error_from_ref(RuntimeErrorRef);
diff --git a/Framework/Foundation/src/BacktraceHelpers.cxx b/Framework/Foundation/src/BacktraceHelpers.cxx
new file mode 100644
index 0000000000000..cf37464f7e0bd
--- /dev/null
+++ b/Framework/Foundation/src/BacktraceHelpers.cxx
@@ -0,0 +1,125 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/BacktraceHelpers.h"
+#include <climits>
+#include <cstddef>
+#include <cstdio>
+#include <cstdlib>
+#include <unistd.h>
+#include <cxxabi.h>
+#include <execinfo.h>
+
+namespace o2::framework
+{
+
+void BacktraceHelpers::demangled_backtrace_symbols(void** stackTrace, unsigned int stackDepth, int fd)
+{
+  char** stackStrings;
+  stackStrings = backtrace_symbols(stackTrace, stackDepth);
+  char exe[PATH_MAX];
+  bool hasExe = false;
+  int exeSize = 0;
+
+#if __linux__
+  exeSize = readlink("/proc/self/exe", exe, PATH_MAX);
+  if (exeSize == -1) {
+    dprintf(fd, "Unable to detect exectuable name\n");
+    hasExe = false;
+  } else {
+    dprintf(fd, "Executable is %.*s\n", exeSize, exe);
+    hasExe = true;
+  }
+#endif
+
+  for (size_t i = 1; i < stackDepth; i++) {
+
+    size_t sz = 64000; // 64K ought to be enough for our templates...
+    char* function = static_cast<char*>(malloc(sz));
+    char *begin = nullptr, *end = nullptr;
+    // find the last space and address offset surrounding the mangled name
+#if __APPLE__
+    for (char* j = stackStrings[i]; *j; ++j) {
+      if (*j == ' ' && *(j + 1) != '+') {
+        begin = j;
+      } else if (*j == ' ' && *(j + 1) == '+') {
+        end = j;
+        break;
+      }
+    }
+    bool tryAddr2Line = false;
+#else
+    for (char* j = stackStrings[i]; j && *j; ++j) {
+      if (*j == '(') {
+        begin = j;
+      } else if (*j == '+') {
+        end = j;
+        break;
+      }
+    }
+    bool tryAddr2Line = true;
+#endif
+    if (begin && end) {
+      *begin++ = '\0';
+      *end = '\0';
+      // found our mangled name, now in [begin, end)
+
+      int status;
+      char* ret = abi::__cxa_demangle(begin, function, &sz, &status);
+      if (ret) {
+        // return value may be a realloc() of the input
+        function = ret;
+        dprintf(fd, "    %s: %s\n", stackStrings[i], function);
+        tryAddr2Line = false;
+      }
+    }
+    if (tryAddr2Line) {
+      // didn't find the mangled name, just print the whole line
+      dprintf(fd, "    %s: ", stackStrings[i]);
+      if (stackTrace[i] && hasExe) {
+        char syscom[4096 + PATH_MAX];
+
+        // Find c++filt from the environment
+        // This is needed for platforms where we still need c++filt -r
+        char const* cxxfilt = getenv("CXXFILT");
+        if (cxxfilt == nullptr) {
+          cxxfilt = "c++filt";
+        }
+        // Do the same for addr2line, just in case we wanted to pass some options
+        char const* addr2line = getenv("ADDR2LINE");
+        if (addr2line == nullptr) {
+          addr2line = "addr2line";
+        }
+        snprintf(syscom, 4096, "%s %p -p -s -f -e %.*s 2>/dev/null | %s ", addr2line, stackTrace[i], exeSize, exe, cxxfilt); // last parameter is the name of this app
+
+        FILE* fp;
+        char path[1024];
+
+        fp = popen(syscom, "r");
+        if (fp == nullptr) {
+          dprintf(fd, "-- no source could be retrieved --\n");
+          continue;
+        }
+
+        while (fgets(path, sizeof(path) - 1, fp) != nullptr) {
+          dprintf(fd, "    %s", path);
+        }
+
+        pclose(fp);
+      } else {
+        dprintf(fd, "-- no source avaliable --\n");
+      }
+    }
+    free(function);
+  }
+  free(stackStrings); // malloc()ed by backtrace_symbols
+  fsync(fd);
+}
+} // namespace o2::framework
diff --git a/Framework/Foundation/src/RuntimeError.cxx b/Framework/Foundation/src/RuntimeError.cxx
index 747b339cc6dbb..6f31fb40b86f5 100644
--- a/Framework/Foundation/src/RuntimeError.cxx
+++ b/Framework/Foundation/src/RuntimeError.cxx
@@ -15,11 +15,11 @@
 #include <climits>
 #include <atomic>
 #include <cstdarg>
-#include <execinfo.h>
+#include <cstring>
 #include <unistd.h>
 #include <cstdlib>
-#include <cstring>
 #include <cxxabi.h>
+#include <execinfo.h>
 #include <stdexcept>
 
 namespace o2::framework
@@ -71,7 +71,7 @@ RuntimeErrorRef runtime_error_f(const char* format, ...)
   va_start(args, format);
   vsnprintf(gError[i].what, RuntimeError::MAX_RUNTIME_ERROR_SIZE, format, args);
   va_end(args);
-  gError[i].maxBacktrace = canDumpBacktrace() ? backtrace(gError[i].backtrace, RuntimeError::MAX_BACKTRACE_SIZE) : 0;
+  gError[i].maxBacktrace = canDumpBacktrace() ? backtrace(gError[i].backtrace, BacktraceHelpers::MAX_BACKTRACE_SIZE) : 0;
   return RuntimeErrorRef{i};
 }
 
@@ -83,7 +83,7 @@ RuntimeErrorRef runtime_error(const char* s)
     ++i;
   }
   strncpy(gError[i].what, s, RuntimeError::MAX_RUNTIME_ERROR_SIZE);
-  gError[i].maxBacktrace = canDumpBacktrace() ? backtrace(gError[i].backtrace, RuntimeError::MAX_BACKTRACE_SIZE) : 0;
+  gError[i].maxBacktrace = canDumpBacktrace() ? backtrace(gError[i].backtrace, BacktraceHelpers::MAX_BACKTRACE_SIZE) : 0;
   return RuntimeErrorRef{i};
 }
 
@@ -92,107 +92,4 @@ void throw_error(RuntimeErrorRef ref)
   throw ref;
 }
 
-void demangled_backtrace_symbols(void** stackTrace, unsigned int stackDepth, int fd)
-{
-  char** stackStrings;
-  stackStrings = backtrace_symbols(stackTrace, stackDepth);
-  char exe[PATH_MAX];
-  bool hasExe = false;
-  int exeSize = 0;
-
-#if __linux__
-  exeSize = readlink("/proc/self/exe", exe, PATH_MAX);
-  if (exeSize == -1) {
-    dprintf(fd, "Unable to detect exectuable name\n");
-    hasExe = false;
-  } else {
-    dprintf(fd, "Executable is %.*s\n", exeSize, exe);
-    hasExe = true;
-  }
-#endif
-
-  for (size_t i = 1; i < stackDepth; i++) {
-
-    size_t sz = 64000; // 64K ought to be enough for our templates...
-    char* function = static_cast<char*>(malloc(sz));
-    char *begin = nullptr, *end = nullptr;
-    // find the last space and address offset surrounding the mangled name
-#if __APPLE__
-    for (char* j = stackStrings[i]; *j; ++j) {
-      if (*j == ' ' && *(j + 1) != '+') {
-        begin = j;
-      } else if (*j == ' ' && *(j + 1) == '+') {
-        end = j;
-        break;
-      }
-    }
-    bool tryAddr2Line = false;
-#else
-    for (char* j = stackStrings[i]; j && *j; ++j) {
-      if (*j == '(') {
-        begin = j;
-      } else if (*j == '+') {
-        end = j;
-        break;
-      }
-    }
-    bool tryAddr2Line = true;
-#endif
-    if (begin && end) {
-      *begin++ = '\0';
-      *end = '\0';
-      // found our mangled name, now in [begin, end)
-
-      int status;
-      char* ret = abi::__cxa_demangle(begin, function, &sz, &status);
-      if (ret) {
-        // return value may be a realloc() of the input
-        function = ret;
-        dprintf(fd, "    %s: %s\n", stackStrings[i], function);
-        tryAddr2Line = false;
-      }
-    }
-    if (tryAddr2Line) {
-      // didn't find the mangled name, just print the whole line
-      dprintf(fd, "    %s: ", stackStrings[i]);
-      if (stackTrace[i] && hasExe) {
-        char syscom[4096 + PATH_MAX];
-
-        // Find c++filt from the environment
-        // This is needed for platforms where we still need c++filt -r
-        char const* cxxfilt = getenv("CXXFILT");
-        if (cxxfilt == nullptr) {
-          cxxfilt = "c++filt";
-        }
-        // Do the same for addr2line, just in case we wanted to pass some options
-        char const* addr2line = getenv("ADDR2LINE");
-        if (addr2line == nullptr) {
-          addr2line = "addr2line";
-        }
-        snprintf(syscom, 4096, "%s %p -p -s -f -e %.*s 2>/dev/null | %s ", addr2line, stackTrace[i], exeSize, exe, cxxfilt); // last parameter is the name of this app
-
-        FILE* fp;
-        char path[1024];
-
-        fp = popen(syscom, "r");
-        if (fp == nullptr) {
-          dprintf(fd, "-- no source could be retrieved --\n");
-          continue;
-        }
-
-        while (fgets(path, sizeof(path) - 1, fp) != nullptr) {
-          dprintf(fd, "    %s", path);
-        }
-
-        pclose(fp);
-      } else {
-        dprintf(fd, "-- no source avaliable --\n");
-      }
-    }
-    free(function);
-  }
-  free(stackStrings); // malloc()ed by backtrace_symbols
-  fsync(fd);
-}
-
 } // namespace o2::framework
diff --git a/Framework/Foundation/test/test_RuntimeError.cxx b/Framework/Foundation/test/test_RuntimeError.cxx
index a628e84175bd0..cc0efa801908a 100644
--- a/Framework/Foundation/test/test_RuntimeError.cxx
+++ b/Framework/Foundation/test/test_RuntimeError.cxx
@@ -11,6 +11,7 @@
 
 #include <catch_amalgamated.hpp>
 #include "Framework/RuntimeError.h"
+#include "Framework/BacktraceHelpers.h"
 #include <unistd.h>
 #include <execinfo.h>
 
@@ -33,7 +34,7 @@ TEST_CASE("TestRuntimeError")
     REQUIRE(strncmp(err.what, "foo", 3) == 0);
 #ifdef DPL_ENABLE_BACKTRACE
     backtrace_symbols_fd(err.backtrace, err.maxBacktrace, STDERR_FILENO);
-    o2::framework::demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
+    o2::framework::BacktraceHelpers::demangled_backtrace_symbols(err.backtrace, err.maxBacktrace, STDERR_FILENO);
 #endif
   }
 }

From a95000a97128df13fb392f19a48dd346c1fd3fba Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 26 Jul 2024 11:47:06 +0200
Subject: [PATCH 593/726] DPL: drop obsolete TRACY_PORT

---
 Framework/Core/include/Framework/DeviceInfo.h | 2 --
 Framework/Core/include/Framework/DriverInfo.h | 2 --
 Framework/Core/src/runDataProcessing.cxx      | 5 -----
 3 files changed, 9 deletions(-)

diff --git a/Framework/Core/include/Framework/DeviceInfo.h b/Framework/Core/include/Framework/DeviceInfo.h
index 49f96d90d87fb..9c789eaa71516 100644
--- a/Framework/Core/include/Framework/DeviceInfo.h
+++ b/Framework/Core/include/Framework/DeviceInfo.h
@@ -80,8 +80,6 @@ struct DeviceInfo {
   boost::property_tree::ptree currentConfig;
   /// Current provenance for the configuration keys
   boost::property_tree::ptree currentProvenance;
-  /// Port to use to connect to tracy profiler
-  short tracyPort;
   /// Timestamp of the last signal received
   size_t lastSignal;
   /// An incremental number for the state of the device
diff --git a/Framework/Core/include/Framework/DriverInfo.h b/Framework/Core/include/Framework/DriverInfo.h
index 45c4c85a32d01..41b868150e047 100644
--- a/Framework/Core/include/Framework/DriverInfo.h
+++ b/Framework/Core/include/Framework/DriverInfo.h
@@ -153,8 +153,6 @@ struct DriverInfo {
   unsigned short resourcesMonitoringDumpInterval = 0;
   /// Port used by the websocket control. 0 means not initialised.
   unsigned short port = 0;
-  /// Last port used for tracy
-  short tracyPort = 8086;
   /// The minimum level after which the device will exit with 1
   LogParsingHelpers::LogLevel minFailureLevel = LogParsingHelpers::LogLevel::Fatal;
 
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 6b95a073ed21d..dbad0658412d5 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -691,8 +691,6 @@ void spawnDevice(uv_loop_t* loop,
   auto& spec = specs[ref.index];
   auto& execution = executions[ref.index];
 
-  driverInfo.tracyPort++;
-
   for (auto& service : spec.services) {
     if (service.preFork != nullptr) {
       service.preFork(serviceRegistry, DeviceConfig{varmap});
@@ -738,8 +736,6 @@ void spawnDevice(uv_loop_t* loop,
     dup2(childFds[ref.index].childstdout[1], STDOUT_FILENO);
     dup2(childFds[ref.index].childstdout[1], STDERR_FILENO);
 
-    auto portS = std::to_string(driverInfo.tracyPort);
-    setenv("TRACY_PORT", portS.c_str(), 1);
     for (auto& service : spec.services) {
       if (service.postForkChild != nullptr) {
         service.postForkChild(serviceRegistry);
@@ -792,7 +788,6 @@ void spawnDevice(uv_loop_t* loop,
                          .readyToQuit = false,
                          .inputChannelMetricsViewIndex = Metric2DViewIndex{"oldest_possible_timeslice", 0, 0, {}},
                          .outputChannelMetricsViewIndex = Metric2DViewIndex{"oldest_possible_output", 0, 0, {}},
-                         .tracyPort = driverInfo.tracyPort,
                          .lastSignal = uv_hrtime() - 10000000});
   // create the offset using uv_hrtime
   timespec now;

From cf5c5b130ccbfeecf85595e31ed7488ecbb86d22 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 26 Jul 2024 11:48:40 +0200
Subject: [PATCH 594/726] Allow CTP scalers to work also on mac

---
 Detectors/CTP/workflowScalers/CMakeLists.txt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Detectors/CTP/workflowScalers/CMakeLists.txt b/Detectors/CTP/workflowScalers/CMakeLists.txt
index fe91ee2fec28b..a31774ac66d69 100644
--- a/Detectors/CTP/workflowScalers/CMakeLists.txt
+++ b/Detectors/CTP/workflowScalers/CMakeLists.txt
@@ -8,7 +8,6 @@
 # In applying this license CERN does not waive the privileges and immunities
 # granted to it by virtue of its status as an Intergovernmental Organization
 # or submit itself to any jurisdiction.
-if (NOT APPLE)
 o2_add_library(CTPWorkflowScalers
                SOURCES src/ctpCCDBManager.cxx
                SOURCES src/RunManager.cxx
@@ -23,7 +22,6 @@ o2_add_executable(
                   SOURCES src/ctp-proxy.cxx
                   PUBLIC_LINK_LIBRARIES O2::DCStestWorkflow
                   O2::CTPWorkflowScalers)
-endif()
 o2_add_executable(
                   qc-proxy
                   COMPONENT_NAME ctp

From 211d7429256a14ea88dd46d425ff2d1dcd66c895 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 26 Jul 2024 12:04:14 +0200
Subject: [PATCH 595/726] Limit number of file descriptors which might be
 closed

---
 Framework/Core/src/runDataProcessing.cxx | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 6b95a073ed21d..29e013faeaf71 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -723,8 +723,11 @@ void spawnDevice(uv_loop_t* loop,
     struct rlimit rlim;
     getrlimit(RLIMIT_NOFILE, &rlim);
     // We close all FD, but the one which are actually
-    // used to communicate with the driver.
-    int rlim_cur = rlim.rlim_cur;
+    // used to communicate with the driver. This is a bad
+    // idea in the first place, because rlim_cur could be huge
+    // FIXME: I should understand which one is really to be closed and use
+    // CLOEXEC on it.
+    int rlim_cur = std::min((int)rlim.rlim_cur, 10000);
     for (int i = 0; i < rlim_cur; ++i) {
       if (childFds[ref.index].childstdin[0] == i) {
         continue;

From 49a0ad3b871df4ba657aa6be48a9279363f3c857 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 26 Jul 2024 14:51:46 +0200
Subject: [PATCH 596/726] DPL: provide stacktrace for errors also on Linux, if
 requested.

---
 Framework/Foundation/include/Framework/Signpost.h | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index 1ddaaa6fb793f..5e844ce01b20c 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -234,7 +234,9 @@ inline _o2_signpost_id_t _o2_signpost_id_make_with_pointer(_o2_log_t* log, void
 #include <cstdarg>
 #include <cstdio>
 #include <cstring>
+#include <execinfo.h>
 #include "Framework/RuntimeError.h"
+#include "Framework/BacktraceHelpers.h"
 void _o2_signpost_interval_end_v(_o2_log_t* log, _o2_signpost_id_t id, char const* name, char const* const format, va_list args);
 
 // returns true if the push was successful, false if the stack was full
@@ -377,6 +379,11 @@ void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* n
   vsnprintf(prebuffer + s, 4096 - s, format, args);
   va_end(args);
   O2_LOG_MACRO("%s", prebuffer);
+  if (log->stacktrace > 1) {
+    void* traces[o2::framework::BacktraceHelpers::MAX_BACKTRACE_SIZE];
+    int maxBacktrace = backtrace(traces, o2::framework::BacktraceHelpers::MAX_BACKTRACE_SIZE);
+    o2::framework::BacktraceHelpers::demangled_backtrace_symbols(traces, maxBacktrace, STDERR_FILENO);
+  }
 }
 
 // This will look at the slot in the log associated to the ID.

From d1e4faf210ffcdce693a2f678155e11562a8abd2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 7 Aug 2024 10:55:57 +0200
Subject: [PATCH 597/726] DPL: add test for backtrace on error

---
 Framework/Foundation/test/test_SignpostLogger.cxx | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Framework/Foundation/test/test_SignpostLogger.cxx b/Framework/Foundation/test/test_SignpostLogger.cxx
index ed917ea7cadf5..e5e401dd8f854 100644
--- a/Framework/Foundation/test/test_SignpostLogger.cxx
+++ b/Framework/Foundation/test/test_SignpostLogger.cxx
@@ -19,6 +19,7 @@
 #include <iostream>
 
 O2_DECLARE_LOG(test_Signpost2, "my category2");
+O2_DECLARE_DYNAMIC_STACKTRACE_LOG(SignpostStacktrace);
 
 int main(int argc, char** argv)
 {
@@ -57,4 +58,9 @@ int main(int argc, char** argv)
   O2_SIGNPOST_START(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will see, because we turned them on");
   O2_SIGNPOST_END(test_SignpostDynamic, id, "Test category", "This is dynamic signpost which you will see, because we turned them on");
 #endif
+
+  // Test stacktraces
+  O2_SIGNPOST_ID_GENERATE(idStacktrace, SignpostStacktrace);
+  O2_LOG_ENABLE(SignpostStacktrace);
+  O2_SIGNPOST_EVENT_EMIT_ERROR(SignpostStacktrace, idStacktrace, "Test category", "An error with stacktrace %d \n", 1);
 }

From 02615e8a9ddef1c8367952494604f4719120e925 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 7 Aug 2024 10:55:57 +0200
Subject: [PATCH 598/726] DPL: improve backtrace handling

In case demangling does not work, still try with addr2line. In case
that fails as well, print what we have from backtrace.
---
 Framework/Foundation/src/BacktraceHelpers.cxx | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/Framework/Foundation/src/BacktraceHelpers.cxx b/Framework/Foundation/src/BacktraceHelpers.cxx
index cf37464f7e0bd..a7e8a4b90d700 100644
--- a/Framework/Foundation/src/BacktraceHelpers.cxx
+++ b/Framework/Foundation/src/BacktraceHelpers.cxx
@@ -39,6 +39,7 @@ void BacktraceHelpers::demangled_backtrace_symbols(void** stackTrace, unsigned i
   }
 #endif
 
+  // We skip the level 0, which is the actual logging function.
   for (size_t i = 1; i < stackDepth; i++) {
 
     size_t sz = 64000; // 64K ought to be enough for our templates...
@@ -67,17 +68,20 @@ void BacktraceHelpers::demangled_backtrace_symbols(void** stackTrace, unsigned i
     bool tryAddr2Line = true;
 #endif
     if (begin && end) {
+      // The first byte is a ' ' which we need to skip.
       *begin++ = '\0';
       *end = '\0';
       // found our mangled name, now in [begin, end)
 
       int status;
       char* ret = abi::__cxa_demangle(begin, function, &sz, &status);
-      if (ret) {
+      if (status == 0) {
         // return value may be a realloc() of the input
         function = ret;
         dprintf(fd, "    %s: %s\n", stackStrings[i], function);
         tryAddr2Line = false;
+      } else {
+        tryAddr2Line = true;
       }
     }
     if (tryAddr2Line) {
@@ -88,12 +92,12 @@ void BacktraceHelpers::demangled_backtrace_symbols(void** stackTrace, unsigned i
 
         // Find c++filt from the environment
         // This is needed for platforms where we still need c++filt -r
-        char const* cxxfilt = getenv("CXXFILT");
+        static char const* cxxfilt = getenv("CXXFILT");
         if (cxxfilt == nullptr) {
           cxxfilt = "c++filt";
         }
         // Do the same for addr2line, just in case we wanted to pass some options
-        char const* addr2line = getenv("ADDR2LINE");
+        static char const* addr2line = getenv("ADDR2LINE");
         if (addr2line == nullptr) {
           addr2line = "addr2line";
         }
@@ -104,7 +108,10 @@ void BacktraceHelpers::demangled_backtrace_symbols(void** stackTrace, unsigned i
 
         fp = popen(syscom, "r");
         if (fp == nullptr) {
-          dprintf(fd, "-- no source could be retrieved --\n");
+          // We could not run addr2line, print whatever we have.
+          // Also free function, which is allocated at the beginning of the loop.
+          dprintf(fd, "%s\n", begin);
+          free(function);
           continue;
         }
 
@@ -114,7 +121,7 @@ void BacktraceHelpers::demangled_backtrace_symbols(void** stackTrace, unsigned i
 
         pclose(fp);
       } else {
-        dprintf(fd, "-- no source avaliable --\n");
+        dprintf(fd, "%s\n", begin);
       }
     }
     free(function);

From fcd84755f183a2958504b63236d5589191cd901b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 10 Aug 2024 16:30:24 +0200
Subject: [PATCH 599/726] DPL: do not flush metrics from threads

This avoids having to have a mutex for the flushing.
---
 Framework/Core/src/CommonServices.cxx | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 39c17599abb9d..0e84333afe115 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -783,8 +783,13 @@ O2_DECLARE_DYNAMIC_LOG(monitoring_service);
 /// This will flush metrics only once every second.
 auto flushMetrics(ServiceRegistryRef registry, DataProcessingStats& stats) -> void
 {
+  // Flushing metrics should only happen on main thread to avoid
+  // having to have a mutex for the communication with the driver.
   O2_SIGNPOST_ID_GENERATE(sid, monitoring_service);
   O2_SIGNPOST_START(monitoring_service, sid, "flush", "flushing metrics");
+  if (registry.isMainThread() == false) {
+    LOGP(fatal, "Flushing metrics should only happen on the main thread.");
+  }
   auto& monitoring = registry.get<Monitoring>();
   auto& relayer = registry.get<DataRelayer>();
 
@@ -1071,13 +1076,9 @@ o2::framework::ServiceSpec CommonServices::dataProcessingStats()
       return ServiceHandle{TypeIdHelpers::uniqueId<DataProcessingStats>(), stats};
     },
     .configure = noConfiguration(),
-    .preProcessing = [](ProcessingContext& context, void* service) {
-      auto* stats = (DataProcessingStats*)service;
-      flushMetrics(context.services(), *stats); },
     .postProcessing = [](ProcessingContext& context, void* service) {
       auto* stats = (DataProcessingStats*)service;
-      stats->updateStats({(short)ProcessingStatsId::PERFORMED_COMPUTATIONS, DataProcessingStats::Op::Add, 1});
-      flushMetrics(context.services(), *stats); },
+      stats->updateStats({(short)ProcessingStatsId::PERFORMED_COMPUTATIONS, DataProcessingStats::Op::Add, 1}); },
     .preDangling = [](DanglingContext& context, void* service) {
        auto* stats = (DataProcessingStats*)service;
        sendRelayerMetrics(context.services(), *stats);
@@ -1090,9 +1091,6 @@ o2::framework::ServiceSpec CommonServices::dataProcessingStats()
       auto* stats = (DataProcessingStats*)service;
       sendRelayerMetrics(context.services(), *stats);
       flushMetrics(context.services(), *stats); },
-    .postDispatching = [](ProcessingContext& context, void* service) {
-      auto* stats = (DataProcessingStats*)service;
-      flushMetrics(context.services(), *stats); },
     .preLoop = [](ServiceRegistryRef ref, void* service) {
       auto* stats = (DataProcessingStats*)service;
       flushMetrics(ref, *stats); },

From f8bbb33d214f27bd1cca80c6757c4b66da2b38fe Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 10 Aug 2024 16:33:29 +0200
Subject: [PATCH 600/726] Make it explicit we are currently using a signed char

On x86 Linux `char` is assumed to be signed, however that's an implementation detail.

This makes sure we are explicit about the sign-ness.
---
 .../ITS/base/include/ITSBase/GeometryTGeo.h   | 28 +++++++++----------
 .../PHOS/reconstruction/src/Clusterer.cxx     |  8 +++---
 .../TRD/base/include/TRDBase/ChamberStatus.h  |  4 +--
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/Detectors/ITSMFT/ITS/base/include/ITSBase/GeometryTGeo.h b/Detectors/ITSMFT/ITS/base/include/ITSBase/GeometryTGeo.h
index 6172a02286abd..6d10a263408af 100644
--- a/Detectors/ITSMFT/ITS/base/include/ITSBase/GeometryTGeo.h
+++ b/Detectors/ITSMFT/ITS/base/include/ITSBase/GeometryTGeo.h
@@ -399,20 +399,20 @@ class GeometryTGeo : public o2::itsmft::GeometryTGeo
  protected:
   static constexpr int MAXLAYERS = 15; ///< max number of active layers
 
-  Int_t mNumberOfLayers;                        ///< number of layers
-  Int_t mNumberOfHalfBarrels;                   ///< number of halfbarrels
-  std::vector<int> mNumberOfStaves;             ///< number of staves/layer(layer)
-  std::vector<int> mNumberOfHalfStaves;         ///< the number of substaves/stave(layer)
-  std::vector<int> mNumberOfModules;            ///< number of modules/substave(layer)
-  std::vector<int> mNumberOfChipsPerModule;     ///< number of chips per module (group of chips on substaves)
-  std::vector<int> mNumberOfChipRowsPerModule;  ///< number of chips rows per module (relevant for OB modules)
-  std::vector<int> mNumberOfChipsPerHalfStave;  ///< number of chips per substave
-  std::vector<int> mNumberOfChipsPerStave;      ///< number of chips per stave
-  std::vector<int> mNumberOfChipsPerHalfBarrel; ///< number of chips per halfbarrel
-  std::vector<int> mNumberOfChipsPerLayer;      ///< number of chips per stave
-  std::vector<int> mLastChipIndex;              ///< max ID of the detctor in the layer
-  std::array<bool, MAXLAYERS> mIsLayerITS3;     ///< flag with the information of the ITS version (ITS2 or ITS3)
-  std::array<char, MAXLAYERS> mLayerToWrapper;  ///< Layer to wrapper correspondence
+  Int_t mNumberOfLayers;                         ///< number of layers
+  Int_t mNumberOfHalfBarrels;                    ///< number of halfbarrels
+  std::vector<int> mNumberOfStaves;              ///< number of staves/layer(layer)
+  std::vector<int> mNumberOfHalfStaves;          ///< the number of substaves/stave(layer)
+  std::vector<int> mNumberOfModules;             ///< number of modules/substave(layer)
+  std::vector<int> mNumberOfChipsPerModule;      ///< number of chips per module (group of chips on substaves)
+  std::vector<int> mNumberOfChipRowsPerModule;   ///< number of chips rows per module (relevant for OB modules)
+  std::vector<int> mNumberOfChipsPerHalfStave;   ///< number of chips per substave
+  std::vector<int> mNumberOfChipsPerStave;       ///< number of chips per stave
+  std::vector<int> mNumberOfChipsPerHalfBarrel;  ///< number of chips per halfbarrel
+  std::vector<int> mNumberOfChipsPerLayer;       ///< number of chips per stave
+  std::vector<int> mLastChipIndex;               ///< max ID of the detctor in the layer
+  std::array<bool, MAXLAYERS> mIsLayerITS3;      ///< flag with the information of the ITS version (ITS2 or ITS3)
+  std::array<int8_t, MAXLAYERS> mLayerToWrapper; ///< Layer to wrapper correspondence
 
   std::vector<float> mCacheRefX;     ///< sensors tracking plane reference X
   std::vector<float> mCacheRefAlpha; ///< sensors tracking plane reference alpha
diff --git a/Detectors/PHOS/reconstruction/src/Clusterer.cxx b/Detectors/PHOS/reconstruction/src/Clusterer.cxx
index 08b8ddfd234a5..a994b4df3fc62 100644
--- a/Detectors/PHOS/reconstruction/src/Clusterer.cxx
+++ b/Detectors/PHOS/reconstruction/src/Clusterer.cxx
@@ -235,7 +235,7 @@ void Clusterer::makeUnfolding(Cluster& clu, std::vector<Cluster>& clusters, std:
     return;
   }
 
-  char nMax = getNumberOfLocalMax(clu, cluelements);
+  int8_t nMax = getNumberOfLocalMax(clu, cluelements);
   if (nMax > 1) {
     unfoldOneCluster(clu, nMax, clusters, cluelements);
   } else {
@@ -250,7 +250,7 @@ void Clusterer::makeUnfolding(Cluster& clu, std::vector<Cluster>& clusters, std:
   }
 }
 //____________________________________________________________________________
-void Clusterer::unfoldOneCluster(Cluster& iniClu, char nMax, std::vector<Cluster>& clusters, std::vector<CluElement>& cluelements)
+void Clusterer::unfoldOneCluster(Cluster& iniClu, int8_t nMax, std::vector<Cluster>& clusters, std::vector<CluElement>& cluelements)
 {
   // Performs the unfolding of a cluster with nMax overlapping showers
   // Parameters: iniClu cluster to be unfolded
@@ -668,8 +668,8 @@ char Clusterer::getNumberOfLocalMax(Cluster& clu, std::vector<CluElement>& cluel
           }
         }
       } // if areneighbours
-    }   // digit j
-  }     // digit i
+    } // digit j
+  } // digit i
 
   int iDigitN = 0;
   for (std::size_t i = 0; i < mIsLocalMax.size(); i++) {
diff --git a/Detectors/TRD/base/include/TRDBase/ChamberStatus.h b/Detectors/TRD/base/include/TRDBase/ChamberStatus.h
index b5e1afd9d98f9..ab0128f0a4b11 100644
--- a/Detectors/TRD/base/include/TRDBase/ChamberStatus.h
+++ b/Detectors/TRD/base/include/TRDBase/ChamberStatus.h
@@ -46,7 +46,7 @@ class ChamberStatus
   ChamberStatus() = default;
   ~ChamberStatus() = default;
   //
-  char getStatus(int det) const { return mStatus[det]; }
+  int8_t getStatus(int det) const { return mStatus[det]; }
   void setStatus(int det, char bit);
   void setRawStatus(int det, char status) { mStatus[det] = status; };
   bool isGood(int det) const { return (mStatus[det] & Good); }
@@ -61,7 +61,7 @@ class ChamberStatus
   TH2D* plotBadCalibrated(int sm, int rphi); // Plot calibration status for sm and halfchamberside
   TH2D* plot(int sm);                        // Plot mStatus for sm
  private:
-  std::array<char, constants::MAXCHAMBER> mStatus{};
+  std::array<int8_t, constants::MAXCHAMBER> mStatus{};
   ClassDefNV(ChamberStatus, 1);
 };
 } // namespace trd

From f11e2700a15b85cda0c47326d1a6ff9fdc098d63 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 10 Aug 2024 16:38:50 +0200
Subject: [PATCH 601/726] Make AsyncQueue thread safe

Mutex should avoid concurrent appending and flushing of the queue.

It would be probably better to use a lockless priority queue, good enough for now.
---
 Framework/Core/src/AsyncQueue.cxx | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/Framework/Core/src/AsyncQueue.cxx b/Framework/Core/src/AsyncQueue.cxx
index 36c0c17009f82..bce2f31e57de7 100644
--- a/Framework/Core/src/AsyncQueue.cxx
+++ b/Framework/Core/src/AsyncQueue.cxx
@@ -15,10 +15,13 @@
 
 O2_DECLARE_DYNAMIC_LOG(async_queue);
 
+std::mutex gAsyncQueueMutex;
+
 namespace o2::framework
 {
 auto AsyncQueueHelpers::create(AsyncQueue& queue, AsyncTaskSpec spec) -> AsyncTaskId
 {
+  std::lock_guard<std::mutex> guard(gAsyncQueueMutex);
   AsyncTaskId id;
   id.value = queue.prototypes.size();
   queue.prototypes.push_back(spec);
@@ -32,11 +35,14 @@ auto AsyncQueueHelpers::post(AsyncQueue& queue, AsyncTaskId id, AsyncCallback ta
   taskToPost.id = id;
   taskToPost.timeslice = timeslice;
   taskToPost.debounce = debounce;
+
+  std::lock_guard<std::mutex> guard(gAsyncQueueMutex);
   queue.tasks.push_back(taskToPost);
 }
 
 auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> void
 {
+  std::lock_guard<std::mutex> guard(gAsyncQueueMutex);
   if (queue.tasks.empty()) {
     return;
   }
@@ -126,6 +132,7 @@ auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> vo
 
 auto AsyncQueueHelpers::reset(AsyncQueue& queue) -> void
 {
+  std::lock_guard<std::mutex> guard(gAsyncQueueMutex);
   queue.tasks.clear();
   queue.iteration = 0;
 }

From 00aad41ea5c46c0f69fbcac00b675978bb02bac4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 12 Aug 2024 18:22:35 +0200
Subject: [PATCH 602/726] Protect against missing vertexer (Cosmics)

---
 Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx b/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx
index fd4c5bdc56486..3a2016897e8b7 100644
--- a/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx
+++ b/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx
@@ -239,7 +239,7 @@ void ITSTrackingInterface::run(framework::ProcessingContext& pc)
   LOG(info) << fmt::format(" - Vertex seeding total elapsed time: {} ms for {} ({} + {}) vertices found in {}/{} ROFs",
                            vertexerElapsedTime,
                            mTimeFrame->getPrimaryVerticesNum(),
-                           mTimeFrame->getTotVertIteration()[0],
+                           mTimeFrame->getTotVertIteration().empty() == false ? mTimeFrame->getTotVertIteration()[0] : 0,
                            o2::its::VertexerParamConfig::Instance().nIterations > 1 ? mTimeFrame->getTotVertIteration()[1] : 0,
                            rofspan.size() - mTimeFrame->getNoVertexROF(),
                            rofspan.size());
@@ -381,4 +381,4 @@ void ITSTrackingInterface::setTraitsFromProvider(VertexerTraits* vertexerTraits,
 template void ITSTrackingInterface::run<true>(framework::ProcessingContext& pc);
 template void ITSTrackingInterface::run<false>(framework::ProcessingContext& pc);
 } // namespace its
-} // namespace o2
\ No newline at end of file
+} // namespace o2

From ada5fb42d52518eeec872012a65f33a0630bf1a7 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 10 Aug 2024 16:33:29 +0200
Subject: [PATCH 603/726] Make it explicit we are currently using a signed char

On x86 Linux `char` is assumed to be signed, however that's an implementation detail.

This makes sure we are explicit about the sign-ness.
---
 .../ITS/base/include/ITSBase/GeometryTGeo.h   | 28 +++++++++----------
 .../PHOS/reconstruction/src/Clusterer.cxx     |  8 +++---
 .../TRD/base/include/TRDBase/ChamberStatus.h  |  4 +--
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/Detectors/ITSMFT/ITS/base/include/ITSBase/GeometryTGeo.h b/Detectors/ITSMFT/ITS/base/include/ITSBase/GeometryTGeo.h
index 6172a02286abd..6d10a263408af 100644
--- a/Detectors/ITSMFT/ITS/base/include/ITSBase/GeometryTGeo.h
+++ b/Detectors/ITSMFT/ITS/base/include/ITSBase/GeometryTGeo.h
@@ -399,20 +399,20 @@ class GeometryTGeo : public o2::itsmft::GeometryTGeo
  protected:
   static constexpr int MAXLAYERS = 15; ///< max number of active layers
 
-  Int_t mNumberOfLayers;                        ///< number of layers
-  Int_t mNumberOfHalfBarrels;                   ///< number of halfbarrels
-  std::vector<int> mNumberOfStaves;             ///< number of staves/layer(layer)
-  std::vector<int> mNumberOfHalfStaves;         ///< the number of substaves/stave(layer)
-  std::vector<int> mNumberOfModules;            ///< number of modules/substave(layer)
-  std::vector<int> mNumberOfChipsPerModule;     ///< number of chips per module (group of chips on substaves)
-  std::vector<int> mNumberOfChipRowsPerModule;  ///< number of chips rows per module (relevant for OB modules)
-  std::vector<int> mNumberOfChipsPerHalfStave;  ///< number of chips per substave
-  std::vector<int> mNumberOfChipsPerStave;      ///< number of chips per stave
-  std::vector<int> mNumberOfChipsPerHalfBarrel; ///< number of chips per halfbarrel
-  std::vector<int> mNumberOfChipsPerLayer;      ///< number of chips per stave
-  std::vector<int> mLastChipIndex;              ///< max ID of the detctor in the layer
-  std::array<bool, MAXLAYERS> mIsLayerITS3;     ///< flag with the information of the ITS version (ITS2 or ITS3)
-  std::array<char, MAXLAYERS> mLayerToWrapper;  ///< Layer to wrapper correspondence
+  Int_t mNumberOfLayers;                         ///< number of layers
+  Int_t mNumberOfHalfBarrels;                    ///< number of halfbarrels
+  std::vector<int> mNumberOfStaves;              ///< number of staves/layer(layer)
+  std::vector<int> mNumberOfHalfStaves;          ///< the number of substaves/stave(layer)
+  std::vector<int> mNumberOfModules;             ///< number of modules/substave(layer)
+  std::vector<int> mNumberOfChipsPerModule;      ///< number of chips per module (group of chips on substaves)
+  std::vector<int> mNumberOfChipRowsPerModule;   ///< number of chips rows per module (relevant for OB modules)
+  std::vector<int> mNumberOfChipsPerHalfStave;   ///< number of chips per substave
+  std::vector<int> mNumberOfChipsPerStave;       ///< number of chips per stave
+  std::vector<int> mNumberOfChipsPerHalfBarrel;  ///< number of chips per halfbarrel
+  std::vector<int> mNumberOfChipsPerLayer;       ///< number of chips per stave
+  std::vector<int> mLastChipIndex;               ///< max ID of the detctor in the layer
+  std::array<bool, MAXLAYERS> mIsLayerITS3;      ///< flag with the information of the ITS version (ITS2 or ITS3)
+  std::array<int8_t, MAXLAYERS> mLayerToWrapper; ///< Layer to wrapper correspondence
 
   std::vector<float> mCacheRefX;     ///< sensors tracking plane reference X
   std::vector<float> mCacheRefAlpha; ///< sensors tracking plane reference alpha
diff --git a/Detectors/PHOS/reconstruction/src/Clusterer.cxx b/Detectors/PHOS/reconstruction/src/Clusterer.cxx
index 08b8ddfd234a5..a994b4df3fc62 100644
--- a/Detectors/PHOS/reconstruction/src/Clusterer.cxx
+++ b/Detectors/PHOS/reconstruction/src/Clusterer.cxx
@@ -235,7 +235,7 @@ void Clusterer::makeUnfolding(Cluster& clu, std::vector<Cluster>& clusters, std:
     return;
   }
 
-  char nMax = getNumberOfLocalMax(clu, cluelements);
+  int8_t nMax = getNumberOfLocalMax(clu, cluelements);
   if (nMax > 1) {
     unfoldOneCluster(clu, nMax, clusters, cluelements);
   } else {
@@ -250,7 +250,7 @@ void Clusterer::makeUnfolding(Cluster& clu, std::vector<Cluster>& clusters, std:
   }
 }
 //____________________________________________________________________________
-void Clusterer::unfoldOneCluster(Cluster& iniClu, char nMax, std::vector<Cluster>& clusters, std::vector<CluElement>& cluelements)
+void Clusterer::unfoldOneCluster(Cluster& iniClu, int8_t nMax, std::vector<Cluster>& clusters, std::vector<CluElement>& cluelements)
 {
   // Performs the unfolding of a cluster with nMax overlapping showers
   // Parameters: iniClu cluster to be unfolded
@@ -668,8 +668,8 @@ char Clusterer::getNumberOfLocalMax(Cluster& clu, std::vector<CluElement>& cluel
           }
         }
       } // if areneighbours
-    }   // digit j
-  }     // digit i
+    } // digit j
+  } // digit i
 
   int iDigitN = 0;
   for (std::size_t i = 0; i < mIsLocalMax.size(); i++) {
diff --git a/Detectors/TRD/base/include/TRDBase/ChamberStatus.h b/Detectors/TRD/base/include/TRDBase/ChamberStatus.h
index b5e1afd9d98f9..ab0128f0a4b11 100644
--- a/Detectors/TRD/base/include/TRDBase/ChamberStatus.h
+++ b/Detectors/TRD/base/include/TRDBase/ChamberStatus.h
@@ -46,7 +46,7 @@ class ChamberStatus
   ChamberStatus() = default;
   ~ChamberStatus() = default;
   //
-  char getStatus(int det) const { return mStatus[det]; }
+  int8_t getStatus(int det) const { return mStatus[det]; }
   void setStatus(int det, char bit);
   void setRawStatus(int det, char status) { mStatus[det] = status; };
   bool isGood(int det) const { return (mStatus[det] & Good); }
@@ -61,7 +61,7 @@ class ChamberStatus
   TH2D* plotBadCalibrated(int sm, int rphi); // Plot calibration status for sm and halfchamberside
   TH2D* plot(int sm);                        // Plot mStatus for sm
  private:
-  std::array<char, constants::MAXCHAMBER> mStatus{};
+  std::array<int8_t, constants::MAXCHAMBER> mStatus{};
   ClassDefNV(ChamberStatus, 1);
 };
 } // namespace trd

From 15b0758157af979943875bc930329e3ac84b5227 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 12 Aug 2024 18:22:58 +0200
Subject: [PATCH 604/726] Attempt at fixing signed  char for PHOS

---
 .../include/DataFormatsPHOS/CalibParams.h     | 14 +++----
 .../Detectors/PHOS/src/BadChannelsMap.cxx     |  8 ++--
 .../Detectors/PHOS/src/CalibParams.cxx        | 40 +++++++++----------
 .../PHOS/base/include/PHOSBase/Geometry.h     | 29 ++++++--------
 Detectors/PHOS/base/src/Geometry.cxx          | 38 +++++++++---------
 Detectors/PHOS/base/src/Mapping.cxx           |  2 +-
 .../include/PHOSReconstruction/Clusterer.h    |  6 +--
 7 files changed, 67 insertions(+), 70 deletions(-)

diff --git a/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/CalibParams.h b/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/CalibParams.h
index ca23a8ed16205..cdb885e1974d8 100644
--- a/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/CalibParams.h
+++ b/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/CalibParams.h
@@ -61,12 +61,12 @@ class CalibParams
   /// \param 2D(64,56) histogram with calibration coefficients
   /// \param module number
   /// \return Is successful
-  bool setGain(TH2* h, char module);
+  bool setGain(TH2* h, int8_t module);
 
   /// \brief Get High Gain to Low Gain ratio calibration coefficients
   /// \param cellID Absolute ID of cell
   /// \return High Gain to Low Gain ratio of the cell
-  float getHGLGRatio(short cellID) const { return mHGLGRatio.at(cellID - OFFSET); }
+  [[nodiscard]] float getHGLGRatio(short cellID) const { return mHGLGRatio.at(cellID - OFFSET); }
 
   /// \brief Set High Gain to Low Gain ratio
   /// \param cellID Absolute ID of cell
@@ -77,12 +77,12 @@ class CalibParams
   /// \param 2D(64,56) histogram with High Gain to Low Gain ratio
   /// \param module number
   /// \return Is successful
-  bool setHGLGRatio(TH2* h, char module);
+  bool setHGLGRatio(TH2* h, int8_t module);
 
   /// \brief Get High Gain time calibration coefficients
   /// \param cellID Absolute ID of cell
   /// \return high gain time calibration coefficient of the cell
-  float getHGTimeCalib(short cellID) const { return mHGTimeCalib.at(cellID - OFFSET); }
+  [[nodiscard]] float getHGTimeCalib(short cellID) const { return mHGTimeCalib.at(cellID - OFFSET); }
 
   /// \brief Set High Gain time calibration coefficient
   /// \param cellID Absolute ID of cell
@@ -93,12 +93,12 @@ class CalibParams
   /// \param 2D(64,56) histogram with calibration coefficients
   /// \param module number
   /// \return Is successful
-  bool setHGTimeCalib(TH2* h, char module);
+  bool setHGTimeCalib(TH2* h, int8_t module);
 
   /// \brief Get Low Gain time calibration coefficient
   /// \param cellID Absolute ID of cell
   /// \return low gain time calibration coefficient of the cell
-  float getLGTimeCalib(short cellID) const { return mLGTimeCalib.at(cellID - OFFSET); }
+  [[nodiscard]] float getLGTimeCalib(short cellID) const { return mLGTimeCalib.at(cellID - OFFSET); }
 
   /// \brief Set time calibration coefficient
   /// \param cellID Absolute ID of cell
@@ -109,7 +109,7 @@ class CalibParams
   /// \param 2D(64,56) histogram with calibration coefficients
   /// \param module number
   /// \return Is successful
-  bool setLGTimeCalib(TH2* h, char module);
+  bool setLGTimeCalib(TH2* h, int8_t module);
 
  private:
   static constexpr short NCHANNELS = 12544;  ///< Number of channels = 14336-1792
diff --git a/DataFormats/Detectors/PHOS/src/BadChannelsMap.cxx b/DataFormats/Detectors/PHOS/src/BadChannelsMap.cxx
index a2be62ab8544f..8cde0c6b9a2d2 100644
--- a/DataFormats/Detectors/PHOS/src/BadChannelsMap.cxx
+++ b/DataFormats/Detectors/PHOS/src/BadChannelsMap.cxx
@@ -62,15 +62,15 @@ void BadChannelsMap::getHistogramRepresentation(char module, TH2* h) const
   }
 
   h->Reset();
-  char relid[3] = {module, 1, 1};
+  int8_t relid[3] = {module, 1, 1};
   short absId;
-  char xmin = 1;
+  int8_t xmin = 1;
   if (module == 1) {
     xmin = 33;
   }
-  for (char ix = xmin; ix <= MAXX; ix++) {
+  for (int8_t ix = xmin; ix <= MAXX; ix++) {
     relid[1] = ix;
-    for (char iz = 1; iz <= MAXZ; iz++) {
+    for (int8_t iz = 1; iz <= MAXZ; iz++) {
       relid[2] = iz;
       if (o2::phos::Geometry::relToAbsNumbering(relid, absId)) {
         if (!isChannelGood(absId)) {
diff --git a/DataFormats/Detectors/PHOS/src/CalibParams.cxx b/DataFormats/Detectors/PHOS/src/CalibParams.cxx
index 7894c8e8c0095..4a52f8a7ae52b 100644
--- a/DataFormats/Detectors/PHOS/src/CalibParams.cxx
+++ b/DataFormats/Detectors/PHOS/src/CalibParams.cxx
@@ -29,10 +29,10 @@ CalibParams::CalibParams(int /*dummy*/)
   mLGTimeCalib.fill(0.);
 }
 
-bool CalibParams::setGain(TH2* h, char module)
+bool CalibParams::setGain(TH2* h, int8_t module)
 {
-  const char MAXX = 64,
-             MAXZ = 56;
+  const int8_t MAXX = 64,
+               MAXZ = 56;
   if (module < 1 || module > 4) {
     LOG(error) << "module " << module << "does not exist";
     return false;
@@ -48,7 +48,7 @@ bool CalibParams::setGain(TH2* h, char module)
     return false;
   }
 
-  char relid[3] = {module, 1, 1};
+  int8_t relid[3] = {module, 1, 1};
   short absId;
   for (char ix = 1; ix <= MAXX; ix++) {
     relid[1] = ix;
@@ -66,9 +66,9 @@ bool CalibParams::setGain(TH2* h, char module)
   return true;
 }
 
-bool CalibParams::setHGLGRatio(TH2* h, char module)
+bool CalibParams::setHGLGRatio(TH2* h, int8_t module)
 {
-  const char MAXX = 64,
+  const int8_t MAXX = 64,
              MAXZ = 56;
   if (module < 1 || module > 4) {
     LOG(error) << "module " << module << "does not exist";
@@ -84,11 +84,11 @@ bool CalibParams::setHGLGRatio(TH2* h, char module)
     return false;
   }
 
-  char relid[3] = {module, 1, 1};
+  int8_t relid[3] = {module, 1, 1};
   short absId;
-  for (char ix = 1; ix <= MAXX; ix++) {
+  for (int8_t ix = 1; ix <= MAXX; ix++) {
     relid[1] = ix;
-    for (char iz = 1; iz <= MAXZ; iz++) {
+    for (int8_t iz = 1; iz <= MAXZ; iz++) {
       relid[2] = iz;
 
       if (o2::phos::Geometry::relToAbsNumbering(relid, absId)) {
@@ -102,9 +102,9 @@ bool CalibParams::setHGLGRatio(TH2* h, char module)
   return true;
 }
 
-bool CalibParams::setHGTimeCalib(TH2* h, char module)
+bool CalibParams::setHGTimeCalib(TH2* h, int8_t module)
 {
-  const char MAXX = 64,
+  const int8_t MAXX = 64,
              MAXZ = 56;
   if (module < 1 || module > 4) {
     LOG(error) << "module " << module << "does not exist";
@@ -120,11 +120,11 @@ bool CalibParams::setHGTimeCalib(TH2* h, char module)
     return false;
   }
 
-  char relid[3] = {module, 1, 1};
+  int8_t relid[3] = {module, 1, 1};
   short absId;
-  for (char ix = 1; ix <= MAXX; ix++) {
+  for (int8_t ix = 1; ix <= MAXX; ix++) {
     relid[1] = ix;
-    for (char iz = 1; iz <= MAXZ; iz++) {
+    for (int8_t iz = 1; iz <= MAXZ; iz++) {
       relid[2] = iz;
 
       if (o2::phos::Geometry::relToAbsNumbering(relid, absId)) {
@@ -138,10 +138,10 @@ bool CalibParams::setHGTimeCalib(TH2* h, char module)
   return true;
 }
 
-bool CalibParams::setLGTimeCalib(TH2* h, char module)
+bool CalibParams::setLGTimeCalib(TH2* h, int8_t module)
 {
-  const char MAXX = 64,
-             MAXZ = 56;
+  const int8_t MAXX = 64,
+               MAXZ = 56;
   if (module < 1 || module > 4) {
     LOG(error) << "module " << module << "does not exist";
     return false;
@@ -156,11 +156,11 @@ bool CalibParams::setLGTimeCalib(TH2* h, char module)
     return false;
   }
 
-  char relid[3] = {module, 1, 1};
+  int8_t relid[3] = {module, 1, 1};
   short absId;
-  for (char ix = 1; ix <= MAXX; ix++) {
+  for (int8_t ix = 1; ix <= MAXX; ix++) {
     relid[1] = ix;
-    for (char iz = 1; iz <= MAXZ; iz++) {
+    for (int8_t iz = 1; iz <= MAXZ; iz++) {
       relid[2] = iz;
 
       if (o2::phos::Geometry::relToAbsNumbering(relid, absId)) {
diff --git a/Detectors/PHOS/base/include/PHOSBase/Geometry.h b/Detectors/PHOS/base/include/PHOSBase/Geometry.h
index a0ca5c9c9f778..fb94c2bd80bfb 100644
--- a/Detectors/PHOS/base/include/PHOSBase/Geometry.h
+++ b/Detectors/PHOS/base/include/PHOSBase/Geometry.h
@@ -22,9 +22,7 @@
 #include <TMath.h>
 #include <array>
 
-namespace o2
-{
-namespace phos
+namespace o2::phos
 {
 class Geometry
 {
@@ -97,27 +95,27 @@ class Geometry
   /// \param moduleNumber: module number
   /// \param strip: strip number
   //  \param cell: cell in strip number
-  static short relToAbsId(char moduleNumber, int strip, int cell);
+  static short relToAbsId(int8_t moduleNumber, int strip, int cell);
   // Converts the absolute numbering into the following array
   //  relid[0] = PHOS Module number 1:module
   //  relid[1] = Row number inside a PHOS module (Phi coordinate)
   //  relid[2] = Column number inside a PHOS module (Z coordinate)
-  static bool absToRelNumbering(short absId, char* relid);
-  static char absIdToModule(short absId);
+  static bool absToRelNumbering(short absId, int8_t * relid);
+  static int8_t absIdToModule(short absId);
   static void absIdToRelPosInModule(short absId, float& x, float& z);
-  static void relPosToRelId(short module, float x, float z, char* relId);
-  static bool relToAbsNumbering(const char* RelId, short& AbsId);
+  static void relPosToRelId(short module, float x, float z, int8_t* relId);
+  static bool relToAbsNumbering(const int8_t* RelId, short& AbsId);
 
   // Converters for TRU tiles
-  static bool truAbsToRelNumbering(short truId, short trigType, char* relid);
-  static short truRelToAbsNumbering(const char* relId, short trigType);
-  static short relPosToTruId(char mod, float x, float z, short trigType);
+  static bool truAbsToRelNumbering(short truId, short trigType, int8_t* relid);
+  static short truRelToAbsNumbering(const int8_t* relId, short trigType);
+  static short relPosToTruId(int8_t mod, float x, float z, short trigType);
 
   // local position to absId
-  static void relPosToAbsId(char module, float x, float z, short& absId);
+  static void relPosToAbsId(int8_t module, float x, float z, short& absId);
 
   // convert local position in module to global position in ALICE
-  void local2Global(char module, float x, float z, TVector3& globaPos) const;
+  void local2Global(int8_t module, float x, float z, TVector3& globaPos) const;
 
   // calculate impact position on PHOS
   bool impactOnPHOS(const TVector3& vtx, const TVector3& p,
@@ -129,9 +127,9 @@ class Geometry
     return absId >= 0 && absId <= getTotalNCells();
   } // TODO: evaluate from real geometry
 
-  const std::string& GetName() const { return mGeoName; }
+  [[nodiscard]] const std::string& GetName() const { return mGeoName; }
 
-  const TGeoHMatrix* getAlignmentMatrix(int mod) const { return &(mPHOS[mod]); }
+  [[nodiscard]] const TGeoHMatrix* getAlignmentMatrix(int mod) const { return &(mPHOS[mod]); }
 
  private:
   static constexpr float CELLSTEP = 2.25;
@@ -143,6 +141,5 @@ class Geometry
 
   ClassDefNV(Geometry, 1);
 };
-} // namespace phos
 } // namespace o2
 #endif
diff --git a/Detectors/PHOS/base/src/Geometry.cxx b/Detectors/PHOS/base/src/Geometry.cxx
index f9adb3044ca1f..ff32f0f0eb087 100644
--- a/Detectors/PHOS/base/src/Geometry.cxx
+++ b/Detectors/PHOS/base/src/Geometry.cxx
@@ -46,7 +46,7 @@ Geometry::Geometry(const std::string_view name) : mGeoName(name)
   fin.Close();
 }
 
-short Geometry::relToAbsId(char moduleNumber, int strip, int cell)
+short Geometry::relToAbsId(int8_t moduleNumber, int strip, int cell)
 {
   // calculates absolute cell Id from moduleNumber, strip (number) and cell (number)
   // PHOS layout parameters:
@@ -62,7 +62,7 @@ short Geometry::relToAbsId(char moduleNumber, int strip, int cell)
          (cell & 1 ? 1 : 0);
 }
 
-bool Geometry::absToRelNumbering(short absId, char* relid)
+bool Geometry::absToRelNumbering(short absId, int8_t* relid)
 {
   // Converts the absolute numbering into the following array
   //  relid[0] = PHOS Module number 1:fNModules
@@ -80,7 +80,7 @@ bool Geometry::absToRelNumbering(short absId, char* relid)
 
   return true;
 }
-bool Geometry::truAbsToRelNumbering(short truId, short trigType, char* relid)
+bool Geometry::truAbsToRelNumbering(short truId, short trigType, int8_t* relid)
 {
   // convert trigger cell Id 1..224*14 to relId (same relId schema as for readout channels)
   // short trigType=0 for 2x2, short trigType=1 for 4x4 trigger
@@ -115,7 +115,7 @@ bool Geometry::truAbsToRelNumbering(short truId, short trigType, char* relid)
   }
   return true;
 }
-short Geometry::truRelToAbsNumbering(const char* relId, short trigType)
+short Geometry::truRelToAbsNumbering(const int8_t* relId, short trigType)
 {
   // Convert position in PHOS module to TRU id for 4x4 or 2x2 tiles
 
@@ -141,14 +141,14 @@ short Geometry::truRelToAbsNumbering(const char* relId, short trigType)
     return getTotalNCells() + absId + 1;
   }
 }
-short Geometry::relPosToTruId(char mod, float x, float z, short trigType)
+short Geometry::relPosToTruId(int8_t mod, float x, float z, short trigType)
 {
   // tranform local cluster coordinates to truId
-  char relid[3] = {mod, static_cast<char>(ceil(x / CELLSTEP + 32.499)), static_cast<char>(ceil(z / CELLSTEP + 28.499))};
+  int8_t relid[3] = {mod, static_cast<int8_t>(ceil(x / CELLSTEP + 32.499)), static_cast<int8_t>(ceil(z / CELLSTEP + 28.499))};
   return truRelToAbsNumbering(relid, trigType);
 }
 
-char Geometry::absIdToModule(short absId)
+int8_t Geometry::absIdToModule(short absId)
 {
   const short nZ = 56;
   const short nPhi = 64;
@@ -168,15 +168,15 @@ int Geometry::areNeighbours(short absId1, short absId2)
   // The order of d1 and d2 is important: first (d1) should be a digit already in a cluster
   //                                      which is compared to a digit (d2)  not yet in a cluster
 
-  char relid1[3];
+  int8_t relid1[3];
   absToRelNumbering(absId1, relid1);
 
-  char relid2[3];
+  int8_t relid2[3];
   absToRelNumbering(absId2, relid2);
 
   if (relid1[0] == relid2[0]) { // inside the same PHOS module
-    char rowdiff = TMath::Abs(relid1[1] - relid2[1]);
-    char coldiff = TMath::Abs(relid1[2] - relid2[2]);
+    int8_t rowdiff = TMath::Abs(relid1[1] - relid2[1]);
+    int8_t coldiff = TMath::Abs(relid1[2] - relid2[2]);
 
     if (coldiff + rowdiff <= 1) { // Common side
       return 1;
@@ -198,13 +198,13 @@ int Geometry::areNeighbours(short absId1, short absId2)
 void Geometry::absIdToRelPosInModule(short absId, float& x, float& z)
 {
 
-  char relid[3];
+  int8_t relid[3];
   absToRelNumbering(absId, relid);
 
   x = (relid[1] - 32 - 0.5) * CELLSTEP;
   z = (relid[2] - 28 - 0.5) * CELLSTEP;
 }
-bool Geometry::relToAbsNumbering(const char* relId, short& absId)
+bool Geometry::relToAbsNumbering(const int8_t* relId, short& absId)
 {
   const short nZ = 56;   // nStripZ * nCellsZInStrip
   const short nPhi = 64; // nStripZ * nCellsZInStrip
@@ -217,21 +217,21 @@ bool Geometry::relToAbsNumbering(const char* relId, short& absId)
   return true;
 }
 // local position to absId
-void Geometry::relPosToAbsId(char module, float x, float z, short& absId)
+void Geometry::relPosToAbsId(int8_t module, float x, float z, short& absId)
 {
   // adding 32.5 instead of 32.499 leads to (rare) rounding errors before ceil()
-  char relid[3] = {module, static_cast<char>(ceil(x / CELLSTEP + 32.499)), static_cast<char>(ceil(z / CELLSTEP + 28.499))};
+  int8_t relid[3] = {module, static_cast<int8_t>(ceil(x / CELLSTEP + 32.499)), static_cast<int8_t>(ceil(z / CELLSTEP + 28.499))};
   relToAbsNumbering(relid, absId);
 }
-void Geometry::relPosToRelId(short module, float x, float z, char* relId)
+void Geometry::relPosToRelId(short module, float x, float z, int8_t* relId)
 {
   relId[0] = module;
-  relId[1] = static_cast<char>(ceil(x / CELLSTEP + 32.499));
-  relId[2] = static_cast<char>(ceil(z / CELLSTEP + 28.499));
+  relId[1] = static_cast<int8_t>(ceil(x / CELLSTEP + 32.499));
+  relId[2] = static_cast<int8_t>(ceil(z / CELLSTEP + 28.499));
 }
 
 // convert local position in module to global position in ALICE
-void Geometry::local2Global(char module, float x, float z, TVector3& globaPos) const
+void Geometry::local2Global(int8_t module, float x, float z, TVector3& globaPos) const
 {
   // constexpr float shiftY=-10.76; Run2
   constexpr float shiftY = -1.26; // Depth-optimized
diff --git a/Detectors/PHOS/base/src/Mapping.cxx b/Detectors/PHOS/base/src/Mapping.cxx
index e2af3d902c239..73e85f5220fef 100644
--- a/Detectors/PHOS/base/src/Mapping.cxx
+++ b/Detectors/PHOS/base/src/Mapping.cxx
@@ -195,7 +195,7 @@ Mapping::ErrorStatus Mapping::setMapping()
 
         short absId;
         if (caloFlag < 2) { // readout channels
-          char relid[3] = {static_cast<char>(m + 1), static_cast<char>(row + 1), static_cast<char>(col + 1)};
+          int8_t relid[3] = {static_cast<int8_t>(m + 1), static_cast<int8_t>(row + 1), static_cast<int8_t>(col + 1)};
           Geometry::relToAbsNumbering(relid, absId);
         } else { // TRU channels: internal storage of TRU channesl absId-NCHANNELS
           if (isTRUReadoutchannel(hwAddress)) {
diff --git a/Detectors/PHOS/reconstruction/include/PHOSReconstruction/Clusterer.h b/Detectors/PHOS/reconstruction/include/PHOSReconstruction/Clusterer.h
index 1ff47ac0cc144..66ac901a50a18 100644
--- a/Detectors/PHOS/reconstruction/include/PHOSReconstruction/Clusterer.h
+++ b/Detectors/PHOS/reconstruction/include/PHOSReconstruction/Clusterer.h
@@ -70,7 +70,7 @@ class Clusterer
     // L1phase correction
     float shift = 0;
     if (!mSkipL1phase) {
-      char relid[3];
+      int8_t relid[3];
       o2::phos::Geometry::absToRelNumbering(absId, relid);
       int ddl = (relid[0] - 1) * 4 + (relid[1] - 1) / 16 - 2;
       int l1 = (mL1phase >> (ddl * 2)) & 3; // extract 2 bits corresponding to this ddl
@@ -90,7 +90,7 @@ class Clusterer
   // Test Bad map
   inline bool isBadChannel(short absId) { return (!mBadMap->isChannelGood(absId)); }
 
-  char getNumberOfLocalMax(Cluster& clu, std::vector<CluElement>& cluel);
+  int8_t getNumberOfLocalMax(Cluster& clu, std::vector<CluElement>& cluel);
   void evalAll(Cluster& clu, std::vector<CluElement>& cluel) const;
   void evalLabels(std::vector<Cluster>& clusters, std::vector<CluElement>& cluel,
                   const o2::dataformats::MCTruthContainer<MCLabel>* dmc,
@@ -99,7 +99,7 @@ class Clusterer
   double showerShape(double r2, double& deriv); // Parameterization of EM shower
 
   void makeUnfolding(Cluster& clu, std::vector<Cluster>& clusters, std::vector<o2::phos::CluElement>& cluel); // unfold cluster with few local maxima
-  void unfoldOneCluster(Cluster& iniClu, char nMax, std::vector<Cluster>& clusters, std::vector<CluElement>& cluelements);
+  void unfoldOneCluster(Cluster& iniClu, int8_t nMax, std::vector<Cluster>& clusters, std::vector<CluElement>& cluelements);
 
  protected:
   static constexpr short NLOCMAX = 30; // Maximal number of local maxima in cluster

From 00118a090ea6c58f960287ba28841c78918ab755 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 12 Aug 2024 23:40:10 +0200
Subject: [PATCH 605/726] More signed char fixes

---
 .../CPV/include/DataFormatsCPV/Cluster.h      | 20 +++++++++----------
 DataFormats/Detectors/CPV/src/Cluster.cxx     |  4 ++--
 .../PHOS/include/DataFormatsPHOS/Cluster.h    | 18 ++++++++---------
 DataFormats/Detectors/PHOS/src/TriggerMap.cxx |  4 ++--
 .../Align/include/Align/AlignableVolume.h     |  2 +-
 .../PHOS/calib/src/PHOSEnergyCalibrator.cxx   |  2 +-
 .../PHOS/calib/src/PHOSL1phaseCalibrator.cxx  |  4 ++--
 .../PHOS/reconstruction/src/AltroDecoder.cxx  |  2 +-
 .../PHOS/reconstruction/src/Clusterer.cxx     |  6 +++---
 Detectors/PHOS/simulation/src/Digitizer.cxx   |  4 ++--
 Detectors/TPC/base/include/TPCBase/Mapper.h   |  4 ++--
 .../TRD/base/include/TRDBase/ChamberStatus.h  |  4 ++--
 Detectors/TRD/base/src/ChamberStatus.cxx      |  2 +-
 13 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/DataFormats/Detectors/CPV/include/DataFormatsCPV/Cluster.h b/DataFormats/Detectors/CPV/include/DataFormatsCPV/Cluster.h
index 97fb973c7951d..dc84b3425b768 100644
--- a/DataFormats/Detectors/CPV/include/DataFormatsCPV/Cluster.h
+++ b/DataFormats/Detectors/CPV/include/DataFormatsCPV/Cluster.h
@@ -65,26 +65,26 @@ class Cluster
     posX = mLocalPosX;
     posZ = mLocalPosZ;
   }
-  unsigned char getMultiplicity() const { return mMulDigit; } // gets the number of digits making this recpoint
+  [[nodiscard]] uint8_t getMultiplicity() const { return mMulDigit; } // gets the number of digits making this recpoint
                                                               // 0: was no unfolging, -1: unfolding failed
-  char getModule() const { return mModule; }                  // CPV module of a current cluster
+  [[nodiscard]] int8_t getModule() const { return mModule; }                  // CPV module of a current cluster
 
   // 0: was no unfolging, -1: unfolding failed
-  void setNExMax(char nmax = 1) { mNExMax = nmax; }
-  char getNExMax() const { return mNExMax; } // Number of maxima found in cluster in unfolding:
+  void setNExMax(int8_t nmax = 1) { mNExMax = nmax; }
+  [[nodiscard]] int8_t getNExMax() const { return mNExMax; } // Number of maxima found in cluster in unfolding:
                                              // 0: was no unfolging, -1: unfolding failed
 
   // raw access for CTF encoding
-  uint16_t getPackedPosX() const { return uint16_t((mLocalPosX - kMinX) / kStepX); }
+  [[nodiscard]] uint16_t getPackedPosX() const { return uint16_t((mLocalPosX - kMinX) / kStepX); }
   void setPackedPosX(uint16_t v) { mLocalPosX = kMinX + kStepX * v; }
 
-  uint16_t getPackedPosZ() const { return uint16_t((mLocalPosZ - kMinZ) / kStepZ); }
+  [[nodiscard]] uint16_t getPackedPosZ() const { return uint16_t((mLocalPosZ - kMinZ) / kStepZ); }
   void setPackedPosZ(uint16_t v) { mLocalPosZ = kMinZ + kStepZ * v; }
 
-  uint8_t getPackedEnergy() const { return uint8_t(std::min(255, int((mEnergy > 100.) ? (log(mEnergy - 63.) / kStepE) : mEnergy))); }
+  [[nodiscard]] uint8_t getPackedEnergy() const { return uint8_t(std::min(255, int((mEnergy > 100.) ? (log(mEnergy - 63.) / kStepE) : mEnergy))); }
   void setPackedEnergy(uint8_t v) { mEnergy = ((v > 100) ? (exp(kStepE * v) + 63.) : (v * 1.)); }
 
-  uint8_t getPackedClusterStatus() const
+  [[nodiscard]] uint8_t getPackedClusterStatus() const
   {
     CluStatus s = {0};
     s.multiplicity = std::min(mMulDigit, static_cast<unsigned char>(31)); // 5 bits available
@@ -110,8 +110,8 @@ class Cluster
 
  protected:
   unsigned char mMulDigit = 0; ///< Digit nultiplicity
-  char mModule = 0;            ///< Module number
-  char mNExMax = -1;           ///< number of (Ex-)maxima before unfolding
+  int8_t mModule = 0;            ///< Module number
+  int8_t mNExMax = -1;           ///< number of (Ex-)maxima before unfolding
   float mLocalPosX = 0.;       ///< Center of gravity position in local module coordunates (phi direction)
   float mLocalPosZ = 0.;       ///< Center of gravity position in local module coordunates (z direction)
   float mEnergy = 0.;          ///< full energy of a cluster
diff --git a/DataFormats/Detectors/CPV/src/Cluster.cxx b/DataFormats/Detectors/CPV/src/Cluster.cxx
index eeeae9998d181..af0888a99cf34 100644
--- a/DataFormats/Detectors/CPV/src/Cluster.cxx
+++ b/DataFormats/Detectors/CPV/src/Cluster.cxx
@@ -22,8 +22,8 @@ bool Cluster::operator<(const Cluster& other) const
 {
   // Compares two Clusters according to their position in the CPV modules
 
-  char cpvmod1 = getModule();
-  char cpvmod2 = other.getModule();
+  int8_t cpvmod1 = getModule();
+  int8_t cpvmod2 = other.getModule();
   if (cpvmod1 != cpvmod2) {
     return cpvmod1 < cpvmod2;
   }
diff --git a/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/Cluster.h b/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/Cluster.h
index 1c06f51bd1630..ec0aa42e0b23b 100644
--- a/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/Cluster.h
+++ b/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/Cluster.h
@@ -86,17 +86,17 @@ class Cluster
   int getMultiplicity() const { return mLastCluElement - mFirstCluElement; } // gets the number of digits making this cluster
 
   // 0: was no unfolging, -1: unfolding failed
-  void setNExMax(char nmax = 1) { mNExMax = nmax; }
-  char getNExMax() const { return mNExMax; }  // Number of maxima found in cluster in unfolding:
+  void setNExMax(int8_t nmax = 1) { mNExMax = nmax; }
+  int8_t getNExMax() const { return mNExMax; }  // Number of maxima found in cluster in unfolding:
                                               // 0: was no unfolging, -1: unfolding failed
-  char module() const { return mModule; }     // PHOS module of a current cluster
-  void setModule(char mod) { mModule = mod; } // set PHOS module of a current cluster
+  int8_t module() const { return mModule; }     // PHOS module of a current cluster
+  void setModule(int8_t mod) { mModule = mod; } // set PHOS module of a current cluster
 
   float getTime() const { return mTime; }
   void setTime(float t) { mTime = t; }
 
-  char firedTrigger() const { return mFiredTrigger; }
-  void setFiredTrigger(char t) { mFiredTrigger = t; }
+  int8_t firedTrigger() const { return mFiredTrigger; }
+  void setFiredTrigger(int8_t t) { mFiredTrigger = t; }
 
   /// \brief Method to add digit to a cluster
   void addDigit() { mLastCluElement++; }
@@ -110,9 +110,9 @@ class Cluster
   // std::vector<Digit>::const_iterator BinarySearch(const std::vector<Digit>* container, Digit& element);
 
  protected:
-  char mModule = 0;               ///< Module number
-  char mNExMax = -1;              ///< number of (Ex-)maxima before unfolding
-  char mFiredTrigger = 0;         ///< matched with PHOS trigger: 0 no match, bit 1 with 2x2, bit 2 with 4x4
+  int8_t mModule = 0;               ///< Module number
+  int8_t mNExMax = -1;              ///< number of (Ex-)maxima before unfolding
+  int8_t mFiredTrigger = 0;         ///< matched with PHOS trigger: 0 no match, bit 1 with 2x2, bit 2 with 4x4
   uint32_t mFirstCluElement = -1; ///< index of the first contributing CluElement in a list
   uint32_t mLastCluElement = -1;  ///< index of the last contributing CluElement in a list
   float mLocalPosX = 0.;          ///< Center of gravity position in local module coordunates (phi direction)
diff --git a/DataFormats/Detectors/PHOS/src/TriggerMap.cxx b/DataFormats/Detectors/PHOS/src/TriggerMap.cxx
index ee5e92cd159d7..f1ce10255187b 100644
--- a/DataFormats/Detectors/PHOS/src/TriggerMap.cxx
+++ b/DataFormats/Detectors/PHOS/src/TriggerMap.cxx
@@ -95,7 +95,7 @@ float TriggerMap::L0triggerProbability(float e, short ddl) const
 }
 bool TriggerMap::isFiredMC2x2(float a, short module, short ix, short iz) const
 {
-  char truRelId[3] = {char(module), char(ix), char(iz)};
+  int8_t truRelId[3] = {int8_t(module), int8_t(ix), int8_t(iz)};
   short tileId = Geometry::truRelToAbsNumbering(truRelId, 0); // 0 for 2x2 TriggerMap
   short iTRU = module * 4 + ix / 16 - 6;
   return isGood2x2(tileId) && try2x2(a, iTRU);
@@ -103,7 +103,7 @@ bool TriggerMap::isFiredMC2x2(float a, short module, short ix, short iz) const
 
 bool TriggerMap::isFiredMC4x4(float a, short module, short ix, short iz) const
 {
-  char truRelId[3] = {char(module), char(ix), char(iz)};
+  int8_t truRelId[3] = {int8_t(module), int8_t(ix), int8_t(iz)};
   short tileId = Geometry::truRelToAbsNumbering(truRelId, 1);
   short iTRU = module * 4 + ix / 16 - 6;
   return isGood4x4(tileId) && try4x4(a, iTRU);
diff --git a/Detectors/Align/include/Align/AlignableVolume.h b/Detectors/Align/include/Align/AlignableVolume.h
index 15b077eb2700f..b09d08fc0f3fc 100644
--- a/Detectors/Align/include/Align/AlignableVolume.h
+++ b/Detectors/Align/include/Align/AlignableVolume.h
@@ -233,7 +233,7 @@ class AlignableVolume : public DOFSet
   bool mIsDummy = false;    // placeholder (e.g. inactive), used to have the numbering corresponding to position in the container
   bool mIsDummyEnvelope = false; // some volumes are dummy envelopes for their children
   //
-  char mNDOFGeomFree = 0;   // number of free geom degrees of freedom
+  int8_t mNDOFGeomFree = 0;   // number of free geom degrees of freedom
   uint8_t mConstrChild = 0; // bitpattern for constraints on children corrections
   //
   AlignableVolume* mParent = nullptr; // parent volume
diff --git a/Detectors/PHOS/calib/src/PHOSEnergyCalibrator.cxx b/Detectors/PHOS/calib/src/PHOSEnergyCalibrator.cxx
index a5a2c9c320fa9..1d96783df61d0 100644
--- a/Detectors/PHOS/calib/src/PHOSEnergyCalibrator.cxx
+++ b/Detectors/PHOS/calib/src/PHOSEnergyCalibrator.cxx
@@ -149,7 +149,7 @@ void PHOSEnergySlot::fillTimeMassHisto(const Cluster& clu, const gsl::span<const
     if (ce.isHG) {
       if (ce.energy > mEminHGTime) {
         mHistos->fill(ETCalibHistos::kTimeHGPerCell, absId, ce.time);
-        char relid[3];
+        int8_t relid[3];
         Geometry::absToRelNumbering(absId, relid);
         int ddl = (relid[0] - 1) * 4 + (relid[1] - 1) / 16 - 2;
         mHistos->fill(ETCalibHistos::kTimeDDL, int(ddl * 4 + mEvBC % 4), ce.time);
diff --git a/Detectors/PHOS/calib/src/PHOSL1phaseCalibrator.cxx b/Detectors/PHOS/calib/src/PHOSL1phaseCalibrator.cxx
index eaf9173bd1ddb..abe7f4578e756 100644
--- a/Detectors/PHOS/calib/src/PHOSL1phaseCalibrator.cxx
+++ b/Detectors/PHOS/calib/src/PHOSL1phaseCalibrator.cxx
@@ -84,7 +84,7 @@ void PHOSL1phaseSlot::fill(const gsl::span<const Cell>& cells, const gsl::span<c
         t = c.getTime() - mCalibParams->getLGTimeCalib(absId);
       }
       if (e > mEmin && t > mTimeMin && t < mTimeMax) {
-        char relid[3];
+        int8_t relid[3];
         o2::phos::Geometry::absToRelNumbering(absId, relid);
         int ddl = (relid[0] - 1) * 4 + (relid[1] - 1) / 16 - 2;
         for (int b = 0; b < 4; b++) {
@@ -266,4 +266,4 @@ void PHOSL1phaseCalibrator::endOfStream()
     }
     mL1phase |= (bestB << (2 * d));
   }
-}
\ No newline at end of file
+}
diff --git a/Detectors/PHOS/reconstruction/src/AltroDecoder.cxx b/Detectors/PHOS/reconstruction/src/AltroDecoder.cxx
index 78cd76a258fee..00d2292744800 100644
--- a/Detectors/PHOS/reconstruction/src/AltroDecoder.cxx
+++ b/Detectors/PHOS/reconstruction/src/AltroDecoder.cxx
@@ -400,7 +400,7 @@ bool AltroDecoder::matchTruDigits(const Cell& cTruFlag, float& sumAmp)
     }
   }
   if (cTruFlag.getType() == TRU4x4) { // direct match of channel ID
-    char relid[3];
+    int8_t relid[3];
     Geometry::truAbsToRelNumbering(cTruFlag.getTRUId(), 1, relid); // 1 for 4x4 trigger
     bool found = false;
     sumAmp = 0.;
diff --git a/Detectors/PHOS/reconstruction/src/Clusterer.cxx b/Detectors/PHOS/reconstruction/src/Clusterer.cxx
index a994b4df3fc62..c412ad6843d0d 100644
--- a/Detectors/PHOS/reconstruction/src/Clusterer.cxx
+++ b/Detectors/PHOS/reconstruction/src/Clusterer.cxx
@@ -610,11 +610,11 @@ void Clusterer::evalAll(Cluster& clu, std::vector<CluElement>& cluel) const
   clu.setElipsAxis(lambdaShort, lambdaLong);
 
   // Test trigger
-  char relId[3];
+  int8_t relId[3];
   Geometry::relPosToRelId(clu.module(), localPosX, localPosZ, relId);
 
   for (auto& trd : mTrigger) {
-    char trurelid[3];
+    int8_t trurelid[3];
     short trtype = trd.is2x2Tile() ? 0 : 1;
     Geometry::truAbsToRelNumbering(trd.getAbsId(), trtype, trurelid);
 
@@ -635,7 +635,7 @@ void Clusterer::evalAll(Cluster& clu, std::vector<CluElement>& cluel) const
   }
 }
 //____________________________________________________________________________
-char Clusterer::getNumberOfLocalMax(Cluster& clu, std::vector<CluElement>& cluel)
+int8_t Clusterer::getNumberOfLocalMax(Cluster& clu, std::vector<CluElement>& cluel)
 {
   // Calculates the number of local maxima in the cluster using LocalMaxCut as the minimum
   // energy difference between maximum and surrounding digits
diff --git a/Detectors/PHOS/simulation/src/Digitizer.cxx b/Detectors/PHOS/simulation/src/Digitizer.cxx
index 9d248d1d41045..db62fa102b7e1 100644
--- a/Detectors/PHOS/simulation/src/Digitizer.cxx
+++ b/Detectors/PHOS/simulation/src/Digitizer.cxx
@@ -188,7 +188,7 @@ void Digitizer::processHits(const std::vector<Hit>* hits, const std::vector<Digi
     short xmin = (module == 1) ? 33 : 1;
     for (short ix = xmin; ix < 64; ix++) {
       for (short iz = 1; iz < 56; iz++) {
-        char relId[3] = {char(module), char(ix), char(iz)};
+        int8_t relId[3] = {int8_t(module), int8_t(ix), int8_t(iz)};
         short tileId = Geometry::truRelToAbsNumbering(relId, 0);
         if (!mTrigUtils->isGood2x2(tileId)) {
           continue;
@@ -247,7 +247,7 @@ void Digitizer::processHits(const std::vector<Hit>* hits, const std::vector<Digi
     if (mTrig4x4) {
       for (short ix = xmin; ix < 62; ix++) {
         for (short iz = 1; iz < 54; iz++) {
-          char relId[3] = {char(module), char(ix), char(iz)};
+          int8_t relId[3] = {int8_t(module), int8_t(ix), int8_t(iz)};
           short tileId = Geometry::truRelToAbsNumbering(relId, 1); // 1 for 4x4 trigger
           if (!mTrigUtils->isGood4x4(tileId)) {
             continue;
diff --git a/Detectors/TPC/base/include/TPCBase/Mapper.h b/Detectors/TPC/base/include/TPCBase/Mapper.h
index cee3d76db85b2..2ffb815fffb12 100644
--- a/Detectors/TPC/base/include/TPCBase/Mapper.h
+++ b/Detectors/TPC/base/include/TPCBase/Mapper.h
@@ -376,8 +376,8 @@ class Mapper
   /// Convert sector, row, pad to global pad row in sector and pad number
   const PadPos getGlobalPadPos(const PadROCPos& padROC) const
   {
-    const char globalRow = padROC.getRow() + (padROC.getROCType() == RocType::OROC) * mNumberOfPadRowsIROC;
-    const char pad = padROC.getPad();
+    const int8_t globalRow = padROC.getRow() + (padROC.getROCType() == RocType::OROC) * mNumberOfPadRowsIROC;
+    const int8_t pad = padROC.getPad();
     return PadPos(globalRow, pad);
   }
 
diff --git a/Detectors/TRD/base/include/TRDBase/ChamberStatus.h b/Detectors/TRD/base/include/TRDBase/ChamberStatus.h
index ab0128f0a4b11..fed4a2f435132 100644
--- a/Detectors/TRD/base/include/TRDBase/ChamberStatus.h
+++ b/Detectors/TRD/base/include/TRDBase/ChamberStatus.h
@@ -47,8 +47,8 @@ class ChamberStatus
   ~ChamberStatus() = default;
   //
   int8_t getStatus(int det) const { return mStatus[det]; }
-  void setStatus(int det, char bit);
-  void setRawStatus(int det, char status) { mStatus[det] = status; };
+  void setStatus(int det, int8_t bit);
+  void setRawStatus(int det, int8_t status) { mStatus[det] = status; };
   bool isGood(int det) const { return (mStatus[det] & Good); }
   bool isNoData(int det) const { return (mStatus[det] & NoData); }
   bool isNoDataSideA(int det) const { return (mStatus[det] & NoDataHalfChamberSideA); }
diff --git a/Detectors/TRD/base/src/ChamberStatus.cxx b/Detectors/TRD/base/src/ChamberStatus.cxx
index c2a14fdeb5c30..db76aef024ddd 100644
--- a/Detectors/TRD/base/src/ChamberStatus.cxx
+++ b/Detectors/TRD/base/src/ChamberStatus.cxx
@@ -21,7 +21,7 @@
 
 using namespace o2::trd;
 
-void ChamberStatus::setStatus(int det, char bit)
+void ChamberStatus::setStatus(int det, int8_t bit)
 {
 
   //

From 8d5109fa172ba88d01238901aadc194d191c0925 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 14 Aug 2024 11:12:30 +0200
Subject: [PATCH 606/726] Simplify builder holders

---
 .../Core/include/Framework/TableBuilder.h     | 74 ++++---------------
 1 file changed, 16 insertions(+), 58 deletions(-)

diff --git a/Framework/Core/include/Framework/TableBuilder.h b/Framework/Core/include/Framework/TableBuilder.h
index 7de45f58d2e62..10dbfd7379cd2 100644
--- a/Framework/Core/include/Framework/TableBuilder.h
+++ b/Framework/Core/include/Framework/TableBuilder.h
@@ -462,10 +462,10 @@ struct CachedInsertion {
   int pos = 0;
 };
 
-template <size_t I, typename T, template <typename U> typename InsertionPolicy>
-struct BuilderHolder : InsertionPolicy<T> {
+template <size_t I, typename T, typename P>
+struct BuilderHolder : P {
   static constexpr size_t index = I;
-  using Policy = InsertionPolicy<T>;
+  using Policy = P;
   using ArrowType = typename detail::ConversionTraits<T>::ArrowType;
   using BuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType;
 
@@ -562,60 +562,18 @@ template <class T, std::size_t N>
 struct is_bounded_array<std::array<T, N>> : std::true_type {
 };
 
-template <size_t I, typename T>
-struct HolderTrait {
-
-  using Holder = BuilderHolder<I, T, DirectInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, int8_t> {
-  using Holder = BuilderHolder<I, int8_t, CachedInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, uint8_t> {
-  using Holder = BuilderHolder<I, uint8_t, CachedInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, uint16_t> {
-  using Holder = BuilderHolder<I, uint16_t, CachedInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, int16_t> {
-  using Holder = BuilderHolder<I, int16_t, CachedInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, int> {
-  using Holder = BuilderHolder<I, int, CachedInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, float> {
-  using Holder = BuilderHolder<I, float, CachedInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, double> {
-  using Holder = BuilderHolder<I, double, CachedInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, unsigned int> {
-  using Holder = BuilderHolder<I, unsigned int, CachedInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, uint64_t> {
-  using Holder = BuilderHolder<I, uint64_t, CachedInsertion>;
+template<typename T>
+concept BulkInsertable = requires(T a)
+{
+    { 
+      std::is_integral_v<std::decay_t<T>> && !std::is_same_v<bool, std::decay_t<T>> };
 };
 
-template <size_t I>
-struct HolderTrait<I, int64_t> {
-  using Holder = BuilderHolder<I, int64_t, CachedInsertion>;
+template <typename T>
+struct InsertionTrait {
+  static consteval DirectInsertion<T> policy() requires (!BulkInsertable<T>);
+  static consteval CachedInsertion<T> policy() requires (BulkInsertable<T>);
+  using Policy = decltype(policy());
 };
 
 /// Helper function to convert a brace-initialisable struct to
@@ -645,7 +603,7 @@ template <typename... ARGS>
 constexpr auto makeHolderTypes()
 {
   return []<std::size_t... Is>(std::index_sequence<Is...>) {
-    return std::tuple(typename HolderTrait<Is, ARGS>::Holder(arrow::default_memory_pool())...);
+    return std::tuple(BuilderHolder<Is, ARGS, typename InsertionTrait<ARGS>::Policy>(arrow::default_memory_pool())...);
   }(std::make_index_sequence<sizeof...(ARGS)>{});
 }
 
@@ -653,7 +611,7 @@ template <typename... ARGS>
 auto makeHolders(arrow::MemoryPool* pool, size_t nRows)
 {
   return [pool, nRows]<std::size_t... Is>(std::index_sequence<Is...>) {
-    return new std::tuple(typename HolderTrait<Is, ARGS>::Holder(pool, nRows)...);
+    return new std::tuple(BuilderHolder<Is, ARGS, typename InsertionTrait<ARGS>::Policy>(pool, nRows)...);
   }(std::make_index_sequence<sizeof...(ARGS)>{});
 }
 
@@ -668,7 +626,7 @@ class TableBuilder
   static void throwError(RuntimeErrorRef const& ref);
 
   template <typename... ARGS>
-  using HoldersTuple = typename std::tuple<typename HolderTrait<0, ARGS>::Holder...>;
+  using HoldersTuple = typename std::tuple<BuilderHolder<0, ARGS, typename InsertionTrait<ARGS>::Policy>...>;
 
   template <typename... ARGS>
   using HoldersTupleIndexed = decltype(makeHolderTypes<ARGS...>());

From 925dbcdc4c7a9fa0d2a5d4f523de2e12da0c29ae Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 15 Aug 2024 11:04:59 +0200
Subject: [PATCH 607/726] Simplify builder holders

---
 .../Core/include/Framework/TableBuilder.h     | 75 ++++---------------
 1 file changed, 16 insertions(+), 59 deletions(-)

diff --git a/Framework/Core/include/Framework/TableBuilder.h b/Framework/Core/include/Framework/TableBuilder.h
index 7de45f58d2e62..0df83bdfb0f53 100644
--- a/Framework/Core/include/Framework/TableBuilder.h
+++ b/Framework/Core/include/Framework/TableBuilder.h
@@ -33,6 +33,7 @@
 #include <memory>
 #include <tuple>
 #include <type_traits>
+#include <concepts>
 
 namespace arrow
 {
@@ -462,10 +463,10 @@ struct CachedInsertion {
   int pos = 0;
 };
 
-template <size_t I, typename T, template <typename U> typename InsertionPolicy>
-struct BuilderHolder : InsertionPolicy<T> {
+template <size_t I, typename T, typename P>
+struct BuilderHolder : P {
   static constexpr size_t index = I;
-  using Policy = InsertionPolicy<T>;
+  using Policy = P;
   using ArrowType = typename detail::ConversionTraits<T>::ArrowType;
   using BuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType;
 
@@ -562,60 +563,16 @@ template <class T, std::size_t N>
 struct is_bounded_array<std::array<T, N>> : std::true_type {
 };
 
-template <size_t I, typename T>
-struct HolderTrait {
-
-  using Holder = BuilderHolder<I, T, DirectInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, int8_t> {
-  using Holder = BuilderHolder<I, int8_t, CachedInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, uint8_t> {
-  using Holder = BuilderHolder<I, uint8_t, CachedInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, uint16_t> {
-  using Holder = BuilderHolder<I, uint16_t, CachedInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, int16_t> {
-  using Holder = BuilderHolder<I, int16_t, CachedInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, int> {
-  using Holder = BuilderHolder<I, int, CachedInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, float> {
-  using Holder = BuilderHolder<I, float, CachedInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, double> {
-  using Holder = BuilderHolder<I, double, CachedInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, unsigned int> {
-  using Holder = BuilderHolder<I, unsigned int, CachedInsertion>;
-};
-
-template <size_t I>
-struct HolderTrait<I, uint64_t> {
-  using Holder = BuilderHolder<I, uint64_t, CachedInsertion>;
-};
+template <typename T>
+concept BulkInsertable = (std::integral<std::decay<T>> && !std::same_as<bool, std::decay_t<T>>);
 
-template <size_t I>
-struct HolderTrait<I, int64_t> {
-  using Holder = BuilderHolder<I, int64_t, CachedInsertion>;
+template <typename T>
+struct InsertionTrait {
+  static consteval DirectInsertion<T> policy()
+    requires(!BulkInsertable<T>);
+  static consteval CachedInsertion<T> policy()
+    requires(BulkInsertable<T>);
+  using Policy = decltype(policy());
 };
 
 /// Helper function to convert a brace-initialisable struct to
@@ -645,7 +602,7 @@ template <typename... ARGS>
 constexpr auto makeHolderTypes()
 {
   return []<std::size_t... Is>(std::index_sequence<Is...>) {
-    return std::tuple(typename HolderTrait<Is, ARGS>::Holder(arrow::default_memory_pool())...);
+    return std::tuple(BuilderHolder<Is, ARGS, typename InsertionTrait<ARGS>::Policy>(arrow::default_memory_pool())...);
   }(std::make_index_sequence<sizeof...(ARGS)>{});
 }
 
@@ -653,7 +610,7 @@ template <typename... ARGS>
 auto makeHolders(arrow::MemoryPool* pool, size_t nRows)
 {
   return [pool, nRows]<std::size_t... Is>(std::index_sequence<Is...>) {
-    return new std::tuple(typename HolderTrait<Is, ARGS>::Holder(pool, nRows)...);
+    return new std::tuple(BuilderHolder<Is, ARGS, typename InsertionTrait<ARGS>::Policy>(pool, nRows)...);
   }(std::make_index_sequence<sizeof...(ARGS)>{});
 }
 
@@ -668,7 +625,7 @@ class TableBuilder
   static void throwError(RuntimeErrorRef const& ref);
 
   template <typename... ARGS>
-  using HoldersTuple = typename std::tuple<typename HolderTrait<0, ARGS>::Holder...>;
+  using HoldersTuple = typename std::tuple<BuilderHolder<0, ARGS, typename InsertionTrait<ARGS>::Policy>...>;
 
   template <typename... ARGS>
   using HoldersTupleIndexed = decltype(makeHolderTypes<ARGS...>());

From f7bd93abdeee4a8397a7c3e6ef23b597f2e658a7 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 15 Aug 2024 16:11:04 +0200
Subject: [PATCH 608/726] Avoid doing one iteration when the tree has no
 entries

---
 Detectors/EMCAL/simulation/src/RawCreator.cxx | 2 +-
 Detectors/PHOS/simulation/src/RawCreator.cxx  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Detectors/EMCAL/simulation/src/RawCreator.cxx b/Detectors/EMCAL/simulation/src/RawCreator.cxx
index 883d907b8858e..386602e32b91c 100644
--- a/Detectors/EMCAL/simulation/src/RawCreator.cxx
+++ b/Detectors/EMCAL/simulation/src/RawCreator.cxx
@@ -122,7 +122,7 @@ int main(int argc, const char** argv)
   rawwriter.init();
 
   // Loop over all entries in the tree, where each tree entry corresponds to a time frame
-  for (auto en : *treereader) {
+  while (treereader->Next()) {
     rawwriter.digitsToRaw(*digitbranch, *triggerbranch);
   }
   rawwriter.getWriter().writeConfFile("EMC", "RAWDATA", o2::utils::Str::concat_string(outputdir, "/EMCraw.cfg"));
diff --git a/Detectors/PHOS/simulation/src/RawCreator.cxx b/Detectors/PHOS/simulation/src/RawCreator.cxx
index 620c9f4def2be..de8bd53741572 100644
--- a/Detectors/PHOS/simulation/src/RawCreator.cxx
+++ b/Detectors/PHOS/simulation/src/RawCreator.cxx
@@ -109,7 +109,7 @@ int main(int argc, const char** argv)
   rawwriter.init();
 
   // Loop over all entries in the tree, where each tree entry corresponds to a time frame
-  for (auto en : *treereader) {
+  while (treereader->Next()) {
     rawwriter.digitsToRaw(*digitbranch, *triggerbranch);
   }
   rawwriter.getWriter().writeConfFile("PHS", "RAWDATA", o2::utils::Str::concat_string(outputdir, "/PHSraw.cfg"));

From c62812abf87f9ffbf2fecee9c318fae586a18293 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 20 Aug 2024 11:48:45 +0200
Subject: [PATCH 609/726] Fix chaining of expendable devices

Obviously for forward channels we need to get the correct channel information.

This probably addresses https://its.cern.ch/jira/browse/O2-5018
---
 Framework/Core/include/Framework/ChannelInfo.h | 1 +
 Framework/Core/src/SendingPolicy.cxx           | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/include/Framework/ChannelInfo.h b/Framework/Core/include/Framework/ChannelInfo.h
index 1f4bd8bc9e90e..530f3fba49c29 100644
--- a/Framework/Core/include/Framework/ChannelInfo.h
+++ b/Framework/Core/include/Framework/ChannelInfo.h
@@ -97,6 +97,7 @@ struct ForwardChannelInfo {
 
 struct ForwardChannelState {
   TimesliceId oldestForChannel = {0};
+  int64_t droppedMessages = 0;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/SendingPolicy.cxx b/Framework/Core/src/SendingPolicy.cxx
index e2c00104a8951..ab166bd8a688e 100644
--- a/Framework/Core/src/SendingPolicy.cxx
+++ b/Framework/Core/src/SendingPolicy.cxx
@@ -174,7 +174,7 @@ std::vector<ForwardingPolicy> ForwardingPolicy::createDefaultPolicies()
             .forward = [](fair::mq::Parts& parts, ChannelIndex channelIndex, ServiceRegistryRef registry) {
               auto &proxy = registry.get<FairMQDeviceProxy>();
               auto *channel = proxy.getForwardChannel(channelIndex);
-              OutputChannelState& state = proxy.getOutputChannelState(channelIndex);
+              ForwardChannelState& state = proxy.getForwardChannelState(channelIndex);
               auto timeout = 1000;
               if (state.droppedMessages > 0) {
                 timeout = 0;

From 9ed5065f9059115d472b60b56209e0fc701e6e96 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 23 Aug 2024 11:29:34 +0200
Subject: [PATCH 610/726] DPL: assert that the dummy sink never depends on
 itself

This is by construction and it's needed to avoid introducing a
non existing dependency due to the fact we check both edges both
(i, j) and (j, i).
---
 Framework/Core/src/TopologyPolicy.cxx | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Framework/Core/src/TopologyPolicy.cxx b/Framework/Core/src/TopologyPolicy.cxx
index ebeb70c7950a3..9389534861f7e 100644
--- a/Framework/Core/src/TopologyPolicy.cxx
+++ b/Framework/Core/src/TopologyPolicy.cxx
@@ -73,6 +73,11 @@ bool expendableDataDeps(DataProcessorSpec const& a, DataProcessorSpec const& b)
 {
   O2_SIGNPOST_ID_GENERATE(sid, topology);
   O2_SIGNPOST_START(topology, sid, "expendableDataDeps", "Checking if %s depends on %s", a.name.c_str(), b.name.c_str());
+  if (a.name.find("internal-dpl-injected-dummy-sink") != std::string::npos &&
+      b.name.find("internal-dpl-injected-dummy-sink") != std::string::npos) {
+    O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "false. Dummy sink never depends on itself.");
+    return false;
+  }
   // We never put anything behind the dummy sink.
   if (b.name.find("internal-dpl-injected-dummy-sink") != std::string::npos) {
     O2_SIGNPOST_END(topology, sid, "expendableDataDeps", "false. %s is dummy sink and it nothing can depend on it.", b.name.c_str());

From 3909a54d33af8a23dc134b90944d06cba7ba6f49 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 28 Aug 2024 09:42:59 +0200
Subject: [PATCH 611/726] DPL: initialize TimingInfo.timeslice

Empty processing will otherwise have it set wrong on End Of Stream
---
 Framework/Core/include/Framework/TimingInfo.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/include/Framework/TimingInfo.h b/Framework/Core/include/Framework/TimingInfo.h
index 6b61b8e4ef5b5..00933a03845eb 100644
--- a/Framework/Core/include/Framework/TimingInfo.h
+++ b/Framework/Core/include/Framework/TimingInfo.h
@@ -23,7 +23,11 @@ namespace o2::framework
 
 struct TimingInfo {
   constexpr static ServiceKind service_kind = ServiceKind::Stream;
-  size_t timeslice;           /// the timeslice associated to current processing
+  size_t timeslice = 0;       /// the timeslice associated to current processing. The default
+                              /// is in general overridden unless the end of stream arrives
+                              /// without any previous processing, so we need to 0 it,
+                              /// and not use -1, which would break the oldest possible timeframe
+                              /// in that case.
   uint32_t firstTForbit = -1; /// the orbit the TF begins
   uint32_t tfCounter = -1;    // the counter associated to a TF
   uint32_t runNumber = -1;

From 71e3343c49cccc234f163f9534fca184ee8fab7e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 2 Sep 2024 10:13:00 +0200
Subject: [PATCH 612/726] DPL: introduce workflow_helpers signpost

---
 Framework/Core/src/WorkflowHelpers.cxx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index da60bdbc6961e..567dcd8cc03a7 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -24,6 +24,7 @@
 #include "Framework/PluginManager.h"
 #include "Framework/DataTakingContext.h"
 #include "Framework/DefaultsHelpers.h"
+#include "Framework/Signpost.h"
 
 #include "Headers/DataHeader.h"
 #include <algorithm>
@@ -34,8 +35,7 @@
 #include <climits>
 #include <thread>
 
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wpedantic"
+O2_DECLARE_DYNAMIC_LOG(workflow_helpers);
 
 namespace o2::framework
 {

From 0a5422ea791c1f777f788f456e460d0db4a1c8d9 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 2 Sep 2024 10:05:17 +0200
Subject: [PATCH 613/726] DPL: internal-dpl-injected-dummy sink should never
 depend on anything

---
 Framework/Core/src/TopologyPolicy.cxx | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/src/TopologyPolicy.cxx b/Framework/Core/src/TopologyPolicy.cxx
index 9389534861f7e..e209c2702031e 100644
--- a/Framework/Core/src/TopologyPolicy.cxx
+++ b/Framework/Core/src/TopologyPolicy.cxx
@@ -162,10 +162,15 @@ TopologyPolicy::DependencyChecker TopologyPolicyHelpers::alwaysDependent()
       O2_SIGNPOST_END(topology, sid, "alwaysDependent", "false. %s and %s are the same.", dependent.name.c_str(), ancestor.name.c_str());
       return false;
     }
-    if (ancestor.name == "internal-dpl-injected-dummy-sink") {
-      O2_SIGNPOST_END(topology, sid, "alwaysDependent", "false. %s is a dummy sink.", ancestor.name.c_str());
+    if (ancestor.name.find("internal-dpl-injected-dummy-sink") != std::string::npos) {
+      O2_SIGNPOST_END(topology, sid, "alwaysDependent", "false. Nothing can depend on %s by policy.", ancestor.name.c_str());
       return false;
     }
+    // We never put anything behind the dummy sink.
+    if (dependent.name.find("internal-dpl-injected-dummy-sink") != std::string::npos) {
+      O2_SIGNPOST_END(topology, sid, "alwaysDependent", "true. %s is always last.", ancestor.name.c_str());
+      return true;
+    }
     const std::regex matcher(".*output-proxy.*");
     // Check if regex applies
     std::cmatch m;

From c008a8c28d2d11d15b9882da86d59104661e0bf2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 2 Sep 2024 10:06:51 +0200
Subject: [PATCH 614/726] DPL: enable DPL_SIGNPOSTS early

This will make sure that we can enable signposts in the driver
---
 Framework/Core/src/runDataProcessing.cxx | 74 ++++++++++++++----------
 1 file changed, 42 insertions(+), 32 deletions(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 2c5ef88fb767c..933eb1dda52cf 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -2725,6 +2725,41 @@ std::string debugTopoInfo(std::vector<DataProcessorSpec> const& specs,
   return out.str();
 }
 
+void enableSignposts(std::string const& signpostsToEnable)
+{
+  static pid_t pid = getpid();
+  if (signpostsToEnable.empty() == true) {
+    auto printAllSignposts = [](char const* name, void* l, void* context) {
+      auto* log = (_o2_log_t*)l;
+      LOGP(detail, "Signpost stream {} disabled. Enable it with o2-log -p {} -a {}", name, pid, (void*)&log->stacktrace);
+      return true;
+    };
+    o2_walk_logs(printAllSignposts, nullptr);
+    return;
+  }
+  auto matchingLogEnabler = [](char const* name, void* l, void* context) {
+    auto* log = (_o2_log_t*)l;
+    auto* selectedName = (char const*)context;
+    std::string prefix = "ch.cern.aliceo2.";
+    if (strcmp(name, (prefix + selectedName).data()) == 0) {
+      LOGP(info, "Enabling signposts for stream \"ch.cern.aliceo2.{}\"", selectedName);
+      _o2_log_set_stacktrace(log, 1);
+      return false;
+    } else {
+      LOGP(info, "Signpost stream \"{}\" disabled. Enable it with o2-log -p {} -a {}", name, pid, (void*)&log->stacktrace);
+    }
+    return true;
+  };
+  // Split signpostsToEnable by comma using strtok_r
+  char* saveptr;
+  char* src = const_cast<char*>(signpostsToEnable.data());
+  auto* token = strtok_r(src, ",", &saveptr);
+  while (token) {
+    o2_walk_logs(matchingLogEnabler, token);
+    token = strtok_r(nullptr, ",", &saveptr);
+  }
+}
+
 // This is a toy executor for the workflow spec
 // What it needs to do is:
 //
@@ -2745,6 +2780,12 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
            std::vector<ConfigParamSpec> const& detectedParams,
            o2::framework::ConfigContext& configContext)
 {
+  // Peek very early in the driver options and look for
+  // signposts, so the we can enable it without going through the whole dance
+  if (getenv("DPL_DRIVER_SIGNPOSTS")) {
+    enableSignposts(getenv("DPL_DRIVER_SIGNPOSTS"));
+  }
+
   std::vector<std::string> currentArgs;
   std::vector<PluginInfo> plugins;
   std::vector<ForwardingPolicy> forwardingPolicies = ForwardingPolicy::createDefaultPolicies();
@@ -3055,38 +3096,7 @@ int doMain(int argc, char** argv, o2::framework::WorkflowSpec const& workflow,
     }
   }
 
-  static pid_t pid = getpid();
-  if (varmap.count("signposts")) {
-    auto signpostsToEnable = varmap["signposts"].as<std::string>();
-    auto matchingLogEnabler = [](char const* name, void* l, void* context) {
-      auto* log = (_o2_log_t*)l;
-      auto* selectedName = (char const*)context;
-      std::string prefix = "ch.cern.aliceo2.";
-      if (strcmp(name, (prefix + selectedName).data()) == 0) {
-        LOGP(info, "Enabling signposts for stream \"ch.cern.aliceo2.{}\"", selectedName);
-        _o2_log_set_stacktrace(log, 1);
-        return false;
-      } else {
-        LOGP(info, "Signpost stream \"{}\" disabled. Enable it with o2-log -p {} -a {}", name, pid, (void*)&log->stacktrace);
-      }
-      return true;
-    };
-    // Split signpostsToEnable by comma using strtok_r
-    char* saveptr;
-    char* src = const_cast<char*>(signpostsToEnable.data());
-    auto* token = strtok_r(src, ",", &saveptr);
-    while (token) {
-      o2_walk_logs(matchingLogEnabler, token);
-      token = strtok_r(nullptr, ",", &saveptr);
-    }
-  } else {
-    auto printAllSignposts = [](char const* name, void* l, void* context) {
-      auto* log = (_o2_log_t*)l;
-      LOGP(detail, "Signpost stream {} disabled. Enable it with o2-log -p {} -a {}", name, pid, (void*)&log->stacktrace);
-      return true;
-    };
-    o2_walk_logs(printAllSignposts, nullptr);
-  }
+  enableSignposts(varmap["signposts"].as<std::string>());
 
   auto evaluateBatchOption = [&varmap]() -> bool {
     if (varmap.count("no-batch") > 0) {

From 4e1d6fe87c769ac02a8477d91638dd39a4a97c5a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 2 Sep 2024 10:08:26 +0200
Subject: [PATCH 615/726] DPL: Debug channel creation

---
 Framework/Core/src/DeviceSpecHelpers.cxx | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index b9552dc093115..8076242bd274e 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -631,6 +631,28 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
     }
     DeviceConnectionId id{edge.producer, edge.consumer, edge.timeIndex, edge.producerTimeIndex, channel.port};
     connections.push_back(id);
+
+    auto& source = workflow[edge.producer];
+
+    O2_SIGNPOST_ID_GENERATE(sid, device_spec_helpers);
+    O2_SIGNPOST_START(device_spec_helpers, sid, "new channels", "Channel %{public}s has been created.", channel.name.c_str());
+    O2_SIGNPOST_ID_GENERATE(iid, device_spec_helpers);
+    O2_SIGNPOST_START(device_spec_helpers, iid, "producer outputs", "Producer %{public}s has the following outputs:", source.name.c_str());
+    for (auto& output : source.outputs) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, iid, "producer outputs", "%{public}s", DataSpecUtils::describe(output).c_str());
+    }
+    O2_SIGNPOST_END(device_spec_helpers, iid, "producer outputs", "");
+    O2_SIGNPOST_START(device_spec_helpers, iid, "producer forwards", "Producer %{public}s has the following forwards:", source.name.c_str());
+    for (auto& forwards : device.forwards) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, iid, "producer forwards", "%{public}s", DataSpecUtils::describe(forwards.matcher).c_str());
+    }
+    O2_SIGNPOST_END(device_spec_helpers, iid, "producer forwards", "");
+    O2_SIGNPOST_START(device_spec_helpers, iid, "consumer inputs", "Consumer %{public}s has the following inputs:", consumer.name.c_str());
+    for (auto& input : consumer.inputs) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, iid, "consumer inputs", "%{public}s", DataSpecUtils::describe(input).c_str());
+    }
+    O2_SIGNPOST_END(device_spec_helpers, iid, "consumer inputs", "");
+    O2_SIGNPOST_END(device_spec_helpers, sid, "new channels", "");
     return channel;
   };
 

From a1ef41d28de994d6130a2cdf74eb53c421212bc0 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 2 Sep 2024 10:12:55 +0200
Subject: [PATCH 616/726] DPL: make sure the suffix does not play a role
 sorting the dummy sink

---
 Framework/Core/src/CompletionPolicy.cxx     | 2 +-
 Framework/Core/src/DataProcessingDevice.cxx | 2 +-
 Framework/Core/src/WorkflowHelpers.cxx      | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/src/CompletionPolicy.cxx b/Framework/Core/src/CompletionPolicy.cxx
index 726148e0a2ae9..9d92fd07e6f5a 100644
--- a/Framework/Core/src/CompletionPolicy.cxx
+++ b/Framework/Core/src/CompletionPolicy.cxx
@@ -26,7 +26,7 @@ std::vector<CompletionPolicy>
 {
   return {
     CompletionPolicyHelpers::consumeWhenAllOrdered("internal-dpl-aod-writer"),
-    CompletionPolicyHelpers::consumeWhenAny("internal-dpl-injected-dummy-sink", [](DeviceSpec const& s) { return s.name == "internal-dpl-injected-dummy-sink"; }),
+    CompletionPolicyHelpers::consumeWhenAny("internal-dpl-injected-dummy-sink", [](DeviceSpec const& s) { return s.name.find("internal-dpl-injected-dummy-sink") != std::string::npos; }),
     CompletionPolicyHelpers::consumeWhenAll()};
 }
 
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 04e83046c0c76..55b22900396e8 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -1096,7 +1096,7 @@ void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceCont
   context.balancingInputs = spec.completionPolicy.balanceChannels;
   // This is needed because the internal injected dummy sink should not
   // try to balance inputs unless the rate limiting is requested.
-  if (enableRateLimiting == false && spec.name == "internal-dpl-injected-dummy-sink") {
+  if (enableRateLimiting == false && spec.name.find("internal-dpl-injected-dummy-sink") != std::string::npos) {
     context.balancingInputs = false;
   }
   if (enableRateLimiting) {
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 567dcd8cc03a7..cd0c20af502f4 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -367,7 +367,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
     // timeframe data.
     bool timeframeSink = hasTimeframeInputs && !hasTimeframeOutputs;
     if (std::stoi(ctx.options().get<std::string>("timeframes-rate-limit-ipcid")) != -1) {
-      if (timeframeSink && processor.name != "internal-dpl-injected-dummy-sink") {
+      if (timeframeSink && processor.name.find("internal-dpl-injected-dummy-sink") == std::string::npos) {
         processor.outputs.push_back(OutputSpec{{"dpl-summary"}, ConcreteDataMatcher{"DPL", "SUMMARY", static_cast<DataAllocator::SubSpecificationType>(runtime_hash(processor.name.c_str()))}});
       }
     }

From 7dc14f09eed3a964eddcebf02c99adab3f430f0e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 2 Sep 2024 10:10:44 +0200
Subject: [PATCH 617/726] DPL: Debug the available outputs

---
 Framework/Core/src/WorkflowHelpers.cxx | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index cd0c20af502f4..d393991cc61dc 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -838,15 +838,23 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
   // Notice that availableOutputsInfo MUST be updated first, since it relies on
   // the size of outputs to be the one before the update.
   auto enumerateAvailableOutputs = [&workflow, &outputs, &availableOutputsInfo]() {
+    O2_SIGNPOST_ID_GENERATE(sid, workflow_helpers);
     for (size_t wi = 0; wi < workflow.size(); ++wi) {
       auto& producer = workflow[wi];
+      if (producer.outputs.empty()) {
+        O2_SIGNPOST_EVENT_EMIT(workflow_helpers, sid, "output enumeration", "No outputs for [%zu] %{public}s", wi, producer.name.c_str());
+      }
+      O2_SIGNPOST_START(workflow_helpers, sid, "output enumeration", "Enumerating outputs for producer [%zu] %{}s public", wi, producer.name.c_str());
 
       for (size_t oi = 0; oi < producer.outputs.size(); ++oi) {
         auto& out = producer.outputs[oi];
         auto uniqueOutputId = outputs.size();
         availableOutputsInfo.emplace_back(LogicalOutputInfo{wi, uniqueOutputId, false});
+        O2_SIGNPOST_EVENT_EMIT(workflow_helpers, sid, "output enumeration", "- [%zu, %zu] %{public}s",
+                               oi, uniqueOutputId, DataSpecUtils::describe(out).c_str());
         outputs.push_back(out);
       }
+      O2_SIGNPOST_END(workflow_helpers, sid, "output enumeration", "");
     }
   };
 

From b1cf2351d49a03baa97370e6a849f31f0ab80ce3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 2 Sep 2024 10:10:44 +0200
Subject: [PATCH 618/726] DPL: debug topology building

---
 Framework/Core/src/WorkflowHelpers.cxx | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index d393991cc61dc..886076e504b5d 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -886,10 +886,17 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
 
   std::vector<bool> matches(constOutputs.size());
   for (size_t consumer = 0; consumer < workflow.size(); ++consumer) {
+    O2_SIGNPOST_ID_GENERATE(sid, workflow_helpers);
+    O2_SIGNPOST_START(workflow_helpers, sid, "input matching", "Matching inputs of consumer [%zu] %{}s public", consumer, workflow[consumer].name.c_str());
     for (size_t input = 0; input < workflow[consumer].inputs.size(); ++input) {
       forwards.clear();
       for (size_t i = 0; i < constOutputs.size(); i++) {
         matches[i] = DataSpecUtils::match(workflow[consumer].inputs[input], constOutputs[i]);
+        if (matches[i]) {
+          O2_SIGNPOST_EVENT_EMIT(workflow_helpers, sid, "output", "Input %{public}s matches %{public}s",
+                                 DataSpecUtils::describe(workflow[consumer].inputs[input]).c_str(),
+                                 DataSpecUtils::describe(constOutputs[i]).c_str());
+        }
       }
 
       for (size_t i = 0; i < availableOutputsInfo.size(); i++) {
@@ -905,7 +912,9 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
         auto uniqueOutputId = oif->outputGlobalIndex;
         for (size_t tpi = 0; tpi < workflow[consumer].maxInputTimeslices; ++tpi) {
           for (size_t ptpi = 0; ptpi < workflow[producer].maxInputTimeslices; ++ptpi) {
-            logicalEdges.emplace_back(DeviceConnectionEdge{producer, consumer, tpi, ptpi, uniqueOutputId, input, oif->forward});
+            O2_SIGNPOST_EVENT_EMIT(workflow_helpers, sid, "output", "Adding edge between %{public}s and %{public}s", workflow[consumer].name.c_str(),
+                                   workflow[producer].name.c_str());
+            logicalEdges.emplace_back(DeviceConnectionEdge{producer, consumer, tpi, ptpi, uniqueOutputId, input, .isForward = oif->forward});
           }
         }
         forwards.push_back(LogicalOutputInfo{consumer, uniqueOutputId, true});
@@ -920,6 +929,7 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
         availableOutputsInfo.push_back(forward);
       }
     }
+    O2_SIGNPOST_END(workflow_helpers, sid, "input matching", "");
   }
 }
 

From 69a27bdff457d2e10483e0d94a362601a7ddf6fb Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 2 Sep 2024 10:15:15 +0200
Subject: [PATCH 619/726] DPL: do not add summary outputs more than once

This seem to be the cause for the spurious circular dependency for
the internal-dpl-injected-dummy-sink.
---
 Framework/Core/src/WorkflowHelpers.cxx | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 886076e504b5d..cf2de544a6e48 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -368,7 +368,22 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
     bool timeframeSink = hasTimeframeInputs && !hasTimeframeOutputs;
     if (std::stoi(ctx.options().get<std::string>("timeframes-rate-limit-ipcid")) != -1) {
       if (timeframeSink && processor.name.find("internal-dpl-injected-dummy-sink") == std::string::npos) {
-        processor.outputs.push_back(OutputSpec{{"dpl-summary"}, ConcreteDataMatcher{"DPL", "SUMMARY", static_cast<DataAllocator::SubSpecificationType>(runtime_hash(processor.name.c_str()))}});
+        O2_SIGNPOST_ID_GENERATE(sid, workflow_helpers);
+        uint32_t hash = runtime_hash(processor.name.c_str());
+        bool hasMatch = false;
+        ConcreteDataMatcher summaryMatcher = ConcreteDataMatcher{"DPL", "SUMMARY", static_cast<DataAllocator::SubSpecificationType>(hash)};
+        for (auto& output : processor.outputs) {
+          if (DataSpecUtils::match(output, summaryMatcher)) {
+            O2_SIGNPOST_EVENT_EMIT(workflow_helpers, sid, "output enumeration", "%{public}s already there in %{public}s",
+                                   DataSpecUtils::describe(output).c_str(), processor.name.c_str());
+            hasMatch = true;
+            break;
+          }
+        }
+        if (!hasMatch) {
+          O2_SIGNPOST_EVENT_EMIT(workflow_helpers, sid, "output enumeration", "Adding DPL/SUMMARY/%d to %{public}s", hash, processor.name.c_str());
+          processor.outputs.push_back(OutputSpec{{"dpl-summary"}, ConcreteDataMatcher{"DPL", "SUMMARY", static_cast<DataAllocator::SubSpecificationType>(hash)}});
+        }
       }
     }
     bool hasConditionOption = false;

From 5656cd10333ec4b5d1a01d30acfc258832850bf4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 2 Sep 2024 10:40:57 +0200
Subject: [PATCH 620/726] DPL: debug topology building

---
 Framework/Core/src/WorkflowHelpers.cxx | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index d393991cc61dc..5b1f5ef766981 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -886,10 +886,17 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
 
   std::vector<bool> matches(constOutputs.size());
   for (size_t consumer = 0; consumer < workflow.size(); ++consumer) {
+    O2_SIGNPOST_ID_GENERATE(sid, workflow_helpers);
+    O2_SIGNPOST_START(workflow_helpers, sid, "input matching", "Matching inputs of consumer [%zu] %{}s public", consumer, workflow[consumer].name.c_str());
     for (size_t input = 0; input < workflow[consumer].inputs.size(); ++input) {
       forwards.clear();
       for (size_t i = 0; i < constOutputs.size(); i++) {
         matches[i] = DataSpecUtils::match(workflow[consumer].inputs[input], constOutputs[i]);
+        if (matches[i]) {
+          O2_SIGNPOST_EVENT_EMIT(workflow_helpers, sid, "output", "Input %{public}s matches %{public}s",
+                                 DataSpecUtils::describe(workflow[consumer].inputs[input]).c_str(),
+                                 DataSpecUtils::describe(constOutputs[i]).c_str());
+        }
       }
 
       for (size_t i = 0; i < availableOutputsInfo.size(); i++) {
@@ -905,6 +912,8 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
         auto uniqueOutputId = oif->outputGlobalIndex;
         for (size_t tpi = 0; tpi < workflow[consumer].maxInputTimeslices; ++tpi) {
           for (size_t ptpi = 0; ptpi < workflow[producer].maxInputTimeslices; ++ptpi) {
+            O2_SIGNPOST_EVENT_EMIT(workflow_helpers, sid, "output", "Adding edge between %{public}s and %{public}s", workflow[consumer].name.c_str(),
+                                   workflow[producer].name.c_str());
             logicalEdges.emplace_back(DeviceConnectionEdge{producer, consumer, tpi, ptpi, uniqueOutputId, input, oif->forward});
           }
         }
@@ -920,6 +929,7 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
         availableOutputsInfo.push_back(forward);
       }
     }
+    O2_SIGNPOST_END(workflow_helpers, sid, "input matching", "");
   }
 }
 

From 2d552d2e066f1b81721100f3291ec9f78a8c9ba2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 2 Sep 2024 10:40:57 +0200
Subject: [PATCH 621/726] DPL: do not add summary outputs more than once

This seem to be the cause for the spurious circular dependency for
the internal-dpl-injected-dummy-sink.
---
 Framework/Core/src/WorkflowHelpers.cxx | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 5b1f5ef766981..f9aa103300813 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -368,7 +368,22 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
     bool timeframeSink = hasTimeframeInputs && !hasTimeframeOutputs;
     if (std::stoi(ctx.options().get<std::string>("timeframes-rate-limit-ipcid")) != -1) {
       if (timeframeSink && processor.name.find("internal-dpl-injected-dummy-sink") == std::string::npos) {
-        processor.outputs.push_back(OutputSpec{{"dpl-summary"}, ConcreteDataMatcher{"DPL", "SUMMARY", static_cast<DataAllocator::SubSpecificationType>(runtime_hash(processor.name.c_str()))}});
+        O2_SIGNPOST_ID_GENERATE(sid, workflow_helpers);
+        uint32_t hash = runtime_hash(processor.name.c_str());
+        bool hasMatch = false;
+        ConcreteDataMatcher summaryMatcher = ConcreteDataMatcher{"DPL", "SUMMARY", static_cast<DataAllocator::SubSpecificationType>(hash)};
+        for (auto& output : processor.outputs) {
+          if (DataSpecUtils::match(output, summaryMatcher)) {
+            O2_SIGNPOST_EVENT_EMIT(workflow_helpers, sid, "output enumeration", "%{public}s already there in %{public}s",
+                                   DataSpecUtils::describe(output).c_str(), processor.name.c_str());
+            hasMatch = true;
+            break;
+          }
+        }
+        if (!hasMatch) {
+          O2_SIGNPOST_EVENT_EMIT(workflow_helpers, sid, "output enumeration", "Adding DPL/SUMMARY/%d to %{public}s", hash, processor.name.c_str());
+          processor.outputs.push_back(OutputSpec{{"dpl-summary"}, ConcreteDataMatcher{"DPL", "SUMMARY", static_cast<DataAllocator::SubSpecificationType>(hash)}});
+        }
       }
     }
     bool hasConditionOption = false;

From a292057370a5eaa2f6cc40adf545980d124e9d98 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 4 Sep 2024 09:15:58 +0200
Subject: [PATCH 622/726] DPL: allow printing stacktrace on signposts

---
 Framework/Core/src/runDataProcessing.cxx | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 933eb1dda52cf..e648d4aef98d7 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -2741,9 +2741,20 @@ void enableSignposts(std::string const& signpostsToEnable)
     auto* log = (_o2_log_t*)l;
     auto* selectedName = (char const*)context;
     std::string prefix = "ch.cern.aliceo2.";
-    if (strcmp(name, (prefix + selectedName).data()) == 0) {
+    auto* last = strchr(selectedName, ':');
+    int maxDepth = 1;
+    if (last) {
+      char* err;
+      maxDepth = strtol(last + 1, &err, 10);
+      if (err) {
+        maxDepth = 1;
+      }
+    }
+
+    auto fullName = prefix + std::string{selectedName, last ? last - selectedName : strlen(selectedName)};
+    if (strncmp(name, fullName.data(), fullName.size()) == 0) {
       LOGP(info, "Enabling signposts for stream \"ch.cern.aliceo2.{}\"", selectedName);
-      _o2_log_set_stacktrace(log, 1);
+      _o2_log_set_stacktrace(log, maxDepth);
       return false;
     } else {
       LOGP(info, "Signpost stream \"{}\" disabled. Enable it with o2-log -p {} -a {}", name, pid, (void*)&log->stacktrace);

From bc279672d0476c7b4ba2296e61a9c5676ff7ead4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 4 Sep 2024 09:19:31 +0200
Subject: [PATCH 623/726] Avoid memory churn when looking up channel

---
 Framework/Core/include/Framework/ExternalFairMQDeviceProxy.h | 4 ++--
 Framework/Core/src/ExternalFairMQDeviceProxy.cxx             | 5 +++--
 .../Core/test/benchmark_ExternalFairMQDeviceProxies.cxx      | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/Framework/Core/include/Framework/ExternalFairMQDeviceProxy.h b/Framework/Core/include/Framework/ExternalFairMQDeviceProxy.h
index 3ff77ae703b9b..415c6c5f702d8 100644
--- a/Framework/Core/include/Framework/ExternalFairMQDeviceProxy.h
+++ b/Framework/Core/include/Framework/ExternalFairMQDeviceProxy.h
@@ -22,7 +22,7 @@ namespace o2::framework
 
 /// A callback function to retrieve the fair::mq::Channel name to be used for sending
 /// messages of the specified OutputSpec
-using ChannelRetriever = std::function<std::string(OutputSpec const&, DataProcessingHeader::StartTime)>;
+using ChannelRetriever = std::function<std::string const&(OutputSpec const&, DataProcessingHeader::StartTime)>;
 /// The callback which actually does the heavy lifting of converting the input data into
 /// DPL messages. The callback is invoked with the following parameters:
 /// @param timingInfo is the timing information of the current timeslice
@@ -144,6 +144,6 @@ DataProcessorSpec specifyFairMQDeviceMultiOutputProxy(char const* label,
                                                       const char* defaultChannelConfig,
                                                       ChannelSelector channelSelector = defaultOutputProxyChannelSelector);
 
-} // namespace o2
+} // namespace o2::framework
 
 #endif // FRAMEWORK_RAWDEVICESOURCE_H
diff --git a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
index 820181ff353ed..239dd5f40e82f 100644
--- a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
+++ b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx
@@ -752,14 +752,15 @@ DataProcessorSpec specifyExternalFairMQDeviceProxy(char const* name,
                         outputChannels = std::move(outputChannels)](ServiceRegistryRef ref, TimingInfo& timingInfo, fair::mq::Parts& inputs, int, size_t ci, bool newRun) -> bool {
       auto* device = ref.get<RawDeviceService>().device();
       // pass a copy of the outputRoutes
-      auto channelRetriever = [&outputRoutes](OutputSpec const& query, DataProcessingHeader::StartTime timeslice) -> std::string {
+      auto channelRetriever = [&outputRoutes](OutputSpec const& query, DataProcessingHeader::StartTime timeslice) -> std::string const& {
+        static std::string emptyChannel = "";
         for (auto& route : outputRoutes) {
           LOG(debug) << "matching: " << DataSpecUtils::describe(query) << " to route " << DataSpecUtils::describe(route.matcher);
           if (DataSpecUtils::match(route.matcher, query) && ((timeslice % route.maxTimeslices) == route.timeslice)) {
             return route.channel;
           }
         }
-        return {""};
+        return emptyChannel;
       };
 
       std::string const& channel = channels[ci];
diff --git a/Framework/Core/test/benchmark_ExternalFairMQDeviceProxies.cxx b/Framework/Core/test/benchmark_ExternalFairMQDeviceProxies.cxx
index c116ed81537ce..de022030155f2 100644
--- a/Framework/Core/test/benchmark_ExternalFairMQDeviceProxies.cxx
+++ b/Framework/Core/test/benchmark_ExternalFairMQDeviceProxies.cxx
@@ -483,7 +483,7 @@ std::vector<DataProcessorSpec> defineDataProcessing(ConfigContext const& config)
     // Note: we want to run both the output and input proxy in the same workflow and thus we need
     // different data identifiers and change the data origin in the forwarding
     OutputSpec query{"PRX", dh->dataDescription, dh->subSpecification};
-    auto channelName = channelRetriever(query, dph->startTime);
+    auto const& channelName = channelRetriever(query, dph->startTime);
     bool isData = DataSpecUtils::match(OutputSpec{"TST", "DATA", 0}, dh->dataOrigin, dh->dataDescription, dh->subSpecification);
     // for the configured data channel we require the channel name, the EOS message containing
     // the forwarded SourceInfoHeader created by the output proxy will be skipped here since the

From a0ddd5dd4fac32d9c575b9ec0333ea1bc61f4515 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 4 Sep 2024 10:32:16 +0200
Subject: [PATCH 624/726] DPL GUI: remove protection for spurious formatting

DebugGUI is now able to detect "%s" and shortcuts the formatting
in that case. This allows us to print colored text without risks.
---
 Framework/GUISupport/src/FrameworkGUIDebugger.cxx | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
index 2fc8c7623c857..a025822a2abcb 100644
--- a/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
+++ b/Framework/GUISupport/src/FrameworkGUIDebugger.cxx
@@ -145,18 +145,12 @@ void displayHistory(const DeviceInfo& info, DeviceControl& control)
       ji = (ji + 1) % historySize;
       continue;
     }
-    // Print matching lines
+    // Print matching lines. Notice we filter twice, once on input, to reduce the
+    // stream, a second time at display time, to avoid showing unrelevant
+    // messages from past.
     if (strstr(line.c_str(), control.logFilter) != nullptr) {
-      auto color = colorForLogLevel(logLevel);
-      // We filter twice, once on input, to reduce the
-      // stream, a second time at display time, to avoid
-      // showing unrelevant messages from past.
       if (logLevel >= control.logLevel) {
-        if (line.find('%', 0) != std::string::npos) {
-          ImGui::TextUnformatted(line.c_str(), line.c_str() + line.size());
-        } else {
-          ImGui::TextColored(color, line.c_str(), line.c_str() + line.size());
-        }
+        ImGui::TextColored(colorForLogLevel(logLevel), "%s", line.c_str());
       }
     }
     ji = (ji + 1) % historySize;

From 45894c7856d497300fe12cd559b200e58294ccbd Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 4 Sep 2024 13:46:01 +0200
Subject: [PATCH 625/726] DPL: allow printing stacktrace on signposts

---
 Framework/Core/src/runDataProcessing.cxx        | 17 ++++++++++++++---
 .../Foundation/include/Framework/Signpost.h     |  3 ++-
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 933eb1dda52cf..4cec8c63fe856 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -2741,9 +2741,20 @@ void enableSignposts(std::string const& signpostsToEnable)
     auto* log = (_o2_log_t*)l;
     auto* selectedName = (char const*)context;
     std::string prefix = "ch.cern.aliceo2.";
-    if (strcmp(name, (prefix + selectedName).data()) == 0) {
-      LOGP(info, "Enabling signposts for stream \"ch.cern.aliceo2.{}\"", selectedName);
-      _o2_log_set_stacktrace(log, 1);
+    auto* last = strchr(selectedName, ':');
+    int maxDepth = 1;
+    if (last) {
+      char* err;
+      maxDepth = strtol(last + 1, &err, 10);
+      if (*(last + 1) == '\0' || *err != '\0') {
+        maxDepth = 1;
+      }
+    }
+
+    auto fullName = prefix + std::string{selectedName, last ? last - selectedName : strlen(selectedName)};
+    if (strncmp(name, fullName.data(), fullName.size()) == 0) {
+      LOGP(info, "Enabling signposts for stream \"{}\" with depth {}.", fullName, maxDepth);
+      _o2_log_set_stacktrace(log, maxDepth);
       return false;
     } else {
       LOGP(info, "Signpost stream \"{}\" disabled. Enable it with o2-log -p {} -a {}", name, pid, (void*)&log->stacktrace);
diff --git a/Framework/Foundation/include/Framework/Signpost.h b/Framework/Foundation/include/Framework/Signpost.h
index 5e844ce01b20c..74368f73c382e 100644
--- a/Framework/Foundation/include/Framework/Signpost.h
+++ b/Framework/Foundation/include/Framework/Signpost.h
@@ -381,7 +381,8 @@ void _o2_signpost_event_emit(_o2_log_t* log, _o2_signpost_id_t id, char const* n
   O2_LOG_MACRO("%s", prebuffer);
   if (log->stacktrace > 1) {
     void* traces[o2::framework::BacktraceHelpers::MAX_BACKTRACE_SIZE];
-    int maxBacktrace = backtrace(traces, o2::framework::BacktraceHelpers::MAX_BACKTRACE_SIZE);
+    // We add one extra frame, because one is for the logging
+    int maxBacktrace = backtrace(traces, (log->stacktrace + 1) < o2::framework::BacktraceHelpers::MAX_BACKTRACE_SIZE ? (log->stacktrace + 1) : o2::framework::BacktraceHelpers::MAX_BACKTRACE_SIZE);
     o2::framework::BacktraceHelpers::demangled_backtrace_symbols(traces, maxBacktrace, STDERR_FILENO);
   }
 }

From 40c705bfc7d6a342129c12a4e4093b935014ca3e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 5 Sep 2024 09:01:29 +0200
Subject: [PATCH 626/726] DPL: move analysis topology helper methods to a
 separate class

On their way to a separate plugin
---
 Framework/Core/CMakeLists.txt                 |   1 +
 .../include/Framework/CommonDataProcessors.h  |  27 +-
 Framework/Core/src/AnalysisSupportHelpers.cxx | 506 ++++++++++++++++++
 Framework/Core/src/AnalysisSupportHelpers.h   |  86 +++
 Framework/Core/src/ArrowSupport.cxx           |   7 +-
 Framework/Core/src/CommonDataProcessors.cxx   | 413 --------------
 Framework/Core/src/WorkflowHelpers.cxx        | 142 +----
 Framework/Core/src/WorkflowHelpers.h          |  18 -
 8 files changed, 624 insertions(+), 576 deletions(-)
 create mode 100644 Framework/Core/src/AnalysisSupportHelpers.cxx
 create mode 100644 Framework/Core/src/AnalysisSupportHelpers.h

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index 4381b6c4fa29d..7ddffda175a0a 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -16,6 +16,7 @@ o2_add_library(Framework
                        src/ArrowSupport.cxx
                        src/ArrowTableSlicingCache.cxx
                        src/AnalysisDataModel.cxx
+                       src/AnalysisSupportHelpers.cxx
                        src/ASoA.cxx
                        src/AsyncQueue.cxx
                        src/AnalysisDataModelHelpers.cxx
diff --git a/Framework/Core/include/Framework/CommonDataProcessors.h b/Framework/Core/include/Framework/CommonDataProcessors.h
index bb97f29edfc42..824386c4d5921 100644
--- a/Framework/Core/include/Framework/CommonDataProcessors.h
+++ b/Framework/Core/include/Framework/CommonDataProcessors.h
@@ -17,31 +17,10 @@
 #include <vector>
 #include <string>
 
-namespace o2::framework
-{
-
-class DataOutputDirector;
-
-struct OutputTaskInfo {
-  uint32_t id;
-  std::string name;
-};
-
-struct OutputObjectInfo {
-  uint32_t id;
-  std::vector<std::string> bindings;
-};
-} // namespace o2::framework
-extern template class std::vector<o2::framework::OutputObjectInfo>;
-extern template class std::vector<o2::framework::OutputTaskInfo>;
 namespace o2::framework
 {
 /// Helpers to create a few general data processors
 struct CommonDataProcessors {
-  /// Match all inputs of kind ATSK and write them to a ROOT file,
-  /// one root file per originating task.
-  static DataProcessorSpec getOutputObjHistSink(std::vector<OutputObjectInfo> const& objmap,
-                                                std::vector<OutputTaskInfo> const& tskmap);
   /// Given the list of @a danglingInputs @return a DataProcessor which does
   /// a binary dump for all the dangling inputs matching the Timeframe
   /// lifetime. @a unmatched will be filled with all the InputSpecs which are
@@ -54,10 +33,6 @@ struct CommonDataProcessors {
   /// @fixme: for now only the dangling inputs are forwarded.
   static DataProcessorSpec getGlobalFairMQSink(std::vector<InputSpec> const& danglingInputs);
 
-  /// writes inputs of kind AOD to file
-  static DataProcessorSpec getGlobalAODSink(std::shared_ptr<DataOutputDirector> dod,
-                                            std::vector<InputSpec> const& outputInputs);
-
   /// @return a dummy DataProcessorSpec which requires all the passed @a InputSpec
   /// and simply discards them. @a rateLimitingChannelConfig is the configuration
   /// for the rate limiting channel, if any required.
@@ -67,4 +42,4 @@ struct CommonDataProcessors {
 
 } // namespace o2::framework
 
-#endif // o2_framework_CommonDataProcessors_H_INCLUDED
+#endif // O2_FRAMEWORK_COMMONDATAPROCESSORS_H_
diff --git a/Framework/Core/src/AnalysisSupportHelpers.cxx b/Framework/Core/src/AnalysisSupportHelpers.cxx
new file mode 100644
index 0000000000000..3613bfedb887a
--- /dev/null
+++ b/Framework/Core/src/AnalysisSupportHelpers.cxx
@@ -0,0 +1,506 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "AnalysisSupportHelpers.h"
+#include "Framework/DataOutputDirector.h"
+#include "Framework/OutputObjHeader.h"
+#include "Framework/ControlService.h"
+#include "Framework/EndOfStreamContext.h"
+#include "Framework/DeviceSpec.h"
+#include "Framework/TableTreeHelpers.h"
+
+#include "TFile.h"
+#include "TTree.h"
+#include "TMap.h"
+#include "TObjString.h"
+
+template class std::vector<o2::framework::OutputObjectInfo>;
+template class std::vector<o2::framework::OutputTaskInfo>;
+
+namespace o2::framework
+{
+
+struct InputObjectRoute {
+  std::string name;
+  uint32_t uniqueId;
+  std::string directory;
+  uint32_t taskHash;
+  OutputObjHandlingPolicy policy;
+  OutputObjSourceType sourceType;
+};
+
+struct InputObject {
+  TClass* kind = nullptr;
+  void* obj = nullptr;
+  std::string name;
+  int count = -1;
+};
+
+void AnalysisSupportHelpers::addMissingOutputsToReader(std::vector<OutputSpec> const& providedOutputs,
+                                                       std::vector<InputSpec> const& requestedInputs,
+                                                       DataProcessorSpec& publisher)
+{
+  auto matchingOutputFor = [](InputSpec const& requested) {
+    return [&requested](OutputSpec const& provided) {
+      return DataSpecUtils::match(requested, provided);
+    };
+  };
+  for (InputSpec const& requested : requestedInputs) {
+    auto provided = std::find_if(providedOutputs.begin(),
+                                 providedOutputs.end(),
+                                 matchingOutputFor(requested));
+
+    if (provided != providedOutputs.end()) {
+      continue;
+    }
+
+    auto inList = std::find_if(publisher.outputs.begin(),
+                               publisher.outputs.end(),
+                               matchingOutputFor(requested));
+    if (inList != publisher.outputs.end()) {
+      continue;
+    }
+
+    auto concrete = DataSpecUtils::asConcreteDataMatcher(requested);
+    publisher.outputs.emplace_back(concrete.origin, concrete.description, concrete.subSpec, requested.lifetime, requested.metadata);
+  }
+}
+
+void AnalysisSupportHelpers::addMissingOutputsToSpawner(std::vector<OutputSpec> const& providedSpecials,
+                                                        std::vector<InputSpec> const& requestedSpecials,
+                                                        std::vector<InputSpec>& requestedAODs,
+                                                        DataProcessorSpec& publisher)
+{
+  for (auto& input : requestedSpecials) {
+    if (std::any_of(providedSpecials.begin(), providedSpecials.end(), [&input](auto const& x) {
+          return DataSpecUtils::match(input, x);
+        })) {
+      continue;
+    }
+    auto concrete = DataSpecUtils::asConcreteDataMatcher(input);
+    publisher.outputs.emplace_back(concrete.origin, concrete.description, concrete.subSpec);
+    for (auto& i : input.metadata) {
+      if ((i.type == VariantType::String) && (i.name.find("input:") != std::string::npos)) {
+        auto spec = DataSpecUtils::fromMetadataString(i.defaultValue.get<std::string>());
+        auto j = std::find(publisher.inputs.begin(), publisher.inputs.end(), spec);
+        if (j == publisher.inputs.end()) {
+          publisher.inputs.push_back(spec);
+        }
+        DataSpecUtils::updateInputList(requestedAODs, std::move(spec));
+      }
+    }
+  }
+}
+
+void AnalysisSupportHelpers::addMissingOutputsToBuilder(std::vector<InputSpec> const& requestedSpecials,
+                                                        std::vector<InputSpec>& requestedAODs,
+                                                        std::vector<InputSpec>& requestedDYNs,
+                                                        DataProcessorSpec& publisher)
+{
+  for (auto& input : requestedSpecials) {
+    auto concrete = DataSpecUtils::asConcreteDataMatcher(input);
+    publisher.outputs.emplace_back(concrete.origin, concrete.description, concrete.subSpec);
+    for (auto& i : input.metadata) {
+      if ((i.type == VariantType::String) && (i.name.find("input:") != std::string::npos)) {
+        auto spec = DataSpecUtils::fromMetadataString(i.defaultValue.get<std::string>());
+        auto j = std::find_if(publisher.inputs.begin(), publisher.inputs.end(), [&](auto x) { return x.binding == spec.binding; });
+        if (j == publisher.inputs.end()) {
+          publisher.inputs.push_back(spec);
+        }
+        if (DataSpecUtils::partialMatch(spec, AODOrigins)) {
+          DataSpecUtils::updateInputList(requestedAODs, std::move(spec));
+        } else if (DataSpecUtils::partialMatch(spec, header::DataOrigin{"DYN"})) {
+          DataSpecUtils::updateInputList(requestedDYNs, std::move(spec));
+        }
+      }
+    }
+  }
+}
+
+const static std::unordered_map<OutputObjHandlingPolicy, std::string> ROOTfileNames = {{OutputObjHandlingPolicy::AnalysisObject, "AnalysisResults.root"},
+                                                                                       {OutputObjHandlingPolicy::QAObject, "QAResults.root"}};
+
+// =============================================================================
+DataProcessorSpec AnalysisSupportHelpers::getOutputObjHistSink(std::vector<OutputObjectInfo> const& objmap, std::vector<OutputTaskInfo> const& tskmap)
+{
+  auto writerFunction = [objmap, tskmap](InitContext& ic) -> std::function<void(ProcessingContext&)> {
+    auto& callbacks = ic.services().get<CallbackService>();
+    auto inputObjects = std::make_shared<std::vector<std::pair<InputObjectRoute, InputObject>>>();
+
+    static TFile* f[OutputObjHandlingPolicy::numPolicies];
+    for (auto i = 0u; i < OutputObjHandlingPolicy::numPolicies; ++i) {
+      f[i] = nullptr;
+    }
+
+    static std::string currentDirectory = "";
+    static std::string currentFile = "";
+
+    auto endofdatacb = [inputObjects](EndOfStreamContext& context) {
+      LOG(debug) << "Writing merged objects and histograms to file";
+      if (inputObjects->empty()) {
+        LOG(error) << "Output object map is empty!";
+        context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
+        return;
+      }
+      for (auto i = 0u; i < OutputObjHandlingPolicy::numPolicies; ++i) {
+        if (f[i] != nullptr) {
+          f[i]->Close();
+        }
+      }
+      LOG(debug) << "All outputs merged in their respective target files";
+      context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
+    };
+
+    callbacks.set<CallbackService::Id::EndOfStream>(endofdatacb);
+    return [inputObjects, objmap, tskmap](ProcessingContext& pc) mutable -> void {
+      auto const& ref = pc.inputs().get("x");
+      if (!ref.header) {
+        LOG(error) << "Header not found";
+        return;
+      }
+      if (!ref.payload) {
+        LOG(error) << "Payload not found";
+        return;
+      }
+      auto datah = o2::header::get<o2::header::DataHeader*>(ref.header);
+      if (!datah) {
+        LOG(error) << "No data header in stack";
+        return;
+      }
+
+      auto objh = o2::header::get<o2::framework::OutputObjHeader*>(ref.header);
+      if (!objh) {
+        LOG(error) << "No output object header in stack";
+        return;
+      }
+
+      InputObject obj;
+      FairInputTBuffer tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
+      tm.InitMap();
+      obj.kind = tm.ReadClass();
+      tm.SetBufferOffset(0);
+      tm.ResetMap();
+      if (obj.kind == nullptr) {
+        LOG(error) << "Cannot read class info from buffer.";
+        return;
+      }
+
+      auto policy = objh->mPolicy;
+      auto sourceType = objh->mSourceType;
+      auto hash = objh->mTaskHash;
+
+      obj.obj = tm.ReadObjectAny(obj.kind);
+      auto* named = static_cast<TNamed*>(obj.obj);
+      obj.name = named->GetName();
+      auto hpos = std::find_if(tskmap.begin(), tskmap.end(), [&](auto&& x) { return x.id == hash; });
+      if (hpos == tskmap.end()) {
+        LOG(error) << "No task found for hash " << hash;
+        return;
+      }
+      auto taskname = hpos->name;
+      auto opos = std::find_if(objmap.begin(), objmap.end(), [&](auto&& x) { return x.id == hash; });
+      if (opos == objmap.end()) {
+        LOG(error) << "No object list found for task " << taskname << " (hash=" << hash << ")";
+        return;
+      }
+      auto objects = opos->bindings;
+      if (std::find(objects.begin(), objects.end(), obj.name) == objects.end()) {
+        LOG(error) << "No object " << obj.name << " in map for task " << taskname;
+        return;
+      }
+      auto nameHash = runtime_hash(obj.name.c_str());
+      InputObjectRoute key{obj.name, nameHash, taskname, hash, policy, sourceType};
+      auto existing = std::find_if(inputObjects->begin(), inputObjects->end(), [&](auto&& x) { return (x.first.uniqueId == nameHash) && (x.first.taskHash == hash); });
+      // If it's the first one, we just add it to the list.
+      if (existing == inputObjects->end()) {
+        obj.count = objh->mPipelineSize;
+        inputObjects->push_back(std::make_pair(key, obj));
+        existing = inputObjects->end() - 1;
+      } else {
+        obj.count = existing->second.count;
+        // Otherwise, we merge it with the existing one.
+        auto merger = existing->second.kind->GetMerge();
+        if (!merger) {
+          LOG(error) << "Already one unmergeable object found for " << obj.name;
+          return;
+        }
+        TList coll;
+        coll.Add(static_cast<TObject*>(obj.obj));
+        merger(existing->second.obj, &coll, nullptr);
+      }
+      // We expect as many objects as the pipeline size, for
+      // a given object name and task hash.
+      existing->second.count -= 1;
+
+      if (existing->second.count != 0) {
+        return;
+      }
+      // Write the object here.
+      auto route = existing->first;
+      auto entry = existing->second;
+      auto file = ROOTfileNames.find(route.policy);
+      if (file == ROOTfileNames.end()) {
+        return;
+      }
+      auto filename = file->second;
+      if (f[route.policy] == nullptr) {
+        f[route.policy] = TFile::Open(filename.c_str(), "RECREATE");
+      }
+      auto nextDirectory = route.directory;
+      if ((nextDirectory != currentDirectory) || (filename != currentFile)) {
+        if (!f[route.policy]->FindKey(nextDirectory.c_str())) {
+          f[route.policy]->mkdir(nextDirectory.c_str());
+        }
+        currentDirectory = nextDirectory;
+        currentFile = filename;
+      }
+
+      // translate the list-structure created by the registry into a directory structure within the file
+      std::function<void(TList*, TDirectory*)> writeListToFile;
+      writeListToFile = [&](TList* list, TDirectory* parentDir) {
+        TIter next(list);
+        TObject* object = nullptr;
+        while ((object = next())) {
+          if (object->InheritsFrom(TList::Class())) {
+            writeListToFile(static_cast<TList*>(object), parentDir->mkdir(object->GetName(), object->GetName(), true));
+          } else {
+            parentDir->WriteObjectAny(object, object->Class(), object->GetName());
+            auto* written = list->Remove(object);
+            delete written;
+          }
+        }
+      };
+
+      TDirectory* currentDir = f[route.policy]->GetDirectory(currentDirectory.c_str());
+      if (route.sourceType == OutputObjSourceType::HistogramRegistrySource) {
+        auto* outputList = static_cast<TList*>(entry.obj);
+        outputList->SetOwner(false);
+
+        // if registry should live in dedicated folder a TNamed object is appended to the list
+        if (outputList->Last() && outputList->Last()->IsA() == TNamed::Class()) {
+          delete outputList->Last();
+          outputList->RemoveLast();
+          currentDir = currentDir->mkdir(outputList->GetName(), outputList->GetName(), true);
+        }
+
+        writeListToFile(outputList, currentDir);
+        outputList->SetOwner();
+        delete outputList;
+        entry.obj = nullptr;
+      } else {
+        currentDir->WriteObjectAny(entry.obj, entry.kind, entry.name.c_str());
+        delete (TObject*)entry.obj;
+        entry.obj = nullptr;
+      }
+    };
+  };
+
+  char const* name = "internal-dpl-aod-global-analysis-file-sink";
+  // Lifetime is sporadic because we do not ask each analysis task to send its
+  // results every timeframe.
+  DataProcessorSpec spec{
+    .name = name,
+    .inputs = {InputSpec("x", DataSpecUtils::dataDescriptorMatcherFrom(header::DataOrigin{"ATSK"}), Lifetime::Sporadic)},
+    .algorithm = {writerFunction},
+  };
+
+  return spec;
+}
+
+// add sink for the AODs
+DataProcessorSpec
+  AnalysisSupportHelpers::getGlobalAODSink(std::shared_ptr<DataOutputDirector> dod,
+                                           std::vector<InputSpec> const& outputInputs)
+{
+
+  auto writerFunction = [dod, outputInputs](InitContext& ic) -> std::function<void(ProcessingContext&)> {
+    LOGP(debug, "======== getGlobalAODSink::Init ==========");
+
+    // find out if any table needs to be saved
+    bool hasOutputsToWrite = false;
+    for (auto& outobj : outputInputs) {
+      auto ds = dod->getDataOutputDescriptors(outobj);
+      if (ds.size() > 0) {
+        hasOutputsToWrite = true;
+        break;
+      }
+    }
+
+    // if nothing needs to be saved then return a trivial functor
+    // this happens when nothing needs to be saved but there are dangling outputs
+    if (!hasOutputsToWrite) {
+      return [](ProcessingContext&) mutable -> void {
+        static bool once = false;
+        if (!once) {
+          LOG(info) << "No AODs to be saved.";
+          once = true;
+        }
+      };
+    }
+
+    // end of data functor is called at the end of the data stream
+    auto endofdatacb = [dod](EndOfStreamContext& context) {
+      dod->closeDataFiles();
+      context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
+    };
+
+    auto& callbacks = ic.services().get<CallbackService>();
+    callbacks.set<CallbackService::Id::EndOfStream>(endofdatacb);
+
+    // prepare map<uint64_t, uint64_t>(startTime, tfNumber)
+    std::map<uint64_t, uint64_t> tfNumbers;
+    std::map<uint64_t, std::string> tfFilenames;
+
+    std::vector<TString> aodMetaDataKeys;
+    std::vector<TString> aodMetaDataVals;
+
+    // this functor is called once per time frame
+    return [dod, tfNumbers, tfFilenames, aodMetaDataKeys, aodMetaDataVals](ProcessingContext& pc) mutable -> void {
+      LOGP(debug, "======== getGlobalAODSink::processing ==========");
+      LOGP(debug, " processing data set with {} entries", pc.inputs().size());
+
+      // return immediately if pc.inputs() is empty. This should never happen!
+      if (pc.inputs().size() == 0) {
+        LOGP(info, "No inputs available!");
+        return;
+      }
+
+      // update tfNumbers
+      uint64_t startTime = 0;
+      uint64_t tfNumber = 0;
+      auto ref = pc.inputs().get("tfn");
+      if (ref.spec && ref.payload) {
+        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref)->startTime;
+        tfNumber = pc.inputs().get<uint64_t>("tfn");
+        tfNumbers.insert(std::pair<uint64_t, uint64_t>(startTime, tfNumber));
+      }
+      // update tfFilenames
+      std::string aodInputFile;
+      auto ref2 = pc.inputs().get("tff");
+      if (ref2.spec && ref2.payload) {
+        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref2)->startTime;
+        aodInputFile = pc.inputs().get<std::string>("tff");
+        tfFilenames.insert(std::pair<uint64_t, std::string>(startTime, aodInputFile));
+      }
+
+      // close all output files if one has reached size limit
+      dod->checkFileSizes();
+
+      // loop over the DataRefs which are contained in pc.inputs()
+      for (const auto& ref : pc.inputs()) {
+        if (!ref.spec) {
+          LOGP(debug, "Invalid input will be skipped!");
+          continue;
+        }
+
+        // get metadata
+        if (DataSpecUtils::partialMatch(*ref.spec, header::DataDescription("AODMetadataKeys"))) {
+          aodMetaDataKeys = pc.inputs().get<std::vector<TString>>(ref.spec->binding);
+        }
+        if (DataSpecUtils::partialMatch(*ref.spec, header::DataDescription("AODMetadataVals"))) {
+          aodMetaDataVals = pc.inputs().get<std::vector<TString>>(ref.spec->binding);
+        }
+
+        // skip non-AOD refs
+        if (!DataSpecUtils::partialMatch(*ref.spec, writableAODOrigins)) {
+          continue;
+        }
+        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref)->startTime;
+
+        // does this need to be saved?
+        auto dh = DataRefUtils::getHeader<header::DataHeader*>(ref);
+        auto tableName = dh->dataDescription.as<std::string>();
+        auto ds = dod->getDataOutputDescriptors(*dh);
+        if (ds.empty()) {
+          continue;
+        }
+
+        // get TF number from startTime
+        auto it = tfNumbers.find(startTime);
+        if (it != tfNumbers.end()) {
+          tfNumber = (it->second / dod->getNumberTimeFramesToMerge()) * dod->getNumberTimeFramesToMerge();
+        } else {
+          LOGP(fatal, "No time frame number found for output with start time {}", startTime);
+          throw std::runtime_error("Processing is stopped!");
+        }
+        // get aod input file from startTime
+        auto it2 = tfFilenames.find(startTime);
+        if (it2 != tfFilenames.end()) {
+          aodInputFile = it2->second;
+        }
+
+        // get the TableConsumer and corresponding arrow table
+        auto msg = pc.inputs().get(ref.spec->binding);
+        if (msg.header == nullptr) {
+          LOGP(error, "No header for message {}:{}", ref.spec->binding, DataSpecUtils::describe(*ref.spec));
+          continue;
+        }
+        auto s = pc.inputs().get<TableConsumer>(ref.spec->binding);
+        auto table = s->asArrowTable();
+        if (!table->Validate().ok()) {
+          LOGP(warning, "The table \"{}\" is not valid and will not be saved!", tableName);
+          continue;
+        }
+        if (table->schema()->fields().empty()) {
+          LOGP(debug, "The table \"{}\" is empty but will be saved anyway!", tableName);
+        }
+
+        // loop over all DataOutputDescriptors
+        // a table can be saved in multiple ways
+        // e.g. different selections of columns to different files
+        for (auto d : ds) {
+          auto fileAndFolder = dod->getFileFolder(d, tfNumber, aodInputFile);
+          auto treename = fileAndFolder.folderName + "/" + d->treename;
+          TableToTree ta2tr(table,
+                            fileAndFolder.file,
+                            treename.c_str());
+
+          // update metadata
+          if (fileAndFolder.file->FindObjectAny("metaData")) {
+            LOGF(debug, "Metadata: target file %s already has metadata, preserving it", fileAndFolder.file->GetName());
+          } else if (!aodMetaDataKeys.empty() && !aodMetaDataVals.empty()) {
+            TMap aodMetaDataMap;
+            for (uint32_t imd = 0; imd < aodMetaDataKeys.size(); imd++) {
+              aodMetaDataMap.Add(new TObjString(aodMetaDataKeys[imd]), new TObjString(aodMetaDataVals[imd]));
+            }
+            fileAndFolder.file->WriteObject(&aodMetaDataMap, "metaData", "Overwrite");
+          }
+
+          if (!d->colnames.empty()) {
+            for (auto& cn : d->colnames) {
+              auto idx = table->schema()->GetFieldIndex(cn);
+              auto col = table->column(idx);
+              auto field = table->schema()->field(idx);
+              if (idx != -1) {
+                ta2tr.addBranch(col, field);
+              }
+            }
+          } else {
+            ta2tr.addAllBranches();
+          }
+          ta2tr.process();
+        }
+      }
+    };
+  }; // end of writerFunction
+
+  // the command line options relevant for the writer are global
+  // see runDataProcessing.h
+  DataProcessorSpec spec{
+    "internal-dpl-aod-writer",
+    outputInputs,
+    Outputs{},
+    AlgorithmSpec(writerFunction),
+    {}};
+
+  return spec;
+}
+} // namespace o2::framework
diff --git a/Framework/Core/src/AnalysisSupportHelpers.h b/Framework/Core/src/AnalysisSupportHelpers.h
new file mode 100644
index 0000000000000..43ce7ab85b96d
--- /dev/null
+++ b/Framework/Core/src/AnalysisSupportHelpers.h
@@ -0,0 +1,86 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef O2_FRAMEWORK_ANALYSISSUPPORTHELPERS_H_
+#define O2_FRAMEWORK_ANALYSISSUPPORTHELPERS_H_
+
+#include "Framework/OutputSpec.h"
+#include "Framework/InputSpec.h"
+#include "Framework/DataProcessorSpec.h"
+#include "Headers/DataHeader.h"
+#include <array>
+
+namespace o2::framework
+{
+static constexpr std::array<header::DataOrigin, 3> AODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}};
+static constexpr std::array<header::DataOrigin, 5> extendedAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}, header::DataOrigin{"AMD"}};
+static constexpr std::array<header::DataOrigin, 4> writableAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}};
+
+class DataOutputDirector;
+
+struct OutputTaskInfo {
+  uint32_t id;
+  std::string name;
+};
+
+struct OutputObjectInfo {
+  uint32_t id;
+  std::vector<std::string> bindings;
+};
+} // namespace o2::framework
+
+extern template class std::vector<o2::framework::OutputObjectInfo>;
+extern template class std::vector<o2::framework::OutputTaskInfo>;
+
+namespace o2::framework
+{
+//
+struct AnalysisContext {
+  std::vector<InputSpec> requestedAODs;
+  std::vector<OutputSpec> providedAODs;
+  std::vector<InputSpec> requestedDYNs;
+  std::vector<OutputSpec> providedDYNs;
+  std::vector<InputSpec> requestedIDXs;
+  std::vector<OutputSpec> providedOutputObjHist;
+  std::vector<InputSpec> spawnerInputs;
+
+  std::vector<OutputTaskInfo> outTskMap;
+  std::vector<OutputObjectInfo> outObjHistMap;
+};
+
+// Helper class to be moved in the AnalysisSupport plugin at some point
+struct AnalysisSupportHelpers {
+  /// Helper functions to add AOD related internal devices.
+  /// FIXME: moved here until we have proper plugin based amendment
+  ///        of device injection
+  static void addMissingOutputsToReader(std::vector<OutputSpec> const& providedOutputs,
+                                        std::vector<InputSpec> const& requestedInputs,
+                                        DataProcessorSpec& publisher);
+  static void addMissingOutputsToSpawner(std::vector<OutputSpec> const& providedSpecials,
+                                         std::vector<InputSpec> const& requestedSpecials,
+                                         std::vector<InputSpec>& requestedAODs,
+                                         DataProcessorSpec& publisher);
+  static void addMissingOutputsToBuilder(std::vector<InputSpec> const& requestedSpecials,
+                                         std::vector<InputSpec>& requestedAODs,
+                                         std::vector<InputSpec>& requestedDYNs,
+                                         DataProcessorSpec& publisher);
+
+  /// Match all inputs of kind ATSK and write them to a ROOT file,
+  /// one root file per originating task.
+  static DataProcessorSpec getOutputObjHistSink(std::vector<OutputObjectInfo> const& objmap,
+                                                std::vector<OutputTaskInfo> const& tskmap);
+  /// writes inputs of kind AOD to file
+  static DataProcessorSpec getGlobalAODSink(std::shared_ptr<DataOutputDirector> dod,
+                                            std::vector<InputSpec> const& outputInputs);
+};
+
+}; // namespace o2::framework
+
+#endif // O2_FRAMEWORK_ANALYSISSUPPORTHELPERS_H_
diff --git a/Framework/Core/src/ArrowSupport.cxx b/Framework/Core/src/ArrowSupport.cxx
index 34916445a3912..5c1a9050c4e40 100644
--- a/Framework/Core/src/ArrowSupport.cxx
+++ b/Framework/Core/src/ArrowSupport.cxx
@@ -30,6 +30,7 @@
 #include "Framework/ServiceMetricsInfo.h"
 #include "WorkflowHelpers.h"
 #include "Framework/WorkflowSpecNode.h"
+#include "AnalysisSupportHelpers.h"
 
 #include "CommonMessageBackendsHelpers.h"
 #include <Monitoring/Monitoring.h>
@@ -445,7 +446,7 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
         // replace AlgorithmSpec
         //  FIXME: it should be made more generic, so it does not need replacement...
         builder->algorithm = readers::AODReaderHelpers::indexBuilderCallback(requestedIDXs);
-        WorkflowHelpers::addMissingOutputsToBuilder(requestedIDXs, requestedAODs, requestedDYNs, *builder);
+        AnalysisSupportHelpers::addMissingOutputsToBuilder(requestedIDXs, requestedAODs, requestedDYNs, *builder);
       }
 
       if (spawner != workflow.end()) {
@@ -480,7 +481,7 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
         // replace AlgorithmSpec
         // FIXME: it should be made more generic, so it does not need replacement...
         spawner->algorithm = readers::AODReaderHelpers::aodSpawnerCallback(spawnerInputs);
-        WorkflowHelpers::addMissingOutputsToSpawner({}, spawnerInputs, requestedAODs, *spawner);
+        AnalysisSupportHelpers::addMissingOutputsToSpawner({}, spawnerInputs, requestedAODs, *spawner);
       }
 
       if (writer != workflow.end()) {
@@ -535,7 +536,7 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
         // add TFNumber and TFFilename as input to the writer
         outputsInputsAOD.emplace_back("tfn", "TFN", "TFNumber");
         outputsInputsAOD.emplace_back("tff", "TFF", "TFFilename");
-        workflow.push_back(CommonDataProcessors::getGlobalAODSink(dod, outputsInputsAOD));
+        workflow.push_back(AnalysisSupportHelpers::getGlobalAODSink(dod, outputsInputsAOD));
       }
       // Move the dummy sink at the end, if needed
       for (size_t i = 0; i < workflow.size(); ++i) {
diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index 9817a57734c1d..d893e16513f40 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -32,7 +32,6 @@
 #include "Framework/Variant.h"
 #include "../../../Algorithm/include/Algorithm/HeaderStack.h"
 #include "Framework/OutputObjHeader.h"
-#include "Framework/TableTreeHelpers.h"
 #include "Framework/StringHelpers.h"
 #include "Framework/ChannelSpec.h"
 #include "Framework/ChannelSpecHelpers.h"
@@ -44,11 +43,6 @@
 #include "WorkflowHelpers.h"
 #include <Monitoring/Monitoring.h>
 
-#include "TFile.h"
-#include "TTree.h"
-#include "TMap.h"
-#include "TObjString.h"
-
 #include <fairmq/Device.h>
 #include <chrono>
 #include <fstream>
@@ -57,417 +51,11 @@
 #include <string>
 #include <thread>
 
-template class std::vector<o2::framework::OutputObjectInfo>;
-template class std::vector<o2::framework::OutputTaskInfo>;
 using namespace o2::framework::data_matcher;
 
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wpedantic"
-
 namespace o2::framework
 {
 
-struct InputObjectRoute {
-  std::string name;
-  uint32_t uniqueId;
-  std::string directory;
-  uint32_t taskHash;
-  OutputObjHandlingPolicy policy;
-  OutputObjSourceType sourceType;
-};
-
-struct InputObject {
-  TClass* kind = nullptr;
-  void* obj = nullptr;
-  std::string name;
-  int count = -1;
-};
-
-const static std::unordered_map<OutputObjHandlingPolicy, std::string> ROOTfileNames = {{OutputObjHandlingPolicy::AnalysisObject, "AnalysisResults.root"},
-                                                                                       {OutputObjHandlingPolicy::QAObject, "QAResults.root"}};
-
-// =============================================================================
-DataProcessorSpec CommonDataProcessors::getOutputObjHistSink(std::vector<OutputObjectInfo> const& objmap, std::vector<OutputTaskInfo> const& tskmap)
-{
-  auto writerFunction = [objmap, tskmap](InitContext& ic) -> std::function<void(ProcessingContext&)> {
-    auto& deviceSpec = ic.services().get<DeviceSpec const>();
-    auto& callbacks = ic.services().get<CallbackService>();
-    auto inputObjects = std::make_shared<std::vector<std::pair<InputObjectRoute, InputObject>>>();
-
-    static TFile* f[OutputObjHandlingPolicy::numPolicies];
-    for (auto i = 0u; i < OutputObjHandlingPolicy::numPolicies; ++i) {
-      f[i] = nullptr;
-    }
-
-    static std::string currentDirectory = "";
-    static std::string currentFile = "";
-
-    auto endofdatacb = [inputObjects](EndOfStreamContext& context) {
-      LOG(debug) << "Writing merged objects and histograms to file";
-      if (inputObjects->empty()) {
-        LOG(error) << "Output object map is empty!";
-        context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
-        return;
-      }
-      for (auto i = 0u; i < OutputObjHandlingPolicy::numPolicies; ++i) {
-        if (f[i] != nullptr) {
-          f[i]->Close();
-        }
-      }
-      LOG(debug) << "All outputs merged in their respective target files";
-      context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
-    };
-
-    callbacks.set<CallbackService::Id::EndOfStream>(endofdatacb);
-    return [inputObjects, objmap, tskmap](ProcessingContext& pc) mutable -> void {
-      auto const& ref = pc.inputs().get("x");
-      if (!ref.header) {
-        LOG(error) << "Header not found";
-        return;
-      }
-      if (!ref.payload) {
-        LOG(error) << "Payload not found";
-        return;
-      }
-      auto datah = o2::header::get<o2::header::DataHeader*>(ref.header);
-      if (!datah) {
-        LOG(error) << "No data header in stack";
-        return;
-      }
-
-      auto objh = o2::header::get<o2::framework::OutputObjHeader*>(ref.header);
-      if (!objh) {
-        LOG(error) << "No output object header in stack";
-        return;
-      }
-
-      InputObject obj;
-      FairInputTBuffer tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
-      tm.InitMap();
-      obj.kind = tm.ReadClass();
-      tm.SetBufferOffset(0);
-      tm.ResetMap();
-      if (obj.kind == nullptr) {
-        LOG(error) << "Cannot read class info from buffer.";
-        return;
-      }
-
-      auto policy = objh->mPolicy;
-      auto sourceType = objh->mSourceType;
-      auto hash = objh->mTaskHash;
-
-      obj.obj = tm.ReadObjectAny(obj.kind);
-      auto* named = static_cast<TNamed*>(obj.obj);
-      obj.name = named->GetName();
-      auto hpos = std::find_if(tskmap.begin(), tskmap.end(), [&](auto&& x) { return x.id == hash; });
-      if (hpos == tskmap.end()) {
-        LOG(error) << "No task found for hash " << hash;
-        return;
-      }
-      auto taskname = hpos->name;
-      auto opos = std::find_if(objmap.begin(), objmap.end(), [&](auto&& x) { return x.id == hash; });
-      if (opos == objmap.end()) {
-        LOG(error) << "No object list found for task " << taskname << " (hash=" << hash << ")";
-        return;
-      }
-      auto objects = opos->bindings;
-      if (std::find(objects.begin(), objects.end(), obj.name) == objects.end()) {
-        LOG(error) << "No object " << obj.name << " in map for task " << taskname;
-        return;
-      }
-      auto nameHash = runtime_hash(obj.name.c_str());
-      InputObjectRoute key{obj.name, nameHash, taskname, hash, policy, sourceType};
-      auto existing = std::find_if(inputObjects->begin(), inputObjects->end(), [&](auto&& x) { return (x.first.uniqueId == nameHash) && (x.first.taskHash == hash); });
-      // If it's the first one, we just add it to the list.
-      if (existing == inputObjects->end()) {
-        obj.count = objh->mPipelineSize;
-        inputObjects->push_back(std::make_pair(key, obj));
-        existing = inputObjects->end() - 1;
-      } else {
-        obj.count = existing->second.count;
-        // Otherwise, we merge it with the existing one.
-        auto merger = existing->second.kind->GetMerge();
-        if (!merger) {
-          LOG(error) << "Already one unmergeable object found for " << obj.name;
-          return;
-        }
-        TList coll;
-        coll.Add(static_cast<TObject*>(obj.obj));
-        merger(existing->second.obj, &coll, nullptr);
-      }
-      // We expect as many objects as the pipeline size, for
-      // a given object name and task hash.
-      existing->second.count -= 1;
-
-      if (existing->second.count != 0) {
-        return;
-      }
-      // Write the object here.
-      auto route = existing->first;
-      auto entry = existing->second;
-      auto file = ROOTfileNames.find(route.policy);
-      if (file == ROOTfileNames.end()) {
-        return;
-      }
-      auto filename = file->second;
-      if (f[route.policy] == nullptr) {
-        f[route.policy] = TFile::Open(filename.c_str(), "RECREATE");
-      }
-      auto nextDirectory = route.directory;
-      if ((nextDirectory != currentDirectory) || (filename != currentFile)) {
-        if (!f[route.policy]->FindKey(nextDirectory.c_str())) {
-          f[route.policy]->mkdir(nextDirectory.c_str());
-        }
-        currentDirectory = nextDirectory;
-        currentFile = filename;
-      }
-
-      // translate the list-structure created by the registry into a directory structure within the file
-      std::function<void(TList*, TDirectory*)> writeListToFile;
-      writeListToFile = [&](TList* list, TDirectory* parentDir) {
-        TIter next(list);
-        TObject* object = nullptr;
-        while ((object = next())) {
-          if (object->InheritsFrom(TList::Class())) {
-            writeListToFile(static_cast<TList*>(object), parentDir->mkdir(object->GetName(), object->GetName(), true));
-          } else {
-            parentDir->WriteObjectAny(object, object->Class(), object->GetName());
-            auto* written = list->Remove(object);
-            delete written;
-          }
-        }
-      };
-
-      TDirectory* currentDir = f[route.policy]->GetDirectory(currentDirectory.c_str());
-      if (route.sourceType == OutputObjSourceType::HistogramRegistrySource) {
-        auto* outputList = static_cast<TList*>(entry.obj);
-        outputList->SetOwner(false);
-
-        // if registry should live in dedicated folder a TNamed object is appended to the list
-        if (outputList->Last() && outputList->Last()->IsA() == TNamed::Class()) {
-          delete outputList->Last();
-          outputList->RemoveLast();
-          currentDir = currentDir->mkdir(outputList->GetName(), outputList->GetName(), true);
-        }
-
-        writeListToFile(outputList, currentDir);
-        outputList->SetOwner();
-        delete outputList;
-        entry.obj = nullptr;
-      } else {
-        currentDir->WriteObjectAny(entry.obj, entry.kind, entry.name.c_str());
-        delete (TObject*)entry.obj;
-        entry.obj = nullptr;
-      }
-    };
-  };
-
-  char const* name = "internal-dpl-aod-global-analysis-file-sink";
-  // Lifetime is sporadic because we do not ask each analysis task to send its
-  // results every timeframe.
-  DataProcessorSpec spec{
-    .name = name,
-    .inputs = {InputSpec("x", DataSpecUtils::dataDescriptorMatcherFrom(header::DataOrigin{"ATSK"}), Lifetime::Sporadic)},
-    .algorithm = {writerFunction},
-  };
-
-  return spec;
-}
-
-enum FileType : int {
-  AOD,
-  DANGLING
-};
-
-// add sink for the AODs
-DataProcessorSpec
-  CommonDataProcessors::getGlobalAODSink(std::shared_ptr<DataOutputDirector> dod,
-                                         std::vector<InputSpec> const& outputInputs)
-{
-
-  auto writerFunction = [dod, outputInputs](InitContext& ic) -> std::function<void(ProcessingContext&)> {
-    LOGP(debug, "======== getGlobalAODSink::Init ==========");
-
-    // find out if any table needs to be saved
-    bool hasOutputsToWrite = false;
-    for (auto& outobj : outputInputs) {
-      auto ds = dod->getDataOutputDescriptors(outobj);
-      if (ds.size() > 0) {
-        hasOutputsToWrite = true;
-        break;
-      }
-    }
-
-    // if nothing needs to be saved then return a trivial functor
-    // this happens when nothing needs to be saved but there are dangling outputs
-    if (!hasOutputsToWrite) {
-      return [](ProcessingContext&) mutable -> void {
-        static bool once = false;
-        if (!once) {
-          LOG(info) << "No AODs to be saved.";
-          once = true;
-        }
-      };
-    }
-
-    // end of data functor is called at the end of the data stream
-    auto endofdatacb = [dod](EndOfStreamContext& context) {
-      dod->closeDataFiles();
-      context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
-    };
-
-    auto& callbacks = ic.services().get<CallbackService>();
-    callbacks.set<CallbackService::Id::EndOfStream>(endofdatacb);
-
-    // prepare map<uint64_t, uint64_t>(startTime, tfNumber)
-    std::map<uint64_t, uint64_t> tfNumbers;
-    std::map<uint64_t, std::string> tfFilenames;
-
-    std::vector<TString> aodMetaDataKeys;
-    std::vector<TString> aodMetaDataVals;
-
-    // this functor is called once per time frame
-    return [dod, tfNumbers, tfFilenames, aodMetaDataKeys, aodMetaDataVals](ProcessingContext& pc) mutable -> void {
-      LOGP(debug, "======== getGlobalAODSink::processing ==========");
-      LOGP(debug, " processing data set with {} entries", pc.inputs().size());
-
-      // return immediately if pc.inputs() is empty. This should never happen!
-      if (pc.inputs().size() == 0) {
-        LOGP(info, "No inputs available!");
-        return;
-      }
-
-      // update tfNumbers
-      uint64_t startTime = 0;
-      uint64_t tfNumber = 0;
-      auto ref = pc.inputs().get("tfn");
-      if (ref.spec && ref.payload) {
-        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref)->startTime;
-        tfNumber = pc.inputs().get<uint64_t>("tfn");
-        tfNumbers.insert(std::pair<uint64_t, uint64_t>(startTime, tfNumber));
-      }
-      // update tfFilenames
-      std::string aodInputFile;
-      auto ref2 = pc.inputs().get("tff");
-      if (ref2.spec && ref2.payload) {
-        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref2)->startTime;
-        aodInputFile = pc.inputs().get<std::string>("tff");
-        tfFilenames.insert(std::pair<uint64_t, std::string>(startTime, aodInputFile));
-      }
-
-      // close all output files if one has reached size limit
-      dod->checkFileSizes();
-
-      // loop over the DataRefs which are contained in pc.inputs()
-      for (const auto& ref : pc.inputs()) {
-        if (!ref.spec) {
-          LOGP(debug, "Invalid input will be skipped!");
-          continue;
-        }
-
-        // get metadata
-        if (DataSpecUtils::partialMatch(*ref.spec, header::DataDescription("AODMetadataKeys"))) {
-          aodMetaDataKeys = pc.inputs().get<std::vector<TString>>(ref.spec->binding);
-        }
-        if (DataSpecUtils::partialMatch(*ref.spec, header::DataDescription("AODMetadataVals"))) {
-          aodMetaDataVals = pc.inputs().get<std::vector<TString>>(ref.spec->binding);
-        }
-
-        // skip non-AOD refs
-        if (!DataSpecUtils::partialMatch(*ref.spec, writableAODOrigins)) {
-          continue;
-        }
-        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref)->startTime;
-
-        // does this need to be saved?
-        auto dh = DataRefUtils::getHeader<header::DataHeader*>(ref);
-        auto tableName = dh->dataDescription.as<std::string>();
-        auto ds = dod->getDataOutputDescriptors(*dh);
-        if (ds.empty()) {
-          continue;
-        }
-
-        // get TF number from startTime
-        auto it = tfNumbers.find(startTime);
-        if (it != tfNumbers.end()) {
-          tfNumber = (it->second / dod->getNumberTimeFramesToMerge()) * dod->getNumberTimeFramesToMerge();
-        } else {
-          LOGP(fatal, "No time frame number found for output with start time {}", startTime);
-          throw std::runtime_error("Processing is stopped!");
-        }
-        // get aod input file from startTime
-        auto it2 = tfFilenames.find(startTime);
-        if (it2 != tfFilenames.end()) {
-          aodInputFile = it2->second;
-        }
-
-        // get the TableConsumer and corresponding arrow table
-        auto msg = pc.inputs().get(ref.spec->binding);
-        if (msg.header == nullptr) {
-          LOGP(error, "No header for message {}:{}", ref.spec->binding, DataSpecUtils::describe(*ref.spec));
-          continue;
-        }
-        auto s = pc.inputs().get<TableConsumer>(ref.spec->binding);
-        auto table = s->asArrowTable();
-        if (!table->Validate().ok()) {
-          LOGP(warning, "The table \"{}\" is not valid and will not be saved!", tableName);
-          continue;
-        }
-        if (table->schema()->fields().empty()) {
-          LOGP(debug, "The table \"{}\" is empty but will be saved anyway!", tableName);
-        }
-
-        // loop over all DataOutputDescriptors
-        // a table can be saved in multiple ways
-        // e.g. different selections of columns to different files
-        for (auto d : ds) {
-          auto fileAndFolder = dod->getFileFolder(d, tfNumber, aodInputFile);
-          auto treename = fileAndFolder.folderName + "/" + d->treename;
-          TableToTree ta2tr(table,
-                            fileAndFolder.file,
-                            treename.c_str());
-
-          // update metadata
-          if (fileAndFolder.file->FindObjectAny("metaData")) {
-            LOGF(debug, "Metadata: target file %s already has metadata, preserving it", fileAndFolder.file->GetName());
-          } else if (!aodMetaDataKeys.empty() && !aodMetaDataVals.empty()) {
-            TMap aodMetaDataMap;
-            for (uint32_t imd = 0; imd < aodMetaDataKeys.size(); imd++) {
-              aodMetaDataMap.Add(new TObjString(aodMetaDataKeys[imd]), new TObjString(aodMetaDataVals[imd]));
-            }
-            fileAndFolder.file->WriteObject(&aodMetaDataMap, "metaData", "Overwrite");
-          }
-
-          if (!d->colnames.empty()) {
-            for (auto& cn : d->colnames) {
-              auto idx = table->schema()->GetFieldIndex(cn);
-              auto col = table->column(idx);
-              auto field = table->schema()->field(idx);
-              if (idx != -1) {
-                ta2tr.addBranch(col, field);
-              }
-            }
-          } else {
-            ta2tr.addAllBranches();
-          }
-          ta2tr.process();
-        }
-      }
-    };
-  }; // end of writerFunction
-
-  // the command line options relevant for the writer are global
-  // see runDataProcessing.h
-  DataProcessorSpec spec{
-    "internal-dpl-aod-writer",
-    outputInputs,
-    Outputs{},
-    AlgorithmSpec(writerFunction),
-    {}};
-
-  return spec;
-}
-
 DataProcessorSpec
   CommonDataProcessors::getGlobalFileSink(std::vector<InputSpec> const& danglingOutputInputs,
                                           std::vector<InputSpec>& unmatched)
@@ -639,5 +227,4 @@ AlgorithmSpec CommonDataProcessors::wrapWithRateLimiting(AlgorithmSpec spec)
   });
 }
 
-#pragma GCC diagnostic pop
 } // namespace o2::framework
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index f9aa103300813..30246492b8b65 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -9,6 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 #include "WorkflowHelpers.h"
+#include "AnalysisSupportHelpers.h"
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/AODReaderHelpers.h"
 #include "Framework/ConfigParamsHelper.h"
@@ -125,87 +126,6 @@ std::vector<TopoIndexInfo>
   return S;
 }
 
-void WorkflowHelpers::addMissingOutputsToReader(std::vector<OutputSpec> const& providedOutputs,
-                                                std::vector<InputSpec> const& requestedInputs,
-                                                DataProcessorSpec& publisher)
-{
-  auto matchingOutputFor = [](InputSpec const& requested) {
-    return [&requested](OutputSpec const& provided) {
-      return DataSpecUtils::match(requested, provided);
-    };
-  };
-  for (InputSpec const& requested : requestedInputs) {
-    auto provided = std::find_if(providedOutputs.begin(),
-                                 providedOutputs.end(),
-                                 matchingOutputFor(requested));
-
-    if (provided != providedOutputs.end()) {
-      continue;
-    }
-
-    auto inList = std::find_if(publisher.outputs.begin(),
-                               publisher.outputs.end(),
-                               matchingOutputFor(requested));
-    if (inList != publisher.outputs.end()) {
-      continue;
-    }
-
-    auto concrete = DataSpecUtils::asConcreteDataMatcher(requested);
-    publisher.outputs.emplace_back(OutputSpec{concrete.origin, concrete.description, concrete.subSpec, requested.lifetime, requested.metadata});
-  }
-}
-
-void WorkflowHelpers::addMissingOutputsToSpawner(std::vector<OutputSpec> const& providedSpecials,
-                                                 std::vector<InputSpec> const& requestedSpecials,
-                                                 std::vector<InputSpec>& requestedAODs,
-                                                 DataProcessorSpec& publisher)
-{
-  for (auto& input : requestedSpecials) {
-    if (std::any_of(providedSpecials.begin(), providedSpecials.end(), [&input](auto const& x) {
-          return DataSpecUtils::match(input, x);
-        })) {
-      continue;
-    }
-    auto concrete = DataSpecUtils::asConcreteDataMatcher(input);
-    publisher.outputs.emplace_back(OutputSpec{concrete.origin, concrete.description, concrete.subSpec});
-    for (auto& i : input.metadata) {
-      if ((i.type == VariantType::String) && (i.name.find("input:") != std::string::npos)) {
-        auto spec = DataSpecUtils::fromMetadataString(i.defaultValue.get<std::string>());
-        auto j = std::find(publisher.inputs.begin(), publisher.inputs.end(), spec);
-        if (j == publisher.inputs.end()) {
-          publisher.inputs.push_back(spec);
-        }
-        DataSpecUtils::updateInputList(requestedAODs, std::move(spec));
-      }
-    }
-  }
-}
-
-void WorkflowHelpers::addMissingOutputsToBuilder(std::vector<InputSpec> const& requestedSpecials,
-                                                 std::vector<InputSpec>& requestedAODs,
-                                                 std::vector<InputSpec>& requestedDYNs,
-                                                 DataProcessorSpec& publisher)
-{
-  for (auto& input : requestedSpecials) {
-    auto concrete = DataSpecUtils::asConcreteDataMatcher(input);
-    publisher.outputs.emplace_back(OutputSpec{concrete.origin, concrete.description, concrete.subSpec});
-    for (auto& i : input.metadata) {
-      if ((i.type == VariantType::String) && (i.name.find("input:") != std::string::npos)) {
-        auto spec = DataSpecUtils::fromMetadataString(i.defaultValue.get<std::string>());
-        auto j = std::find_if(publisher.inputs.begin(), publisher.inputs.end(), [&](auto x) { return x.binding == spec.binding; });
-        if (j == publisher.inputs.end()) {
-          publisher.inputs.push_back(spec);
-        }
-        if (DataSpecUtils::partialMatch(spec, AODOrigins)) {
-          DataSpecUtils::updateInputList(requestedAODs, std::move(spec));
-        } else if (DataSpecUtils::partialMatch(spec, header::DataOrigin{"DYN"})) {
-          DataSpecUtils::updateInputList(requestedDYNs, std::move(spec));
-        }
-      }
-    }
-  }
-}
-
 // get the default value for condition-backend
 std::string defaultConditionBackend()
 {
@@ -321,24 +241,15 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
     aodReader.options.emplace_back(ConfigParamSpec{"channel-config", VariantType::String, rateLimitingChannelConfigInput, {"how many timeframes can be in flight at the same time"}});
   }
 
-  std::vector<InputSpec> requestedAODs;
-  std::vector<OutputSpec> providedAODs;
-  std::vector<InputSpec> requestedDYNs;
-  std::vector<OutputSpec> providedDYNs;
-  std::vector<InputSpec> requestedIDXs;
-
+  AnalysisContext ac;
   std::vector<InputSpec> requestedCCDBs;
   std::vector<OutputSpec> providedCCDBs;
-  std::vector<OutputSpec> providedOutputObjHist;
-
-  std::vector<OutputTaskInfo> outTskMap;
-  std::vector<OutputObjectInfo> outObjHistMap;
 
   for (size_t wi = 0; wi < workflow.size(); ++wi) {
     auto& processor = workflow[wi];
     auto name = processor.name;
     auto hash = runtime_hash(name.c_str());
-    outTskMap.push_back({hash, name});
+    ac.outTskMap.push_back({hash, name});
 
     std::string prefix = "internal-dpl-";
     if (processor.inputs.empty() && processor.name.compare(0, prefix.size(), prefix) != 0) {
@@ -435,13 +346,13 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
           break;
       }
       if (DataSpecUtils::partialMatch(input, AODOrigins)) {
-        DataSpecUtils::updateInputList(requestedAODs, InputSpec{input});
+        DataSpecUtils::updateInputList(ac.requestedAODs, InputSpec{input});
       }
       if (DataSpecUtils::partialMatch(input, header::DataOrigin{"DYN"})) {
-        DataSpecUtils::updateInputList(requestedDYNs, InputSpec{input});
+        DataSpecUtils::updateInputList(ac.requestedDYNs, InputSpec{input});
       }
       if (DataSpecUtils::partialMatch(input, header::DataOrigin{"IDX"})) {
-        DataSpecUtils::updateInputList(requestedIDXs, InputSpec{input});
+        DataSpecUtils::updateInputList(ac.requestedIDXs, InputSpec{input});
       }
     }
 
@@ -449,14 +360,14 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
 
     for (auto& output : processor.outputs) {
       if (DataSpecUtils::partialMatch(output, AODOrigins)) {
-        providedAODs.emplace_back(output);
+        ac.providedAODs.emplace_back(output);
       } else if (DataSpecUtils::partialMatch(output, header::DataOrigin{"DYN"})) {
-        providedDYNs.emplace_back(output);
+        ac.providedDYNs.emplace_back(output);
       } else if (DataSpecUtils::partialMatch(output, header::DataOrigin{"ATSK"})) {
-        providedOutputObjHist.emplace_back(output);
-        auto it = std::find_if(outObjHistMap.begin(), outObjHistMap.end(), [&](auto&& x) { return x.id == hash; });
-        if (it == outObjHistMap.end()) {
-          outObjHistMap.push_back({hash, {output.binding.value}});
+        ac.providedOutputObjHist.emplace_back(output);
+        auto it = std::find_if(ac.outObjHistMap.begin(), ac.outObjHistMap.end(), [&](auto&& x) { return x.id == hash; });
+        if (it == ac.outObjHistMap.end()) {
+          ac.outObjHistMap.push_back({hash, {output.binding.value}});
         } else {
           it->bindings.push_back(output.binding.value);
         }
@@ -469,12 +380,11 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
 
   auto inputSpecLessThan = [](InputSpec const& lhs, InputSpec const& rhs) { return DataSpecUtils::describe(lhs) < DataSpecUtils::describe(rhs); };
   auto outputSpecLessThan = [](OutputSpec const& lhs, OutputSpec const& rhs) { return DataSpecUtils::describe(lhs) < DataSpecUtils::describe(rhs); };
-  std::sort(requestedDYNs.begin(), requestedDYNs.end(), inputSpecLessThan);
-  std::sort(providedDYNs.begin(), providedDYNs.end(), outputSpecLessThan);
-  std::vector<InputSpec> spawnerInputs;
-  for (auto& input : requestedDYNs) {
-    if (std::none_of(providedDYNs.begin(), providedDYNs.end(), [&input](auto const& x) { return DataSpecUtils::match(input, x); })) {
-      spawnerInputs.emplace_back(input);
+  std::sort(ac.requestedDYNs.begin(), ac.requestedDYNs.end(), inputSpecLessThan);
+  std::sort(ac.providedDYNs.begin(), ac.providedDYNs.end(), outputSpecLessThan);
+  for (auto& input : ac.requestedDYNs) {
+    if (std::none_of(ac.providedDYNs.begin(), ac.providedDYNs.end(), [&input](auto const& x) { return DataSpecUtils::match(input, x); })) {
+      ac.spawnerInputs.emplace_back(input);
     }
   }
 
@@ -482,21 +392,21 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
     "internal-dpl-aod-spawner",
     {},
     {},
-    readers::AODReaderHelpers::aodSpawnerCallback(spawnerInputs),
+    readers::AODReaderHelpers::aodSpawnerCallback(ac.spawnerInputs),
     {}};
 
   DataProcessorSpec indexBuilder{
     "internal-dpl-aod-index-builder",
     {},
     {},
-    readers::AODReaderHelpers::indexBuilderCallback(requestedIDXs),
+    readers::AODReaderHelpers::indexBuilderCallback(ac.requestedIDXs),
     {}};
 
-  addMissingOutputsToBuilder(requestedIDXs, requestedAODs, requestedDYNs, indexBuilder);
-  addMissingOutputsToSpawner({}, spawnerInputs, requestedAODs, aodSpawner);
+  AnalysisSupportHelpers::addMissingOutputsToBuilder(ac.requestedIDXs, ac.requestedAODs, ac.requestedDYNs, indexBuilder);
+  AnalysisSupportHelpers::addMissingOutputsToSpawner({}, ac.spawnerInputs, ac.requestedAODs, aodSpawner);
 
-  addMissingOutputsToReader(providedAODs, requestedAODs, aodReader);
-  addMissingOutputsToReader(providedCCDBs, requestedCCDBs, ccdbBackend);
+  AnalysisSupportHelpers::addMissingOutputsToReader(ac.providedAODs, ac.requestedAODs, aodReader);
+  AnalysisSupportHelpers::addMissingOutputsToReader(providedCCDBs, requestedCCDBs, ccdbBackend);
 
   std::vector<DataProcessorSpec> extraSpecs;
 
@@ -662,8 +572,8 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
 
   // This is to inject a file sink so that any dangling ATSK object is written
   // to a ROOT file.
-  if (providedOutputObjHist.empty() == false) {
-    auto rootSink = CommonDataProcessors::getOutputObjHistSink(outObjHistMap, outTskMap);
+  if (ac.providedOutputObjHist.empty() == false) {
+    auto rootSink = AnalysisSupportHelpers::getOutputObjHistSink(ac.outObjHistMap, ac.outTskMap);
     extraSpecs.push_back(rootSink);
   }
 
@@ -694,7 +604,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
     // add TFNumber and TFFilename as input to the writer
     outputsInputsAOD.emplace_back(InputSpec{"tfn", "TFN", "TFNumber"});
     outputsInputsAOD.emplace_back(InputSpec{"tff", "TFF", "TFFilename"});
-    auto fileSink = CommonDataProcessors::getGlobalAODSink(dod, outputsInputsAOD);
+    auto fileSink = AnalysisSupportHelpers::getGlobalAODSink(dod, outputsInputsAOD);
     extraSpecs.push_back(fileSink);
 
     auto it = std::find_if(outputsInputs.begin(), outputsInputs.end(), [](InputSpec& spec) -> bool {
diff --git a/Framework/Core/src/WorkflowHelpers.h b/Framework/Core/src/WorkflowHelpers.h
index 055cae1d6011b..b20249b99edc8 100644
--- a/Framework/Core/src/WorkflowHelpers.h
+++ b/Framework/Core/src/WorkflowHelpers.h
@@ -22,9 +22,6 @@
 
 namespace o2::framework
 {
-static constexpr std::array<header::DataOrigin, 3> AODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}};
-static constexpr std::array<header::DataOrigin, 5> extendedAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}, header::DataOrigin{"AMD"}};
-static constexpr std::array<header::DataOrigin, 4> writableAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}};
 
 inline static std::string debugWorkflow(std::vector<DataProcessorSpec> const& specs)
 {
@@ -185,21 +182,6 @@ struct WorkflowHelpers {
   // @a ctx the context for the configuration phase
   static void injectServiceDevices(WorkflowSpec& workflow, ConfigContext const& ctx);
 
-  /// Helper functions to add AOD related internal devices.
-  /// FIXME: moved here until we have proper plugin based amendment
-  ///        of device injection
-  static void addMissingOutputsToReader(std::vector<OutputSpec> const& providedOutputs,
-                                        std::vector<InputSpec> const& requestedInputs,
-                                        DataProcessorSpec& publisher);
-  static void addMissingOutputsToSpawner(std::vector<OutputSpec> const& providedSpecials,
-                                         std::vector<InputSpec> const& requestedSpecials,
-                                         std::vector<InputSpec>& requestedAODs,
-                                         DataProcessorSpec& publisher);
-  static void addMissingOutputsToBuilder(std::vector<InputSpec> const& requestedSpecials,
-                                         std::vector<InputSpec>& requestedAODs,
-                                         std::vector<InputSpec>& requestedDYNs,
-                                         DataProcessorSpec& publisher);
-
   // Final adjustments to @a workflow after service devices have been injected.
   static void adjustTopology(WorkflowSpec& workflow, ConfigContext const& ctx);
 

From 81560c4253e0aed77d659abb0357633ed64e7ede Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 16 Sep 2024 16:43:27 +0200
Subject: [PATCH 627/726] DPL Analysis: make sure VLAs use only one basket

Otherwise it seems to create issues with ROOT bulk reading if the baskets
are split across multiple clusters.
---
 Framework/Core/include/Framework/TableTreeHelpers.h | 5 +++--
 Framework/Core/src/TableTreeHelpers.cxx             | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/include/Framework/TableTreeHelpers.h b/Framework/Core/include/Framework/TableTreeHelpers.h
index e12f4a38a973b..ccc7035ba3435 100644
--- a/Framework/Core/include/Framework/TableTreeHelpers.h
+++ b/Framework/Core/include/Framework/TableTreeHelpers.h
@@ -83,8 +83,9 @@ class ColumnToBranch
   ColumnToBranch(ColumnToBranch const& other) = delete;
   ColumnToBranch(ColumnToBranch&& other) = delete;
   void at(const int64_t* pos);
-  int fieldSize() const { return mFieldSize; }
-  char const* branchName() const { return mBranchName.c_str(); }
+  [[nodiscard]] int fieldSize() const { return mFieldSize; }
+  [[nodiscard]] int columnEntries() const { return mColumn->length(); }
+  [[nodiscard]] char const* branchName() const { return mBranchName.c_str(); }
 
  private:
   void accessChunk();
diff --git a/Framework/Core/src/TableTreeHelpers.cxx b/Framework/Core/src/TableTreeHelpers.cxx
index e6e6a1349e619..082535bdbb9f3 100644
--- a/Framework/Core/src/TableTreeHelpers.cxx
+++ b/Framework/Core/src/TableTreeHelpers.cxx
@@ -454,7 +454,7 @@ std::shared_ptr<TTree> TableToTree::process()
   }
 
   for (auto& reader : mColumnReaders) {
-    int idealBasketSize = 1024 + reader->fieldSize() * mRows; // minimal additional size needed, otherwise we get 2 baskets
+    int idealBasketSize = 1024 + reader->fieldSize() * reader->columnEntries(); // minimal additional size needed, otherwise we get 2 baskets
     int basketSize = std::max(32000, idealBasketSize);        // keep a minimum value
     mTree->SetBasketSize(reader->branchName(), basketSize);
   }

From 2d34451c7279311603d1b238b618775d7e585c9a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 18 Sep 2024 09:11:29 +0200
Subject: [PATCH 628/726] DPL: Drop generic code chose between unique and
 shared pointers

Just use an if constexpr nowadays...
---
 Framework/Core/CMakeLists.txt                 |  1 -
 Framework/Core/include/Framework/TypeTraits.h | 27 -----------
 Framework/Core/src/runDataProcessing.cxx      |  5 +--
 Framework/Core/test/test_PtrHelpers.cxx       | 45 -------------------
 4 files changed, 1 insertion(+), 77 deletions(-)
 delete mode 100644 Framework/Core/test/test_PtrHelpers.cxx

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index 7ddffda175a0a..8c2d9650776c0 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -229,7 +229,6 @@ add_executable(o2-test-framework-core
               test/test_OptionsHelpers.cxx
               test/test_OverrideLabels.cxx
               test/test_O2DataModelHelpers.cxx
-              test/test_PtrHelpers.cxx
               test/test_RootConfigParamHelpers.cxx
               test/test_Services.cxx
               test/test_StringHelpers.cxx
diff --git a/Framework/Core/include/Framework/TypeTraits.h b/Framework/Core/include/Framework/TypeTraits.h
index 6938b87238903..19ca548835cdd 100644
--- a/Framework/Core/include/Framework/TypeTraits.h
+++ b/Framework/Core/include/Framework/TypeTraits.h
@@ -164,32 +164,5 @@ struct has_root_setowner<
     void>> : public std::true_type {
 };
 
-/// Helper class to deal with the case we are creating the first instance of a
-/// (possibly) shared resource.
-///
-/// works for both:
-///
-/// std::shared_ptr<Base> storage = make_matching<decltype(storage), Concrete1>(args...);
-///
-/// or
-///
-/// std::unique_ptr<Base> storage = make_matching<decltype(storage), Concrete1>(args...);
-///
-/// Useful to deal with those who cannot make up their mind about ownership.
-/// ;-)
-template <typename HOLDER, typename T, typename... ARGS>
-static std::enable_if_t<sizeof(std::declval<HOLDER>().unique()) != 0, HOLDER>
-  make_matching(ARGS&&... args)
-{
-  return std::make_shared<T>(std::forward<ARGS>(args)...);
-}
-
-template <typename HOLDER, typename T, typename... ARGS>
-static std::enable_if_t<sizeof(std::declval<HOLDER>().get_deleter()) != 0, HOLDER>
-  make_matching(ARGS&&... args)
-{
-  return std::make_unique<T>(std::forward<ARGS>(args)...);
-}
-
 } // namespace o2::framework
 #endif // FRAMEWORK_TYPETRAITS_H
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 4cec8c63fe856..7d298998d0563 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1098,10 +1098,7 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
     serviceRef.registerService(ServiceRegistryHelpers::handleForService<DeviceContext>(deviceContext.get()));
     serviceRef.registerService(ServiceRegistryHelpers::handleForService<DriverConfig const>(&driverConfig));
 
-    // The decltype stuff is to be able to compile with both new and old
-    // FairMQ API (one which uses a shared_ptr, the other one a unique_ptr.
-    decltype(r.fDevice) device;
-    device = make_matching<decltype(device), DataProcessingDevice>(ref, serviceRegistry, processingPolicies);
+    auto device = std::make_unique<DataProcessingDevice>(ref, serviceRegistry, processingPolicies);
 
     serviceRef.get<RawDeviceService>().setDevice(device.get());
     r.fDevice = std::move(device);
diff --git a/Framework/Core/test/test_PtrHelpers.cxx b/Framework/Core/test/test_PtrHelpers.cxx
deleted file mode 100644
index 01486c9371749..0000000000000
--- a/Framework/Core/test/test_PtrHelpers.cxx
+++ /dev/null
@@ -1,45 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include <catch_amalgamated.hpp>
-#include "Framework/TypeTraits.h"
-#include <memory>
-
-/// exclude from doxygen, TODO: we might want to do this on a higher level
-/// because of doxygen's autolinking of references, all 'A' are displayed as
-/// reference to this struct.
-/// @cond
-struct Base {
-  int v;
-};
-struct A : public Base {
-  A(int v_, int va_) : Base{v_}, va{va_} {}
-  int va;
-};
-struct B : public Base {
-  B(int v_, int vb_) : Base{v_}, vb{vb_} {}
-  int vb;
-};
-
-TEST_CASE("MatchingPtrMaker")
-{
-
-  std::shared_ptr<Base> s = std::move(o2::framework::make_matching<decltype(s), A>(1, 3));
-  std::unique_ptr<Base> u = std::move(o2::framework::make_matching<decltype(u), B>(2, 4));
-
-  REQUIRE(s != nullptr);
-  REQUIRE(u != nullptr);
-  REQUIRE(s->v == 1);
-  REQUIRE(u->v == 2);
-  REQUIRE(static_cast<A*>(s.get())->va == 3);
-  REQUIRE(static_cast<B*>(u.get())->vb == 4);
-}
-// @endcond

From d15478ccac672cac2b1cc5c9709871dea6efde9d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 19 Sep 2024 13:05:24 +0200
Subject: [PATCH 629/726] DPL: AsyncQueue without std::function

---
 Framework/Core/include/Framework/AsyncQueue.h |  74 ++++++++---
 Framework/Core/src/AsyncQueue.cxx             |  17 +--
 Framework/Core/src/CommonServices.cxx         | 118 +++++++++++-------
 Framework/Core/src/DataProcessingDevice.cxx   |  61 +++++----
 Framework/Core/test/test_AsyncQueue.cxx       |  91 +++++++++-----
 5 files changed, 235 insertions(+), 126 deletions(-)

diff --git a/Framework/Core/include/Framework/AsyncQueue.h b/Framework/Core/include/Framework/AsyncQueue.h
index 6945c2a5ab1da..d214b9bfc09ba 100644
--- a/Framework/Core/include/Framework/AsyncQueue.h
+++ b/Framework/Core/include/Framework/AsyncQueue.h
@@ -12,49 +12,91 @@
 #define O2_FRAMEWORK_ASYNCQUUE_H_
 
 #include "Framework/TimesliceSlot.h"
-#include <functional>
 #include <string>
 #include <vector>
 
 namespace o2::framework
 {
 
-struct AsyncTaskSpec {
-  std::string name;
-  // Its priority compared to the other tasks
-  int score = 0;
-};
-
 /// The position of the TaskSpec in the prototypes
+/// Up to 127 different kind of tasks are allowed per queue
 struct AsyncTaskId {
-  int value = -1;
+  int16_t value = -1;
 };
 
 /// An actuatual task to be executed
 struct AsyncTask {
-  // The task to be executed. Id can be used as unique
-  // id for the signpost in the async_queue stream.
-  std::function<void(size_t id)> task;
-  // The associated task spec
-  AsyncTaskId id = {-1};
+  // The timeslice after which this callback should be executed.
   TimesliceId timeslice = {TimesliceId::INVALID};
   // Only the task with the highest debounce value will be executed
-  int debounce = 0;
+  // The associated task spec. Notice that the context
+  // is stored separately so that we do not need to
+  // manage lifetimes of std::functions
+  AsyncTaskId id = {-1};
+  // Debounce value.
+  int8_t debounce = 0;
   bool runnable = false;
+  // Some unuser integer
+  int32_t unused = 0;
+  // The callback to be executed. id can be used as unique
+  // id for the signpost in the async_queue stream.
+  // Context is provided by the userdata attached to the
+  // task the moment we post it.
+  // Notice that we do not store the task in the prototype
+  // because we do support at the moment coalescing two
+  // different callbaks with the same id.
+  void (*callback)(AsyncTask& task, size_t id);
+  // Some user data e.g. to decode what comes next
+  // This can either be used via the .data pointer
+  // or by asking a cast to the appropriate type via
+  // the user() method.
+  void* data[5] = {nullptr, nullptr, nullptr, nullptr, nullptr};
+
+  // Helper to return userdata
+  template <typename T>
+  T& user()
+  {
+    static_assert(sizeof(T) <= 5 * sizeof(void*), "User object does not fit user data");
+    return *(T*)data;
+  }
+
+  // Helper to set userdata
+  // @return a reference to this task modified. This is meant to be used like:
+  //
+  // AsyncQueueHelpers::post(queue, AsyncTask{.id = myTask}.user(Context{.contextValue}));
+  //
+  // Coupled with the other one:
+  //
+  // task.user<Context>().contextValue;
+  //
+  // it can be used to mimick capturing lambdas
+  template <typename T>
+  AsyncTask& user(T&& value)
+  {
+    static_assert(sizeof(T) <= 5 * sizeof(void*), "User object does not fit user data");
+    new (&data[0])(T){value};
+    return *this;
+  }
+};
+
+struct AsyncTaskSpec {
+  std::string name;
+  // Its priority compared to the other tasks
+  int score = 0;
 };
 
 struct AsyncQueue {
   std::vector<AsyncTaskSpec> prototypes;
   std::vector<AsyncTask> tasks;
   size_t iteration = 0;
+  AsyncQueue();
 };
 
 struct AsyncQueueHelpers {
-  using AsyncCallback = std::function<void(size_t)>;
   static AsyncTaskId create(AsyncQueue& queue, AsyncTaskSpec spec);
   // Schedule a task with @a taskId to be executed whenever the timeslice
   // is past timeslice. If debounce is provided, only execute the task
-  static void post(AsyncQueue& queue, AsyncTaskId taskId, AsyncCallback task, TimesliceId timeslice, int64_t debounce = 0);
+  static void post(AsyncQueue& queue, AsyncTask const& task);
   /// Run all the tasks which are older than the oldestPossible timeslice
   /// executing them by:
   /// 1. sorting the tasks by timeslice
diff --git a/Framework/Core/src/AsyncQueue.cxx b/Framework/Core/src/AsyncQueue.cxx
index 36c0c17009f82..e0e44da9cd19c 100644
--- a/Framework/Core/src/AsyncQueue.cxx
+++ b/Framework/Core/src/AsyncQueue.cxx
@@ -17,6 +17,10 @@ O2_DECLARE_DYNAMIC_LOG(async_queue);
 
 namespace o2::framework
 {
+AsyncQueue::AsyncQueue()
+{
+}
+
 auto AsyncQueueHelpers::create(AsyncQueue& queue, AsyncTaskSpec spec) -> AsyncTaskId
 {
   AsyncTaskId id;
@@ -25,14 +29,9 @@ auto AsyncQueueHelpers::create(AsyncQueue& queue, AsyncTaskSpec spec) -> AsyncTa
   return id;
 }
 
-auto AsyncQueueHelpers::post(AsyncQueue& queue, AsyncTaskId id, AsyncCallback task, TimesliceId timeslice, int64_t debounce) -> void
+auto AsyncQueueHelpers::post(AsyncQueue& queue, AsyncTask const& task) -> void
 {
-  AsyncTask taskToPost;
-  taskToPost.task = task;
-  taskToPost.id = id;
-  taskToPost.timeslice = timeslice;
-  taskToPost.debounce = debounce;
-  queue.tasks.push_back(taskToPost);
+  queue.tasks.push_back(task);
 }
 
 auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> void
@@ -85,6 +84,8 @@ auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> vo
     }
   }
   // Keep only the tasks with the highest debounce value for a given id
+  // For this reason I need to keep the callback in the task itself, because
+  // two different callbacks with the same id will be coalesced.
   auto newEnd = std::unique(order.begin(), order.end(), [&queue](int a, int b) {
     return queue.tasks[a].runnable == queue.tasks[b].runnable && queue.tasks[a].id.value == queue.tasks[b].id.value && queue.tasks[a].debounce >= 0 && queue.tasks[b].debounce >= 0;
   });
@@ -111,7 +112,7 @@ auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> vo
       O2_SIGNPOST_EVENT_EMIT(async_queue, opid, "run", "Running task %{public}s (%d) for timeslice %zu",
                              queue.prototypes[queue.tasks[i].id.value].name.c_str(), i,
                              queue.tasks[i].timeslice.value);
-      queue.tasks[i].task(opid.value);
+      queue.tasks[i].callback(queue.tasks[i], opid.value);
       O2_SIGNPOST_EVENT_EMIT(async_queue, opid, "run", "Done running %d", i);
     }
   }
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 13b02c74839b9..9192cbd0c965d 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -529,12 +529,72 @@ o2::framework::ServiceSpec CommonServices::ccdbSupportSpec()
       } },
     .kind = ServiceKind::Global};
 }
+struct DecongestionContext {
+  ServiceRegistryRef ref;
+  TimesliceIndex::OldestOutputInfo oldestPossibleOutput;
+};
+
+auto decongestionCallback = [](AsyncTask& task, size_t id) -> void {
+  auto& oldestPossibleOutput = task.user<DecongestionContext>().oldestPossibleOutput;
+  auto& ref = task.user<DecongestionContext>().ref;
+
+  auto& decongestion = ref.get<DecongestionService>();
+  auto& proxy = ref.get<FairMQDeviceProxy>();
+
+  O2_SIGNPOST_ID_GENERATE(cid, async_queue);
+  cid.value = id;
+  if (decongestion.lastTimeslice >= oldestPossibleOutput.timeslice.value) {
+    O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice", "Not sending already sent value: %" PRIu64 "> %" PRIu64,
+                           decongestion.lastTimeslice, (uint64_t)oldestPossibleOutput.timeslice.value);
+    return;
+  }
+  O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice", "Running oldest possible timeslice %" PRIu64 " propagation.",
+                         (uint64_t)oldestPossibleOutput.timeslice.value);
+  DataProcessingHelpers::broadcastOldestPossibleTimeslice(ref, oldestPossibleOutput.timeslice.value);
+
+  for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
+    auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
+    auto& state = proxy.getForwardChannelState(ChannelIndex{fi});
+    // TODO: this we could cache in the proxy at the bind moment.
+    if (info.channelType != ChannelAccountingType::DPL) {
+      O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
+      continue;
+    }
+    if (DataProcessingHelpers::sendOldestPossibleTimeframe(ref, info, state, oldestPossibleOutput.timeslice.value)) {
+      O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice",
+                             "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
+                             info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
+    }
+  }
+  decongestion.lastTimeslice = oldestPossibleOutput.timeslice.value;
+};
+
+auto decongestionCallbackOrdered = [](AsyncTask& task, size_t id) -> void {
+  auto& oldestPossibleOutput = task.user<DecongestionContext>().oldestPossibleOutput;
+  auto& ref = task.user<DecongestionContext>().ref;
+
+  auto& decongestion = ref.get<DecongestionService>();
+  auto& state = ref.get<DeviceState>();
+  auto& timesliceIndex = ref.get<TimesliceIndex>();
+  O2_SIGNPOST_ID_GENERATE(cid, async_queue);
+  int64_t oldNextTimeslice = decongestion.nextTimeslice;
+  decongestion.nextTimeslice = std::max(decongestion.nextTimeslice, (int64_t)oldestPossibleOutput.timeslice.value);
+  if (oldNextTimeslice != decongestion.nextTimeslice) {
+    if (state.transitionHandling != TransitionHandlingState::NoTransition && DefaultsHelpers::onlineDeploymentMode()) {
+      O2_SIGNPOST_EVENT_EMIT_WARN(async_queue, cid, "oldest_possible_timeslice", "Stop transition requested. Some Lifetime::Timeframe data got dropped starting at %" PRIi64, oldNextTimeslice);
+    } else {
+      O2_SIGNPOST_EVENT_EMIT_ERROR(async_queue, cid, "oldest_possible_timeslice", "Some Lifetime::Timeframe data got dropped starting at %" PRIi64, oldNextTimeslice);
+    }
+    timesliceIndex.rescan();
+  }
+};
 
 // Decongestion service
 // If we do not have any Timeframe input, it means we must be creating timeslices
 // in order and that we should propagate the oldest possible timeslice at the end
 // of each processing step.
-o2::framework::ServiceSpec CommonServices::decongestionSpec()
+o2::framework::ServiceSpec
+  CommonServices::decongestionSpec()
 {
   return ServiceSpec{
     .name = "decongestion",
@@ -548,7 +608,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
         }
       }
       auto& queue = services.get<AsyncQueue>();
-      decongestion->oldestPossibleTimesliceTask = AsyncQueueHelpers::create(queue, {"oldest-possible-timeslice", 100});
+      decongestion->oldestPossibleTimesliceTask = AsyncQueueHelpers::create(queue, {.name = "oldest-possible-timeslice", .score = 100});
       return ServiceHandle{TypeIdHelpers::uniqueId<DecongestionService>(), decongestion, ServiceKind::Serial};
     },
     .postForwarding = [](ProcessingContext& ctx, void* service) {
@@ -632,7 +692,6 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
       auto& decongestion = services.get<DecongestionService>();
       auto& relayer = services.get<DataRelayer>();
       auto& timesliceIndex = services.get<TimesliceIndex>();
-      auto& proxy = services.get<FairMQDeviceProxy>();
       O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, &decongestion);
       O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Received oldest possible timeframe %" PRIu64 " from channel %d",
                              (uint64_t)oldestPossibleTimeslice, channel.value);
@@ -650,59 +709,22 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
         return;
       }
       auto& queue = services.get<AsyncQueue>();
-      const auto& spec = services.get<DeviceSpec const>();
       const auto& state = services.get<DeviceState>();
-      auto *device = services.get<RawDeviceService>().device();
       /// We use the oldest possible timeslice to debounce, so that only the latest one
       /// at the end of one iteration is sent.
       O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Queueing oldest possible timeslice %" PRIu64 " propagation for execution.",
                              (uint64_t)oldestPossibleOutput.timeslice.value);
       AsyncQueueHelpers::post(
-        queue, decongestion.oldestPossibleTimesliceTask, [ref = services, oldestPossibleOutput, &decongestion, &proxy, &spec, device, &timesliceIndex](size_t id) {
-          O2_SIGNPOST_ID_GENERATE(cid, async_queue);
-          cid.value = id;
-          if (decongestion.lastTimeslice >= oldestPossibleOutput.timeslice.value) {
-            O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice", "Not sending already sent value: %" PRIu64 "> %" PRIu64,
-                decongestion.lastTimeslice, (uint64_t)oldestPossibleOutput.timeslice.value);
-            return;
-          }
-          O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice", "Running oldest possible timeslice %" PRIu64 " propagation.",
-                                 (uint64_t)oldestPossibleOutput.timeslice.value);
-          DataProcessingHelpers::broadcastOldestPossibleTimeslice(ref, oldestPossibleOutput.timeslice.value);
-
-          for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
-            auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
-            auto& state = proxy.getForwardChannelState(ChannelIndex{fi});
-            // TODO: this we could cache in the proxy at the bind moment.
-            if (info.channelType != ChannelAccountingType::DPL) {
-              O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
-              continue;
-            }
-            if (DataProcessingHelpers::sendOldestPossibleTimeframe(ref, info, state, oldestPossibleOutput.timeslice.value)) {
-              O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice",
-                                     "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
-                                     info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
-            }
-          }
-          decongestion.lastTimeslice = oldestPossibleOutput.timeslice.value;
-        },
-        TimesliceId{oldestPossibleTimeslice}, -1);
+        queue, AsyncTask{ .timeslice = TimesliceId{oldestPossibleTimeslice}, 
+                .id = decongestion.oldestPossibleTimesliceTask, 
+                .debounce = -1, .callback = decongestionCallback}
+            .user<DecongestionContext>(DecongestionContext{.ref = services, .oldestPossibleOutput = oldestPossibleOutput}));
+
       if (decongestion.orderedCompletionPolicyActive) {
         AsyncQueueHelpers::post(
-          queue, decongestion.oldestPossibleTimesliceTask, [ref = services, oldestPossibleOutput, &decongestion, &proxy, &spec, &state, device, &timesliceIndex](size_t id) {
-            O2_SIGNPOST_ID_GENERATE(cid, async_queue);
-            int64_t oldNextTimeslice = decongestion.nextTimeslice;
-            decongestion.nextTimeslice = std::max(decongestion.nextTimeslice, (int64_t)oldestPossibleOutput.timeslice.value);
-            if (oldNextTimeslice != decongestion.nextTimeslice) {
-              if (state.transitionHandling != TransitionHandlingState::NoTransition && DefaultsHelpers::onlineDeploymentMode()) {
-                O2_SIGNPOST_EVENT_EMIT_WARN(async_queue, cid, "oldest_possible_timeslice", "Stop transition requested. Some Lifetime::Timeframe data got dropped starting at %" PRIi64, oldNextTimeslice);
-              } else {
-                O2_SIGNPOST_EVENT_EMIT_ERROR(async_queue, cid, "oldest_possible_timeslice", "Some Lifetime::Timeframe data got dropped starting at %" PRIi64, oldNextTimeslice);
-              }
-              timesliceIndex.rescan();
-            }
-          },
-          TimesliceId{oldestPossibleOutput.timeslice.value}, -1);
+          queue, AsyncTask{.timeslice = TimesliceId{oldestPossibleOutput.timeslice.value},.id = decongestion.oldestPossibleTimesliceTask,  .debounce = -1, 
+          .callback = decongestionCallbackOrdered}
+          .user<DecongestionContext>({.ref = services, .oldestPossibleOutput = oldestPossibleOutput}));
       } },
     .kind = ServiceKind::Serial};
 }
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 1c46ee9ae0ac6..3e85d3214abab 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -636,6 +636,39 @@ static auto toBeforwardedMessageSet = [](std::vector<ChannelIndex>& cachedForwar
   return cachedForwardingChoices.empty() == false;
 };
 
+struct DecongestionContext {
+  ServiceRegistryRef ref;
+  TimesliceIndex::OldestOutputInfo oldestTimeslice;
+};
+
+auto decongestionCallbackLate = [](AsyncTask& task, size_t aid) -> void {
+  auto& oldestTimeslice = task.user<DecongestionContext>().oldestTimeslice;
+  auto& ref = task.user<DecongestionContext>().ref;
+
+  auto& decongestion = ref.get<DecongestionService>();
+  auto& proxy = ref.get<FairMQDeviceProxy>();
+  if (oldestTimeslice.timeslice.value <= decongestion.lastTimeslice) {
+    LOG(debug) << "Not sending already sent oldest possible timeslice " << oldestTimeslice.timeslice.value;
+    return;
+  }
+  for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
+    auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
+    auto& state = proxy.getForwardChannelState(ChannelIndex{fi});
+    O2_SIGNPOST_ID_GENERATE(aid, async_queue);
+    // TODO: this we could cache in the proxy at the bind moment.
+    if (info.channelType != ChannelAccountingType::DPL) {
+      O2_SIGNPOST_EVENT_EMIT(async_queue, aid, "forwardInputsCallback", "Skipping channel %{public}s because it's not a DPL channel",
+                             info.name.c_str());
+
+      continue;
+    }
+    if (DataProcessingHelpers::sendOldestPossibleTimeframe(ref, info, state, oldestTimeslice.timeslice.value)) {
+      O2_SIGNPOST_EVENT_EMIT(async_queue, aid, "forwardInputsCallback", "Forwarding to channel %{public}s oldest possible timeslice %zu, prio 20",
+                             info.name.c_str(), oldestTimeslice.timeslice.value);
+    }
+  }
+};
+
 // This is how we do the forwarding, i.e. we push
 // the inputs which are shared between this device and others
 // to the next one in the daisy chain.
@@ -721,36 +754,12 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
 
   auto& asyncQueue = registry.get<AsyncQueue>();
   auto& decongestion = registry.get<DecongestionService>();
-  auto& timesliceIndex = registry.get<TimesliceIndex>();
   O2_SIGNPOST_ID_GENERATE(aid, async_queue);
   O2_SIGNPOST_EVENT_EMIT(async_queue, aid, "forwardInputs", "Queuing forwarding oldestPossible %zu", oldestTimeslice.timeslice.value);
-  AsyncQueueHelpers::post(
-    asyncQueue, decongestion.oldestPossibleTimesliceTask, [&proxy, &decongestion, registry, oldestTimeslice, &timesliceIndex](size_t aid) {
-      // DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, oldestTimeslice.timeslice.value);
-      if (oldestTimeslice.timeslice.value <= decongestion.lastTimeslice) {
-        LOG(debug) << "Not sending already sent oldest possible timeslice " << oldestTimeslice.timeslice.value;
-        return;
-      }
-      for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
-        auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
-        auto& state = proxy.getForwardChannelState(ChannelIndex{fi});
-        O2_SIGNPOST_ID_GENERATE(aid, async_queue);
-        // TODO: this we could cache in the proxy at the bind moment.
-        if (info.channelType != ChannelAccountingType::DPL) {
-          O2_SIGNPOST_EVENT_EMIT(async_queue, aid, "forwardInputsCallback", "Skipping channel %{public}s because it's not a DPL channel",
-                                 info.name.c_str());
-
-          continue;
-        }
-        if (DataProcessingHelpers::sendOldestPossibleTimeframe(registry, info, state, oldestTimeslice.timeslice.value)) {
-          O2_SIGNPOST_EVENT_EMIT(async_queue, aid, "forwardInputsCallback", "Forwarding to channel %{public}s oldest possible timeslice %zu, prio 20",
-                                 info.name.c_str(), oldestTimeslice.timeslice.value);
-        }
-      }
-    },
-    oldestTimeslice.timeslice, -1);
+  AsyncQueueHelpers::post(asyncQueue, {.timeslice = oldestTimeslice.timeslice, .id = decongestion.oldestPossibleTimesliceTask, .debounce = -1, .callback = decongestionCallbackLate});
   O2_SIGNPOST_END(forwarding, sid, "forwardInputs", "Forwarding done");
 };
+
 extern volatile int region_read_global_dummy_variable;
 volatile int region_read_global_dummy_variable;
 
diff --git a/Framework/Core/test/test_AsyncQueue.cxx b/Framework/Core/test/test_AsyncQueue.cxx
index 635eb131cc666..7103bb087bb87 100644
--- a/Framework/Core/test/test_AsyncQueue.cxx
+++ b/Framework/Core/test/test_AsyncQueue.cxx
@@ -8,10 +8,13 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
-
 #include <catch_amalgamated.hpp>
 #include "Framework/AsyncQueue.h"
 
+struct TaskContext {
+  int& count;
+};
+
 /// Test debouncing functionality. The same task cannot be executed more than once
 /// in a given run.
 TEST_CASE("TestDebouncing")
@@ -20,11 +23,11 @@ TEST_CASE("TestDebouncing")
   AsyncQueue queue;
   auto taskId = AsyncQueueHelpers::create(queue, {.name = "test", .score = 10});
   // Push two tasks on the queue with the same id
-  auto count = 0;
-  AsyncQueueHelpers::post(
-    queue, taskId, [&count](size_t) { count += 1; }, TimesliceId{0}, 10);
-  AsyncQueueHelpers::post(
-    queue, taskId, [&count](size_t) { count += 2; }, TimesliceId{1}, 20);
+  int count = 0;
+  AsyncQueueHelpers::post(queue,
+                          AsyncTask{.timeslice = TimesliceId{0}, .id = taskId, .debounce = 10, .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 1; }}.user<TaskContext>({.count = count}));
+  AsyncQueueHelpers::post(queue,
+                          AsyncTask{.timeslice = TimesliceId{1}, .id = taskId, .debounce = 20, .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 2; }}.user<TaskContext>({.count = count}));
   AsyncQueueHelpers::run(queue, TimesliceId{2});
   REQUIRE(count == 2);
 }
@@ -37,11 +40,9 @@ TEST_CASE("TestPriority")
   auto taskId1 = AsyncQueueHelpers::create(queue, {.name = "test1", .score = 10});
   auto taskId2 = AsyncQueueHelpers::create(queue, {.name = "test2", .score = 20});
   // Push two tasks on the queue with the same id
-  auto count = 0;
-  AsyncQueueHelpers::post(
-    queue, taskId1, [&count](size_t) { count += 10; }, TimesliceId{0});
-  AsyncQueueHelpers::post(
-    queue, taskId2, [&count](size_t) { count /= 10; }, TimesliceId{0});
+  int count = 0;
+  AsyncQueueHelpers::post(queue, AsyncTask{.timeslice = TimesliceId{0}, .id = taskId1, .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 10; }}.user<TaskContext>({.count = count}));
+  AsyncQueueHelpers::post(queue, AsyncTask{.timeslice = TimesliceId{0}, .id = taskId2, .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count /= 10; }}.user<TaskContext>({.count = count}));
   AsyncQueueHelpers::run(queue, TimesliceId{2});
   REQUIRE(count == 10);
 }
@@ -56,9 +57,9 @@ TEST_CASE("TestOldestTimeslice")
   // Push two tasks on the queue with the same id
   auto count = 0;
   AsyncQueueHelpers::post(
-    queue, taskId1, [&count](size_t) { count += 10; }, TimesliceId{1});
+    queue, AsyncTask{.timeslice = TimesliceId{1}, .id = taskId1, .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 10; }}.user<TaskContext>({.count = count}));
   AsyncQueueHelpers::post(
-    queue, taskId2, [&count](size_t) { count += 20; }, TimesliceId{0});
+    queue, AsyncTask{.timeslice = TimesliceId{0}, .id = taskId2, .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 20; }}.user<TaskContext>({.count = count}));
   AsyncQueueHelpers::run(queue, TimesliceId{0});
   REQUIRE(count == 20);
   AsyncQueueHelpers::run(queue, TimesliceId{0});
@@ -72,16 +73,28 @@ TEST_CASE("TestOldestTimesliceWithBounce")
 {
   using namespace o2::framework;
   AsyncQueue queue;
-  auto taskId1 = AsyncQueueHelpers::create(queue, {.name = "test1", .score = 10});
-  auto taskId2 = AsyncQueueHelpers::create(queue, {.name = "test2", .score = 20});
   // Push two tasks on the queue with the same id
   auto count = 0;
-  AsyncQueueHelpers::post(
-    queue, taskId1, [&count](size_t) { count += 10; }, TimesliceId{2});
-  AsyncQueueHelpers::post(
-    queue, taskId2, [&count](size_t) { count += 20; }, TimesliceId{1}, 10);
-  AsyncQueueHelpers::post(
-    queue, taskId2, [&count](size_t) { count += 30; }, TimesliceId{1}, 20);
+  auto taskId1 = AsyncQueueHelpers::create(queue, {.name = "test1", .score = 10});
+  auto taskId2 = AsyncQueueHelpers::create(queue, {.name = "test2", .score = 20});
+  REQUIRE(taskId1.value != taskId2.value);
+  AsyncQueueHelpers::post(queue, AsyncTask{
+                                   .timeslice = TimesliceId{2},
+                                   .id = taskId1,
+                                   .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 10; }}
+                                   .user<TaskContext>({.count = count}));
+  AsyncQueueHelpers::post(queue, AsyncTask{
+                                   .timeslice = TimesliceId{1},
+                                   .id = taskId2,
+                                   .debounce = 10,
+                                   .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 20; }}
+                                   .user<TaskContext>({.count = count}));
+  AsyncQueueHelpers::post(queue, AsyncTask{
+                                   .timeslice = TimesliceId{1},
+                                   .id = taskId2,
+                                   .debounce = 20,
+                                   .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 30; }}
+                                   .user<TaskContext>({.count = count}));
   AsyncQueueHelpers::run(queue, TimesliceId{0});
   REQUIRE(count == 0);
   REQUIRE(queue.tasks.size() == 3);
@@ -98,16 +111,27 @@ TEST_CASE("TestOldestTimesliceWithNegativeBounce")
 {
   using namespace o2::framework;
   AsyncQueue queue;
+  int count = 0;
   auto taskId1 = AsyncQueueHelpers::create(queue, {.name = "test1", .score = 10});
   auto taskId2 = AsyncQueueHelpers::create(queue, {.name = "test2", .score = 20});
   // Push two tasks on the queue with the same id
-  auto count = 0;
   AsyncQueueHelpers::post(
-    queue, taskId1, [&count](size_t) { count += 10; }, TimesliceId{2});
+    queue, AsyncTask{.timeslice = TimesliceId{2},
+                     .id = taskId1,
+                     .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 10; }}
+             .user<TaskContext>({.count = count}));
   AsyncQueueHelpers::post(
-    queue, taskId2, [&count](size_t) { count += 20; }, TimesliceId{1}, -10);
+    queue, AsyncTask{.timeslice = TimesliceId{1},
+                     .id = taskId2,
+                     .debounce = -10,
+                     .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 20; }}
+             .user<TaskContext>({.count = count}));
   AsyncQueueHelpers::post(
-    queue, taskId2, [&count](size_t) { count += 30; }, TimesliceId{1}, -20);
+    queue, AsyncTask{.timeslice = TimesliceId{1},
+                     .id = taskId2,
+                     .debounce = -20,
+                     .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 30; }}
+             .user<TaskContext>({.count = count}));
   AsyncQueueHelpers::run(queue, TimesliceId{0});
   REQUIRE(count == 0);
   REQUIRE(queue.tasks.size() == 3);
@@ -126,15 +150,26 @@ TEST_CASE("TestOldestTimeslicePerTimeslice")
   AsyncQueue queue;
   auto taskId1 = AsyncQueueHelpers::create(queue, {.name = "test1", .score = 10});
   // Push two tasks on the queue with the same id
-  auto count = 0;
+  int count = 0;
   AsyncQueueHelpers::post(
-    queue, taskId1, [&count](size_t) { count += 10; }, TimesliceId{1});
+    queue, AsyncTask{.timeslice = TimesliceId{1},
+                     .id = taskId1,
+                     .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 10; }}
+             .user<TaskContext>({.count = count}));
+  // The size of the queue is only updated once a flush happens
+  REQUIRE(queue.tasks.size() == 0);
+  AsyncQueueHelpers::flushPending(queue);
   REQUIRE(queue.tasks.size() == 1);
   AsyncQueueHelpers::run(queue, TimesliceId{0});
   REQUIRE(queue.tasks.size() == 1);
   REQUIRE(count == 0);
   AsyncQueueHelpers::post(
-    queue, taskId1, [&count](size_t) { count += 20; }, TimesliceId{2});
+    queue, AsyncTask{.timeslice = TimesliceId{2},
+                     .id = taskId1,
+                     .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 20; }}
+             .user<TaskContext>({.count = count}));
+  REQUIRE(queue.tasks.size() == 1);
+  AsyncQueueHelpers::flushPending(queue);
   REQUIRE(queue.tasks.size() == 2);
   AsyncQueueHelpers::run(queue, TimesliceId{1});
   REQUIRE(queue.tasks.size() == 1);

From 11febb531fc5b1d885a702e9d616143d38284462 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 19 Sep 2024 13:05:24 +0200
Subject: [PATCH 630/726] Import x9 library for blazing fast interthread
 message passing

---
 Framework/Foundation/3rdparty/CMakeLists.txt  |   7 +
 Framework/Foundation/3rdparty/x9/LICENSE      |  24 +
 Framework/Foundation/3rdparty/x9/README.md    |  68 +++
 .../Foundation/3rdparty/x9/examples/README.md | 241 ++++++++
 .../3rdparty/x9/examples/run_examples.sh      |  26 +
 .../3rdparty/x9/examples/x9_example_1.c       | 113 ++++
 .../3rdparty/x9/examples/x9_example_2.c       | 197 +++++++
 .../3rdparty/x9/examples/x9_example_3.c       | 207 +++++++
 .../3rdparty/x9/examples/x9_example_4.c       | 158 ++++++
 .../3rdparty/x9/examples/x9_example_5.c       | 161 ++++++
 .../3rdparty/x9/examples/x9_example_6.c       | 159 ++++++
 .../3rdparty/x9/profiling/README.md           | 106 ++++
 .../3rdparty/x9/profiling/compile_profiler.sh |   3 +
 .../3rdparty/x9/profiling/x9_profiler.c       | 524 ++++++++++++++++++
 Framework/Foundation/3rdparty/x9/x9.c         | 424 ++++++++++++++
 Framework/Foundation/3rdparty/x9/x9.h         | 166 ++++++
 16 files changed, 2584 insertions(+)
 create mode 100644 Framework/Foundation/3rdparty/x9/LICENSE
 create mode 100644 Framework/Foundation/3rdparty/x9/README.md
 create mode 100644 Framework/Foundation/3rdparty/x9/examples/README.md
 create mode 100755 Framework/Foundation/3rdparty/x9/examples/run_examples.sh
 create mode 100644 Framework/Foundation/3rdparty/x9/examples/x9_example_1.c
 create mode 100644 Framework/Foundation/3rdparty/x9/examples/x9_example_2.c
 create mode 100644 Framework/Foundation/3rdparty/x9/examples/x9_example_3.c
 create mode 100644 Framework/Foundation/3rdparty/x9/examples/x9_example_4.c
 create mode 100644 Framework/Foundation/3rdparty/x9/examples/x9_example_5.c
 create mode 100644 Framework/Foundation/3rdparty/x9/examples/x9_example_6.c
 create mode 100644 Framework/Foundation/3rdparty/x9/profiling/README.md
 create mode 100755 Framework/Foundation/3rdparty/x9/profiling/compile_profiler.sh
 create mode 100644 Framework/Foundation/3rdparty/x9/profiling/x9_profiler.c
 create mode 100644 Framework/Foundation/3rdparty/x9/x9.c
 create mode 100644 Framework/Foundation/3rdparty/x9/x9.h

diff --git a/Framework/Foundation/3rdparty/CMakeLists.txt b/Framework/Foundation/3rdparty/CMakeLists.txt
index 5001ad38ed6b5..1a490278a8d37 100644
--- a/Framework/Foundation/3rdparty/CMakeLists.txt
+++ b/Framework/Foundation/3rdparty/CMakeLists.txt
@@ -16,5 +16,12 @@ o2_add_library(Catch2
                TARGETVARNAME targetName
                PUBLIC_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}/catch2)
 
+o2_add_library(X9
+               SOURCES x9/x9.c
+               TARGETVARNAME targetName
+               PUBLIC_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}/x9)
+
 install(FILES ${CMAKE_CURRENT_LIST_DIR}/catch2/catch_amalgamated.hpp
         DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+install(FILES ${CMAKE_CURRENT_LIST_DIR}/x9/x9.h
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
diff --git a/Framework/Foundation/3rdparty/x9/LICENSE b/Framework/Foundation/3rdparty/x9/LICENSE
new file mode 100644
index 0000000000000..7b90e16605089
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/LICENSE
@@ -0,0 +1,24 @@
+BSD 2-Clause License
+
+Copyright (c) 2023, Diogo Flores
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/Framework/Foundation/3rdparty/x9/README.md b/Framework/Foundation/3rdparty/x9/README.md
new file mode 100644
index 0000000000000..1b7ead68961eb
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/README.md
@@ -0,0 +1,68 @@
+X9
+---
+
+*Note: I am currently working on v2.0 which will bring further performance 
+gains and flexibility to the user at the (unfortunate) cost of breaking the 
+current API. I expect to release v2.0 by September/October 2024 and for it to be the 
+last major/API-breaking change to X9.*
+
+X9 is a low level high performance message passing library, based on a
+lock-free ring buffer implemented with atomic variables, for low latency
+multithreading work.  
+It allows for multiple producers/consumers to concurrently access the same
+underlying ring buffer and provides both spinning (busy loop) and non-blocking
+read and write functions.
+
+The library is based on three concepts:
+
+- A message, which is a user defined struct.
+- A `x9_inbox`, which is where messages are both written to and read from.
+- A `x9_node`, which is an abstraction that unifies x9_inbox(es).
+
+The library provides multiple functions to both read from and write to a
+`x9_inbox`, as the right choice depends on the user needs.  
+Refer to _x9.h_, where all public functions are properly documented and their
+use cases explained, and the examples folder for comprehensive examples of
+different architectures.
+
+Enabling `X9_DEBUG` at compile time will print to stdout the reason why the
+functions `x9_inbox_is_valid` and `x9_node_is_valid` returned 'false' (if they
+indeed returned 'false'), or why `x9_select_inbox_from_node` did not return a
+valid `x9_inbox`.
+
+To use the library just link with x9.c and include x9.h where necessary.
+
+X9 is as generic, performant and intuitive as C allows, without forcing the
+user to any sort of build system preprocessor hell, pseudo-C macro based
+library, or worse.  
+It was originally written in the context of an high-frequency-trading system
+that this author developed, and was made public in June of 2023.  
+It is released under the BSD-2-Clause license, with the purpose of serving
+others and other programs.  
+
+Benchmarks
+---
+
+- Single producer and single consumer transmitting 100M messages via a single 
+`x9_inbox`.  
+- Run on Intel 11900k (cpu and ram untuned).  
+- _Msg size_ expressed in bytes, and _Inbox size_ in number of slots in the 
+ring buffer.   
+- _(See /profiling for how to run your own tests)_  
+
+```
+Inbox size | Msg size | Time (secs) | Msgs/second
+-------------------------------------------------
+      1024 |       16 |        4.00 |      25.01M
+      1024 |       32 |        4.03 |      24.82M
+      1024 |       64 |        4.17 |      23.99M
+-------------------------------------------------
+      2048 |       16 |        3.90 |      25.63M
+      2048 |       32 |        3.96 |      25.26M
+      2048 |       64 |        4.09 |      24.43M
+-------------------------------------------------
+      4096 |       16 |        3.86 |      25.88M
+      4096 |       32 |        3.92 |      25.50M
+      4096 |       64 |        4.05 |      24.67M
+-------------------------------------------------
+```
diff --git a/Framework/Foundation/3rdparty/x9/examples/README.md b/Framework/Foundation/3rdparty/x9/examples/README.md
new file mode 100644
index 0000000000000..c2525d76cd585
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/examples/README.md
@@ -0,0 +1,241 @@
+Notes: 
+--- 
+
+- Inbox N is always associated with a message of type N, hence to inbox 1
+producers will write messages of type 1 and consumers will read messages of
+type 1, and so on.
+
+- Each producer/consumer runs in a unique thread.
+
+- All x9_inbox sizes equal 4. This is not ideal for performance and should not
+be used as a guideline. The reason why I use such a small buffer is because
+I wanted to saturate the inbox and make it much more likely to trigger a data 
+race (which there is none).
+
+- All examples/tests can be run by executing ./run_examples.sh
+
+```
+──────▷  write to
+─ ─ ─ ▷  read from
+```
+
+Examples
+---
+```
+x9_example_1.c:
+
+ One producer
+ One consumer
+ One message type
+ 
+ ┌────────┐       ┏━━━━━━━━┓       ┌────────┐
+ │Producer│──────▷┃ inbox  ┃◁ ─ ─ ─│Consumer│
+ └────────┘       ┗━━━━━━━━┛       └────────┘
+ 
+ This example showcases the simplest (multi-threading) pattern.
+ 
+ Data structures used:
+  - x9_inbox
+ 
+ Functions used:
+  - x9_create_inbox
+  - x9_inbox_is_valid
+  - x9_write_to_inbox_spin
+  - x9_read_from_inbox_spin
+  - x9_free_inbox
+ 
+ Test is considered passed iff:
+  - None of the threads stall and exit cleanly after doing the work.
+  - All messages sent by the producer(s) are received and asserted to be
+  valid by the consumer(s).
+ ```
+-------------------------------------------------------------------------------
+```
+x9_example_2.c
+
+ Two producers.
+ One consumer and producer simultaneously.
+ One consumer.
+
+ ┌────────┐      ┏━━━━━━━━┓                      ┏━━━━━━━━┓
+ │Producer│─────▷┃        ┃      ┌────────┐      ┃        ┃
+ └────────┘      ┃        ┃      │Consumer│      ┃        ┃      ┌────────┐
+                 ┃inbox 1 ┃◁ ─ ─ │  and   │─────▷┃inbox 2 ┃◁ ─ ─ │Consumer│
+ ┌────────┐      ┃        ┃      │Producer│      ┃        ┃      └────────┘
+ │Producer│─────▷┃        ┃      └────────┘      ┃        ┃
+ └────────┘      ┗━━━━━━━━┛                      ┗━━━━━━━━┛
+
+ This example showcase using multiple threads to write to the same inbox,
+ using multiple message types, the 'x9_node' abstraction, and
+ respective create/free and select functions.
+
+ Data structures used:
+  - x9_inbox
+  - x9_node 
+
+ Functions used:
+  - x9_create_inbox
+  - x9_inbox_is_valid
+  - x9_create_node
+  - x9_node_is_valid
+  - x9_select_inbox_from_node
+  - x9_write_to_inbox_spin
+  - x9_read_from_inbox_spin
+  - x9_free_node_and_attached_inboxes
+
+ Test is considered passed iff:
+  - None of the threads stall and exit cleanly after doing the work.
+  - All messages sent by the producer(s) are received and asserted to be
+  valid by the consumer(s).
+```
+-------------------------------------------------------------------------------
+```
+x9_example_3.c
+
+ Two producers and simultaneously consumers.
+ 
+ ┌────────┐       ┏━━━━━━━━┓       ┌────────┐
+ │Producer│──────▷┃inbox 1 ┃◁ ─ ─ ─│Producer│
+ │        │       ┗━━━━━━━━┛       │        │
+ │  and   │                        │  and   │
+ │        │       ┏━━━━━━━━┓       │        │
+ │Consumer│─ ─ ─ ▷┃inbox 2 ┃◁──────│Consumer│
+ └────────┘       ┗━━━━━━━━┛       └────────┘
+
+ This example showcases the use of: 'x9_write_to_inbox'
+ and 'x9_read_from_inbox', which, unlike 'x9_write_to_inbox_spin' and
+ 'x9_read_from_inbox_spin' do not block until are able to write/read a msg.
+
+ Data structures used:
+  - x9_inbox
+  - x9_node
+
+ Functions used:
+  - x9_create_inbox
+  - x9_inbox_is_valid
+  - x9_create_node
+  - x9_node_is_valid
+  - x9_select_inbox_from_node
+  - x9_write_to_inbox
+  - x9_read_from_inbox
+  - x9_free_node_and_attached_inboxes
+
+ Test is considered passed iff:
+  - None of the threads stall and exit cleanly after doing the work.
+  - All messages sent by the producer(s) are received and asserted to be
+  valid by the consumer(s).
+```
+-------------------------------------------------------------------------------
+```
+x9_example_4.c
+
+ One producer broadcasts the same message to three inboxes.
+ Three consumers read from each inbox.
+ One message type.
+ 
+                  ┏━━━━━━━━┓      ┌────────┐
+              ┌──▷┃ inbox  ┃◁─ ─ ─│Consumer│
+              │   ┗━━━━━━━━┛      └────────┘
+ ┌────────┐   │   ┏━━━━━━━━┓      ┌────────┐
+ │Producer│───┼──▷┃ inbox  ┃◁─ ─ ─│Consumer│
+ └────────┘   │   ┗━━━━━━━━┛      └────────┘
+              │   ┏━━━━━━━━┓      ┌────────┐
+              └──▷┃ inbox  ┃◁─ ─ ─│Consumer│
+                  ┗━━━━━━━━┛      └────────┘
+
+
+ This example showcases the use of 'x9_broadcast_msg_to_all_node_inboxes'.
+
+ Data structures used:
+  - x9_inbox
+  - x9_node
+
+ Functions used:
+  - x9_create_inbox
+  - x9_inbox_is_valid
+  - x9_create_node
+  - x9_node_is_valid
+  - x9_broadcast_msg_to_all_node_inboxes
+  - x9_read_from_inbox_spin
+  - x9_free_node_and_attached_inboxes
+
+ IMPORTANT:
+  - All inboxes  must receive messages of the same type (or at least of the
+  same size) that its being broadcasted.
+
+ Test is considered passed iff:
+  - None of the threads stall and exit cleanly after doing the work.
+  - All messages sent by the producer(s) are received and asserted to be
+  valid by the consumer(s).
+```
+-------------------------------------------------------------------------------
+```
+x9_example_5.c
+
+ Three producers.
+ Three consumers reading concurrently from the same inbox.
+ One message type.
+
+                                   ┌────────┐
+ ┌────────┐       ┏━━━━━━━━┓    ─ ─│Consumer│
+ │Producer│──────▷┃        ┃   │   └────────┘
+ ├────────┤       ┃        ┃       ┌────────┐
+ │Producer│──────▷┃ inbox  ┃◁──┤─ ─│Consumer│
+ ├────────┤       ┃        ┃       └────────┘
+ │Producer│──────▷┃        ┃   │   ┌────────┐
+ └────────┘       ┗━━━━━━━━┛    ─ ─│Consumer│
+                                   └────────┘
+
+ This example showcases the use of 'x9_read_from_shared_inbox'.
+
+ Data structures used:
+  - x9_inbox
+ 
+ Functions used:
+  - x9_create_inbox
+  - x9_inbox_is_valid
+  - x9_write_to_inbox_spin
+  - x9_read_from_shared_inbox
+  - x9_free_inbox
+
+ Test is considered passed iff:
+  - None of the threads stall and exit cleanly after doing the work.
+  - All messages sent by the producer(s) are received and asserted to be
+  valid by the consumer(s).
+  - Each consumer processes at least one message.
+```
+-------------------------------------------------------------------------------
+```
+x9_example_6.c
+
+ One producer
+ Two consumers reading from the same inbox concurrently (busy loop).
+ One message type.
+
+                  ┏━━━━━━━━┓       ┌────────┐
+                  ┃        ┃    ─ ─│Consumer│
+ ┌────────┐       ┃        ┃   │   └────────┘
+ │Producer│──────▷┃ inbox  ┃◁ ─
+ └────────┘       ┃        ┃   │   ┌────────┐
+                  ┃        ┃    ─ ─│Consumer│
+                  ┗━━━━━━━━┛       └────────┘
+
+ This example showcases the use of 'x9_read_from_shared_inbox_spin'.
+
+ Data structures used:
+  - x9_inbox
+
+ Functions used:
+  - x9_create_inbox
+  - x9_inbox_is_valid
+  - x9_write_to_inbox_spin
+  - x9_read_from_shared_inbox_spin
+  - x9_free_inbox
+
+ Test is considered passed iff:
+  - None of the threads stall and exit cleanly after doing the work.
+  - All messages sent by the producer(s) are received and asserted to be
+  valid by the consumer(s).
+  - Each consumer processes at least one message.
+```
+-------------------------------------------------------------------------------
diff --git a/Framework/Foundation/3rdparty/x9/examples/run_examples.sh b/Framework/Foundation/3rdparty/x9/examples/run_examples.sh
new file mode 100755
index 0000000000000..6719bd9b4bb86
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/examples/run_examples.sh
@@ -0,0 +1,26 @@
+#/bin/bash
+
+echo "- Running examples with GCC with \"-fsanitize=thread,undefined\" enabled.";
+gcc -Wextra -Wall -Werror -pedantic -O3 -march=native x9_example_1.c ../x9.c -o X9_TEST_1 -fsanitize=thread,undefined -D X9_DEBUG
+gcc -Wextra -Wall -Werror -pedantic -O3 -march=native x9_example_2.c ../x9.c -o X9_TEST_2 -fsanitize=thread,undefined -D X9_DEBUG
+gcc -Wextra -Wall -Werror -pedantic -O3 -march=native x9_example_3.c ../x9.c -o X9_TEST_3 -fsanitize=thread,undefined -D X9_DEBUG
+gcc -Wextra -Wall -Werror -pedantic -O3 -march=native x9_example_4.c ../x9.c -o X9_TEST_4 -fsanitize=thread,undefined -D X9_DEBUG
+gcc -Wextra -Wall -Werror -pedantic -O3 -march=native x9_example_5.c ../x9.c -o X9_TEST_5 -fsanitize=thread,undefined -D X9_DEBUG
+gcc -Wextra -Wall -Werror -pedantic -O3 -march=native x9_example_6.c ../x9.c -o X9_TEST_6 -fsanitize=thread,undefined -D X9_DEBUG
+
+./X9_TEST_1; ./X9_TEST_2; ./X9_TEST_3; ./X9_TEST_4; ./X9_TEST_5; ./X9_TEST_6
+rm X9_TEST_1 X9_TEST_2 X9_TEST_3 X9_TEST_4 X9_TEST_5 X9_TEST_6
+
+echo ""
+echo "- Running examples with clang with \"-fsanitize=address,undefined,leak\" enabled.";
+
+clang -Wextra -Wall -Werror -O3 -march=native x9_example_1.c ../x9.c -o X9_TEST_1 -fsanitize=address,undefined,leak -D X9_DEBUG
+clang -Wextra -Wall -Werror -O3 -march=native x9_example_2.c ../x9.c -o X9_TEST_2 -fsanitize=address,undefined,leak -D X9_DEBUG
+clang -Wextra -Wall -Werror -O3 -march=native x9_example_3.c ../x9.c -o X9_TEST_3 -fsanitize=address,undefined,leak -D X9_DEBUG
+clang -Wextra -Wall -Werror -O3 -march=native x9_example_4.c ../x9.c -o X9_TEST_4 -fsanitize=address,undefined,leak -D X9_DEBUG
+clang -Wextra -Wall -Werror -O3 -march=native x9_example_5.c ../x9.c -o X9_TEST_5 -fsanitize=address,undefined,leak -D X9_DEBUG
+clang -Wextra -Wall -Werror -O3 -march=native x9_example_6.c ../x9.c -o X9_TEST_6 -fsanitize=address,undefined,leak -D X9_DEBUG
+
+./X9_TEST_1; ./X9_TEST_2; ./X9_TEST_3; ./X9_TEST_4; ./X9_TEST_5; ./X9_TEST_6
+rm X9_TEST_1 X9_TEST_2 X9_TEST_3 X9_TEST_4 X9_TEST_5 X9_TEST_6 
+
diff --git a/Framework/Foundation/3rdparty/x9/examples/x9_example_1.c b/Framework/Foundation/3rdparty/x9/examples/x9_example_1.c
new file mode 100644
index 0000000000000..6498ac2a76da7
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/examples/x9_example_1.c
@@ -0,0 +1,113 @@
+/* x9_example_1.c:
+ *
+ *  One producer
+ *  One consumer
+ *  One message type
+ *
+ *  ┌────────┐       ┏━━━━━━━━┓       ┌────────┐
+ *  │Producer│──────▷┃ inbox  ┃◁ ─ ─ ─│Consumer│
+ *  └────────┘       ┗━━━━━━━━┛       └────────┘
+ *
+ *  This example showcases the simplest (multi-threading) pattern.
+ *
+ *  Data structures used:
+ *   - x9_inbox
+ *
+ *  Functions used:
+ *   - x9_create_inbox
+ *   - x9_inbox_is_valid
+ *   - x9_write_to_inbox_spin
+ *   - x9_read_from_inbox_spin
+ *   - x9_free_inbox
+ *
+ *  Test is considered passed iff:
+ *   - None of the threads stall and exit cleanly after doing the work.
+ *   - All messages sent by the producer(s) are received and asserted to be
+ *   valid by the consumer(s).
+ */
+
+#include <assert.h>  /* assert */
+#include <pthread.h> /* pthread_t, pthread functions */
+#include <stdio.h>   /* printf */
+#include <stdlib.h>  /* rand, RAND_MAX */
+
+#include "../x9.h"
+
+/* Both producer and consumer loops, would commonly be infinite loops, but for
+ * the purpose of testing a reasonable NUMBER_OF_MESSAGES is defined. */
+#define NUMBER_OF_MESSAGES 1000000
+
+typedef struct {
+  x9_inbox* inbox;
+} th_struct;
+
+typedef struct {
+  int a;
+  int b;
+  int sum;
+} msg;
+
+static int random_int(int const min, int const max) {
+  return min + rand() / (RAND_MAX / (max - min + 1) + 1);
+}
+
+static inline void fill_msg_1(msg* const msg) {
+  msg->a   = random_int(0, 10);
+  msg->b   = random_int(0, 10);
+  msg->sum = msg->a + msg->b;
+}
+
+static void* producer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {0};
+  for (uint64_t k = 0; k != NUMBER_OF_MESSAGES; ++k) {
+    fill_msg_1(&m);
+    x9_write_to_inbox_spin(data->inbox, sizeof(msg), &m);
+  }
+  return 0;
+}
+
+static void* consumer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {0};
+  for (uint64_t k = 0; k != NUMBER_OF_MESSAGES; ++k) {
+    x9_read_from_inbox_spin(data->inbox, sizeof(msg), &m);
+    assert(m.sum == (m.a + m.b));
+  }
+  return 0;
+}
+
+int main(void) {
+  /* Seed random generator */
+  srand((uint32_t)time(0));
+
+  /* Create inbox */
+  x9_inbox* const inbox = x9_create_inbox(4, "ibx_1", sizeof(msg));
+
+  /* Using assert to simplify code for presentation purpose. */
+  assert(x9_inbox_is_valid(inbox));
+
+  /* Producer */
+  pthread_t producer_th     = {0};
+  th_struct producer_struct = {.inbox = inbox};
+
+  /* Consumer */
+  pthread_t consumer_th     = {0};
+  th_struct consumer_struct = {.inbox = inbox};
+
+  /* Launch threads */
+  pthread_create(&producer_th, NULL, producer_fn, &producer_struct);
+  pthread_create(&consumer_th, NULL, consumer_fn, &consumer_struct);
+
+  /* Join them */
+  pthread_join(consumer_th, NULL);
+  pthread_join(producer_th, NULL);
+
+  /* Cleanup */
+  x9_free_inbox(inbox);
+
+  printf("TEST PASSED: x9_example_1.c\n");
+  return EXIT_SUCCESS;
+}
diff --git a/Framework/Foundation/3rdparty/x9/examples/x9_example_2.c b/Framework/Foundation/3rdparty/x9/examples/x9_example_2.c
new file mode 100644
index 0000000000000..411c7c8128779
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/examples/x9_example_2.c
@@ -0,0 +1,197 @@
+/* x9_example_2.c
+ *
+ *  Two producers.
+ *  One consumer and producer simultaneously.
+ *  One consumer.
+ *
+ * ┌────────┐      ┏━━━━━━━━┓                      ┏━━━━━━━━┓
+ * │Producer│─────▷┃        ┃      ┌────────┐      ┃        ┃
+ * └────────┘      ┃        ┃      │Consumer│      ┃        ┃      ┌────────┐
+ *                 ┃inbox 1 ┃◁ ─ ─ │  and   │─────▷┃inbox 2 ┃◁ ─ ─ │Consumer│
+ * ┌────────┐      ┃        ┃      │Producer│      ┃        ┃      └────────┘
+ * │Producer│─────▷┃        ┃      └────────┘      ┃        ┃
+ * └────────┘      ┗━━━━━━━━┛                      ┗━━━━━━━━┛
+ *
+ *  This example showcase using multiple threads to write to the same inbox,
+ *  using multiple message types, the 'x9_node' abstraction, and
+ *  respective create/free and select functions.
+ *
+ *  Data structures used:
+ *   - x9_inbox
+ *   - x9_node
+ *
+ *  Functions used:
+ *   - x9_create_inbox
+ *   - x9_inbox_is_valid
+ *   - x9_create_node
+ *   - x9_node_is_valid
+ *   - x9_select_inbox_from_node
+ *   - x9_write_to_inbox_spin
+ *   - x9_read_from_inbox_spin
+ *   - x9_free_node_and_attached_inboxes
+ *
+ *  Test is considered passed iff:
+ *   - None of the threads stall and exit cleanly after doing the work.
+ *   - All messages sent by the producer(s) are received and asserted to be
+ *   valid by the consumer(s).
+ */
+
+#include <assert.h>  /* assert */
+#include <pthread.h> /* pthread_t, pthread functions */
+#include <stdio.h>   /* printf */
+#include <stdlib.h>  /* rand, RAND_MAX */
+
+#include "../x9.h"
+
+/* Both producer and consumer loops, would commonly be infinite loops, but for
+ * the purpose of testing a reasonable NUMBER_OF_MESSAGES is defined. */
+#define NUMBER_OF_MESSAGES 1000000
+
+#define NUMBER_OF_PRODUCER_THREADS 2
+
+typedef struct {
+  x9_node* node;
+} th_struct;
+
+typedef struct {
+  int a;
+  int b;
+  int sum;
+
+} msg_type_1;
+
+typedef struct {
+  int x;
+  int y;
+  int sum;
+  int product;
+} msg_type_2;
+
+static inline int random_int(int const min, int const max) {
+  return min + rand() / (RAND_MAX / (max - min + 1) + 1);
+}
+
+static inline void fill_msg_type_1(msg_type_1 msg[const static 1]) {
+  msg->a   = random_int(0, 10);
+  msg->b   = random_int(0, 10);
+  msg->sum = msg->a + msg->b;
+}
+
+static inline void fill_msg_type_2(msg_type_2       to[const static 1],
+                                   msg_type_1 const from[const static 1]) {
+  to->x       = from->a;
+  to->y       = from->b;
+  to->sum     = from->sum;
+  to->product = (from->a * from->b);
+}
+
+static void* producer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  x9_inbox* const destination = x9_select_inbox_from_node(data->node, "ibx_1");
+  assert(x9_inbox_is_valid(destination));
+
+  msg_type_1 msg = {0};
+  for (uint64_t k = 0; k != NUMBER_OF_MESSAGES; ++k) {
+    fill_msg_type_1(&msg);
+    x9_write_to_inbox_spin(destination, sizeof(msg_type_1), &msg);
+  }
+  return 0;
+}
+
+static void* producer_consumer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  x9_inbox* const inbox = x9_select_inbox_from_node(data->node, "ibx_1");
+
+  assert(x9_inbox_is_valid(inbox));
+
+  x9_inbox* const destination = x9_select_inbox_from_node(data->node, "ibx_2");
+
+  assert(x9_inbox_is_valid(destination));
+
+  msg_type_1 incoming_msg = {0};
+  msg_type_2 outgoing_msg = {0};
+  for (uint64_t k = 0; k != (NUMBER_OF_MESSAGES * NUMBER_OF_PRODUCER_THREADS);
+       ++k) {
+    x9_read_from_inbox_spin(inbox, sizeof(msg_type_1), &incoming_msg);
+    assert(incoming_msg.sum == (incoming_msg.a + incoming_msg.b));
+    fill_msg_type_2(&outgoing_msg, &incoming_msg);
+    x9_write_to_inbox_spin(destination, sizeof(msg_type_2), &outgoing_msg);
+  }
+  return 0;
+}
+
+static void* consumer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  x9_inbox* const inbox = x9_select_inbox_from_node(data->node, "ibx_2");
+
+  assert(x9_inbox_is_valid(inbox));
+
+  msg_type_2 msg = {0};
+  for (uint64_t k = 0; k != (NUMBER_OF_MESSAGES * NUMBER_OF_PRODUCER_THREADS);
+       ++k) {
+    x9_read_from_inbox_spin(inbox, sizeof(msg_type_2), &msg);
+    assert(msg.sum == (msg.x + msg.y));
+    assert(msg.product == (msg.x * msg.y));
+  }
+  return 0;
+}
+
+int main(void) {
+  /* Seed random generator */
+  srand((uint32_t)time(0));
+
+  /* Create inboxes */
+  x9_inbox* const inbox_msg_type_1 =
+      x9_create_inbox(4, "ibx_1", sizeof(msg_type_1));
+
+  x9_inbox* const inbox_msg_type_2 =
+      x9_create_inbox(4, "ibx_2", sizeof(msg_type_2));
+
+  /* Using asserts to simplify code for presentation purpose. */
+  assert(x9_inbox_is_valid(inbox_msg_type_1));
+  assert(x9_inbox_is_valid(inbox_msg_type_2));
+
+  /* Create node */
+  x9_node* const node =
+      x9_create_node("my_node", 2, inbox_msg_type_1, inbox_msg_type_2);
+
+  /* Assert - Same reason as above. */
+  assert(x9_node_is_valid(node));
+
+  /* Producers */
+  pthread_t producer_th_1     = {0};
+  th_struct producer_1_struct = {.node = node};
+
+  pthread_t producer_th_2     = {0};
+  th_struct producer_2_struct = {.node = node};
+
+  /* Producer/Consumer */
+  pthread_t producer_consumer_th = {0};
+  th_struct prod_cons_struct     = {.node = node};
+
+  /* Consumer */
+  pthread_t consumer_th     = {0};
+  th_struct consumer_struct = {.node = node};
+
+  /* Launch threads */
+  pthread_create(&producer_th_1, NULL, producer_fn, &producer_1_struct);
+  pthread_create(&producer_th_2, NULL, producer_fn, &producer_2_struct);
+  pthread_create(&consumer_th, NULL, consumer_fn, &consumer_struct);
+  pthread_create(&producer_consumer_th, NULL, &producer_consumer_fn,
+                 &prod_cons_struct);
+
+  /* Join them */
+  pthread_join(consumer_th, NULL);
+  pthread_join(producer_consumer_th, NULL);
+  pthread_join(producer_th_1, NULL);
+  pthread_join(producer_th_2, NULL);
+
+  /* Cleanup */
+  x9_free_node_and_attached_inboxes(node);
+
+  printf("TEST PASSED: x9_example_2.c\n");
+  return EXIT_SUCCESS;
+}
diff --git a/Framework/Foundation/3rdparty/x9/examples/x9_example_3.c b/Framework/Foundation/3rdparty/x9/examples/x9_example_3.c
new file mode 100644
index 0000000000000..c10824e208c3c
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/examples/x9_example_3.c
@@ -0,0 +1,207 @@
+/* x9_example_3.c
+ *
+ *  Two producers and simultaneously consumers.
+ *
+ *  ┌────────┐       ┏━━━━━━━━┓       ┌────────┐
+ *  │Producer│──────▷┃inbox 1 ┃◁ ─ ─ ─│Producer│
+ *  │        │       ┗━━━━━━━━┛       │        │
+ *  │  and   │                        │  and   │
+ *  │        │       ┏━━━━━━━━┓       │        │
+ *  │Consumer│─ ─ ─ ▷┃inbox 2 ┃◁──────│Consumer│
+ *  └────────┘       ┗━━━━━━━━┛       └────────┘
+ *
+ *  This example showcases the use of: 'x9_write_to_inbox'
+ *  and 'x9_read_from_inbox', which, unlike 'x9_write_to_inbox_spin' and
+ *  'x9_read_from_inbox_spin' do not block until are able to write/read a msg.
+ *
+ *  Data structures used:
+ *   - x9_inbox
+ *   - x9_node
+ *
+ *  Functions used:
+ *   - x9_create_inbox
+ *   - x9_inbox_is_valid
+ *   - x9_create_node
+ *   - x9_node_is_valid
+ *   - x9_select_inbox_from_node
+ *   - x9_write_to_inbox
+ *   - x9_read_from_inbox
+ *   - x9_free_node_and_attached_inboxes
+ *
+ *  Test is considered passed iff:
+ *   - None of the threads stall and exit cleanly after doing the work.
+ *   - All messages sent by the producer(s) are received and asserted to be
+ *   valid by the consumer(s).
+ */
+
+#include <assert.h>  /* assert */
+#include <math.h>    /* fabs */
+#include <pthread.h> /* pthread_t, pthread functions */
+#include <stdio.h>   /* printf */
+#include <stdlib.h>  /* rand, RAND_MAX */
+
+#include "../x9.h"
+
+/* Both producer and consumer loops, would commonly be infinite loops, but for
+ * the purpose of testing a reasonable NUMBER_OF_MESSAGES is defined. */
+#define NUMBER_OF_MESSAGES 1000000
+
+typedef struct {
+  x9_node* node;
+} th_struct;
+
+typedef struct {
+  int a;
+  int b;
+  int sum;
+} msg_type_1;
+
+typedef struct {
+  double x;
+  double y;
+  double product;
+} msg_type_2;
+
+static inline int random_int(int const min, int const max) {
+  return min + rand() / (RAND_MAX / (max - min + 1) + 1);
+}
+
+static inline double random_double(int const min, int const max) {
+  return ((double)(random_int(min, max) * 1.0));
+}
+
+static inline void fill_msg_type_1(msg_type_1* const msg) {
+  msg->a   = random_int(0, 10);
+  msg->b   = random_int(0, 10);
+  msg->sum = msg->a + msg->b;
+}
+
+static inline void fill_msg_type_2(msg_type_2* const msg) {
+  msg->x       = random_double(0, 10);
+  msg->y       = random_double(0, 10);
+  msg->product = msg->x * msg->y;
+}
+
+static void* producer_1_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  x9_inbox* const write_inbox = x9_select_inbox_from_node(data->node, "ibx_1");
+  assert(write_inbox);
+
+  x9_inbox* const read_inbox = x9_select_inbox_from_node(data->node, "ibx_2");
+  assert(read_inbox);
+
+  uint64_t msgs_read = {0};
+  uint64_t msgs_sent = {0};
+
+  msg_type_2 incoming_msg = {0};
+  msg_type_1 outgoing_msg = {0};
+
+  for (;;) {
+    if ((msgs_read == NUMBER_OF_MESSAGES) &&
+        (msgs_sent == NUMBER_OF_MESSAGES)) {
+      break;
+    }
+
+    if (msgs_sent != NUMBER_OF_MESSAGES) {
+      fill_msg_type_1(&outgoing_msg);
+      if (x9_write_to_inbox(write_inbox, sizeof(msg_type_1), &outgoing_msg)) {
+        ++msgs_sent;
+      }
+    }
+
+    if (msgs_read != NUMBER_OF_MESSAGES) {
+      if (x9_read_from_inbox(read_inbox, sizeof(msg_type_2), &incoming_msg)) {
+        ++msgs_read;
+        assert(fabs(incoming_msg.product - (incoming_msg.x * incoming_msg.y)) <
+               0.1);
+      }
+    }
+  }
+
+  return 0;
+}
+
+static void* producer_2_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  x9_inbox* const write_inbox = x9_select_inbox_from_node(data->node, "ibx_2");
+  assert(write_inbox);
+
+  x9_inbox* const read_inbox = x9_select_inbox_from_node(data->node, "ibx_1");
+  assert(read_inbox);
+
+  uint64_t msgs_read = {0};
+  uint64_t msgs_sent = {0};
+
+  msg_type_1 incoming_msg = {0};
+  msg_type_2 outgoing_msg = {0};
+
+  for (;;) {
+    if ((msgs_read == NUMBER_OF_MESSAGES) &&
+        (msgs_sent == NUMBER_OF_MESSAGES)) {
+      break;
+    }
+
+    if (msgs_read != NUMBER_OF_MESSAGES) {
+      if (x9_read_from_inbox(read_inbox, sizeof(msg_type_1), &incoming_msg)) {
+        ++msgs_read;
+        assert(incoming_msg.sum == (incoming_msg.a + incoming_msg.b));
+      }
+    }
+
+    if (msgs_sent != NUMBER_OF_MESSAGES) {
+      fill_msg_type_2(&outgoing_msg);
+      if (x9_write_to_inbox(write_inbox, sizeof(msg_type_2), &outgoing_msg)) {
+        ++msgs_sent;
+      }
+    }
+  }
+
+  return 0;
+}
+
+int main(void) {
+  /* Seed random generator */
+  srand((uint32_t)time(0));
+
+  /* Create inboxes */
+  x9_inbox* const inbox_msg_type_1 =
+      x9_create_inbox(4, "ibx_1", sizeof(msg_type_1));
+
+  x9_inbox* const inbox_msg_type_2 =
+      x9_create_inbox(4, "ibx_2", sizeof(msg_type_2));
+
+  /* Using asserts to simplify code for presentation purpose. */
+  assert(x9_inbox_is_valid(inbox_msg_type_1));
+  assert(x9_inbox_is_valid(inbox_msg_type_2));
+
+  /* Create node */
+  x9_node* const node =
+      x9_create_node("my_node", 2, inbox_msg_type_1, inbox_msg_type_2);
+
+  /* Asserts - Same reason as above.*/
+  assert(x9_node_is_valid(node));
+
+  /* Producer 1 (left on diagram) */
+  pthread_t producer_1_th     = {0};
+  th_struct producer_1_struct = {.node = node};
+
+  /* Producer 2 (right on diagram) */
+  pthread_t producer_2_th     = {0};
+  th_struct producer_2_struct = {.node = node};
+
+  /* Launch threads */
+  pthread_create(&producer_1_th, NULL, producer_1_fn, &producer_1_struct);
+  pthread_create(&producer_2_th, NULL, producer_2_fn, &producer_2_struct);
+
+  /* Join them */
+  pthread_join(producer_1_th, NULL);
+  pthread_join(producer_2_th, NULL);
+
+  /* Cleanup */
+  x9_free_node_and_attached_inboxes(node);
+
+  printf("TEST PASSED: x9_example_3.c\n");
+  return EXIT_SUCCESS;
+}
diff --git a/Framework/Foundation/3rdparty/x9/examples/x9_example_4.c b/Framework/Foundation/3rdparty/x9/examples/x9_example_4.c
new file mode 100644
index 0000000000000..a4a096bebce94
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/examples/x9_example_4.c
@@ -0,0 +1,158 @@
+/* x9_example_4.c
+ *
+ *  One producer broadcasts the same message to three inboxes.
+ *  Three consumers read from each inbox.
+ *  One message type.
+ *
+ *                   ┏━━━━━━━━┓      ┌────────┐
+ *               ┌──▷┃ inbox  ┃◁─ ─ ─│Consumer│
+ *               │   ┗━━━━━━━━┛      └────────┘
+ *  ┌────────┐   │   ┏━━━━━━━━┓      ┌────────┐
+ *  │Producer│───┼──▷┃ inbox  ┃◁─ ─ ─│Consumer│
+ *  └────────┘   │   ┗━━━━━━━━┛      └────────┘
+ *               │   ┏━━━━━━━━┓      ┌────────┐
+ *               └──▷┃ inbox  ┃◁─ ─ ─│Consumer│
+ *                   ┗━━━━━━━━┛      └────────┘
+ *
+ *
+ *  This example showcases the use of 'x9_broadcast_msg_to_all_node_inboxes'.
+ *
+ *  Data structures used:
+ *   - x9_inbox
+ *   - x9_node
+ *
+ *  Functions used:
+ *   - x9_create_inbox
+ *   - x9_inbox_is_valid
+ *   - x9_create_node
+ *   - x9_node_is_valid
+ *   - x9_broadcast_msg_to_all_node_inboxes
+ *   - x9_read_from_inbox_spin
+ *   - x9_free_node_and_attached_inboxes
+ *
+ *  IMPORTANT:
+ *   - All inboxes  must receive messages of the same type (or at least of the
+ *   same size) that its being broadcasted.
+ *
+ *  Test is considered passed iff:
+ *   - None of the threads stall and exit cleanly after doing the work.
+ *   - All messages sent by the producer(s) are received and asserted to be
+ *   valid by the consumer(s).
+ */
+
+#include <assert.h>  /* assert */
+#include <pthread.h> /* pthread_t, pthread functions */
+#include <stdio.h>   /* printf */
+#include <stdlib.h>  /* rand, RAND_MAX */
+
+#include "../x9.h"
+
+/* Both producer and consumer loops, would commonly be infinite loops, but for
+ * the purpose of testing a reasonable NUMBER_OF_MESSAGES is defined. */
+#define NUMBER_OF_MESSAGES 1000000
+
+typedef struct {
+  x9_node* node;
+  char*    inbox_to_consume_from;
+} th_struct;
+
+typedef struct {
+  int a;
+  int b;
+  int sum;
+} msg;
+
+static inline int random_int(int const min, int const max) {
+  return min + rand() / (RAND_MAX / (max - min + 1) + 1);
+}
+
+static inline void fill_msg_type(msg* const m) {
+  m->a   = random_int(0, 10);
+  m->b   = random_int(0, 10);
+  m->sum = m->a + m->b;
+}
+
+static void* producer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {0};
+  for (uint_fast64_t k = 0; k != NUMBER_OF_MESSAGES; ++k) {
+    fill_msg_type(&m);
+    x9_broadcast_msg_to_all_node_inboxes(data->node, sizeof(msg), &m);
+  }
+  return 0;
+}
+
+static void* consumer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  x9_inbox* const inbox =
+      x9_select_inbox_from_node(data->node, data->inbox_to_consume_from);
+  assert(x9_inbox_is_valid(inbox));
+
+  msg m = {0};
+
+  for (uint64_t k = 0; k != NUMBER_OF_MESSAGES; ++k) {
+    x9_read_from_inbox_spin(inbox, sizeof(msg), &m);
+    assert(m.sum == (m.a + m.b));
+  }
+  return 0;
+}
+
+int main(void) {
+  /* Seed random generator */
+  srand((uint32_t)time(0));
+
+  /* Create inboxes */
+  x9_inbox* const inbox_consumer_1 = x9_create_inbox(4, "ibx_1", sizeof(msg));
+  x9_inbox* const inbox_consumer_2 = x9_create_inbox(4, "ibx_2", sizeof(msg));
+  x9_inbox* const inbox_consumer_3 = x9_create_inbox(4, "ibx_3", sizeof(msg));
+
+  /* Using asserts to simplify code for presentation purpose. */
+  assert(x9_inbox_is_valid(inbox_consumer_1));
+  assert(x9_inbox_is_valid(inbox_consumer_2));
+  assert(x9_inbox_is_valid(inbox_consumer_3));
+
+  /* Create node */
+  x9_node* const node = x9_create_node("my_node", 3, inbox_consumer_1,
+                                       inbox_consumer_2, inbox_consumer_3);
+
+  /* Assert - Same reason as above. */
+  assert(x9_node_is_valid(node));
+
+  /* Producer */
+  pthread_t producer_th     = {0};
+  th_struct producer_struct = {.node = node};
+
+  /* Consumer 1 */
+  pthread_t consumer_1_th     = {0};
+  th_struct consumer_1_struct = {.node                  = node,
+                                 .inbox_to_consume_from = "ibx_1"};
+  /* Consumer 2 */
+  pthread_t consumer_2_th     = {0};
+  th_struct consumer_2_struct = {.node                  = node,
+                                 .inbox_to_consume_from = "ibx_2"};
+
+  /* Consumer 3 */
+  pthread_t consumer_3_th     = {0};
+  th_struct consumer_3_struct = {.node                  = node,
+                                 .inbox_to_consume_from = "ibx_3"};
+
+  /* Launch threads */
+  pthread_create(&producer_th, NULL, producer_fn, &producer_struct);
+  pthread_create(&consumer_1_th, NULL, consumer_fn, &consumer_1_struct);
+  pthread_create(&consumer_2_th, NULL, consumer_fn, &consumer_2_struct);
+  pthread_create(&consumer_3_th, NULL, consumer_fn, &consumer_3_struct);
+
+  /* Join them */
+  pthread_join(producer_th, NULL);
+  pthread_join(consumer_1_th, NULL);
+  pthread_join(consumer_2_th, NULL);
+  pthread_join(consumer_3_th, NULL);
+
+  /* Cleanup */
+  x9_free_node_and_attached_inboxes(node);
+
+  printf("TEST PASSED: x9_example_4.c\n");
+  return EXIT_SUCCESS;
+}
diff --git a/Framework/Foundation/3rdparty/x9/examples/x9_example_5.c b/Framework/Foundation/3rdparty/x9/examples/x9_example_5.c
new file mode 100644
index 0000000000000..9f310feed6b90
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/examples/x9_example_5.c
@@ -0,0 +1,161 @@
+/* x9_example_5.c
+ *
+ *  Three producers.
+ *  Three consumers reading concurrently from the same inbox.
+ *  One message type.
+ *
+ *                                    ┌────────┐
+ *  ┌────────┐       ┏━━━━━━━━┓    ─ ─│Consumer│
+ *  │Producer│──────▷┃        ┃   │   └────────┘
+ *  ├────────┤       ┃        ┃       ┌────────┐
+ *  │Producer│──────▷┃ inbox  ┃◁──┤─ ─│Consumer│
+ *  ├────────┤       ┃        ┃       └────────┘
+ *  │Producer│──────▷┃        ┃   │   ┌────────┐
+ *  └────────┘       ┗━━━━━━━━┛    ─ ─│Consumer│
+ *                                    └────────┘
+ *
+ *  This example showcases the use of 'x9_read_from_shared_inbox'.
+ *
+ *  Data structures used:
+ *   - x9_inbox
+ *
+ *  Functions used:
+ *   - x9_create_inbox
+ *   - x9_inbox_is_valid
+ *   - x9_write_to_inbox_spin
+ *   - x9_read_from_shared_inbox
+ *   - x9_free_inbox
+ *
+ *  Test is considered passed iff:
+ *   - None of the threads stall and exit cleanly after doing the work.
+ *   - All messages sent by the producer(s) are received and asserted to be
+ *   valid by the consumer(s).
+ *   - Each consumer processes at least one message.
+ */
+
+#include <assert.h>    /* assert */
+#include <pthread.h>   /* pthread_t, pthread functions */
+#include <stdbool.h>   /* bool */
+#include <stdint.h>    /* uint64_t */
+#include <stdio.h>     /* printf */
+#include <stdlib.h>    /* rand, RAND_MAX */
+
+#include "../x9.h"
+
+/* Both producer and consumer loops, would commonly be infinite loops, but for
+ * the purpose of testing a reasonable NUMBER_OF_MESSAGES is defined. */
+#define NUMBER_OF_MESSAGES 1000000
+
+#define NUMBER_OF_PRODUCER_THREADS 3
+
+typedef struct {
+  x9_inbox* inbox;
+  uint64_t  msgs_read;
+} th_struct;
+
+typedef struct {
+  int  a;
+  int  b;
+  int  sum;
+  bool last_message;
+  char pad[3];
+} msg;
+
+static inline int random_int(int const min, int const max) {
+  return min + rand() / (RAND_MAX / (max - min + 1) + 1);
+}
+
+static inline void fill_msg_type(msg* const m) {
+  m->a   = random_int(0, 10);
+  m->b   = random_int(0, 10);
+  m->sum = m->a + m->b;
+}
+
+static void* producer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {0};
+  for (uint_fast64_t k = 0; k != NUMBER_OF_MESSAGES; ++k) {
+    fill_msg_type(&m);
+    if (k == (NUMBER_OF_MESSAGES - 1)) { m.last_message = true; }
+    x9_write_to_inbox_spin(data->inbox, sizeof(msg), &m);
+  }
+  return 0;
+}
+
+static void* consumer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {0};
+  for (;;) {
+    if (x9_read_from_shared_inbox(data->inbox, sizeof(msg), &m)) {
+      assert(m.sum == (m.a + m.b));
+      ++data->msgs_read;
+      if (m.last_message) { return 0; }
+    }
+  }
+}
+
+int main(void) {
+  /* Seed random generator */
+  srand((uint32_t)time(0));
+
+  /* Create inbox */
+  x9_inbox* const inbox = x9_create_inbox(4, "ibx", sizeof(msg));
+
+  /* Using assert to simplify code for presentation purpose. */
+  assert(x9_inbox_is_valid(inbox));
+
+  /* Producers */
+  pthread_t producer_1_th     = {0};
+  th_struct producer_1_struct = {.inbox = inbox};
+
+  pthread_t producer_2_th     = {0};
+  th_struct producer_2_struct = {.inbox = inbox};
+
+  pthread_t producer_3_th     = {0};
+  th_struct producer_3_struct = {.inbox = inbox};
+
+  /* Consumers */
+  pthread_t consumer_1_th     = {0};
+  th_struct consumer_1_struct = {.inbox = inbox};
+
+  pthread_t consumer_2_th     = {0};
+  th_struct consumer_2_struct = {.inbox = inbox};
+
+  pthread_t consumer_3_th     = {0};
+  th_struct consumer_3_struct = {.inbox = inbox};
+
+  /* Launch threads */
+  pthread_create(&producer_1_th, NULL, producer_fn, &producer_1_struct);
+  pthread_create(&producer_2_th, NULL, producer_fn, &producer_2_struct);
+  pthread_create(&producer_3_th, NULL, producer_fn, &producer_3_struct);
+  pthread_create(&consumer_1_th, NULL, consumer_fn, &consumer_1_struct);
+  pthread_create(&consumer_2_th, NULL, consumer_fn, &consumer_2_struct);
+  pthread_create(&consumer_3_th, NULL, consumer_fn, &consumer_3_struct);
+
+  /* Join them */
+  pthread_join(producer_1_th, NULL);
+  pthread_join(producer_2_th, NULL);
+  pthread_join(producer_3_th, NULL);
+  pthread_join(consumer_1_th, NULL);
+  pthread_join(consumer_2_th, NULL);
+  pthread_join(consumer_3_th, NULL);
+
+  /* Assert that all of the consumers read from the shared inbox. */
+  assert(consumer_1_struct.msgs_read > 0);
+  assert(consumer_2_struct.msgs_read > 0);
+  assert(consumer_3_struct.msgs_read > 0);
+
+  /* Assert that the total number of messages read == (NUMBER_OF_MESSAGES *
+   * NUMBER_OF_PRODUCER_THREADS) */
+  assert((NUMBER_OF_MESSAGES * NUMBER_OF_PRODUCER_THREADS) ==
+         (consumer_1_struct.msgs_read + consumer_2_struct.msgs_read +
+          consumer_3_struct.msgs_read));
+
+  /* Cleanup */
+  x9_free_inbox(inbox);
+
+  printf("TEST PASSED: x9_example_5.c\n");
+  return EXIT_SUCCESS;
+}
diff --git a/Framework/Foundation/3rdparty/x9/examples/x9_example_6.c b/Framework/Foundation/3rdparty/x9/examples/x9_example_6.c
new file mode 100644
index 0000000000000..1faad026e1feb
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/examples/x9_example_6.c
@@ -0,0 +1,159 @@
+/* x9_example_6.c
+ *
+ *  One producer
+ *  Two consumers reading from the same inbox concurrently (busy loop).
+ *  One message type.
+ *
+ *                   ┏━━━━━━━━┓       ┌────────┐
+ *                   ┃        ┃    ─ ─│Consumer│
+ *  ┌────────┐       ┃        ┃   │   └────────┘
+ *  │Producer│──────▷┃ inbox  ┃◁ ─
+ *  └────────┘       ┃        ┃   │   ┌────────┐
+ *                   ┃        ┃    ─ ─│Consumer│
+ *                   ┗━━━━━━━━┛       └────────┘
+ *
+ *  This example showcases the use of 'x9_read_from_shared_inbox_spin'.
+ *
+ *  Data structures used:
+ *   - x9_inbox
+ *
+ *  Functions used:
+ *   - x9_create_inbox
+ *   - x9_inbox_is_valid
+ *   - x9_write_to_inbox_spin
+ *   - x9_read_from_shared_inbox_spin
+ *   - x9_free_inbox
+ *
+ *  Test is considered passed iff:
+ *   - None of the threads stall and exit cleanly after doing the work.
+ *   - All messages sent by the producer(s) are received and asserted to be
+ *   valid by the consumer(s).
+ *   - Each consumer processes at least one message.
+ */
+
+#include <assert.h>    /* assert */
+#include <pthread.h>   /* pthread_t, pthread functions */
+#include <stdatomic.h> /* atomic_* */
+#include <stdbool.h>   /* bool */
+#include <stdint.h>    /* uint64_t */
+#include <stdio.h>     /* printf */
+#include <stdlib.h>    /* rand, RAND_MAX */
+
+#include "../x9.h"
+
+/* Both producer and consumer loops, would commonly be infinite loops, but for
+ * the purpose of testing a reasonable NUMBER_OF_MESSAGES is defined. */
+#define NUMBER_OF_MESSAGES 1000000
+
+typedef struct {
+  x9_inbox* inbox;
+  uint64_t  msgs_read;
+} th_struct;
+
+typedef struct {
+  int  a;
+  int  b;
+  int  sum;
+  bool last_message;
+  char pad[3];
+} msg;
+
+static inline int random_int(int const min, int const max) {
+  return min + rand() / (RAND_MAX / (max - min + 1) + 1);
+}
+
+static inline void fill_msg_type(msg* const m) {
+  m->a   = random_int(0, 10);
+  m->b   = random_int(0, 10);
+  m->sum = m->a + m->b;
+}
+
+static void* producer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {0};
+  for (uint_fast64_t k = 0; k != (NUMBER_OF_MESSAGES); ++k) {
+    fill_msg_type(&m);
+    if (k == (NUMBER_OF_MESSAGES - 1)) { m.last_message = true; }
+    x9_write_to_inbox_spin(data->inbox, sizeof(msg), &m);
+  }
+  return 0;
+}
+
+static void* consumer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+  msg        m    = {0};
+  for (;;) {
+    x9_read_from_shared_inbox_spin(data->inbox, sizeof(msg), &m);
+    assert(m.sum == (m.a + m.b));
+
+    /* The First thread to read the 'last message' writes the same message back
+     * to the inbox, so in case the second thread had already entered
+     * 'x9_read_from_shared_inbox_spin' function, it will be able to read the
+     * message and do a clean exit.
+     * If the second thread, at the time that the first thread set
+     * 'g_read_last= true' to hasn't entered 'x9_write_to_inbox_spin', then it
+     * will check the if(g_read_last) condition above and exit.
+     * The first case would equal NUMBER_OF_MESSAGES + 1 read.
+     * The second case, NUMBER_OF_MESSAGES.
+     * Given the implementation of 'x9_read_from_shared_inbox_spin', this is
+     * the only way to get a clean exit without killing the second thread. */
+    if (m.last_message) {
+      x9_write_to_inbox_spin(data->inbox, sizeof(msg), &m);
+      ++data->msgs_read;
+      return 0;
+    }
+    ++data->msgs_read;
+  }
+}
+
+int main(void) {
+  /* Seed random generator */
+  srand((uint32_t)time(0));
+
+  /* Create inbox */
+  x9_inbox* const inbox = x9_create_inbox(4, "ibx", sizeof(msg));
+
+  /* Using assert to simplify code for presentation purpose */
+  assert(x9_inbox_is_valid(inbox));
+
+  /* Producer */
+  pthread_t producer_th     = {0};
+  th_struct producer_struct = {.inbox = inbox};
+
+  /* Consumers */
+  pthread_t consumer_1_th     = {0};
+  th_struct consumer_1_struct = {.inbox = inbox};
+
+  pthread_t consumer_2_th     = {0};
+  th_struct consumer_2_struct = {.inbox = inbox};
+
+  /* Launch threads */
+  pthread_create(&producer_th, NULL, producer_fn, &producer_struct);
+  pthread_create(&consumer_1_th, NULL, consumer_fn, &consumer_1_struct);
+  pthread_create(&consumer_2_th, NULL, consumer_fn, &consumer_2_struct);
+
+  /* Join them */
+  pthread_join(producer_th, NULL);
+  pthread_join(consumer_1_th, NULL);
+  pthread_join(consumer_2_th, NULL);
+
+  /* Assert that all of the consumers read from the shared inbox. */
+  assert(consumer_1_struct.msgs_read > 0);
+  assert(consumer_2_struct.msgs_read > 0);
+
+  /* Assert that the total number of messages read == NUMBER_OF_MESSAGES or
+   * NUMBER_OF_MESSAGES + 1 */
+  assert((NUMBER_OF_MESSAGES + 1) ==
+             (consumer_1_struct.msgs_read + consumer_2_struct.msgs_read) ||
+         (NUMBER_OF_MESSAGES) ==
+             (consumer_1_struct.msgs_read + consumer_2_struct.msgs_read)
+
+  );
+
+  /* Cleanup */
+  x9_free_inbox(inbox);
+
+  printf("TEST PASSED: x9_example_6.c\n");
+  return EXIT_SUCCESS;
+}
diff --git a/Framework/Foundation/3rdparty/x9/profiling/README.md b/Framework/Foundation/3rdparty/x9/profiling/README.md
new file mode 100644
index 0000000000000..c10f3f1c658cc
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/profiling/README.md
@@ -0,0 +1,106 @@
+Notes:
+---
+
+```
+x9_profiler.c
+
+- One producer thread
+- One consumer thread
+- One x9_inbox
+
+┌────────┐       ┏━━━━━━━━┓       ┌────────┐
+│Producer│──────▷┃ inbox  ┃◁ ─ ─ ─│Consumer│
+└────────┘       ┗━━━━━━━━┛       └────────┘
+```
+
+`x9_profiler.c` allows to test the performance of the library given different
+parameters, such as the _inbox size_, _message size_ and in which _cpu cores_
+the test is performed.  
+It does so by having the producer/writer write _N messages_ to the _inbox_ and
+the consumer/reader read them.  
+All of these parameters are user defined and passed to the program as command 
+line arguments, as shown below.
+
+Additionally, two different types of tests can be run:
+- **--test 1** should be used to get an idea of the raw performance of the 
+library since it calls `x9_write_to_inbox_spin` and `x9_read_from_inbox_spin` 
+in the background, which is ideal for low latency systems.
+- **--test 2** uses the non spinning version of the functions above, and while
+it will be slower, it allows to understand the _hit ratio_ of
+both the producer and consumer.
+
+_Hit ratio_ is defined as the number of messages the writer(reader) wrote(read)
+divided by the number of times it attempted to write(read), and can be helpful 
+when deciding in which _cpu cores_ to run specific threads.
+
+Overall, both tests are useful for fine-tuning which _inbox size_ to use and 
+gaining a better understanding of the expected performance when transmitting 
+_messages_ of a given type (i.e. sizeof(some_type)).  
+Moreover, they can be used to measure the impact of overclocking 
+(on specific cores or as a whole).
+
+- When `--n_its` > 1  the results presented will be the _median_ of all 
+iterations.
+- The writer will run on the first core of the values passed to
+`--run_in_cores` and the reader in the second.
+
+The program can be compiled with: _./compile_profiler.sh_ and run as follows:
+
+```
+Example (test 1):
+
+$ ./X9_PROF \
+  --test 1 \
+  --inboxes_szs 1024,2048,4096 \
+  --msgs_szs 16,32,64 \
+  --n_msgs 100000000 \
+  --n_its 1 \
+  --run_in_cores 2,4
+
+
+Inbox size | Msg size | Time (secs) | Msgs/Second
+-------------------------------------------------
+      1024 |       16 |        8.82 |      11.33M
+      1024 |       32 |        8.70 |      11.50M
+      1024 |       64 |        8.65 |      11.56M
+-------------------------------------------------
+      2048 |       16 |        9.10 |      10.99M
+      2048 |       32 |        9.53 |      10.49M
+      2048 |       64 |        9.44 |      10.60M
+-------------------------------------------------
+      4096 |       16 |        9.08 |      11.01M
+      4096 |       32 |        9.53 |      10.50M
+      4096 |       64 |        9.43 |      10.60M
+-------------------------------------------------
+```
+
+```
+Example (test 2):
+
+$ ./X9_PROF \
+  --test 2 \
+  --inboxes_szs 1024,2048,4096 \
+  --msgs_szs 16,32,64,128 \
+  --n_msgs 100000000 \
+  --n_its 1 \
+  --run_in_cores 2,4
+
+
+Inbox size | Msg size | Time (secs) | Msgs/second | Writer hit ratio | Reader hit ratio
+---------------------------------------------------------------------------------------
+      1024 |       16 |       12.69 |       7.88M |          100.00% |           69.92%
+      1024 |       32 |       13.00 |       7.69M |          100.00% |           56.46%
+      1024 |       64 |       13.79 |       7.25M |           46.85% |           99.88%
+      1024 |      128 |       11.53 |       8.68M |          100.00% |           67.11%
+---------------------------------------------------------------------------------------
+      2048 |       16 |       11.92 |       8.39M |          100.00% |           68.22%
+      2048 |       32 |       11.65 |       8.58M |          100.00% |           82.29%
+      2048 |       64 |       10.05 |       9.95M |           81.91% |           99.81%
+      2048 |      128 |       11.64 |       8.59M |          100.00% |           78.21%
+---------------------------------------------------------------------------------------
+      4096 |       16 |       12.19 |       8.21M |          100.00% |           79.13%
+      4096 |       32 |       12.75 |       7.84M |          100.00% |           65.95%
+      4096 |       64 |       12.10 |       8.26M |          100.00% |           67.25%
+      4096 |      128 |       11.61 |       8.61M |          100.00% |           78.29%
+---------------------------------------------------------------------------------------
+```
diff --git a/Framework/Foundation/3rdparty/x9/profiling/compile_profiler.sh b/Framework/Foundation/3rdparty/x9/profiling/compile_profiler.sh
new file mode 100755
index 0000000000000..0f89b0e9acdc1
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/profiling/compile_profiler.sh
@@ -0,0 +1,3 @@
+#/bin/bash
+
+gcc -Wextra -Wall -Werror -pedantic -flto -O3 -march=native x9_profiler.c ../x9.c -o X9_PROF
diff --git a/Framework/Foundation/3rdparty/x9/profiling/x9_profiler.c b/Framework/Foundation/3rdparty/x9/profiling/x9_profiler.c
new file mode 100644
index 0000000000000..d596dd816791d
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/profiling/x9_profiler.c
@@ -0,0 +1,524 @@
+/* x9_profiler.c:
+ *
+ *  One producer
+ *  One consumer
+ *  One message type
+ *
+ *  ┌────────┐       ┏━━━━━━━━┓       ┌────────┐
+ *  │Producer│──────▷┃ inbox  ┃◁ ─ ─ ─│Consumer│
+ *  └────────┘       ┗━━━━━━━━┛       └────────┘
+ *
+ *  '--test 1' uses 'x9_write_to_inbox_spin' and 'x9_read_from_inbox_spin'
+ *  '--test 2' uses x9_read_from_inbox and 'x9_read_from_inbox'
+ *
+ *  The advantage of '--test 2' is that, given its non spinning nature,
+ *  it's possible to gather more performance metrics.
+ */
+
+#define _GNU_SOURCE  /* cpu_*, pthread_setaffinity_np */
+#include <assert.h>  /* assert */
+#include <getopt.h>  /* required_argument, getopt_long */
+#include <pthread.h> /* pthread_t, pthread functions */
+#include <stdint.h>  /* uint8_t, uint64_t, int64_t */
+#include <stdio.h>   /* printf */
+#include <stdlib.h>  /* qsort, rand, RAND_MAX */
+#include <string.h>  /* memset, strcmp */
+#include <unistd.h>  /* sysconf, _SC_NPROCESSORS_ONLN */
+
+#include "../x9.h"
+
+typedef enum { HEADER = 1, SEPARATOR } stdout_output;
+
+#define ARG(arg_name) (!strcmp(long_options[option_idx].name, arg_name))
+
+typedef struct vector {
+  uint64_t size;
+  uint64_t used;
+  int64_t* data;
+} vector;
+
+typedef struct {
+  vector* inboxes_sizes;
+  vector* msgs_sizes;
+  vector* run_in_cores;
+  int64_t n_messages;
+  int64_t n_iterations;
+  int64_t test;
+} perf_config;
+
+typedef struct {
+  double time_secs;
+  double writer_hit_ratio;
+  double reader_hit_ratio;
+} perf_results;
+
+typedef struct {
+  x9_inbox* inbox;
+  uint64_t  msg_sz;
+  uint64_t  n_msgs;
+  double    writer_hit_ratio;
+  double    reader_hit_ratio;
+} th_struct;
+
+typedef struct {
+  uint8_t* a;
+} msg;
+
+__attribute__((noreturn)) static void abort_test(char const* const msg) {
+  printf("%s\n", msg);
+  abort();
+}
+
+static vector* vector_init(uint64_t const sz) {
+  if ((sz % 2)) { abort_test("ERROR: vector sz must be % 2 == 0"); }
+  vector* const v = calloc(1, sizeof(vector));
+  if (NULL == v) { abort_test("ERROR: failed to allocate vector."); }
+  v->data = calloc(1, sz * sizeof(*v->data));
+  if (NULL == v->data) { abort_test("ERROR: failed to allocate v->data."); }
+  v->size = sz;
+  return v;
+}
+
+static void vector_insert(vector* const v, int64_t value) {
+  if (v->used == v->size) {
+    v->size *= 2;
+    v->data = realloc(v->data, v->size * sizeof(*v->data));
+    if (NULL == v->data) { abort_test("ERROR: realloc failed."); }
+  }
+  v->data[v->used++] = value;
+}
+
+static void vector_free(vector* const v) {
+  free(v->data);
+  free(v);
+}
+
+static int random_int(int const min, int const max) {
+  return min + rand() / (RAND_MAX / (max - min + 1) + 1);
+}
+
+static void* producer_fn_test_1(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {.a = calloc(data->msg_sz, sizeof(uint8_t))};
+  if (NULL == m.a) { abort_test("ERROR: failed to allocate msg buffer"); }
+
+  for (uint64_t k = 0; k != data->n_msgs; ++k) {
+    int32_t const random_val = random_int(1, 9);
+    memset(m.a, random_val, data->msg_sz);
+    x9_write_to_inbox_spin(data->inbox, data->msg_sz, m.a);
+  }
+  free(m.a);
+  return 0;
+}
+
+static void* consumer_fn_test_1(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {.a = calloc(data->msg_sz, sizeof(uint8_t))};
+  if (NULL == m.a) { abort_test("ERROR: failed to allocate msg buffer"); }
+
+  for (uint64_t k = 0; k != data->n_msgs; ++k) {
+    x9_read_from_inbox_spin(data->inbox, data->msg_sz, m.a);
+    assert(m.a[(data->msg_sz - 1)] == (m.a[0]));
+  }
+  free(m.a);
+  return 0;
+}
+
+static void* producer_fn_test_2(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {.a = calloc(data->msg_sz, sizeof(uint8_t))};
+  if (NULL == m.a) { abort_test("ERROR: failed to allocate msg buffer"); }
+
+  uint64_t write_attempts = 0;
+  uint64_t msgs_written   = 0;
+
+  for (;;) {
+    if (msgs_written == data->n_msgs) { break; }
+    int32_t const random_val = random_int(1, 9);
+    memset(m.a, random_val, data->msg_sz);
+    if (x9_write_to_inbox(data->inbox, data->msg_sz, m.a)) { ++msgs_written; }
+    ++write_attempts;
+  }
+
+  data->writer_hit_ratio = (double)msgs_written / (double)write_attempts;
+  free(m.a);
+  return 0;
+}
+
+static void* consumer_fn_test_2(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {.a = calloc(data->msg_sz, sizeof(uint8_t))};
+  if (NULL == m.a) { abort_test("ERROR: failed to allocate msg buffer"); }
+
+  uint64_t read_attempts = 0;
+  uint64_t msgs_read     = 0;
+
+  for (;;) {
+    if (msgs_read == data->n_msgs) { break; }
+    if (x9_read_from_inbox(data->inbox, data->msg_sz, m.a)) {
+      ++msgs_read;
+      assert(m.a[(data->msg_sz - 1)] == (m.a[0]));
+    }
+    ++read_attempts;
+  }
+
+  data->reader_hit_ratio = (double)msgs_read / (double)read_attempts;
+  free(m.a);
+  return 0;
+}
+
+static perf_results run_test(uint64_t const ibx_sz,
+                             uint64_t const msg_sz,
+                             uint64_t const n_msgs,
+                             uint64_t const first_core,
+                             uint64_t const second_core,
+                             uint64_t const test
+
+) {
+  /* Create inbox */
+  x9_inbox* const inbox = x9_create_inbox(ibx_sz, "ibx_1", msg_sz);
+
+  /* Confirm that it's valid */
+  if (!(x9_inbox_is_valid(inbox))) {
+    abort_test("ERROR: x9_inbox is invalid");
+  }
+
+  /* Producer */
+  pthread_t      producer_th   = {0};
+  pthread_attr_t producer_attr = {0};
+  pthread_attr_init(&producer_attr);
+  th_struct producer_struct = {
+      .inbox = inbox, .msg_sz = msg_sz, .n_msgs = n_msgs};
+
+  /* Consumer */
+  pthread_t      consumer_th   = {0};
+  pthread_attr_t consumer_attr = {0};
+  pthread_attr_init(&consumer_attr);
+  th_struct consumer_struct = {
+      .inbox = inbox, .msg_sz = msg_sz, .n_msgs = n_msgs};
+
+  /* Set affinity */
+  cpu_set_t f_core = {0};
+  CPU_ZERO(&f_core);
+  CPU_SET(first_core, &f_core);
+
+  cpu_set_t s_core = {0};
+  CPU_ZERO(&s_core);
+  CPU_SET(second_core, &s_core);
+
+  pthread_attr_setaffinity_np(&producer_attr, sizeof(cpu_set_t), &f_core);
+  pthread_attr_setaffinity_np(&consumer_attr, sizeof(cpu_set_t), &s_core);
+
+  /* Start timer */
+  struct timespec tic = {0};
+  clock_gettime(CLOCK_MONOTONIC, &tic);
+
+  /* Launch threads */
+  if (1 == test) {
+    pthread_create(&consumer_th, &consumer_attr, consumer_fn_test_1,
+                   &consumer_struct);
+    pthread_create(&producer_th, &producer_attr, producer_fn_test_1,
+                   &producer_struct);
+
+  } else {
+    pthread_create(&consumer_th, &consumer_attr, consumer_fn_test_2,
+                   &consumer_struct);
+    pthread_create(&producer_th, &producer_attr, producer_fn_test_2,
+                   &producer_struct);
+  }
+
+  /* Join them */
+  pthread_join(consumer_th, NULL);
+  pthread_join(producer_th, NULL);
+
+  /* Stop timer */
+  struct timespec toc = {0};
+  clock_gettime(CLOCK_MONOTONIC, &toc);
+
+  uint64_t const before =
+      ((uint64_t)tic.tv_sec * 1000000000UL) + (uint64_t)tic.tv_nsec;
+  uint64_t const after =
+      ((uint64_t)toc.tv_sec * 1000000000UL) + (uint64_t)toc.tv_nsec;
+
+  /* Cleanup */
+  pthread_attr_destroy(&producer_attr);
+  pthread_attr_destroy(&consumer_attr);
+  x9_free_inbox(inbox);
+
+  return (perf_results){.time_secs        = (double)(after - before) / 1e9,
+                        .writer_hit_ratio = producer_struct.writer_hit_ratio,
+                        .reader_hit_ratio = consumer_struct.reader_hit_ratio};
+}
+
+static void parse_array_arguments(char* restrict const args,
+                                  vector* const write_to) {
+  char*             args_start = args;
+  char const* const args_end   = args + strlen(args);
+
+  for (; args_start < args_end;) {
+    char* begin = args_start;
+    char* end   = args_start;
+    for (; end != args_end; ++end) {
+      if (',' == *end) { break; }
+    }
+    args_start      = end + 1;
+    int64_t const n = strtoll(begin, &end, 10);
+    vector_insert(write_to, n);
+  }
+}
+
+static perf_config* parse_command_line_args(int argc, char** argv) {
+  perf_config* config = calloc(1, sizeof(perf_config));
+  if (NULL == config) {
+    abort_test("ERROR: failed to allocate 'perf_config'");
+  }
+
+  for (;;) {
+    int                  option_idx     = 0;
+    static struct option long_options[] = {
+        {"inboxes_szs", required_argument, 0, 0},
+        {"msgs_szs", required_argument, 0, 0},
+        {"n_msgs", required_argument, 0, 0},
+        {"n_its", required_argument, 0, 0},
+        {"run_in_cores", required_argument, 0, 0},
+        {"test", required_argument, 0, 0},
+        {0, 0, 0, 0}
+
+    };
+
+    int const c = getopt_long(argc, argv, "", long_options, &option_idx);
+    if (-1 == c) { break; }
+
+    switch (c) {
+      case 0:
+        if (ARG("inboxes_szs")) {
+          config->inboxes_sizes = vector_init(8);
+
+          parse_array_arguments(optarg, config->inboxes_sizes);
+
+          if (!config->inboxes_sizes->data[0]) {
+            abort_test(
+                "ERROR: test requires at least one value for "
+                "'--inboxes_sizes'");
+          }
+
+          for (uint64_t k = 0; k != config->inboxes_sizes->used; ++k) {
+            int64_t const n = config->inboxes_sizes->data[k];
+            if (!((n > 0) && ((n % 2) == 0))) {
+              abort_test(
+                  "ERROR: '--inboxes_sizes' values must be > 0 and % 2 == "
+                  "0");
+            }
+          }
+        }
+        if (ARG("msgs_szs")) {
+          config->msgs_sizes = vector_init(8);
+          parse_array_arguments(optarg, config->msgs_sizes);
+
+          if (!config->msgs_sizes->data[0]) {
+            abort_test(
+                "ERROR: test requires at least one value for "
+                "'--msgs_sizes'");
+          }
+
+          for (uint64_t k = 0; k != config->msgs_sizes->used; ++k) {
+            int64_t const n = config->msgs_sizes->data[k];
+            if (!(n > 0)) {
+              abort_test("ERROR: '--msgs_sizes' values must be > 0");
+            }
+          }
+        }
+
+        if (ARG("run_in_cores")) {
+          config->run_in_cores = vector_init(8);
+          parse_array_arguments(optarg, config->run_in_cores);
+
+          if (config->run_in_cores->used != 2) {
+            abort_test("ERROR: --run_in_cores requires two values.");
+          }
+
+          int64_t const n_cores = sysconf(_SC_NPROCESSORS_ONLN);
+
+          if ((config->run_in_cores->data[0] < 0) ||
+              config->run_in_cores->data[0] > n_cores ||
+              (config->run_in_cores->data[1] < 0) ||
+              config->run_in_cores->data[1] > n_cores) {
+            char abort_msg[128] = {0};
+            sprintf(abort_msg,
+                    "ERROR: '--run_in_cores' values must be between 0 "
+                    "and %ld",
+                    n_cores);
+            abort_test(abort_msg);
+          }
+        }
+
+        if (ARG("n_msgs")) {
+          int64_t const n = atoll(optarg);
+          if (!(n > 0)) { abort_test("ERROR: '--n_msgs' value must be > 0"); }
+          config->n_messages = n;
+        }
+
+        if (ARG("n_its")) {
+          int64_t const n = atoll(optarg);
+          if (!(n > 0)) { abort_test("ERROR: '--n_its' value must be > 0"); }
+          config->n_iterations = n;
+        }
+
+        if (ARG("test")) {
+          int64_t n = atoll(optarg);
+          if (!((n > 0) && (n < 3))) {
+            abort_test("ERROR: '--test' value must be either '1' or '2'");
+          }
+          config->test = n;
+        }
+    }
+  }
+
+  if (!config->test || !config->inboxes_sizes || !config->msgs_sizes ||
+      !config->n_messages || !config->n_iterations || !config->run_in_cores) {
+    abort_test("ERROR: missing command line arguments.");
+  }
+
+  if (1 == config->test) {
+    if (config->run_in_cores->data[0] == config->run_in_cores->data[1]) {
+      abort_test(
+          "ERROR: for '--test 1' the values of '--run_in_cores' can not be "
+          "equal because there's no sched_yield())'");
+    }
+  }
+  return config;
+}
+
+static void free_perf_config(perf_config* config) {
+  vector_free(config->inboxes_sizes);
+  vector_free(config->msgs_sizes);
+  vector_free(config->run_in_cores);
+  free(config);
+}
+
+static void print_to_stdout(perf_config const* const config,
+                            stdout_output const      what_to_print) {
+  char const* const i_sz     = "Inbox size";
+  char const* const sep      = " | ";
+  char const* const m_sz     = "Msg size";
+  char const* const time     = "Time (secs)";
+  char const* const m_sec    = "Msgs/second";
+  char const* const prod_hit = "Writer hit ratio";
+  char const* const cons_hit = "Reader hit ratio";
+
+  uint64_t const test_1_sep_len = strlen(i_sz) + strlen(m_sz) + strlen(time) +
+                                  strlen(m_sec) + (3 * strlen(sep));
+
+  uint64_t const test_2_sep_len = strlen(i_sz) + strlen(m_sz) + strlen(time) +
+                                  strlen(m_sec) + strlen(prod_hit) +
+                                  strlen(cons_hit) + (5 * strlen(sep));
+
+  if (HEADER == what_to_print) {
+    if (1 == config->test) {
+      printf("\n%s%s%s%s%s%s%s\n", i_sz, sep, m_sz, sep, time, sep, m_sec);
+
+    } else {
+      printf("\n%s%s%s%s%s%s%s%s%s%s%s\n", i_sz, sep, m_sz, sep, time, sep,
+             m_sec, sep, prod_hit, sep, cons_hit);
+    }
+  }
+  if (1 == config->test) {
+    for (uint64_t k = 0; k != test_1_sep_len; ++k) { fputs("-", stdout); }
+  } else {
+    for (uint64_t k = 0; k != test_2_sep_len; ++k) { fputs("-", stdout); }
+  }
+  puts("");
+}
+
+static int cmp(const void* a, const void* b) {
+  return (*(const double*)a > *(const double*)b)   ? 1
+         : (*(const double*)a < *(const double*)b) ? -1
+                                                   : 0;
+}
+
+static double calculate_median(uint64_t const sz, double* const arr) {
+  qsort(arr, sz, sizeof(double), cmp);
+  return ((sz % 2) == 0) ? ((arr[sz / 2 - 1] + arr[sz / 2]) / 2) : arr[sz / 2];
+}
+
+int main(int argc, char** argv) {
+  /* Seed random generator */
+  srand((uint32_t)time(0));
+
+  perf_config* config = parse_command_line_args(argc, argv);
+
+  print_to_stdout(config, HEADER);
+
+  double* time_secs = calloc((uint64_t)config->n_iterations, sizeof(double));
+  if (NULL == time_secs) {
+    abort_test("ERROR: failed to allocate 'time_secs'");
+  }
+
+  double* writer_hit_ratio =
+      calloc((uint64_t)config->n_iterations, sizeof(double));
+  if (NULL == writer_hit_ratio) {
+    abort_test("ERROR: failed to allocate 'writer_hit_ratio'");
+  }
+
+  double* reader_hit_ratio =
+      calloc((uint64_t)config->n_iterations, sizeof(double));
+  if (NULL == reader_hit_ratio) {
+    abort_test("ERROR: failed to allocate 'reader_hit_ratio'");
+  }
+
+  for (uint64_t k = 0; k != config->inboxes_sizes->used; ++k) {
+    if (config->inboxes_sizes->data[k]) {
+      for (uint64_t j = 0; j != config->msgs_sizes->used; ++j) {
+        if (config->msgs_sizes->data[j]) {
+          for (uint64_t it = 0; it != (uint64_t)config->n_iterations; ++it) {
+            perf_results results =
+                run_test((uint64_t)config->inboxes_sizes->data[k],
+                         (uint64_t)config->msgs_sizes->data[j],
+                         (uint64_t)config->n_messages,
+                         (uint64_t)config->run_in_cores->data[0],
+                         (uint64_t)config->run_in_cores->data[1],
+                         (uint64_t)config->test);
+
+            time_secs[it]        = results.time_secs;
+            writer_hit_ratio[it] = results.writer_hit_ratio;
+            reader_hit_ratio[it] = results.reader_hit_ratio;
+          }
+
+          double const median_secs =
+              calculate_median((uint64_t)config->n_iterations, time_secs);
+
+          printf("%10ld | ", config->inboxes_sizes->data[k]);
+          printf("%8ld | ", config->msgs_sizes->data[j]);
+          median_secs > 1 ? printf("%*.2f | ", 11, median_secs)
+                          : printf("%*.4f | ", 11, median_secs);
+          printf("%*.2fM", 10,
+                 ((double)config->n_messages / median_secs) / 1e6);
+
+          if (2 == config->test) {
+            double const median_writer_hit = calculate_median(
+                (uint64_t)config->n_iterations, writer_hit_ratio);
+
+            double const median_reader_hit = calculate_median(
+                (uint64_t)config->n_iterations, reader_hit_ratio);
+
+            printf(" |%*.2f%% | ", 16, median_writer_hit * 100);
+            printf("%*.2f%%", 15, median_reader_hit * 100);
+          }
+        }
+        puts("");
+      }
+      print_to_stdout(config, SEPARATOR);
+    }
+  }
+
+  puts("");
+  free(time_secs);
+  free(writer_hit_ratio);
+  free(reader_hit_ratio);
+  free_perf_config(config);
+  return EXIT_SUCCESS;
+}
diff --git a/Framework/Foundation/3rdparty/x9/x9.c b/Framework/Foundation/3rdparty/x9/x9.c
new file mode 100644
index 0000000000000..46d3d126de9f4
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/x9.c
@@ -0,0 +1,424 @@
+/*
+   X9 - high performance message passing library.
+   Copyright (c) 2023, Diogo Flores
+
+   BSD 2-Clause License (https://opensource.org/license/bsd-2-clause/)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at: diogoxflores@gmail.com
+*/
+
+#pragma GCC diagnostic ignored "-Wpadded"
+
+#include "x9.h"
+
+#include <assert.h>    /* assert */
+#include <immintrin.h> /* _mm_pause */
+#include <stdarg.h>    /* va_* */
+#include <stdatomic.h> /* atomic_* */
+#include <stdbool.h>   /* bool */
+#include <stdio.h>     /* printf */
+#include <stdlib.h>    /* aligned_alloc, calloc */
+#include <string.h>    /* strcmp */
+#include <string.h>    /* memcpy */
+
+/* CPU cache line size */
+#define X9_CL_SIZE       64
+#define X9_ALIGN_TO_CL() __attribute__((__aligned__(X9_CL_SIZE)))
+
+#ifdef X9_DEBUG
+static void x9_print_error_msg(char const* const error_msg) {
+  printf("X9_ERROR: %s\n", error_msg);
+  fflush(stdout);
+  return;
+}
+#endif
+
+/* --- Internal types --- */
+
+typedef struct {
+  _Atomic(bool) slot_has_data;
+  _Atomic(bool) msg_written;
+  _Atomic(bool) shared;
+  char const    pad[5];
+} x9_msg_header;
+
+typedef struct x9_inbox_internal {
+  _Atomic(uint64_t) read_idx  X9_ALIGN_TO_CL();
+  _Atomic(uint64_t) write_idx X9_ALIGN_TO_CL();
+  uint64_t sz                 X9_ALIGN_TO_CL();
+  uint64_t                    msg_sz;
+  uint64_t                    constant;
+  void*                       msgs;
+  char*                       name;
+  char                        pad[24];
+} x9_inbox;
+
+typedef struct x9_node_internal {
+  x9_inbox** inboxes;
+  uint64_t   n_inboxes;
+  char*      name;
+} x9_node;
+
+/* --- Internal functions --- */
+
+static inline uint64_t x9_load_idx(x9_inbox* const inbox,
+                                   bool const      read_idx) {
+  /* From paper: Faster Remainder by Direct Computation, Lemire et al */
+  register uint64_t const low_bits =
+      inbox->constant *
+      (read_idx ? atomic_load_explicit(&inbox->read_idx, __ATOMIC_RELAXED)
+                : atomic_load_explicit(&inbox->write_idx, __ATOMIC_RELAXED));
+  return ((__uint128_t)low_bits * inbox->sz) >> 64;
+}
+
+static inline uint64_t x9_increment_idx(x9_inbox* const inbox,
+                                        bool const      read_idx) {
+  /* From paper: Faster Remainder by Direct Computation, Lemire et al */
+  register uint64_t const low_bits =
+      inbox->constant *
+      (read_idx
+           ? atomic_fetch_add_explicit(&inbox->read_idx, 1, __ATOMIC_RELAXED)
+           : atomic_fetch_add_explicit(&inbox->write_idx, 1,
+                                       __ATOMIC_RELAXED));
+  return ((__uint128_t)low_bits * inbox->sz) >> 64;
+}
+
+static inline void* x9_header_ptr(x9_inbox const* const inbox,
+                                  uint64_t const        idx) {
+  return &((char*)inbox->msgs)[idx * (inbox->msg_sz + sizeof(x9_msg_header))];
+}
+
+/* --- Public functions --- */
+
+x9_inbox* x9_create_inbox(uint64_t const sz,
+                          char const* restrict const name,
+                          uint64_t const msg_sz) {
+  if (!((sz > 0) && !(sz % 2))) { goto inbox_incorrect_size; }
+
+  x9_inbox* inbox = aligned_alloc(X9_CL_SIZE, sizeof(x9_inbox));
+  if (NULL == inbox) { goto inbox_allocation_failed; }
+  memset(inbox, 0, sizeof(x9_inbox));
+
+  uint64_t const name_len = strlen(name);
+  char*          ibx_name = calloc(name_len + 1, sizeof(char));
+  if (NULL == ibx_name) { goto inbox_name_allocation_failed; }
+  memcpy(ibx_name, name, name_len);
+
+  void* msgs = calloc(sz, msg_sz + sizeof(x9_msg_header));
+  if (NULL == msgs) { goto inbox_msgs_allocation_failed; }
+
+  inbox->constant = UINT64_C(0xFFFFFFFFFFFFFFFF) / sz + 1;
+  inbox->name     = ibx_name;
+  inbox->msgs     = msgs;
+  inbox->sz       = sz;
+  inbox->msg_sz   = msg_sz;
+  return inbox;
+
+inbox_incorrect_size:
+#ifdef X9_DEBUG
+  x9_print_error_msg("INBOX_INCORRECT_SIZE");
+#endif
+  return NULL;
+
+inbox_allocation_failed:
+#ifdef X9_DEBUG
+  x9_print_error_msg("INBOX_ALLOCATION_FAILED");
+#endif
+  return NULL;
+
+inbox_name_allocation_failed:
+#ifdef X9_DEBUG
+  x9_print_error_msg("INBOX_NAME_ALLOCATION_FAILED");
+#endif
+  free(inbox);
+  return NULL;
+
+inbox_msgs_allocation_failed:
+#ifdef X9_DEBUG
+  x9_print_error_msg("INBOX_MSGS_ALLOCATION_FAILED");
+#endif
+  free(ibx_name);
+  free(inbox);
+  return NULL;
+}
+
+bool x9_inbox_is_valid(x9_inbox const* const inbox) {
+  return !(NULL == inbox);
+}
+
+bool x9_inbox_name_is(x9_inbox const* const inbox,
+                      char const* restrict const cmp) {
+  return !strcmp(inbox->name, cmp) ? true : false;
+}
+
+void x9_free_inbox(x9_inbox* const inbox) {
+  free(inbox->msgs);
+  free(inbox->name);
+  free(inbox);
+}
+
+x9_inbox* x9_select_inbox_from_node(x9_node const* const node,
+                                    char const* restrict const name) {
+  for (uint64_t k = 0; k != node->n_inboxes; ++k) {
+    if (x9_inbox_name_is(node->inboxes[k], name)) { return node->inboxes[k]; }
+  }
+#ifdef X9_DEBUG
+  x9_print_error_msg("NODE_DOES_NOT_CONTAIN_INBOX");
+#endif
+  return NULL;
+}
+
+x9_node* x9_create_node(char* restrict const name,
+                        uint64_t const n_inboxes,
+                        ...) {
+  if (!(n_inboxes > 0)) { goto node_incorrect_definition; }
+
+  x9_node* node = calloc(1, sizeof(x9_node));
+  if (NULL == node) { goto node_allocation_failed; }
+
+  uint64_t const name_len  = strlen(name);
+  char*          node_name = calloc(name_len + 1, sizeof(char));
+  if (NULL == node_name) { goto node_name_allocation_failed; }
+  memcpy(node_name, name, name_len);
+  node->name = node_name;
+
+  va_list argp = {0};
+  va_start(argp, n_inboxes);
+
+  x9_inbox** inboxes = calloc(n_inboxes, sizeof(x9_inbox));
+  if (NULL == inboxes) { goto node_inboxes_allocation_failed; }
+  node->inboxes   = inboxes;
+  node->n_inboxes = n_inboxes;
+
+  for (uint64_t k = 0; k != n_inboxes; ++k) {
+    x9_inbox* inbox = va_arg(argp, x9_inbox*);
+    if (k) {
+      for (uint64_t j = 0; j != k; ++j) {
+        if (inbox == node->inboxes[j]) {
+          goto node_multiple_equal_inboxes;
+        } else {
+          node->inboxes[k] = inbox;
+        }
+      }
+    } else {
+      node->inboxes[k] = inbox;
+    }
+  }
+  va_end(argp);
+
+  return node;
+
+node_incorrect_definition:
+#ifdef X9_DEBUG
+  x9_print_error_msg("NODE_INCORRECT_DEFINITION");
+#endif
+  return NULL;
+
+node_allocation_failed:
+#ifdef X9_DEBUG
+  x9_print_error_msg("NODE_ALLOCATION_FAILED");
+#endif
+  return NULL;
+
+node_name_allocation_failed:
+#ifdef X9_DEBUG
+  x9_print_error_msg("NODE_NAME_ALLOCATION_FAILED");
+#endif
+  free(node);
+  return NULL;
+
+node_inboxes_allocation_failed:
+#ifdef X9_DEBUG
+  x9_print_error_msg("NODE_INBOXES_ALLOCATION_FAILED");
+#endif
+  va_end(argp);
+  free(node_name);
+  free(node);
+  return NULL;
+
+node_multiple_equal_inboxes:
+#ifdef X9_DEBUG
+  x9_print_error_msg("NODE_MULTIPLE_EQUAL_INBOXES");
+#endif
+  va_end(argp);
+  free(node_name);
+  free(node->inboxes);
+  free(node);
+  return NULL;
+}
+
+bool x9_node_is_valid(x9_node const* const node) { return !(NULL == node); }
+
+bool x9_node_name_is(x9_node const* const node,
+                     char const* restrict const cmp) {
+  return !strcmp(node->name, cmp) ? true : false;
+}
+
+void x9_free_node(x9_node* const node) {
+  free(node->name);
+  free(node->inboxes);
+  free(node);
+}
+
+void x9_free_node_and_attached_inboxes(x9_node* const node) {
+  for (uint64_t k = 0; k != node->n_inboxes; ++k) {
+    if (NULL != node->inboxes[k]) {
+      x9_free_inbox(node->inboxes[k]);
+      node->inboxes[k] = NULL;
+    }
+  }
+  x9_free_node(node);
+}
+
+bool x9_write_to_inbox(x9_inbox* const inbox,
+                       uint64_t const  msg_sz,
+                       void const* restrict const msg) {
+  bool                          f      = false;
+  register uint64_t const       idx    = x9_load_idx(inbox, false);
+  register x9_msg_header* const header = x9_header_ptr(inbox, idx);
+
+  if (atomic_compare_exchange_strong_explicit(&header->slot_has_data, &f, true,
+                                              __ATOMIC_ACQUIRE,
+                                              __ATOMIC_RELAXED)) {
+    memcpy((char*)header + sizeof(x9_msg_header), msg, msg_sz);
+    atomic_fetch_add_explicit(&inbox->write_idx, 1, __ATOMIC_RELEASE);
+    atomic_store_explicit(&header->msg_written, true, __ATOMIC_RELEASE);
+    return true;
+  }
+  return false;
+}
+
+void x9_write_to_inbox_spin(x9_inbox* const inbox,
+                            uint64_t const  msg_sz,
+                            void const* restrict const msg) {
+  for (;;) {
+    bool                          f      = false;
+    register uint64_t const       idx    = x9_increment_idx(inbox, false);
+    register x9_msg_header* const header = x9_header_ptr(inbox, idx);
+    if (atomic_compare_exchange_weak_explicit(&header->slot_has_data, &f, true,
+                                              __ATOMIC_ACQUIRE,
+                                              __ATOMIC_RELAXED)) {
+      memcpy((char*)header + sizeof(x9_msg_header), msg, msg_sz);
+      atomic_store_explicit(&header->msg_written, true, __ATOMIC_RELEASE);
+      break;
+    }
+  }
+}
+
+void x9_broadcast_msg_to_all_node_inboxes(x9_node const* const node,
+                                          uint64_t const       msg_sz,
+                                          void const* restrict const msg) {
+  for (uint64_t k = 0; k != node->n_inboxes; ++k) {
+    x9_write_to_inbox_spin(node->inboxes[k], msg_sz, msg);
+  }
+}
+
+bool x9_read_from_inbox(x9_inbox* const inbox,
+                        uint64_t const  msg_sz,
+                        void* restrict const outparam) {
+  register uint64_t const       idx    = x9_load_idx(inbox, true);
+  register x9_msg_header* const header = x9_header_ptr(inbox, idx);
+
+  if (atomic_load_explicit(&header->slot_has_data, __ATOMIC_RELAXED)) {
+    if (atomic_load_explicit(&header->msg_written, __ATOMIC_ACQUIRE)) {
+      memcpy(outparam, (char*)header + sizeof(x9_msg_header), msg_sz);
+      atomic_store_explicit(&header->msg_written, false, __ATOMIC_RELAXED);
+      atomic_store_explicit(&header->slot_has_data, false, __ATOMIC_RELEASE);
+      atomic_fetch_add_explicit(&inbox->read_idx, 1, __ATOMIC_RELEASE);
+      return true;
+    }
+  }
+  return false;
+}
+
+void x9_read_from_inbox_spin(x9_inbox* const inbox,
+                             uint64_t const  msg_sz,
+                             void* restrict const outparam) {
+  register uint64_t const       idx    = x9_increment_idx(inbox, true);
+  register x9_msg_header* const header = x9_header_ptr(inbox, idx);
+
+  for (;;) {
+    _mm_pause();
+    if (atomic_load_explicit(&header->slot_has_data, __ATOMIC_RELAXED)) {
+      if (atomic_load_explicit(&header->msg_written, __ATOMIC_ACQUIRE)) {
+        memcpy(outparam, (char*)header + sizeof(x9_msg_header), msg_sz);
+        atomic_store_explicit(&header->msg_written, false, __ATOMIC_RELAXED);
+        atomic_store_explicit(&header->slot_has_data, false, __ATOMIC_RELEASE);
+        return;
+      }
+    }
+  }
+}
+
+bool x9_read_from_shared_inbox(x9_inbox* const inbox,
+                               uint64_t const  msg_sz,
+                               void* restrict const outparam) {
+  bool                          f      = false;
+  register uint64_t const       idx    = x9_load_idx(inbox, true);
+  register x9_msg_header* const header = x9_header_ptr(inbox, idx);
+
+  if (atomic_compare_exchange_strong_explicit(
+          &header->shared, &f, true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+    if (atomic_load_explicit(&header->slot_has_data, __ATOMIC_RELAXED)) {
+      if (atomic_load_explicit(&header->msg_written, __ATOMIC_ACQUIRE)) {
+        memcpy(outparam, (char*)header + sizeof(x9_msg_header), msg_sz);
+        atomic_fetch_add_explicit(&inbox->read_idx, 1, __ATOMIC_RELEASE);
+        atomic_store_explicit(&header->msg_written, false, __ATOMIC_RELAXED);
+        atomic_store_explicit(&header->slot_has_data, false, __ATOMIC_RELEASE);
+        atomic_store_explicit(&header->shared, false, __ATOMIC_RELEASE);
+        return true;
+      }
+    }
+    atomic_store_explicit(&header->shared, false, __ATOMIC_RELEASE);
+  }
+  return false;
+}
+
+void x9_read_from_shared_inbox_spin(x9_inbox* const inbox,
+                                    uint64_t const  msg_sz,
+                                    void* restrict const outparam) {
+  for (;;) {
+    bool                          f      = false;
+    register uint64_t const       idx    = x9_increment_idx(inbox, true);
+    register x9_msg_header* const header = x9_header_ptr(inbox, idx);
+
+    if (atomic_compare_exchange_strong_explicit(
+            &header->shared, &f, true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+      if (atomic_load_explicit(&header->slot_has_data, __ATOMIC_RELAXED)) {
+        if (atomic_load_explicit(&header->msg_written, __ATOMIC_ACQUIRE)) {
+          memcpy(outparam, (char*)header + sizeof(x9_msg_header), msg_sz);
+          atomic_store_explicit(&header->msg_written, false, __ATOMIC_RELAXED);
+          atomic_store_explicit(&header->slot_has_data, false,
+                                __ATOMIC_RELEASE);
+          atomic_store_explicit(&header->shared, false, __ATOMIC_RELEASE);
+          return;
+        }
+      }
+      atomic_store_explicit(&header->shared, false, __ATOMIC_RELEASE);
+    }
+  }
+}
+
diff --git a/Framework/Foundation/3rdparty/x9/x9.h b/Framework/Foundation/3rdparty/x9/x9.h
new file mode 100644
index 0000000000000..bd8717916ddc3
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/x9.h
@@ -0,0 +1,166 @@
+/*
+   X9 - high performance message passing library.
+   Copyright (c) 2023, Diogo Flores
+
+   BSD 2-Clause License (https://opensource.org/license/bsd-2-clause/)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at: diogoxflores@gmail.com
+*/
+
+#pragma once
+
+#include <stdbool.h> /* bool */
+#include <stdint.h>  /* uint64_t */
+
+/* --- Opaque types --- */
+
+typedef struct x9_node_internal  x9_node;
+typedef struct x9_inbox_internal x9_inbox;
+
+/* --- Public API --- */
+
+/* Creates a x9_inbox with a buffer of size 'sz', which must be positive and
+ * mod 2 == 0, a 'name', which can be used for comparison by calling
+ * 'x9_inbox_name_is' or for selecting a specific x9_inbox from a x9_node by
+ * calling 'x9_select_inbox_from_node', and a 'msg_sz' which the inbox is
+ * expected to receive.
+ *
+ * Example:
+ *   x9_inbox* inbox = x9_create_inbox(512, "ibx", sizeof(<some struct>));*/
+__attribute__((nonnull)) x9_inbox* x9_create_inbox(
+    uint64_t const sz, char const* restrict const name, uint64_t const msg_sz);
+
+/* Variadic function that creates a 'x9_node', which is an abstraction that
+ * unifies x9_inbox(es).
+ * 'name' can be used for comparison by calling 'x9_node_name_is',
+ * 'n_inboxes' must be > 0 and equal the number of inboxes passed to the
+ * function to be attached to the x9_node.
+ *
+ * Example:
+ *   x9_node* node = x9_create_node("my_node", 3, inbox_1, inbox_2, inbox_3);*/
+__attribute__((nonnull)) x9_node* x9_create_node(char* restrict const name,
+                                                 uint64_t const n_inboxes,
+                                                 ...);
+
+/* Returns 'true' if the 'inbox' is valid, 'false' otherwise.
+ * Should always be called after 'x9_create_inbox' to validate the correct
+ * initialization of the underlying data structure, and after calling
+ * 'x9_select_inbox_from_node' to ensure that the user selected a valid inbox.
+ * If the return value is 'false', and it's unclear why it is so, the user can
+ * enable the 'X9_DEBUG' macro at compile time for precise error information.*/
+bool x9_inbox_is_valid(x9_inbox const* const inbox);
+
+/* Returns 'true' if the 'node' is valid, 'false' otherwise.
+ * Should always be called after 'x9_create_node' to validate the correct
+ * initialization of the underlying data structure.
+ * If the return value is 'false', and it's unclear why it is so, the user can
+ * enable the 'X9_DEBUG' macro at compile time for precise error information.*/
+bool x9_node_is_valid(x9_node const* const node);
+
+/* Selects a x9_inbox from a 'node' by its 'name'.
+ * Returns NULL if the 'node' does not contain an x9_inbox with such 'name'.*/
+__attribute__((nonnull)) x9_inbox* x9_select_inbox_from_node(
+    x9_node const* const node, char const* restrict const name);
+
+/* Returns 'true' if the 'inbox' name == 'cmp', 'false' otherwise.*/
+__attribute__((nonnull)) bool x9_inbox_name_is(x9_inbox const* const inbox,
+                                               char const* restrict const cmp);
+
+/* Returns 'true' if the 'node' name == 'cmp', 'false' otherwise.*/
+__attribute__((nonnull)) bool x9_node_name_is(x9_node const* const node,
+                                              char const* restrict const cmp);
+
+/* Frees the 'inbox' data structure and its internal components. */
+__attribute__((nonnull)) void x9_free_inbox(x9_inbox* const inbox);
+
+/* Frees the 'node' data structure and its internal components. */
+__attribute__((nonnull)) void x9_free_node(x9_node* const node);
+
+/* Frees the 'node' and all the x9_inbox(es) attached to it.
+ * IMPORTANT: should only be used when the attached x9_inbox(es) are not shared
+ * with other nodes.*/
+__attribute__((nonnull)) void x9_free_node_and_attached_inboxes(
+    x9_node* const node);
+
+/* Returns 'true' if a message was read, 'false' otherwise.
+ * If 'true', the message will be written to 'outparam'.
+ * IMPORTANT: Can only be used to read from inboxes where the thread calling
+ * this function is the only thread reading from said 'inbox'.*/
+__attribute__((nonnull)) bool x9_read_from_inbox(
+    x9_inbox* const inbox,
+    uint64_t const  msg_sz,
+    void* restrict const outparam);
+
+/* Reads the next unread message in the 'inbox' to 'outparam'.
+ * Uses spinning, that is, it wil not return until it has read a message, and
+ * it will keep checking if a message was written and try to read it. */
+__attribute__((nonnull)) void x9_read_from_inbox_spin(
+    x9_inbox* const inbox,
+    uint64_t const  msg_sz,
+    void* restrict const outparam);
+
+/* Returns 'true' if a message was read, 'false' otherwise.
+ * If 'true', the msg contents will be written to the 'outparam'.
+ * Use this function when multiple threads read from the same inbox. */
+__attribute__((nonnull)) bool x9_read_from_shared_inbox(
+    x9_inbox* const inbox,
+    uint64_t const  msg_sz,
+    void* restrict const outparam);
+
+/* Reads the next unread message in the 'inbox' to 'outparam'.
+ * Use this function when multiple threads read from the same inbox.
+ * Uses spinning, that is, it wil not return until it has read a message, and
+ * it will keep checking if a message was written and try to read it. */
+__attribute__((nonnull)) void x9_read_from_shared_inbox_spin(
+    x9_inbox* const inbox,
+    uint64_t const  msg_sz,
+    void* restrict const outparam);
+
+/* Returns 'true' if the message was written to the 'inbox', 'false'
+ * otherwise. */
+__attribute__((nonnull)) bool x9_write_to_inbox(
+    x9_inbox* const inbox,
+    uint64_t const  msg_sz,
+    void const* restrict const msg);
+
+/* Writes the 'msg' to the 'inbox'.
+ * Uses spinning, that is, it wil not not return until it has written the
+ * 'msg', and it will keep checking if the destination inbox has a free
+ * slot that it can write the 'msg' to. */
+__attribute__((nonnull)) void x9_write_to_inbox_spin(
+    x9_inbox* const inbox,
+    uint64_t const  msg_sz,
+    void const* restrict const msg);
+
+/* Writes the same 'msg' to all 'node' inboxes.
+ * Calls 'x9_write_to_inbox_spin' in the background.
+ * Users must guarantee that all 'node' inboxes accept messages of the
+ * same type (or at least of the same 'msg_sz') */
+__attribute__((nonnull)) void x9_broadcast_msg_to_all_node_inboxes(
+    x9_node const* const node,
+    uint64_t const       msg_sz,
+    void const* restrict const msg);
+

From 6ee74d1cb20471aaacfc9adbdadb69d61d7abf1b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 19 Sep 2024 13:05:24 +0200
Subject: [PATCH 631/726] Platform independent yielding

---
 Framework/Foundation/3rdparty/x9/x9.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/Framework/Foundation/3rdparty/x9/x9.c b/Framework/Foundation/3rdparty/x9/x9.c
index 46d3d126de9f4..2ca4bb80237b3 100644
--- a/Framework/Foundation/3rdparty/x9/x9.c
+++ b/Framework/Foundation/3rdparty/x9/x9.c
@@ -35,7 +35,12 @@
 #include "x9.h"
 
 #include <assert.h>    /* assert */
+#if defined(__x86_64__) || defined(__i386__)
 #include <immintrin.h> /* _mm_pause */
+#elif defined(__aarch64__)
+#else
+#error Not supported architecture
+#endif
 #include <stdarg.h>    /* va_* */
 #include <stdatomic.h> /* atomic_* */
 #include <stdbool.h>   /* bool */
@@ -361,7 +366,13 @@ void x9_read_from_inbox_spin(x9_inbox* const inbox,
   register x9_msg_header* const header = x9_header_ptr(inbox, idx);
 
   for (;;) {
+#if defined(__x86_64__) || defined(__i386__)
     _mm_pause();
+#elif defined(__aarch64__)
+    __asm__ __volatile__ ("yield");
+#else
+#error Not supported architecture
+#endif
     if (atomic_load_explicit(&header->slot_has_data, __ATOMIC_RELAXED)) {
       if (atomic_load_explicit(&header->msg_written, __ATOMIC_ACQUIRE)) {
         memcpy(outparam, (char*)header + sizeof(x9_msg_header), msg_sz);

From c5855d83a756e88c5f225a8176d5168c9971c628 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 19 Sep 2024 13:05:24 +0200
Subject: [PATCH 632/726] Support using X9 from C++

---
 Framework/Foundation/3rdparty/x9/x9.h | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Framework/Foundation/3rdparty/x9/x9.h b/Framework/Foundation/3rdparty/x9/x9.h
index bd8717916ddc3..229d7adf7dba0 100644
--- a/Framework/Foundation/3rdparty/x9/x9.h
+++ b/Framework/Foundation/3rdparty/x9/x9.h
@@ -32,6 +32,11 @@
 
 #pragma once
 
+#ifdef __cplusplus
+extern "C" {
+#define restrict __restrict__
+#endif
+
 #include <stdbool.h> /* bool */
 #include <stdint.h>  /* uint64_t */
 
@@ -164,3 +169,6 @@ __attribute__((nonnull)) void x9_broadcast_msg_to_all_node_inboxes(
     uint64_t const       msg_sz,
     void const* restrict const msg);
 
+#ifdef __cplusplus
+}
+#endif

From f9065464d27810c50fc8b4feb4b9c52b44b22e49 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 19 Sep 2024 13:05:24 +0200
Subject: [PATCH 633/726] DPL: Use X9 to make AsyncQueue atomic

---
 Framework/Core/CMakeLists.txt                 |  1 +
 Framework/Core/include/Framework/AsyncQueue.h | 11 +++++++
 Framework/Core/src/AsyncQueue.cxx             | 33 ++++++++++++++++++-
 3 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index 8c2d9650776c0..c01ef534eb212 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -163,6 +163,7 @@ o2_add_library(Framework
                                      O2::Headers
                                      O2::MemoryResources
                                      O2::PCG
+                                     O2::X9
                                      RapidJSON::RapidJSON
                                      Arrow::arrow_shared
                                      Microsoft.GSL::GSL
diff --git a/Framework/Core/include/Framework/AsyncQueue.h b/Framework/Core/include/Framework/AsyncQueue.h
index d214b9bfc09ba..ae9df0f79ef25 100644
--- a/Framework/Core/include/Framework/AsyncQueue.h
+++ b/Framework/Core/include/Framework/AsyncQueue.h
@@ -15,6 +15,9 @@
 #include <string>
 #include <vector>
 
+typedef struct x9_inbox_internal x9_inbox;
+typedef struct x9_node_internal x9_node;
+
 namespace o2::framework
 {
 
@@ -89,6 +92,12 @@ struct AsyncQueue {
   std::vector<AsyncTaskSpec> prototypes;
   std::vector<AsyncTask> tasks;
   size_t iteration = 0;
+
+  std::atomic<bool> first = true;
+
+  // Inbox for the message queue used to append
+  // tasks to this queue.
+  x9_inbox* inbox = nullptr;
   AsyncQueue();
 };
 
@@ -104,6 +113,8 @@ struct AsyncQueueHelpers {
   /// 3. only execute the highest (timeslice, debounce) value
   static void run(AsyncQueue& queue, TimesliceId oldestPossibleTimeslice);
 
+  // Flush tasks which were posted but not yet committed to the queue
+  static void flushPending(AsyncQueue& queue);
   /// Reset the queue to its initial state
   static void reset(AsyncQueue& queue);
 };
diff --git a/Framework/Core/src/AsyncQueue.cxx b/Framework/Core/src/AsyncQueue.cxx
index e0e44da9cd19c..c9be88f096c0d 100644
--- a/Framework/Core/src/AsyncQueue.cxx
+++ b/Framework/Core/src/AsyncQueue.cxx
@@ -11,6 +11,7 @@
 
 #include "Framework/AsyncQueue.h"
 #include "Framework/Signpost.h"
+#include "x9.h"
 #include <numeric>
 
 O2_DECLARE_DYNAMIC_LOG(async_queue);
@@ -18,7 +19,9 @@ O2_DECLARE_DYNAMIC_LOG(async_queue);
 namespace o2::framework
 {
 AsyncQueue::AsyncQueue()
+  : inbox(x9_create_inbox(16, "async_queue", sizeof(AsyncTask)))
 {
+  this->inbox = x9_create_inbox(16, "async_queue", sizeof(AsyncTask));
 }
 
 auto AsyncQueueHelpers::create(AsyncQueue& queue, AsyncTaskSpec spec) -> AsyncTaskId
@@ -31,11 +34,39 @@ auto AsyncQueueHelpers::create(AsyncQueue& queue, AsyncTaskSpec spec) -> AsyncTa
 
 auto AsyncQueueHelpers::post(AsyncQueue& queue, AsyncTask const& task) -> void
 {
-  queue.tasks.push_back(task);
+  // Until we do not manage to write to the inbox, keep removing
+  // items from the queue if you are the first one which fails to
+  // write.
+  while (!x9_write_to_inbox(queue.inbox, sizeof(AsyncTask), &task)) {
+    AsyncQueueHelpers::flushPending(queue);
+  }
+}
+
+auto AsyncQueueHelpers::flushPending(AsyncQueue& queue) -> void
+{
+  bool isFirst = true;
+  if (!std::atomic_compare_exchange_strong(&queue.first, &isFirst, false)) {
+    // Not the first, try again.
+    return;
+  }
+  // First thread which does not manage to write to the queue.
+  // Flush it a bit before we try again.
+  AsyncTask toFlush;
+  // This potentially stalls if the inserting tasks are faster to insert
+  // than we are to retrieve. We should probably have a cut-off
+  while (x9_read_from_inbox(queue.inbox, sizeof(AsyncTask), &toFlush)) {
+    queue.tasks.push_back(toFlush);
+  }
+  queue.first = true;
 }
 
 auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> void
 {
+  // We synchronize right before we run to get as many
+  // tasks as possible. Notice we might still miss some
+  // which will have to handled on a subsequent iteration.
+  AsyncQueueHelpers::flushPending(queue);
+
   if (queue.tasks.empty()) {
     return;
   }

From ba66fde5050e304427b61bfa1c98518111447474 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 19 Sep 2024 13:05:25 +0200
Subject: [PATCH 634/726] DPL: Use X9 to make AsyncQueue atomic

---
 Framework/Core/CMakeLists.txt                 |  1 +
 Framework/Core/include/Framework/AsyncQueue.h | 12 +++++++
 Framework/Core/src/AsyncQueue.cxx             | 33 ++++++++++++++++++-
 3 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index 8c2d9650776c0..c01ef534eb212 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -163,6 +163,7 @@ o2_add_library(Framework
                                      O2::Headers
                                      O2::MemoryResources
                                      O2::PCG
+                                     O2::X9
                                      RapidJSON::RapidJSON
                                      Arrow::arrow_shared
                                      Microsoft.GSL::GSL
diff --git a/Framework/Core/include/Framework/AsyncQueue.h b/Framework/Core/include/Framework/AsyncQueue.h
index d214b9bfc09ba..0f25410179c1f 100644
--- a/Framework/Core/include/Framework/AsyncQueue.h
+++ b/Framework/Core/include/Framework/AsyncQueue.h
@@ -14,6 +14,10 @@
 #include "Framework/TimesliceSlot.h"
 #include <string>
 #include <vector>
+#include <atomic>
+
+typedef struct x9_inbox_internal x9_inbox;
+typedef struct x9_node_internal x9_node;
 
 namespace o2::framework
 {
@@ -89,6 +93,12 @@ struct AsyncQueue {
   std::vector<AsyncTaskSpec> prototypes;
   std::vector<AsyncTask> tasks;
   size_t iteration = 0;
+
+  std::atomic<bool> first = true;
+
+  // Inbox for the message queue used to append
+  // tasks to this queue.
+  x9_inbox* inbox = nullptr;
   AsyncQueue();
 };
 
@@ -104,6 +114,8 @@ struct AsyncQueueHelpers {
   /// 3. only execute the highest (timeslice, debounce) value
   static void run(AsyncQueue& queue, TimesliceId oldestPossibleTimeslice);
 
+  // Flush tasks which were posted but not yet committed to the queue
+  static void flushPending(AsyncQueue& queue);
   /// Reset the queue to its initial state
   static void reset(AsyncQueue& queue);
 };
diff --git a/Framework/Core/src/AsyncQueue.cxx b/Framework/Core/src/AsyncQueue.cxx
index e0e44da9cd19c..c9be88f096c0d 100644
--- a/Framework/Core/src/AsyncQueue.cxx
+++ b/Framework/Core/src/AsyncQueue.cxx
@@ -11,6 +11,7 @@
 
 #include "Framework/AsyncQueue.h"
 #include "Framework/Signpost.h"
+#include "x9.h"
 #include <numeric>
 
 O2_DECLARE_DYNAMIC_LOG(async_queue);
@@ -18,7 +19,9 @@ O2_DECLARE_DYNAMIC_LOG(async_queue);
 namespace o2::framework
 {
 AsyncQueue::AsyncQueue()
+  : inbox(x9_create_inbox(16, "async_queue", sizeof(AsyncTask)))
 {
+  this->inbox = x9_create_inbox(16, "async_queue", sizeof(AsyncTask));
 }
 
 auto AsyncQueueHelpers::create(AsyncQueue& queue, AsyncTaskSpec spec) -> AsyncTaskId
@@ -31,11 +34,39 @@ auto AsyncQueueHelpers::create(AsyncQueue& queue, AsyncTaskSpec spec) -> AsyncTa
 
 auto AsyncQueueHelpers::post(AsyncQueue& queue, AsyncTask const& task) -> void
 {
-  queue.tasks.push_back(task);
+  // Until we do not manage to write to the inbox, keep removing
+  // items from the queue if you are the first one which fails to
+  // write.
+  while (!x9_write_to_inbox(queue.inbox, sizeof(AsyncTask), &task)) {
+    AsyncQueueHelpers::flushPending(queue);
+  }
+}
+
+auto AsyncQueueHelpers::flushPending(AsyncQueue& queue) -> void
+{
+  bool isFirst = true;
+  if (!std::atomic_compare_exchange_strong(&queue.first, &isFirst, false)) {
+    // Not the first, try again.
+    return;
+  }
+  // First thread which does not manage to write to the queue.
+  // Flush it a bit before we try again.
+  AsyncTask toFlush;
+  // This potentially stalls if the inserting tasks are faster to insert
+  // than we are to retrieve. We should probably have a cut-off
+  while (x9_read_from_inbox(queue.inbox, sizeof(AsyncTask), &toFlush)) {
+    queue.tasks.push_back(toFlush);
+  }
+  queue.first = true;
 }
 
 auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> void
 {
+  // We synchronize right before we run to get as many
+  // tasks as possible. Notice we might still miss some
+  // which will have to handled on a subsequent iteration.
+  AsyncQueueHelpers::flushPending(queue);
+
   if (queue.tasks.empty()) {
     return;
   }

From 1dacd31b09a51d3a67413277f9007ec04fdf3ebc Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 19 Sep 2024 14:21:51 +0200
Subject: [PATCH 635/726] DPL: AsyncQueue without std::function

---
 Framework/Core/include/Framework/AsyncQueue.h |  74 ++++++++---
 Framework/Core/src/AsyncQueue.cxx             |  17 +--
 Framework/Core/src/CommonServices.cxx         | 118 +++++++++++-------
 Framework/Core/src/DataProcessingDevice.cxx   |  62 +++++----
 Framework/Core/test/test_AsyncQueue.cxx       |  91 +++++++++-----
 5 files changed, 236 insertions(+), 126 deletions(-)

diff --git a/Framework/Core/include/Framework/AsyncQueue.h b/Framework/Core/include/Framework/AsyncQueue.h
index 6945c2a5ab1da..d214b9bfc09ba 100644
--- a/Framework/Core/include/Framework/AsyncQueue.h
+++ b/Framework/Core/include/Framework/AsyncQueue.h
@@ -12,49 +12,91 @@
 #define O2_FRAMEWORK_ASYNCQUUE_H_
 
 #include "Framework/TimesliceSlot.h"
-#include <functional>
 #include <string>
 #include <vector>
 
 namespace o2::framework
 {
 
-struct AsyncTaskSpec {
-  std::string name;
-  // Its priority compared to the other tasks
-  int score = 0;
-};
-
 /// The position of the TaskSpec in the prototypes
+/// Up to 127 different kind of tasks are allowed per queue
 struct AsyncTaskId {
-  int value = -1;
+  int16_t value = -1;
 };
 
 /// An actuatual task to be executed
 struct AsyncTask {
-  // The task to be executed. Id can be used as unique
-  // id for the signpost in the async_queue stream.
-  std::function<void(size_t id)> task;
-  // The associated task spec
-  AsyncTaskId id = {-1};
+  // The timeslice after which this callback should be executed.
   TimesliceId timeslice = {TimesliceId::INVALID};
   // Only the task with the highest debounce value will be executed
-  int debounce = 0;
+  // The associated task spec. Notice that the context
+  // is stored separately so that we do not need to
+  // manage lifetimes of std::functions
+  AsyncTaskId id = {-1};
+  // Debounce value.
+  int8_t debounce = 0;
   bool runnable = false;
+  // Some unuser integer
+  int32_t unused = 0;
+  // The callback to be executed. id can be used as unique
+  // id for the signpost in the async_queue stream.
+  // Context is provided by the userdata attached to the
+  // task the moment we post it.
+  // Notice that we do not store the task in the prototype
+  // because we do support at the moment coalescing two
+  // different callbaks with the same id.
+  void (*callback)(AsyncTask& task, size_t id);
+  // Some user data e.g. to decode what comes next
+  // This can either be used via the .data pointer
+  // or by asking a cast to the appropriate type via
+  // the user() method.
+  void* data[5] = {nullptr, nullptr, nullptr, nullptr, nullptr};
+
+  // Helper to return userdata
+  template <typename T>
+  T& user()
+  {
+    static_assert(sizeof(T) <= 5 * sizeof(void*), "User object does not fit user data");
+    return *(T*)data;
+  }
+
+  // Helper to set userdata
+  // @return a reference to this task modified. This is meant to be used like:
+  //
+  // AsyncQueueHelpers::post(queue, AsyncTask{.id = myTask}.user(Context{.contextValue}));
+  //
+  // Coupled with the other one:
+  //
+  // task.user<Context>().contextValue;
+  //
+  // it can be used to mimick capturing lambdas
+  template <typename T>
+  AsyncTask& user(T&& value)
+  {
+    static_assert(sizeof(T) <= 5 * sizeof(void*), "User object does not fit user data");
+    new (&data[0])(T){value};
+    return *this;
+  }
+};
+
+struct AsyncTaskSpec {
+  std::string name;
+  // Its priority compared to the other tasks
+  int score = 0;
 };
 
 struct AsyncQueue {
   std::vector<AsyncTaskSpec> prototypes;
   std::vector<AsyncTask> tasks;
   size_t iteration = 0;
+  AsyncQueue();
 };
 
 struct AsyncQueueHelpers {
-  using AsyncCallback = std::function<void(size_t)>;
   static AsyncTaskId create(AsyncQueue& queue, AsyncTaskSpec spec);
   // Schedule a task with @a taskId to be executed whenever the timeslice
   // is past timeslice. If debounce is provided, only execute the task
-  static void post(AsyncQueue& queue, AsyncTaskId taskId, AsyncCallback task, TimesliceId timeslice, int64_t debounce = 0);
+  static void post(AsyncQueue& queue, AsyncTask const& task);
   /// Run all the tasks which are older than the oldestPossible timeslice
   /// executing them by:
   /// 1. sorting the tasks by timeslice
diff --git a/Framework/Core/src/AsyncQueue.cxx b/Framework/Core/src/AsyncQueue.cxx
index 36c0c17009f82..e0e44da9cd19c 100644
--- a/Framework/Core/src/AsyncQueue.cxx
+++ b/Framework/Core/src/AsyncQueue.cxx
@@ -17,6 +17,10 @@ O2_DECLARE_DYNAMIC_LOG(async_queue);
 
 namespace o2::framework
 {
+AsyncQueue::AsyncQueue()
+{
+}
+
 auto AsyncQueueHelpers::create(AsyncQueue& queue, AsyncTaskSpec spec) -> AsyncTaskId
 {
   AsyncTaskId id;
@@ -25,14 +29,9 @@ auto AsyncQueueHelpers::create(AsyncQueue& queue, AsyncTaskSpec spec) -> AsyncTa
   return id;
 }
 
-auto AsyncQueueHelpers::post(AsyncQueue& queue, AsyncTaskId id, AsyncCallback task, TimesliceId timeslice, int64_t debounce) -> void
+auto AsyncQueueHelpers::post(AsyncQueue& queue, AsyncTask const& task) -> void
 {
-  AsyncTask taskToPost;
-  taskToPost.task = task;
-  taskToPost.id = id;
-  taskToPost.timeslice = timeslice;
-  taskToPost.debounce = debounce;
-  queue.tasks.push_back(taskToPost);
+  queue.tasks.push_back(task);
 }
 
 auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> void
@@ -85,6 +84,8 @@ auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> vo
     }
   }
   // Keep only the tasks with the highest debounce value for a given id
+  // For this reason I need to keep the callback in the task itself, because
+  // two different callbacks with the same id will be coalesced.
   auto newEnd = std::unique(order.begin(), order.end(), [&queue](int a, int b) {
     return queue.tasks[a].runnable == queue.tasks[b].runnable && queue.tasks[a].id.value == queue.tasks[b].id.value && queue.tasks[a].debounce >= 0 && queue.tasks[b].debounce >= 0;
   });
@@ -111,7 +112,7 @@ auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> vo
       O2_SIGNPOST_EVENT_EMIT(async_queue, opid, "run", "Running task %{public}s (%d) for timeslice %zu",
                              queue.prototypes[queue.tasks[i].id.value].name.c_str(), i,
                              queue.tasks[i].timeslice.value);
-      queue.tasks[i].task(opid.value);
+      queue.tasks[i].callback(queue.tasks[i], opid.value);
       O2_SIGNPOST_EVENT_EMIT(async_queue, opid, "run", "Done running %d", i);
     }
   }
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 13b02c74839b9..9192cbd0c965d 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -529,12 +529,72 @@ o2::framework::ServiceSpec CommonServices::ccdbSupportSpec()
       } },
     .kind = ServiceKind::Global};
 }
+struct DecongestionContext {
+  ServiceRegistryRef ref;
+  TimesliceIndex::OldestOutputInfo oldestPossibleOutput;
+};
+
+auto decongestionCallback = [](AsyncTask& task, size_t id) -> void {
+  auto& oldestPossibleOutput = task.user<DecongestionContext>().oldestPossibleOutput;
+  auto& ref = task.user<DecongestionContext>().ref;
+
+  auto& decongestion = ref.get<DecongestionService>();
+  auto& proxy = ref.get<FairMQDeviceProxy>();
+
+  O2_SIGNPOST_ID_GENERATE(cid, async_queue);
+  cid.value = id;
+  if (decongestion.lastTimeslice >= oldestPossibleOutput.timeslice.value) {
+    O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice", "Not sending already sent value: %" PRIu64 "> %" PRIu64,
+                           decongestion.lastTimeslice, (uint64_t)oldestPossibleOutput.timeslice.value);
+    return;
+  }
+  O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice", "Running oldest possible timeslice %" PRIu64 " propagation.",
+                         (uint64_t)oldestPossibleOutput.timeslice.value);
+  DataProcessingHelpers::broadcastOldestPossibleTimeslice(ref, oldestPossibleOutput.timeslice.value);
+
+  for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
+    auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
+    auto& state = proxy.getForwardChannelState(ChannelIndex{fi});
+    // TODO: this we could cache in the proxy at the bind moment.
+    if (info.channelType != ChannelAccountingType::DPL) {
+      O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
+      continue;
+    }
+    if (DataProcessingHelpers::sendOldestPossibleTimeframe(ref, info, state, oldestPossibleOutput.timeslice.value)) {
+      O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice",
+                             "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
+                             info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
+    }
+  }
+  decongestion.lastTimeslice = oldestPossibleOutput.timeslice.value;
+};
+
+auto decongestionCallbackOrdered = [](AsyncTask& task, size_t id) -> void {
+  auto& oldestPossibleOutput = task.user<DecongestionContext>().oldestPossibleOutput;
+  auto& ref = task.user<DecongestionContext>().ref;
+
+  auto& decongestion = ref.get<DecongestionService>();
+  auto& state = ref.get<DeviceState>();
+  auto& timesliceIndex = ref.get<TimesliceIndex>();
+  O2_SIGNPOST_ID_GENERATE(cid, async_queue);
+  int64_t oldNextTimeslice = decongestion.nextTimeslice;
+  decongestion.nextTimeslice = std::max(decongestion.nextTimeslice, (int64_t)oldestPossibleOutput.timeslice.value);
+  if (oldNextTimeslice != decongestion.nextTimeslice) {
+    if (state.transitionHandling != TransitionHandlingState::NoTransition && DefaultsHelpers::onlineDeploymentMode()) {
+      O2_SIGNPOST_EVENT_EMIT_WARN(async_queue, cid, "oldest_possible_timeslice", "Stop transition requested. Some Lifetime::Timeframe data got dropped starting at %" PRIi64, oldNextTimeslice);
+    } else {
+      O2_SIGNPOST_EVENT_EMIT_ERROR(async_queue, cid, "oldest_possible_timeslice", "Some Lifetime::Timeframe data got dropped starting at %" PRIi64, oldNextTimeslice);
+    }
+    timesliceIndex.rescan();
+  }
+};
 
 // Decongestion service
 // If we do not have any Timeframe input, it means we must be creating timeslices
 // in order and that we should propagate the oldest possible timeslice at the end
 // of each processing step.
-o2::framework::ServiceSpec CommonServices::decongestionSpec()
+o2::framework::ServiceSpec
+  CommonServices::decongestionSpec()
 {
   return ServiceSpec{
     .name = "decongestion",
@@ -548,7 +608,7 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
         }
       }
       auto& queue = services.get<AsyncQueue>();
-      decongestion->oldestPossibleTimesliceTask = AsyncQueueHelpers::create(queue, {"oldest-possible-timeslice", 100});
+      decongestion->oldestPossibleTimesliceTask = AsyncQueueHelpers::create(queue, {.name = "oldest-possible-timeslice", .score = 100});
       return ServiceHandle{TypeIdHelpers::uniqueId<DecongestionService>(), decongestion, ServiceKind::Serial};
     },
     .postForwarding = [](ProcessingContext& ctx, void* service) {
@@ -632,7 +692,6 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
       auto& decongestion = services.get<DecongestionService>();
       auto& relayer = services.get<DataRelayer>();
       auto& timesliceIndex = services.get<TimesliceIndex>();
-      auto& proxy = services.get<FairMQDeviceProxy>();
       O2_SIGNPOST_ID_FROM_POINTER(cid, data_processor_context, &decongestion);
       O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Received oldest possible timeframe %" PRIu64 " from channel %d",
                              (uint64_t)oldestPossibleTimeslice, channel.value);
@@ -650,59 +709,22 @@ o2::framework::ServiceSpec CommonServices::decongestionSpec()
         return;
       }
       auto& queue = services.get<AsyncQueue>();
-      const auto& spec = services.get<DeviceSpec const>();
       const auto& state = services.get<DeviceState>();
-      auto *device = services.get<RawDeviceService>().device();
       /// We use the oldest possible timeslice to debounce, so that only the latest one
       /// at the end of one iteration is sent.
       O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Queueing oldest possible timeslice %" PRIu64 " propagation for execution.",
                              (uint64_t)oldestPossibleOutput.timeslice.value);
       AsyncQueueHelpers::post(
-        queue, decongestion.oldestPossibleTimesliceTask, [ref = services, oldestPossibleOutput, &decongestion, &proxy, &spec, device, &timesliceIndex](size_t id) {
-          O2_SIGNPOST_ID_GENERATE(cid, async_queue);
-          cid.value = id;
-          if (decongestion.lastTimeslice >= oldestPossibleOutput.timeslice.value) {
-            O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice", "Not sending already sent value: %" PRIu64 "> %" PRIu64,
-                decongestion.lastTimeslice, (uint64_t)oldestPossibleOutput.timeslice.value);
-            return;
-          }
-          O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice", "Running oldest possible timeslice %" PRIu64 " propagation.",
-                                 (uint64_t)oldestPossibleOutput.timeslice.value);
-          DataProcessingHelpers::broadcastOldestPossibleTimeslice(ref, oldestPossibleOutput.timeslice.value);
-
-          for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
-            auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
-            auto& state = proxy.getForwardChannelState(ChannelIndex{fi});
-            // TODO: this we could cache in the proxy at the bind moment.
-            if (info.channelType != ChannelAccountingType::DPL) {
-              O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice", "Skipping channel %{public}s", info.name.c_str());
-              continue;
-            }
-            if (DataProcessingHelpers::sendOldestPossibleTimeframe(ref, info, state, oldestPossibleOutput.timeslice.value)) {
-              O2_SIGNPOST_EVENT_EMIT(async_queue, cid, "oldest_possible_timeslice",
-                                     "Forwarding to channel %{public}s oldest possible timeslice %" PRIu64 ", priority %d",
-                                     info.name.c_str(), (uint64_t)oldestPossibleOutput.timeslice.value, 20);
-            }
-          }
-          decongestion.lastTimeslice = oldestPossibleOutput.timeslice.value;
-        },
-        TimesliceId{oldestPossibleTimeslice}, -1);
+        queue, AsyncTask{ .timeslice = TimesliceId{oldestPossibleTimeslice}, 
+                .id = decongestion.oldestPossibleTimesliceTask, 
+                .debounce = -1, .callback = decongestionCallback}
+            .user<DecongestionContext>(DecongestionContext{.ref = services, .oldestPossibleOutput = oldestPossibleOutput}));
+
       if (decongestion.orderedCompletionPolicyActive) {
         AsyncQueueHelpers::post(
-          queue, decongestion.oldestPossibleTimesliceTask, [ref = services, oldestPossibleOutput, &decongestion, &proxy, &spec, &state, device, &timesliceIndex](size_t id) {
-            O2_SIGNPOST_ID_GENERATE(cid, async_queue);
-            int64_t oldNextTimeslice = decongestion.nextTimeslice;
-            decongestion.nextTimeslice = std::max(decongestion.nextTimeslice, (int64_t)oldestPossibleOutput.timeslice.value);
-            if (oldNextTimeslice != decongestion.nextTimeslice) {
-              if (state.transitionHandling != TransitionHandlingState::NoTransition && DefaultsHelpers::onlineDeploymentMode()) {
-                O2_SIGNPOST_EVENT_EMIT_WARN(async_queue, cid, "oldest_possible_timeslice", "Stop transition requested. Some Lifetime::Timeframe data got dropped starting at %" PRIi64, oldNextTimeslice);
-              } else {
-                O2_SIGNPOST_EVENT_EMIT_ERROR(async_queue, cid, "oldest_possible_timeslice", "Some Lifetime::Timeframe data got dropped starting at %" PRIi64, oldNextTimeslice);
-              }
-              timesliceIndex.rescan();
-            }
-          },
-          TimesliceId{oldestPossibleOutput.timeslice.value}, -1);
+          queue, AsyncTask{.timeslice = TimesliceId{oldestPossibleOutput.timeslice.value},.id = decongestion.oldestPossibleTimesliceTask,  .debounce = -1, 
+          .callback = decongestionCallbackOrdered}
+          .user<DecongestionContext>({.ref = services, .oldestPossibleOutput = oldestPossibleOutput}));
       } },
     .kind = ServiceKind::Serial};
 }
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 1c46ee9ae0ac6..f3fe328e78a06 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -636,6 +636,39 @@ static auto toBeforwardedMessageSet = [](std::vector<ChannelIndex>& cachedForwar
   return cachedForwardingChoices.empty() == false;
 };
 
+struct DecongestionContext {
+  ServiceRegistryRef ref;
+  TimesliceIndex::OldestOutputInfo oldestTimeslice;
+};
+
+auto decongestionCallbackLate = [](AsyncTask& task, size_t aid) -> void {
+  auto& oldestTimeslice = task.user<DecongestionContext>().oldestTimeslice;
+  auto& ref = task.user<DecongestionContext>().ref;
+
+  auto& decongestion = ref.get<DecongestionService>();
+  auto& proxy = ref.get<FairMQDeviceProxy>();
+  if (oldestTimeslice.timeslice.value <= decongestion.lastTimeslice) {
+    LOG(debug) << "Not sending already sent oldest possible timeslice " << oldestTimeslice.timeslice.value;
+    return;
+  }
+  for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
+    auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
+    auto& state = proxy.getForwardChannelState(ChannelIndex{fi});
+    O2_SIGNPOST_ID_GENERATE(aid, async_queue);
+    // TODO: this we could cache in the proxy at the bind moment.
+    if (info.channelType != ChannelAccountingType::DPL) {
+      O2_SIGNPOST_EVENT_EMIT(async_queue, aid, "forwardInputsCallback", "Skipping channel %{public}s because it's not a DPL channel",
+                             info.name.c_str());
+
+      continue;
+    }
+    if (DataProcessingHelpers::sendOldestPossibleTimeframe(ref, info, state, oldestTimeslice.timeslice.value)) {
+      O2_SIGNPOST_EVENT_EMIT(async_queue, aid, "forwardInputsCallback", "Forwarding to channel %{public}s oldest possible timeslice %zu, prio 20",
+                             info.name.c_str(), oldestTimeslice.timeslice.value);
+    }
+  }
+};
+
 // This is how we do the forwarding, i.e. we push
 // the inputs which are shared between this device and others
 // to the next one in the daisy chain.
@@ -721,36 +754,13 @@ static auto forwardInputs = [](ServiceRegistryRef registry, TimesliceSlot slot,
 
   auto& asyncQueue = registry.get<AsyncQueue>();
   auto& decongestion = registry.get<DecongestionService>();
-  auto& timesliceIndex = registry.get<TimesliceIndex>();
   O2_SIGNPOST_ID_GENERATE(aid, async_queue);
   O2_SIGNPOST_EVENT_EMIT(async_queue, aid, "forwardInputs", "Queuing forwarding oldestPossible %zu", oldestTimeslice.timeslice.value);
-  AsyncQueueHelpers::post(
-    asyncQueue, decongestion.oldestPossibleTimesliceTask, [&proxy, &decongestion, registry, oldestTimeslice, &timesliceIndex](size_t aid) {
-      // DataProcessingHelpers::broadcastOldestPossibleTimeslice(proxy, oldestTimeslice.timeslice.value);
-      if (oldestTimeslice.timeslice.value <= decongestion.lastTimeslice) {
-        LOG(debug) << "Not sending already sent oldest possible timeslice " << oldestTimeslice.timeslice.value;
-        return;
-      }
-      for (int fi = 0; fi < proxy.getNumForwardChannels(); fi++) {
-        auto& info = proxy.getForwardChannelInfo(ChannelIndex{fi});
-        auto& state = proxy.getForwardChannelState(ChannelIndex{fi});
-        O2_SIGNPOST_ID_GENERATE(aid, async_queue);
-        // TODO: this we could cache in the proxy at the bind moment.
-        if (info.channelType != ChannelAccountingType::DPL) {
-          O2_SIGNPOST_EVENT_EMIT(async_queue, aid, "forwardInputsCallback", "Skipping channel %{public}s because it's not a DPL channel",
-                                 info.name.c_str());
-
-          continue;
-        }
-        if (DataProcessingHelpers::sendOldestPossibleTimeframe(registry, info, state, oldestTimeslice.timeslice.value)) {
-          O2_SIGNPOST_EVENT_EMIT(async_queue, aid, "forwardInputsCallback", "Forwarding to channel %{public}s oldest possible timeslice %zu, prio 20",
-                                 info.name.c_str(), oldestTimeslice.timeslice.value);
-        }
-      }
-    },
-    oldestTimeslice.timeslice, -1);
+  AsyncQueueHelpers::post(asyncQueue, AsyncTask{.timeslice = oldestTimeslice.timeslice, .id = decongestion.oldestPossibleTimesliceTask, .debounce = -1, .callback = decongestionCallbackLate}
+                                        .user<DecongestionContext>({.ref = registry, .oldestTimeslice = oldestTimeslice}));
   O2_SIGNPOST_END(forwarding, sid, "forwardInputs", "Forwarding done");
 };
+
 extern volatile int region_read_global_dummy_variable;
 volatile int region_read_global_dummy_variable;
 
diff --git a/Framework/Core/test/test_AsyncQueue.cxx b/Framework/Core/test/test_AsyncQueue.cxx
index 635eb131cc666..7103bb087bb87 100644
--- a/Framework/Core/test/test_AsyncQueue.cxx
+++ b/Framework/Core/test/test_AsyncQueue.cxx
@@ -8,10 +8,13 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
-
 #include <catch_amalgamated.hpp>
 #include "Framework/AsyncQueue.h"
 
+struct TaskContext {
+  int& count;
+};
+
 /// Test debouncing functionality. The same task cannot be executed more than once
 /// in a given run.
 TEST_CASE("TestDebouncing")
@@ -20,11 +23,11 @@ TEST_CASE("TestDebouncing")
   AsyncQueue queue;
   auto taskId = AsyncQueueHelpers::create(queue, {.name = "test", .score = 10});
   // Push two tasks on the queue with the same id
-  auto count = 0;
-  AsyncQueueHelpers::post(
-    queue, taskId, [&count](size_t) { count += 1; }, TimesliceId{0}, 10);
-  AsyncQueueHelpers::post(
-    queue, taskId, [&count](size_t) { count += 2; }, TimesliceId{1}, 20);
+  int count = 0;
+  AsyncQueueHelpers::post(queue,
+                          AsyncTask{.timeslice = TimesliceId{0}, .id = taskId, .debounce = 10, .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 1; }}.user<TaskContext>({.count = count}));
+  AsyncQueueHelpers::post(queue,
+                          AsyncTask{.timeslice = TimesliceId{1}, .id = taskId, .debounce = 20, .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 2; }}.user<TaskContext>({.count = count}));
   AsyncQueueHelpers::run(queue, TimesliceId{2});
   REQUIRE(count == 2);
 }
@@ -37,11 +40,9 @@ TEST_CASE("TestPriority")
   auto taskId1 = AsyncQueueHelpers::create(queue, {.name = "test1", .score = 10});
   auto taskId2 = AsyncQueueHelpers::create(queue, {.name = "test2", .score = 20});
   // Push two tasks on the queue with the same id
-  auto count = 0;
-  AsyncQueueHelpers::post(
-    queue, taskId1, [&count](size_t) { count += 10; }, TimesliceId{0});
-  AsyncQueueHelpers::post(
-    queue, taskId2, [&count](size_t) { count /= 10; }, TimesliceId{0});
+  int count = 0;
+  AsyncQueueHelpers::post(queue, AsyncTask{.timeslice = TimesliceId{0}, .id = taskId1, .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 10; }}.user<TaskContext>({.count = count}));
+  AsyncQueueHelpers::post(queue, AsyncTask{.timeslice = TimesliceId{0}, .id = taskId2, .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count /= 10; }}.user<TaskContext>({.count = count}));
   AsyncQueueHelpers::run(queue, TimesliceId{2});
   REQUIRE(count == 10);
 }
@@ -56,9 +57,9 @@ TEST_CASE("TestOldestTimeslice")
   // Push two tasks on the queue with the same id
   auto count = 0;
   AsyncQueueHelpers::post(
-    queue, taskId1, [&count](size_t) { count += 10; }, TimesliceId{1});
+    queue, AsyncTask{.timeslice = TimesliceId{1}, .id = taskId1, .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 10; }}.user<TaskContext>({.count = count}));
   AsyncQueueHelpers::post(
-    queue, taskId2, [&count](size_t) { count += 20; }, TimesliceId{0});
+    queue, AsyncTask{.timeslice = TimesliceId{0}, .id = taskId2, .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 20; }}.user<TaskContext>({.count = count}));
   AsyncQueueHelpers::run(queue, TimesliceId{0});
   REQUIRE(count == 20);
   AsyncQueueHelpers::run(queue, TimesliceId{0});
@@ -72,16 +73,28 @@ TEST_CASE("TestOldestTimesliceWithBounce")
 {
   using namespace o2::framework;
   AsyncQueue queue;
-  auto taskId1 = AsyncQueueHelpers::create(queue, {.name = "test1", .score = 10});
-  auto taskId2 = AsyncQueueHelpers::create(queue, {.name = "test2", .score = 20});
   // Push two tasks on the queue with the same id
   auto count = 0;
-  AsyncQueueHelpers::post(
-    queue, taskId1, [&count](size_t) { count += 10; }, TimesliceId{2});
-  AsyncQueueHelpers::post(
-    queue, taskId2, [&count](size_t) { count += 20; }, TimesliceId{1}, 10);
-  AsyncQueueHelpers::post(
-    queue, taskId2, [&count](size_t) { count += 30; }, TimesliceId{1}, 20);
+  auto taskId1 = AsyncQueueHelpers::create(queue, {.name = "test1", .score = 10});
+  auto taskId2 = AsyncQueueHelpers::create(queue, {.name = "test2", .score = 20});
+  REQUIRE(taskId1.value != taskId2.value);
+  AsyncQueueHelpers::post(queue, AsyncTask{
+                                   .timeslice = TimesliceId{2},
+                                   .id = taskId1,
+                                   .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 10; }}
+                                   .user<TaskContext>({.count = count}));
+  AsyncQueueHelpers::post(queue, AsyncTask{
+                                   .timeslice = TimesliceId{1},
+                                   .id = taskId2,
+                                   .debounce = 10,
+                                   .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 20; }}
+                                   .user<TaskContext>({.count = count}));
+  AsyncQueueHelpers::post(queue, AsyncTask{
+                                   .timeslice = TimesliceId{1},
+                                   .id = taskId2,
+                                   .debounce = 20,
+                                   .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 30; }}
+                                   .user<TaskContext>({.count = count}));
   AsyncQueueHelpers::run(queue, TimesliceId{0});
   REQUIRE(count == 0);
   REQUIRE(queue.tasks.size() == 3);
@@ -98,16 +111,27 @@ TEST_CASE("TestOldestTimesliceWithNegativeBounce")
 {
   using namespace o2::framework;
   AsyncQueue queue;
+  int count = 0;
   auto taskId1 = AsyncQueueHelpers::create(queue, {.name = "test1", .score = 10});
   auto taskId2 = AsyncQueueHelpers::create(queue, {.name = "test2", .score = 20});
   // Push two tasks on the queue with the same id
-  auto count = 0;
   AsyncQueueHelpers::post(
-    queue, taskId1, [&count](size_t) { count += 10; }, TimesliceId{2});
+    queue, AsyncTask{.timeslice = TimesliceId{2},
+                     .id = taskId1,
+                     .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 10; }}
+             .user<TaskContext>({.count = count}));
   AsyncQueueHelpers::post(
-    queue, taskId2, [&count](size_t) { count += 20; }, TimesliceId{1}, -10);
+    queue, AsyncTask{.timeslice = TimesliceId{1},
+                     .id = taskId2,
+                     .debounce = -10,
+                     .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 20; }}
+             .user<TaskContext>({.count = count}));
   AsyncQueueHelpers::post(
-    queue, taskId2, [&count](size_t) { count += 30; }, TimesliceId{1}, -20);
+    queue, AsyncTask{.timeslice = TimesliceId{1},
+                     .id = taskId2,
+                     .debounce = -20,
+                     .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 30; }}
+             .user<TaskContext>({.count = count}));
   AsyncQueueHelpers::run(queue, TimesliceId{0});
   REQUIRE(count == 0);
   REQUIRE(queue.tasks.size() == 3);
@@ -126,15 +150,26 @@ TEST_CASE("TestOldestTimeslicePerTimeslice")
   AsyncQueue queue;
   auto taskId1 = AsyncQueueHelpers::create(queue, {.name = "test1", .score = 10});
   // Push two tasks on the queue with the same id
-  auto count = 0;
+  int count = 0;
   AsyncQueueHelpers::post(
-    queue, taskId1, [&count](size_t) { count += 10; }, TimesliceId{1});
+    queue, AsyncTask{.timeslice = TimesliceId{1},
+                     .id = taskId1,
+                     .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 10; }}
+             .user<TaskContext>({.count = count}));
+  // The size of the queue is only updated once a flush happens
+  REQUIRE(queue.tasks.size() == 0);
+  AsyncQueueHelpers::flushPending(queue);
   REQUIRE(queue.tasks.size() == 1);
   AsyncQueueHelpers::run(queue, TimesliceId{0});
   REQUIRE(queue.tasks.size() == 1);
   REQUIRE(count == 0);
   AsyncQueueHelpers::post(
-    queue, taskId1, [&count](size_t) { count += 20; }, TimesliceId{2});
+    queue, AsyncTask{.timeslice = TimesliceId{2},
+                     .id = taskId1,
+                     .callback = [](AsyncTask& task, size_t) { task.user<TaskContext>().count += 20; }}
+             .user<TaskContext>({.count = count}));
+  REQUIRE(queue.tasks.size() == 1);
+  AsyncQueueHelpers::flushPending(queue);
   REQUIRE(queue.tasks.size() == 2);
   AsyncQueueHelpers::run(queue, TimesliceId{1});
   REQUIRE(queue.tasks.size() == 1);

From b0ead00c078e8038ab9aba8a3853e052cd94664f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 19 Sep 2024 14:21:51 +0200
Subject: [PATCH 636/726] Import x9 library for blazing fast interthread
 message passing

---
 Framework/Foundation/3rdparty/CMakeLists.txt  |   7 +
 Framework/Foundation/3rdparty/x9/LICENSE      |  24 +
 Framework/Foundation/3rdparty/x9/README.md    |  68 +++
 .../Foundation/3rdparty/x9/examples/README.md | 241 ++++++++
 .../3rdparty/x9/examples/run_examples.sh      |  26 +
 .../3rdparty/x9/examples/x9_example_1.c       | 113 ++++
 .../3rdparty/x9/examples/x9_example_2.c       | 197 +++++++
 .../3rdparty/x9/examples/x9_example_3.c       | 207 +++++++
 .../3rdparty/x9/examples/x9_example_4.c       | 158 ++++++
 .../3rdparty/x9/examples/x9_example_5.c       | 161 ++++++
 .../3rdparty/x9/examples/x9_example_6.c       | 159 ++++++
 .../3rdparty/x9/profiling/README.md           | 106 ++++
 .../3rdparty/x9/profiling/compile_profiler.sh |   3 +
 .../3rdparty/x9/profiling/x9_profiler.c       | 524 ++++++++++++++++++
 Framework/Foundation/3rdparty/x9/x9.c         | 424 ++++++++++++++
 Framework/Foundation/3rdparty/x9/x9.h         | 166 ++++++
 16 files changed, 2584 insertions(+)
 create mode 100644 Framework/Foundation/3rdparty/x9/LICENSE
 create mode 100644 Framework/Foundation/3rdparty/x9/README.md
 create mode 100644 Framework/Foundation/3rdparty/x9/examples/README.md
 create mode 100755 Framework/Foundation/3rdparty/x9/examples/run_examples.sh
 create mode 100644 Framework/Foundation/3rdparty/x9/examples/x9_example_1.c
 create mode 100644 Framework/Foundation/3rdparty/x9/examples/x9_example_2.c
 create mode 100644 Framework/Foundation/3rdparty/x9/examples/x9_example_3.c
 create mode 100644 Framework/Foundation/3rdparty/x9/examples/x9_example_4.c
 create mode 100644 Framework/Foundation/3rdparty/x9/examples/x9_example_5.c
 create mode 100644 Framework/Foundation/3rdparty/x9/examples/x9_example_6.c
 create mode 100644 Framework/Foundation/3rdparty/x9/profiling/README.md
 create mode 100755 Framework/Foundation/3rdparty/x9/profiling/compile_profiler.sh
 create mode 100644 Framework/Foundation/3rdparty/x9/profiling/x9_profiler.c
 create mode 100644 Framework/Foundation/3rdparty/x9/x9.c
 create mode 100644 Framework/Foundation/3rdparty/x9/x9.h

diff --git a/Framework/Foundation/3rdparty/CMakeLists.txt b/Framework/Foundation/3rdparty/CMakeLists.txt
index 5001ad38ed6b5..1a490278a8d37 100644
--- a/Framework/Foundation/3rdparty/CMakeLists.txt
+++ b/Framework/Foundation/3rdparty/CMakeLists.txt
@@ -16,5 +16,12 @@ o2_add_library(Catch2
                TARGETVARNAME targetName
                PUBLIC_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}/catch2)
 
+o2_add_library(X9
+               SOURCES x9/x9.c
+               TARGETVARNAME targetName
+               PUBLIC_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}/x9)
+
 install(FILES ${CMAKE_CURRENT_LIST_DIR}/catch2/catch_amalgamated.hpp
         DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+install(FILES ${CMAKE_CURRENT_LIST_DIR}/x9/x9.h
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
diff --git a/Framework/Foundation/3rdparty/x9/LICENSE b/Framework/Foundation/3rdparty/x9/LICENSE
new file mode 100644
index 0000000000000..7b90e16605089
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/LICENSE
@@ -0,0 +1,24 @@
+BSD 2-Clause License
+
+Copyright (c) 2023, Diogo Flores
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/Framework/Foundation/3rdparty/x9/README.md b/Framework/Foundation/3rdparty/x9/README.md
new file mode 100644
index 0000000000000..1b7ead68961eb
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/README.md
@@ -0,0 +1,68 @@
+X9
+---
+
+*Note: I am currently working on v2.0 which will bring further performance 
+gains and flexibility to the user at the (unfortunate) cost of breaking the 
+current API. I expect to release v2.0 by September/October 2024 and for it to be the 
+last major/API-breaking change to X9.*
+
+X9 is a low level high performance message passing library, based on a
+lock-free ring buffer implemented with atomic variables, for low latency
+multithreading work.  
+It allows for multiple producers/consumers to concurrently access the same
+underlying ring buffer and provides both spinning (busy loop) and non-blocking
+read and write functions.
+
+The library is based on three concepts:
+
+- A message, which is a user defined struct.
+- A `x9_inbox`, which is where messages are both written to and read from.
+- A `x9_node`, which is an abstraction that unifies x9_inbox(es).
+
+The library provides multiple functions to both read from and write to a
+`x9_inbox`, as the right choice depends on the user needs.  
+Refer to _x9.h_, where all public functions are properly documented and their
+use cases explained, and the examples folder for comprehensive examples of
+different architectures.
+
+Enabling `X9_DEBUG` at compile time will print to stdout the reason why the
+functions `x9_inbox_is_valid` and `x9_node_is_valid` returned 'false' (if they
+indeed returned 'false'), or why `x9_select_inbox_from_node` did not return a
+valid `x9_inbox`.
+
+To use the library just link with x9.c and include x9.h where necessary.
+
+X9 is as generic, performant and intuitive as C allows, without forcing the
+user to any sort of build system preprocessor hell, pseudo-C macro based
+library, or worse.  
+It was originally written in the context of an high-frequency-trading system
+that this author developed, and was made public in June of 2023.  
+It is released under the BSD-2-Clause license, with the purpose of serving
+others and other programs.  
+
+Benchmarks
+---
+
+- Single producer and single consumer transmitting 100M messages via a single 
+`x9_inbox`.  
+- Run on Intel 11900k (cpu and ram untuned).  
+- _Msg size_ expressed in bytes, and _Inbox size_ in number of slots in the 
+ring buffer.   
+- _(See /profiling for how to run your own tests)_  
+
+```
+Inbox size | Msg size | Time (secs) | Msgs/second
+-------------------------------------------------
+      1024 |       16 |        4.00 |      25.01M
+      1024 |       32 |        4.03 |      24.82M
+      1024 |       64 |        4.17 |      23.99M
+-------------------------------------------------
+      2048 |       16 |        3.90 |      25.63M
+      2048 |       32 |        3.96 |      25.26M
+      2048 |       64 |        4.09 |      24.43M
+-------------------------------------------------
+      4096 |       16 |        3.86 |      25.88M
+      4096 |       32 |        3.92 |      25.50M
+      4096 |       64 |        4.05 |      24.67M
+-------------------------------------------------
+```
diff --git a/Framework/Foundation/3rdparty/x9/examples/README.md b/Framework/Foundation/3rdparty/x9/examples/README.md
new file mode 100644
index 0000000000000..c2525d76cd585
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/examples/README.md
@@ -0,0 +1,241 @@
+Notes: 
+--- 
+
+- Inbox N is always associated with a message of type N, hence to inbox 1
+producers will write messages of type 1 and consumers will read messages of
+type 1, and so on.
+
+- Each producer/consumer runs in a unique thread.
+
+- All x9_inbox sizes equal 4. This is not ideal for performance and should not
+be used as a guideline. The reason why I use such a small buffer is because
+I wanted to saturate the inbox and make it much more likely to trigger a data 
+race (which there is none).
+
+- All examples/tests can be run by executing ./run_examples.sh
+
+```
+──────▷  write to
+─ ─ ─ ▷  read from
+```
+
+Examples
+---
+```
+x9_example_1.c:
+
+ One producer
+ One consumer
+ One message type
+ 
+ ┌────────┐       ┏━━━━━━━━┓       ┌────────┐
+ │Producer│──────▷┃ inbox  ┃◁ ─ ─ ─│Consumer│
+ └────────┘       ┗━━━━━━━━┛       └────────┘
+ 
+ This example showcases the simplest (multi-threading) pattern.
+ 
+ Data structures used:
+  - x9_inbox
+ 
+ Functions used:
+  - x9_create_inbox
+  - x9_inbox_is_valid
+  - x9_write_to_inbox_spin
+  - x9_read_from_inbox_spin
+  - x9_free_inbox
+ 
+ Test is considered passed iff:
+  - None of the threads stall and exit cleanly after doing the work.
+  - All messages sent by the producer(s) are received and asserted to be
+  valid by the consumer(s).
+ ```
+-------------------------------------------------------------------------------
+```
+x9_example_2.c
+
+ Two producers.
+ One consumer and producer simultaneously.
+ One consumer.
+
+ ┌────────┐      ┏━━━━━━━━┓                      ┏━━━━━━━━┓
+ │Producer│─────▷┃        ┃      ┌────────┐      ┃        ┃
+ └────────┘      ┃        ┃      │Consumer│      ┃        ┃      ┌────────┐
+                 ┃inbox 1 ┃◁ ─ ─ │  and   │─────▷┃inbox 2 ┃◁ ─ ─ │Consumer│
+ ┌────────┐      ┃        ┃      │Producer│      ┃        ┃      └────────┘
+ │Producer│─────▷┃        ┃      └────────┘      ┃        ┃
+ └────────┘      ┗━━━━━━━━┛                      ┗━━━━━━━━┛
+
+ This example showcase using multiple threads to write to the same inbox,
+ using multiple message types, the 'x9_node' abstraction, and
+ respective create/free and select functions.
+
+ Data structures used:
+  - x9_inbox
+  - x9_node 
+
+ Functions used:
+  - x9_create_inbox
+  - x9_inbox_is_valid
+  - x9_create_node
+  - x9_node_is_valid
+  - x9_select_inbox_from_node
+  - x9_write_to_inbox_spin
+  - x9_read_from_inbox_spin
+  - x9_free_node_and_attached_inboxes
+
+ Test is considered passed iff:
+  - None of the threads stall and exit cleanly after doing the work.
+  - All messages sent by the producer(s) are received and asserted to be
+  valid by the consumer(s).
+```
+-------------------------------------------------------------------------------
+```
+x9_example_3.c
+
+ Two producers and simultaneously consumers.
+ 
+ ┌────────┐       ┏━━━━━━━━┓       ┌────────┐
+ │Producer│──────▷┃inbox 1 ┃◁ ─ ─ ─│Producer│
+ │        │       ┗━━━━━━━━┛       │        │
+ │  and   │                        │  and   │
+ │        │       ┏━━━━━━━━┓       │        │
+ │Consumer│─ ─ ─ ▷┃inbox 2 ┃◁──────│Consumer│
+ └────────┘       ┗━━━━━━━━┛       └────────┘
+
+ This example showcases the use of: 'x9_write_to_inbox'
+ and 'x9_read_from_inbox', which, unlike 'x9_write_to_inbox_spin' and
+ 'x9_read_from_inbox_spin' do not block until are able to write/read a msg.
+
+ Data structures used:
+  - x9_inbox
+  - x9_node
+
+ Functions used:
+  - x9_create_inbox
+  - x9_inbox_is_valid
+  - x9_create_node
+  - x9_node_is_valid
+  - x9_select_inbox_from_node
+  - x9_write_to_inbox
+  - x9_read_from_inbox
+  - x9_free_node_and_attached_inboxes
+
+ Test is considered passed iff:
+  - None of the threads stall and exit cleanly after doing the work.
+  - All messages sent by the producer(s) are received and asserted to be
+  valid by the consumer(s).
+```
+-------------------------------------------------------------------------------
+```
+x9_example_4.c
+
+ One producer broadcasts the same message to three inboxes.
+ Three consumers read from each inbox.
+ One message type.
+ 
+                  ┏━━━━━━━━┓      ┌────────┐
+              ┌──▷┃ inbox  ┃◁─ ─ ─│Consumer│
+              │   ┗━━━━━━━━┛      └────────┘
+ ┌────────┐   │   ┏━━━━━━━━┓      ┌────────┐
+ │Producer│───┼──▷┃ inbox  ┃◁─ ─ ─│Consumer│
+ └────────┘   │   ┗━━━━━━━━┛      └────────┘
+              │   ┏━━━━━━━━┓      ┌────────┐
+              └──▷┃ inbox  ┃◁─ ─ ─│Consumer│
+                  ┗━━━━━━━━┛      └────────┘
+
+
+ This example showcases the use of 'x9_broadcast_msg_to_all_node_inboxes'.
+
+ Data structures used:
+  - x9_inbox
+  - x9_node
+
+ Functions used:
+  - x9_create_inbox
+  - x9_inbox_is_valid
+  - x9_create_node
+  - x9_node_is_valid
+  - x9_broadcast_msg_to_all_node_inboxes
+  - x9_read_from_inbox_spin
+  - x9_free_node_and_attached_inboxes
+
+ IMPORTANT:
+  - All inboxes  must receive messages of the same type (or at least of the
+  same size) that its being broadcasted.
+
+ Test is considered passed iff:
+  - None of the threads stall and exit cleanly after doing the work.
+  - All messages sent by the producer(s) are received and asserted to be
+  valid by the consumer(s).
+```
+-------------------------------------------------------------------------------
+```
+x9_example_5.c
+
+ Three producers.
+ Three consumers reading concurrently from the same inbox.
+ One message type.
+
+                                   ┌────────┐
+ ┌────────┐       ┏━━━━━━━━┓    ─ ─│Consumer│
+ │Producer│──────▷┃        ┃   │   └────────┘
+ ├────────┤       ┃        ┃       ┌────────┐
+ │Producer│──────▷┃ inbox  ┃◁──┤─ ─│Consumer│
+ ├────────┤       ┃        ┃       └────────┘
+ │Producer│──────▷┃        ┃   │   ┌────────┐
+ └────────┘       ┗━━━━━━━━┛    ─ ─│Consumer│
+                                   └────────┘
+
+ This example showcases the use of 'x9_read_from_shared_inbox'.
+
+ Data structures used:
+  - x9_inbox
+ 
+ Functions used:
+  - x9_create_inbox
+  - x9_inbox_is_valid
+  - x9_write_to_inbox_spin
+  - x9_read_from_shared_inbox
+  - x9_free_inbox
+
+ Test is considered passed iff:
+  - None of the threads stall and exit cleanly after doing the work.
+  - All messages sent by the producer(s) are received and asserted to be
+  valid by the consumer(s).
+  - Each consumer processes at least one message.
+```
+-------------------------------------------------------------------------------
+```
+x9_example_6.c
+
+ One producer
+ Two consumers reading from the same inbox concurrently (busy loop).
+ One message type.
+
+                  ┏━━━━━━━━┓       ┌────────┐
+                  ┃        ┃    ─ ─│Consumer│
+ ┌────────┐       ┃        ┃   │   └────────┘
+ │Producer│──────▷┃ inbox  ┃◁ ─
+ └────────┘       ┃        ┃   │   ┌────────┐
+                  ┃        ┃    ─ ─│Consumer│
+                  ┗━━━━━━━━┛       └────────┘
+
+ This example showcases the use of 'x9_read_from_shared_inbox_spin'.
+
+ Data structures used:
+  - x9_inbox
+
+ Functions used:
+  - x9_create_inbox
+  - x9_inbox_is_valid
+  - x9_write_to_inbox_spin
+  - x9_read_from_shared_inbox_spin
+  - x9_free_inbox
+
+ Test is considered passed iff:
+  - None of the threads stall and exit cleanly after doing the work.
+  - All messages sent by the producer(s) are received and asserted to be
+  valid by the consumer(s).
+  - Each consumer processes at least one message.
+```
+-------------------------------------------------------------------------------
diff --git a/Framework/Foundation/3rdparty/x9/examples/run_examples.sh b/Framework/Foundation/3rdparty/x9/examples/run_examples.sh
new file mode 100755
index 0000000000000..6719bd9b4bb86
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/examples/run_examples.sh
@@ -0,0 +1,26 @@
+#/bin/bash
+
+echo "- Running examples with GCC with \"-fsanitize=thread,undefined\" enabled.";
+gcc -Wextra -Wall -Werror -pedantic -O3 -march=native x9_example_1.c ../x9.c -o X9_TEST_1 -fsanitize=thread,undefined -D X9_DEBUG
+gcc -Wextra -Wall -Werror -pedantic -O3 -march=native x9_example_2.c ../x9.c -o X9_TEST_2 -fsanitize=thread,undefined -D X9_DEBUG
+gcc -Wextra -Wall -Werror -pedantic -O3 -march=native x9_example_3.c ../x9.c -o X9_TEST_3 -fsanitize=thread,undefined -D X9_DEBUG
+gcc -Wextra -Wall -Werror -pedantic -O3 -march=native x9_example_4.c ../x9.c -o X9_TEST_4 -fsanitize=thread,undefined -D X9_DEBUG
+gcc -Wextra -Wall -Werror -pedantic -O3 -march=native x9_example_5.c ../x9.c -o X9_TEST_5 -fsanitize=thread,undefined -D X9_DEBUG
+gcc -Wextra -Wall -Werror -pedantic -O3 -march=native x9_example_6.c ../x9.c -o X9_TEST_6 -fsanitize=thread,undefined -D X9_DEBUG
+
+./X9_TEST_1; ./X9_TEST_2; ./X9_TEST_3; ./X9_TEST_4; ./X9_TEST_5; ./X9_TEST_6
+rm X9_TEST_1 X9_TEST_2 X9_TEST_3 X9_TEST_4 X9_TEST_5 X9_TEST_6
+
+echo ""
+echo "- Running examples with clang with \"-fsanitize=address,undefined,leak\" enabled.";
+
+clang -Wextra -Wall -Werror -O3 -march=native x9_example_1.c ../x9.c -o X9_TEST_1 -fsanitize=address,undefined,leak -D X9_DEBUG
+clang -Wextra -Wall -Werror -O3 -march=native x9_example_2.c ../x9.c -o X9_TEST_2 -fsanitize=address,undefined,leak -D X9_DEBUG
+clang -Wextra -Wall -Werror -O3 -march=native x9_example_3.c ../x9.c -o X9_TEST_3 -fsanitize=address,undefined,leak -D X9_DEBUG
+clang -Wextra -Wall -Werror -O3 -march=native x9_example_4.c ../x9.c -o X9_TEST_4 -fsanitize=address,undefined,leak -D X9_DEBUG
+clang -Wextra -Wall -Werror -O3 -march=native x9_example_5.c ../x9.c -o X9_TEST_5 -fsanitize=address,undefined,leak -D X9_DEBUG
+clang -Wextra -Wall -Werror -O3 -march=native x9_example_6.c ../x9.c -o X9_TEST_6 -fsanitize=address,undefined,leak -D X9_DEBUG
+
+./X9_TEST_1; ./X9_TEST_2; ./X9_TEST_3; ./X9_TEST_4; ./X9_TEST_5; ./X9_TEST_6
+rm X9_TEST_1 X9_TEST_2 X9_TEST_3 X9_TEST_4 X9_TEST_5 X9_TEST_6 
+
diff --git a/Framework/Foundation/3rdparty/x9/examples/x9_example_1.c b/Framework/Foundation/3rdparty/x9/examples/x9_example_1.c
new file mode 100644
index 0000000000000..6498ac2a76da7
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/examples/x9_example_1.c
@@ -0,0 +1,113 @@
+/* x9_example_1.c:
+ *
+ *  One producer
+ *  One consumer
+ *  One message type
+ *
+ *  ┌────────┐       ┏━━━━━━━━┓       ┌────────┐
+ *  │Producer│──────▷┃ inbox  ┃◁ ─ ─ ─│Consumer│
+ *  └────────┘       ┗━━━━━━━━┛       └────────┘
+ *
+ *  This example showcases the simplest (multi-threading) pattern.
+ *
+ *  Data structures used:
+ *   - x9_inbox
+ *
+ *  Functions used:
+ *   - x9_create_inbox
+ *   - x9_inbox_is_valid
+ *   - x9_write_to_inbox_spin
+ *   - x9_read_from_inbox_spin
+ *   - x9_free_inbox
+ *
+ *  Test is considered passed iff:
+ *   - None of the threads stall and exit cleanly after doing the work.
+ *   - All messages sent by the producer(s) are received and asserted to be
+ *   valid by the consumer(s).
+ */
+
+#include <assert.h>  /* assert */
+#include <pthread.h> /* pthread_t, pthread functions */
+#include <stdio.h>   /* printf */
+#include <stdlib.h>  /* rand, RAND_MAX */
+
+#include "../x9.h"
+
+/* Both producer and consumer loops, would commonly be infinite loops, but for
+ * the purpose of testing a reasonable NUMBER_OF_MESSAGES is defined. */
+#define NUMBER_OF_MESSAGES 1000000
+
+typedef struct {
+  x9_inbox* inbox;
+} th_struct;
+
+typedef struct {
+  int a;
+  int b;
+  int sum;
+} msg;
+
+static int random_int(int const min, int const max) {
+  return min + rand() / (RAND_MAX / (max - min + 1) + 1);
+}
+
+static inline void fill_msg_1(msg* const msg) {
+  msg->a   = random_int(0, 10);
+  msg->b   = random_int(0, 10);
+  msg->sum = msg->a + msg->b;
+}
+
+static void* producer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {0};
+  for (uint64_t k = 0; k != NUMBER_OF_MESSAGES; ++k) {
+    fill_msg_1(&m);
+    x9_write_to_inbox_spin(data->inbox, sizeof(msg), &m);
+  }
+  return 0;
+}
+
+static void* consumer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {0};
+  for (uint64_t k = 0; k != NUMBER_OF_MESSAGES; ++k) {
+    x9_read_from_inbox_spin(data->inbox, sizeof(msg), &m);
+    assert(m.sum == (m.a + m.b));
+  }
+  return 0;
+}
+
+int main(void) {
+  /* Seed random generator */
+  srand((uint32_t)time(0));
+
+  /* Create inbox */
+  x9_inbox* const inbox = x9_create_inbox(4, "ibx_1", sizeof(msg));
+
+  /* Using assert to simplify code for presentation purpose. */
+  assert(x9_inbox_is_valid(inbox));
+
+  /* Producer */
+  pthread_t producer_th     = {0};
+  th_struct producer_struct = {.inbox = inbox};
+
+  /* Consumer */
+  pthread_t consumer_th     = {0};
+  th_struct consumer_struct = {.inbox = inbox};
+
+  /* Launch threads */
+  pthread_create(&producer_th, NULL, producer_fn, &producer_struct);
+  pthread_create(&consumer_th, NULL, consumer_fn, &consumer_struct);
+
+  /* Join them */
+  pthread_join(consumer_th, NULL);
+  pthread_join(producer_th, NULL);
+
+  /* Cleanup */
+  x9_free_inbox(inbox);
+
+  printf("TEST PASSED: x9_example_1.c\n");
+  return EXIT_SUCCESS;
+}
diff --git a/Framework/Foundation/3rdparty/x9/examples/x9_example_2.c b/Framework/Foundation/3rdparty/x9/examples/x9_example_2.c
new file mode 100644
index 0000000000000..411c7c8128779
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/examples/x9_example_2.c
@@ -0,0 +1,197 @@
+/* x9_example_2.c
+ *
+ *  Two producers.
+ *  One consumer and producer simultaneously.
+ *  One consumer.
+ *
+ * ┌────────┐      ┏━━━━━━━━┓                      ┏━━━━━━━━┓
+ * │Producer│─────▷┃        ┃      ┌────────┐      ┃        ┃
+ * └────────┘      ┃        ┃      │Consumer│      ┃        ┃      ┌────────┐
+ *                 ┃inbox 1 ┃◁ ─ ─ │  and   │─────▷┃inbox 2 ┃◁ ─ ─ │Consumer│
+ * ┌────────┐      ┃        ┃      │Producer│      ┃        ┃      └────────┘
+ * │Producer│─────▷┃        ┃      └────────┘      ┃        ┃
+ * └────────┘      ┗━━━━━━━━┛                      ┗━━━━━━━━┛
+ *
+ *  This example showcase using multiple threads to write to the same inbox,
+ *  using multiple message types, the 'x9_node' abstraction, and
+ *  respective create/free and select functions.
+ *
+ *  Data structures used:
+ *   - x9_inbox
+ *   - x9_node
+ *
+ *  Functions used:
+ *   - x9_create_inbox
+ *   - x9_inbox_is_valid
+ *   - x9_create_node
+ *   - x9_node_is_valid
+ *   - x9_select_inbox_from_node
+ *   - x9_write_to_inbox_spin
+ *   - x9_read_from_inbox_spin
+ *   - x9_free_node_and_attached_inboxes
+ *
+ *  Test is considered passed iff:
+ *   - None of the threads stall and exit cleanly after doing the work.
+ *   - All messages sent by the producer(s) are received and asserted to be
+ *   valid by the consumer(s).
+ */
+
+#include <assert.h>  /* assert */
+#include <pthread.h> /* pthread_t, pthread functions */
+#include <stdio.h>   /* printf */
+#include <stdlib.h>  /* rand, RAND_MAX */
+
+#include "../x9.h"
+
+/* Both producer and consumer loops, would commonly be infinite loops, but for
+ * the purpose of testing a reasonable NUMBER_OF_MESSAGES is defined. */
+#define NUMBER_OF_MESSAGES 1000000
+
+#define NUMBER_OF_PRODUCER_THREADS 2
+
+typedef struct {
+  x9_node* node;
+} th_struct;
+
+typedef struct {
+  int a;
+  int b;
+  int sum;
+
+} msg_type_1;
+
+typedef struct {
+  int x;
+  int y;
+  int sum;
+  int product;
+} msg_type_2;
+
+static inline int random_int(int const min, int const max) {
+  return min + rand() / (RAND_MAX / (max - min + 1) + 1);
+}
+
+static inline void fill_msg_type_1(msg_type_1 msg[const static 1]) {
+  msg->a   = random_int(0, 10);
+  msg->b   = random_int(0, 10);
+  msg->sum = msg->a + msg->b;
+}
+
+static inline void fill_msg_type_2(msg_type_2       to[const static 1],
+                                   msg_type_1 const from[const static 1]) {
+  to->x       = from->a;
+  to->y       = from->b;
+  to->sum     = from->sum;
+  to->product = (from->a * from->b);
+}
+
+static void* producer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  x9_inbox* const destination = x9_select_inbox_from_node(data->node, "ibx_1");
+  assert(x9_inbox_is_valid(destination));
+
+  msg_type_1 msg = {0};
+  for (uint64_t k = 0; k != NUMBER_OF_MESSAGES; ++k) {
+    fill_msg_type_1(&msg);
+    x9_write_to_inbox_spin(destination, sizeof(msg_type_1), &msg);
+  }
+  return 0;
+}
+
+static void* producer_consumer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  x9_inbox* const inbox = x9_select_inbox_from_node(data->node, "ibx_1");
+
+  assert(x9_inbox_is_valid(inbox));
+
+  x9_inbox* const destination = x9_select_inbox_from_node(data->node, "ibx_2");
+
+  assert(x9_inbox_is_valid(destination));
+
+  msg_type_1 incoming_msg = {0};
+  msg_type_2 outgoing_msg = {0};
+  for (uint64_t k = 0; k != (NUMBER_OF_MESSAGES * NUMBER_OF_PRODUCER_THREADS);
+       ++k) {
+    x9_read_from_inbox_spin(inbox, sizeof(msg_type_1), &incoming_msg);
+    assert(incoming_msg.sum == (incoming_msg.a + incoming_msg.b));
+    fill_msg_type_2(&outgoing_msg, &incoming_msg);
+    x9_write_to_inbox_spin(destination, sizeof(msg_type_2), &outgoing_msg);
+  }
+  return 0;
+}
+
+static void* consumer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  x9_inbox* const inbox = x9_select_inbox_from_node(data->node, "ibx_2");
+
+  assert(x9_inbox_is_valid(inbox));
+
+  msg_type_2 msg = {0};
+  for (uint64_t k = 0; k != (NUMBER_OF_MESSAGES * NUMBER_OF_PRODUCER_THREADS);
+       ++k) {
+    x9_read_from_inbox_spin(inbox, sizeof(msg_type_2), &msg);
+    assert(msg.sum == (msg.x + msg.y));
+    assert(msg.product == (msg.x * msg.y));
+  }
+  return 0;
+}
+
+int main(void) {
+  /* Seed random generator */
+  srand((uint32_t)time(0));
+
+  /* Create inboxes */
+  x9_inbox* const inbox_msg_type_1 =
+      x9_create_inbox(4, "ibx_1", sizeof(msg_type_1));
+
+  x9_inbox* const inbox_msg_type_2 =
+      x9_create_inbox(4, "ibx_2", sizeof(msg_type_2));
+
+  /* Using asserts to simplify code for presentation purpose. */
+  assert(x9_inbox_is_valid(inbox_msg_type_1));
+  assert(x9_inbox_is_valid(inbox_msg_type_2));
+
+  /* Create node */
+  x9_node* const node =
+      x9_create_node("my_node", 2, inbox_msg_type_1, inbox_msg_type_2);
+
+  /* Assert - Same reason as above. */
+  assert(x9_node_is_valid(node));
+
+  /* Producers */
+  pthread_t producer_th_1     = {0};
+  th_struct producer_1_struct = {.node = node};
+
+  pthread_t producer_th_2     = {0};
+  th_struct producer_2_struct = {.node = node};
+
+  /* Producer/Consumer */
+  pthread_t producer_consumer_th = {0};
+  th_struct prod_cons_struct     = {.node = node};
+
+  /* Consumer */
+  pthread_t consumer_th     = {0};
+  th_struct consumer_struct = {.node = node};
+
+  /* Launch threads */
+  pthread_create(&producer_th_1, NULL, producer_fn, &producer_1_struct);
+  pthread_create(&producer_th_2, NULL, producer_fn, &producer_2_struct);
+  pthread_create(&consumer_th, NULL, consumer_fn, &consumer_struct);
+  pthread_create(&producer_consumer_th, NULL, &producer_consumer_fn,
+                 &prod_cons_struct);
+
+  /* Join them */
+  pthread_join(consumer_th, NULL);
+  pthread_join(producer_consumer_th, NULL);
+  pthread_join(producer_th_1, NULL);
+  pthread_join(producer_th_2, NULL);
+
+  /* Cleanup */
+  x9_free_node_and_attached_inboxes(node);
+
+  printf("TEST PASSED: x9_example_2.c\n");
+  return EXIT_SUCCESS;
+}
diff --git a/Framework/Foundation/3rdparty/x9/examples/x9_example_3.c b/Framework/Foundation/3rdparty/x9/examples/x9_example_3.c
new file mode 100644
index 0000000000000..c10824e208c3c
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/examples/x9_example_3.c
@@ -0,0 +1,207 @@
+/* x9_example_3.c
+ *
+ *  Two producers and simultaneously consumers.
+ *
+ *  ┌────────┐       ┏━━━━━━━━┓       ┌────────┐
+ *  │Producer│──────▷┃inbox 1 ┃◁ ─ ─ ─│Producer│
+ *  │        │       ┗━━━━━━━━┛       │        │
+ *  │  and   │                        │  and   │
+ *  │        │       ┏━━━━━━━━┓       │        │
+ *  │Consumer│─ ─ ─ ▷┃inbox 2 ┃◁──────│Consumer│
+ *  └────────┘       ┗━━━━━━━━┛       └────────┘
+ *
+ *  This example showcases the use of: 'x9_write_to_inbox'
+ *  and 'x9_read_from_inbox', which, unlike 'x9_write_to_inbox_spin' and
+ *  'x9_read_from_inbox_spin' do not block until are able to write/read a msg.
+ *
+ *  Data structures used:
+ *   - x9_inbox
+ *   - x9_node
+ *
+ *  Functions used:
+ *   - x9_create_inbox
+ *   - x9_inbox_is_valid
+ *   - x9_create_node
+ *   - x9_node_is_valid
+ *   - x9_select_inbox_from_node
+ *   - x9_write_to_inbox
+ *   - x9_read_from_inbox
+ *   - x9_free_node_and_attached_inboxes
+ *
+ *  Test is considered passed iff:
+ *   - None of the threads stall and exit cleanly after doing the work.
+ *   - All messages sent by the producer(s) are received and asserted to be
+ *   valid by the consumer(s).
+ */
+
+#include <assert.h>  /* assert */
+#include <math.h>    /* fabs */
+#include <pthread.h> /* pthread_t, pthread functions */
+#include <stdio.h>   /* printf */
+#include <stdlib.h>  /* rand, RAND_MAX */
+
+#include "../x9.h"
+
+/* Both producer and consumer loops, would commonly be infinite loops, but for
+ * the purpose of testing a reasonable NUMBER_OF_MESSAGES is defined. */
+#define NUMBER_OF_MESSAGES 1000000
+
+typedef struct {
+  x9_node* node;
+} th_struct;
+
+typedef struct {
+  int a;
+  int b;
+  int sum;
+} msg_type_1;
+
+typedef struct {
+  double x;
+  double y;
+  double product;
+} msg_type_2;
+
+static inline int random_int(int const min, int const max) {
+  return min + rand() / (RAND_MAX / (max - min + 1) + 1);
+}
+
+static inline double random_double(int const min, int const max) {
+  return ((double)(random_int(min, max) * 1.0));
+}
+
+static inline void fill_msg_type_1(msg_type_1* const msg) {
+  msg->a   = random_int(0, 10);
+  msg->b   = random_int(0, 10);
+  msg->sum = msg->a + msg->b;
+}
+
+static inline void fill_msg_type_2(msg_type_2* const msg) {
+  msg->x       = random_double(0, 10);
+  msg->y       = random_double(0, 10);
+  msg->product = msg->x * msg->y;
+}
+
+static void* producer_1_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  x9_inbox* const write_inbox = x9_select_inbox_from_node(data->node, "ibx_1");
+  assert(write_inbox);
+
+  x9_inbox* const read_inbox = x9_select_inbox_from_node(data->node, "ibx_2");
+  assert(read_inbox);
+
+  uint64_t msgs_read = {0};
+  uint64_t msgs_sent = {0};
+
+  msg_type_2 incoming_msg = {0};
+  msg_type_1 outgoing_msg = {0};
+
+  for (;;) {
+    if ((msgs_read == NUMBER_OF_MESSAGES) &&
+        (msgs_sent == NUMBER_OF_MESSAGES)) {
+      break;
+    }
+
+    if (msgs_sent != NUMBER_OF_MESSAGES) {
+      fill_msg_type_1(&outgoing_msg);
+      if (x9_write_to_inbox(write_inbox, sizeof(msg_type_1), &outgoing_msg)) {
+        ++msgs_sent;
+      }
+    }
+
+    if (msgs_read != NUMBER_OF_MESSAGES) {
+      if (x9_read_from_inbox(read_inbox, sizeof(msg_type_2), &incoming_msg)) {
+        ++msgs_read;
+        assert(fabs(incoming_msg.product - (incoming_msg.x * incoming_msg.y)) <
+               0.1);
+      }
+    }
+  }
+
+  return 0;
+}
+
+static void* producer_2_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  x9_inbox* const write_inbox = x9_select_inbox_from_node(data->node, "ibx_2");
+  assert(write_inbox);
+
+  x9_inbox* const read_inbox = x9_select_inbox_from_node(data->node, "ibx_1");
+  assert(read_inbox);
+
+  uint64_t msgs_read = {0};
+  uint64_t msgs_sent = {0};
+
+  msg_type_1 incoming_msg = {0};
+  msg_type_2 outgoing_msg = {0};
+
+  for (;;) {
+    if ((msgs_read == NUMBER_OF_MESSAGES) &&
+        (msgs_sent == NUMBER_OF_MESSAGES)) {
+      break;
+    }
+
+    if (msgs_read != NUMBER_OF_MESSAGES) {
+      if (x9_read_from_inbox(read_inbox, sizeof(msg_type_1), &incoming_msg)) {
+        ++msgs_read;
+        assert(incoming_msg.sum == (incoming_msg.a + incoming_msg.b));
+      }
+    }
+
+    if (msgs_sent != NUMBER_OF_MESSAGES) {
+      fill_msg_type_2(&outgoing_msg);
+      if (x9_write_to_inbox(write_inbox, sizeof(msg_type_2), &outgoing_msg)) {
+        ++msgs_sent;
+      }
+    }
+  }
+
+  return 0;
+}
+
+int main(void) {
+  /* Seed random generator */
+  srand((uint32_t)time(0));
+
+  /* Create inboxes */
+  x9_inbox* const inbox_msg_type_1 =
+      x9_create_inbox(4, "ibx_1", sizeof(msg_type_1));
+
+  x9_inbox* const inbox_msg_type_2 =
+      x9_create_inbox(4, "ibx_2", sizeof(msg_type_2));
+
+  /* Using asserts to simplify code for presentation purpose. */
+  assert(x9_inbox_is_valid(inbox_msg_type_1));
+  assert(x9_inbox_is_valid(inbox_msg_type_2));
+
+  /* Create node */
+  x9_node* const node =
+      x9_create_node("my_node", 2, inbox_msg_type_1, inbox_msg_type_2);
+
+  /* Asserts - Same reason as above.*/
+  assert(x9_node_is_valid(node));
+
+  /* Producer 1 (left on diagram) */
+  pthread_t producer_1_th     = {0};
+  th_struct producer_1_struct = {.node = node};
+
+  /* Producer 2 (right on diagram) */
+  pthread_t producer_2_th     = {0};
+  th_struct producer_2_struct = {.node = node};
+
+  /* Launch threads */
+  pthread_create(&producer_1_th, NULL, producer_1_fn, &producer_1_struct);
+  pthread_create(&producer_2_th, NULL, producer_2_fn, &producer_2_struct);
+
+  /* Join them */
+  pthread_join(producer_1_th, NULL);
+  pthread_join(producer_2_th, NULL);
+
+  /* Cleanup */
+  x9_free_node_and_attached_inboxes(node);
+
+  printf("TEST PASSED: x9_example_3.c\n");
+  return EXIT_SUCCESS;
+}
diff --git a/Framework/Foundation/3rdparty/x9/examples/x9_example_4.c b/Framework/Foundation/3rdparty/x9/examples/x9_example_4.c
new file mode 100644
index 0000000000000..a4a096bebce94
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/examples/x9_example_4.c
@@ -0,0 +1,158 @@
+/* x9_example_4.c
+ *
+ *  One producer broadcasts the same message to three inboxes.
+ *  Three consumers read from each inbox.
+ *  One message type.
+ *
+ *                   ┏━━━━━━━━┓      ┌────────┐
+ *               ┌──▷┃ inbox  ┃◁─ ─ ─│Consumer│
+ *               │   ┗━━━━━━━━┛      └────────┘
+ *  ┌────────┐   │   ┏━━━━━━━━┓      ┌────────┐
+ *  │Producer│───┼──▷┃ inbox  ┃◁─ ─ ─│Consumer│
+ *  └────────┘   │   ┗━━━━━━━━┛      └────────┘
+ *               │   ┏━━━━━━━━┓      ┌────────┐
+ *               └──▷┃ inbox  ┃◁─ ─ ─│Consumer│
+ *                   ┗━━━━━━━━┛      └────────┘
+ *
+ *
+ *  This example showcases the use of 'x9_broadcast_msg_to_all_node_inboxes'.
+ *
+ *  Data structures used:
+ *   - x9_inbox
+ *   - x9_node
+ *
+ *  Functions used:
+ *   - x9_create_inbox
+ *   - x9_inbox_is_valid
+ *   - x9_create_node
+ *   - x9_node_is_valid
+ *   - x9_broadcast_msg_to_all_node_inboxes
+ *   - x9_read_from_inbox_spin
+ *   - x9_free_node_and_attached_inboxes
+ *
+ *  IMPORTANT:
+ *   - All inboxes  must receive messages of the same type (or at least of the
+ *   same size) that its being broadcasted.
+ *
+ *  Test is considered passed iff:
+ *   - None of the threads stall and exit cleanly after doing the work.
+ *   - All messages sent by the producer(s) are received and asserted to be
+ *   valid by the consumer(s).
+ */
+
+#include <assert.h>  /* assert */
+#include <pthread.h> /* pthread_t, pthread functions */
+#include <stdio.h>   /* printf */
+#include <stdlib.h>  /* rand, RAND_MAX */
+
+#include "../x9.h"
+
+/* Both producer and consumer loops, would commonly be infinite loops, but for
+ * the purpose of testing a reasonable NUMBER_OF_MESSAGES is defined. */
+#define NUMBER_OF_MESSAGES 1000000
+
+typedef struct {
+  x9_node* node;
+  char*    inbox_to_consume_from;
+} th_struct;
+
+typedef struct {
+  int a;
+  int b;
+  int sum;
+} msg;
+
+static inline int random_int(int const min, int const max) {
+  return min + rand() / (RAND_MAX / (max - min + 1) + 1);
+}
+
+static inline void fill_msg_type(msg* const m) {
+  m->a   = random_int(0, 10);
+  m->b   = random_int(0, 10);
+  m->sum = m->a + m->b;
+}
+
+static void* producer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {0};
+  for (uint_fast64_t k = 0; k != NUMBER_OF_MESSAGES; ++k) {
+    fill_msg_type(&m);
+    x9_broadcast_msg_to_all_node_inboxes(data->node, sizeof(msg), &m);
+  }
+  return 0;
+}
+
+static void* consumer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  x9_inbox* const inbox =
+      x9_select_inbox_from_node(data->node, data->inbox_to_consume_from);
+  assert(x9_inbox_is_valid(inbox));
+
+  msg m = {0};
+
+  for (uint64_t k = 0; k != NUMBER_OF_MESSAGES; ++k) {
+    x9_read_from_inbox_spin(inbox, sizeof(msg), &m);
+    assert(m.sum == (m.a + m.b));
+  }
+  return 0;
+}
+
+int main(void) {
+  /* Seed random generator */
+  srand((uint32_t)time(0));
+
+  /* Create inboxes */
+  x9_inbox* const inbox_consumer_1 = x9_create_inbox(4, "ibx_1", sizeof(msg));
+  x9_inbox* const inbox_consumer_2 = x9_create_inbox(4, "ibx_2", sizeof(msg));
+  x9_inbox* const inbox_consumer_3 = x9_create_inbox(4, "ibx_3", sizeof(msg));
+
+  /* Using asserts to simplify code for presentation purpose. */
+  assert(x9_inbox_is_valid(inbox_consumer_1));
+  assert(x9_inbox_is_valid(inbox_consumer_2));
+  assert(x9_inbox_is_valid(inbox_consumer_3));
+
+  /* Create node */
+  x9_node* const node = x9_create_node("my_node", 3, inbox_consumer_1,
+                                       inbox_consumer_2, inbox_consumer_3);
+
+  /* Assert - Same reason as above. */
+  assert(x9_node_is_valid(node));
+
+  /* Producer */
+  pthread_t producer_th     = {0};
+  th_struct producer_struct = {.node = node};
+
+  /* Consumer 1 */
+  pthread_t consumer_1_th     = {0};
+  th_struct consumer_1_struct = {.node                  = node,
+                                 .inbox_to_consume_from = "ibx_1"};
+  /* Consumer 2 */
+  pthread_t consumer_2_th     = {0};
+  th_struct consumer_2_struct = {.node                  = node,
+                                 .inbox_to_consume_from = "ibx_2"};
+
+  /* Consumer 3 */
+  pthread_t consumer_3_th     = {0};
+  th_struct consumer_3_struct = {.node                  = node,
+                                 .inbox_to_consume_from = "ibx_3"};
+
+  /* Launch threads */
+  pthread_create(&producer_th, NULL, producer_fn, &producer_struct);
+  pthread_create(&consumer_1_th, NULL, consumer_fn, &consumer_1_struct);
+  pthread_create(&consumer_2_th, NULL, consumer_fn, &consumer_2_struct);
+  pthread_create(&consumer_3_th, NULL, consumer_fn, &consumer_3_struct);
+
+  /* Join them */
+  pthread_join(producer_th, NULL);
+  pthread_join(consumer_1_th, NULL);
+  pthread_join(consumer_2_th, NULL);
+  pthread_join(consumer_3_th, NULL);
+
+  /* Cleanup */
+  x9_free_node_and_attached_inboxes(node);
+
+  printf("TEST PASSED: x9_example_4.c\n");
+  return EXIT_SUCCESS;
+}
diff --git a/Framework/Foundation/3rdparty/x9/examples/x9_example_5.c b/Framework/Foundation/3rdparty/x9/examples/x9_example_5.c
new file mode 100644
index 0000000000000..9f310feed6b90
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/examples/x9_example_5.c
@@ -0,0 +1,161 @@
+/* x9_example_5.c
+ *
+ *  Three producers.
+ *  Three consumers reading concurrently from the same inbox.
+ *  One message type.
+ *
+ *                                    ┌────────┐
+ *  ┌────────┐       ┏━━━━━━━━┓    ─ ─│Consumer│
+ *  │Producer│──────▷┃        ┃   │   └────────┘
+ *  ├────────┤       ┃        ┃       ┌────────┐
+ *  │Producer│──────▷┃ inbox  ┃◁──┤─ ─│Consumer│
+ *  ├────────┤       ┃        ┃       └────────┘
+ *  │Producer│──────▷┃        ┃   │   ┌────────┐
+ *  └────────┘       ┗━━━━━━━━┛    ─ ─│Consumer│
+ *                                    └────────┘
+ *
+ *  This example showcases the use of 'x9_read_from_shared_inbox'.
+ *
+ *  Data structures used:
+ *   - x9_inbox
+ *
+ *  Functions used:
+ *   - x9_create_inbox
+ *   - x9_inbox_is_valid
+ *   - x9_write_to_inbox_spin
+ *   - x9_read_from_shared_inbox
+ *   - x9_free_inbox
+ *
+ *  Test is considered passed iff:
+ *   - None of the threads stall and exit cleanly after doing the work.
+ *   - All messages sent by the producer(s) are received and asserted to be
+ *   valid by the consumer(s).
+ *   - Each consumer processes at least one message.
+ */
+
+#include <assert.h>    /* assert */
+#include <pthread.h>   /* pthread_t, pthread functions */
+#include <stdbool.h>   /* bool */
+#include <stdint.h>    /* uint64_t */
+#include <stdio.h>     /* printf */
+#include <stdlib.h>    /* rand, RAND_MAX */
+
+#include "../x9.h"
+
+/* Both producer and consumer loops, would commonly be infinite loops, but for
+ * the purpose of testing a reasonable NUMBER_OF_MESSAGES is defined. */
+#define NUMBER_OF_MESSAGES 1000000
+
+#define NUMBER_OF_PRODUCER_THREADS 3
+
+typedef struct {
+  x9_inbox* inbox;
+  uint64_t  msgs_read;
+} th_struct;
+
+typedef struct {
+  int  a;
+  int  b;
+  int  sum;
+  bool last_message;
+  char pad[3];
+} msg;
+
+static inline int random_int(int const min, int const max) {
+  return min + rand() / (RAND_MAX / (max - min + 1) + 1);
+}
+
+static inline void fill_msg_type(msg* const m) {
+  m->a   = random_int(0, 10);
+  m->b   = random_int(0, 10);
+  m->sum = m->a + m->b;
+}
+
+static void* producer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {0};
+  for (uint_fast64_t k = 0; k != NUMBER_OF_MESSAGES; ++k) {
+    fill_msg_type(&m);
+    if (k == (NUMBER_OF_MESSAGES - 1)) { m.last_message = true; }
+    x9_write_to_inbox_spin(data->inbox, sizeof(msg), &m);
+  }
+  return 0;
+}
+
+static void* consumer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {0};
+  for (;;) {
+    if (x9_read_from_shared_inbox(data->inbox, sizeof(msg), &m)) {
+      assert(m.sum == (m.a + m.b));
+      ++data->msgs_read;
+      if (m.last_message) { return 0; }
+    }
+  }
+}
+
+int main(void) {
+  /* Seed random generator */
+  srand((uint32_t)time(0));
+
+  /* Create inbox */
+  x9_inbox* const inbox = x9_create_inbox(4, "ibx", sizeof(msg));
+
+  /* Using assert to simplify code for presentation purpose. */
+  assert(x9_inbox_is_valid(inbox));
+
+  /* Producers */
+  pthread_t producer_1_th     = {0};
+  th_struct producer_1_struct = {.inbox = inbox};
+
+  pthread_t producer_2_th     = {0};
+  th_struct producer_2_struct = {.inbox = inbox};
+
+  pthread_t producer_3_th     = {0};
+  th_struct producer_3_struct = {.inbox = inbox};
+
+  /* Consumers */
+  pthread_t consumer_1_th     = {0};
+  th_struct consumer_1_struct = {.inbox = inbox};
+
+  pthread_t consumer_2_th     = {0};
+  th_struct consumer_2_struct = {.inbox = inbox};
+
+  pthread_t consumer_3_th     = {0};
+  th_struct consumer_3_struct = {.inbox = inbox};
+
+  /* Launch threads */
+  pthread_create(&producer_1_th, NULL, producer_fn, &producer_1_struct);
+  pthread_create(&producer_2_th, NULL, producer_fn, &producer_2_struct);
+  pthread_create(&producer_3_th, NULL, producer_fn, &producer_3_struct);
+  pthread_create(&consumer_1_th, NULL, consumer_fn, &consumer_1_struct);
+  pthread_create(&consumer_2_th, NULL, consumer_fn, &consumer_2_struct);
+  pthread_create(&consumer_3_th, NULL, consumer_fn, &consumer_3_struct);
+
+  /* Join them */
+  pthread_join(producer_1_th, NULL);
+  pthread_join(producer_2_th, NULL);
+  pthread_join(producer_3_th, NULL);
+  pthread_join(consumer_1_th, NULL);
+  pthread_join(consumer_2_th, NULL);
+  pthread_join(consumer_3_th, NULL);
+
+  /* Assert that all of the consumers read from the shared inbox. */
+  assert(consumer_1_struct.msgs_read > 0);
+  assert(consumer_2_struct.msgs_read > 0);
+  assert(consumer_3_struct.msgs_read > 0);
+
+  /* Assert that the total number of messages read == (NUMBER_OF_MESSAGES *
+   * NUMBER_OF_PRODUCER_THREADS) */
+  assert((NUMBER_OF_MESSAGES * NUMBER_OF_PRODUCER_THREADS) ==
+         (consumer_1_struct.msgs_read + consumer_2_struct.msgs_read +
+          consumer_3_struct.msgs_read));
+
+  /* Cleanup */
+  x9_free_inbox(inbox);
+
+  printf("TEST PASSED: x9_example_5.c\n");
+  return EXIT_SUCCESS;
+}
diff --git a/Framework/Foundation/3rdparty/x9/examples/x9_example_6.c b/Framework/Foundation/3rdparty/x9/examples/x9_example_6.c
new file mode 100644
index 0000000000000..1faad026e1feb
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/examples/x9_example_6.c
@@ -0,0 +1,159 @@
+/* x9_example_6.c
+ *
+ *  One producer
+ *  Two consumers reading from the same inbox concurrently (busy loop).
+ *  One message type.
+ *
+ *                   ┏━━━━━━━━┓       ┌────────┐
+ *                   ┃        ┃    ─ ─│Consumer│
+ *  ┌────────┐       ┃        ┃   │   └────────┘
+ *  │Producer│──────▷┃ inbox  ┃◁ ─
+ *  └────────┘       ┃        ┃   │   ┌────────┐
+ *                   ┃        ┃    ─ ─│Consumer│
+ *                   ┗━━━━━━━━┛       └────────┘
+ *
+ *  This example showcases the use of 'x9_read_from_shared_inbox_spin'.
+ *
+ *  Data structures used:
+ *   - x9_inbox
+ *
+ *  Functions used:
+ *   - x9_create_inbox
+ *   - x9_inbox_is_valid
+ *   - x9_write_to_inbox_spin
+ *   - x9_read_from_shared_inbox_spin
+ *   - x9_free_inbox
+ *
+ *  Test is considered passed iff:
+ *   - None of the threads stall and exit cleanly after doing the work.
+ *   - All messages sent by the producer(s) are received and asserted to be
+ *   valid by the consumer(s).
+ *   - Each consumer processes at least one message.
+ */
+
+#include <assert.h>    /* assert */
+#include <pthread.h>   /* pthread_t, pthread functions */
+#include <stdatomic.h> /* atomic_* */
+#include <stdbool.h>   /* bool */
+#include <stdint.h>    /* uint64_t */
+#include <stdio.h>     /* printf */
+#include <stdlib.h>    /* rand, RAND_MAX */
+
+#include "../x9.h"
+
+/* Both producer and consumer loops, would commonly be infinite loops, but for
+ * the purpose of testing a reasonable NUMBER_OF_MESSAGES is defined. */
+#define NUMBER_OF_MESSAGES 1000000
+
+typedef struct {
+  x9_inbox* inbox;
+  uint64_t  msgs_read;
+} th_struct;
+
+typedef struct {
+  int  a;
+  int  b;
+  int  sum;
+  bool last_message;
+  char pad[3];
+} msg;
+
+static inline int random_int(int const min, int const max) {
+  return min + rand() / (RAND_MAX / (max - min + 1) + 1);
+}
+
+static inline void fill_msg_type(msg* const m) {
+  m->a   = random_int(0, 10);
+  m->b   = random_int(0, 10);
+  m->sum = m->a + m->b;
+}
+
+static void* producer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {0};
+  for (uint_fast64_t k = 0; k != (NUMBER_OF_MESSAGES); ++k) {
+    fill_msg_type(&m);
+    if (k == (NUMBER_OF_MESSAGES - 1)) { m.last_message = true; }
+    x9_write_to_inbox_spin(data->inbox, sizeof(msg), &m);
+  }
+  return 0;
+}
+
+static void* consumer_fn(void* args) {
+  th_struct* data = (th_struct*)args;
+  msg        m    = {0};
+  for (;;) {
+    x9_read_from_shared_inbox_spin(data->inbox, sizeof(msg), &m);
+    assert(m.sum == (m.a + m.b));
+
+    /* The First thread to read the 'last message' writes the same message back
+     * to the inbox, so in case the second thread had already entered
+     * 'x9_read_from_shared_inbox_spin' function, it will be able to read the
+     * message and do a clean exit.
+     * If the second thread, at the time that the first thread set
+     * 'g_read_last= true' to hasn't entered 'x9_write_to_inbox_spin', then it
+     * will check the if(g_read_last) condition above and exit.
+     * The first case would equal NUMBER_OF_MESSAGES + 1 read.
+     * The second case, NUMBER_OF_MESSAGES.
+     * Given the implementation of 'x9_read_from_shared_inbox_spin', this is
+     * the only way to get a clean exit without killing the second thread. */
+    if (m.last_message) {
+      x9_write_to_inbox_spin(data->inbox, sizeof(msg), &m);
+      ++data->msgs_read;
+      return 0;
+    }
+    ++data->msgs_read;
+  }
+}
+
+int main(void) {
+  /* Seed random generator */
+  srand((uint32_t)time(0));
+
+  /* Create inbox */
+  x9_inbox* const inbox = x9_create_inbox(4, "ibx", sizeof(msg));
+
+  /* Using assert to simplify code for presentation purpose */
+  assert(x9_inbox_is_valid(inbox));
+
+  /* Producer */
+  pthread_t producer_th     = {0};
+  th_struct producer_struct = {.inbox = inbox};
+
+  /* Consumers */
+  pthread_t consumer_1_th     = {0};
+  th_struct consumer_1_struct = {.inbox = inbox};
+
+  pthread_t consumer_2_th     = {0};
+  th_struct consumer_2_struct = {.inbox = inbox};
+
+  /* Launch threads */
+  pthread_create(&producer_th, NULL, producer_fn, &producer_struct);
+  pthread_create(&consumer_1_th, NULL, consumer_fn, &consumer_1_struct);
+  pthread_create(&consumer_2_th, NULL, consumer_fn, &consumer_2_struct);
+
+  /* Join them */
+  pthread_join(producer_th, NULL);
+  pthread_join(consumer_1_th, NULL);
+  pthread_join(consumer_2_th, NULL);
+
+  /* Assert that all of the consumers read from the shared inbox. */
+  assert(consumer_1_struct.msgs_read > 0);
+  assert(consumer_2_struct.msgs_read > 0);
+
+  /* Assert that the total number of messages read == NUMBER_OF_MESSAGES or
+   * NUMBER_OF_MESSAGES + 1 */
+  assert((NUMBER_OF_MESSAGES + 1) ==
+             (consumer_1_struct.msgs_read + consumer_2_struct.msgs_read) ||
+         (NUMBER_OF_MESSAGES) ==
+             (consumer_1_struct.msgs_read + consumer_2_struct.msgs_read)
+
+  );
+
+  /* Cleanup */
+  x9_free_inbox(inbox);
+
+  printf("TEST PASSED: x9_example_6.c\n");
+  return EXIT_SUCCESS;
+}
diff --git a/Framework/Foundation/3rdparty/x9/profiling/README.md b/Framework/Foundation/3rdparty/x9/profiling/README.md
new file mode 100644
index 0000000000000..c10f3f1c658cc
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/profiling/README.md
@@ -0,0 +1,106 @@
+Notes:
+---
+
+```
+x9_profiler.c
+
+- One producer thread
+- One consumer thread
+- One x9_inbox
+
+┌────────┐       ┏━━━━━━━━┓       ┌────────┐
+│Producer│──────▷┃ inbox  ┃◁ ─ ─ ─│Consumer│
+└────────┘       ┗━━━━━━━━┛       └────────┘
+```
+
+`x9_profiler.c` allows to test the performance of the library given different
+parameters, such as the _inbox size_, _message size_ and in which _cpu cores_
+the test is performed.  
+It does so by having the producer/writer write _N messages_ to the _inbox_ and
+the consumer/reader read them.  
+All of these parameters are user defined and passed to the program as command 
+line arguments, as shown below.
+
+Additionally, two different types of tests can be run:
+- **--test 1** should be used to get an idea of the raw performance of the 
+library since it calls `x9_write_to_inbox_spin` and `x9_read_from_inbox_spin` 
+in the background, which is ideal for low latency systems.
+- **--test 2** uses the non spinning version of the functions above, and while
+it will be slower, it allows to understand the _hit ratio_ of
+both the producer and consumer.
+
+_Hit ratio_ is defined as the number of messages the writer(reader) wrote(read)
+divided by the number of times it attempted to write(read), and can be helpful 
+when deciding in which _cpu cores_ to run specific threads.
+
+Overall, both tests are useful for fine-tuning which _inbox size_ to use and 
+gaining a better understanding of the expected performance when transmitting 
+_messages_ of a given type (i.e. sizeof(some_type)).  
+Moreover, they can be used to measure the impact of overclocking 
+(on specific cores or as a whole).
+
+- When `--n_its` > 1  the results presented will be the _median_ of all 
+iterations.
+- The writer will run on the first core of the values passed to
+`--run_in_cores` and the reader in the second.
+
+The program can be compiled with: _./compile_profiler.sh_ and run as follows:
+
+```
+Example (test 1):
+
+$ ./X9_PROF \
+  --test 1 \
+  --inboxes_szs 1024,2048,4096 \
+  --msgs_szs 16,32,64 \
+  --n_msgs 100000000 \
+  --n_its 1 \
+  --run_in_cores 2,4
+
+
+Inbox size | Msg size | Time (secs) | Msgs/Second
+-------------------------------------------------
+      1024 |       16 |        8.82 |      11.33M
+      1024 |       32 |        8.70 |      11.50M
+      1024 |       64 |        8.65 |      11.56M
+-------------------------------------------------
+      2048 |       16 |        9.10 |      10.99M
+      2048 |       32 |        9.53 |      10.49M
+      2048 |       64 |        9.44 |      10.60M
+-------------------------------------------------
+      4096 |       16 |        9.08 |      11.01M
+      4096 |       32 |        9.53 |      10.50M
+      4096 |       64 |        9.43 |      10.60M
+-------------------------------------------------
+```
+
+```
+Example (test 2):
+
+$ ./X9_PROF \
+  --test 2 \
+  --inboxes_szs 1024,2048,4096 \
+  --msgs_szs 16,32,64,128 \
+  --n_msgs 100000000 \
+  --n_its 1 \
+  --run_in_cores 2,4
+
+
+Inbox size | Msg size | Time (secs) | Msgs/second | Writer hit ratio | Reader hit ratio
+---------------------------------------------------------------------------------------
+      1024 |       16 |       12.69 |       7.88M |          100.00% |           69.92%
+      1024 |       32 |       13.00 |       7.69M |          100.00% |           56.46%
+      1024 |       64 |       13.79 |       7.25M |           46.85% |           99.88%
+      1024 |      128 |       11.53 |       8.68M |          100.00% |           67.11%
+---------------------------------------------------------------------------------------
+      2048 |       16 |       11.92 |       8.39M |          100.00% |           68.22%
+      2048 |       32 |       11.65 |       8.58M |          100.00% |           82.29%
+      2048 |       64 |       10.05 |       9.95M |           81.91% |           99.81%
+      2048 |      128 |       11.64 |       8.59M |          100.00% |           78.21%
+---------------------------------------------------------------------------------------
+      4096 |       16 |       12.19 |       8.21M |          100.00% |           79.13%
+      4096 |       32 |       12.75 |       7.84M |          100.00% |           65.95%
+      4096 |       64 |       12.10 |       8.26M |          100.00% |           67.25%
+      4096 |      128 |       11.61 |       8.61M |          100.00% |           78.29%
+---------------------------------------------------------------------------------------
+```
diff --git a/Framework/Foundation/3rdparty/x9/profiling/compile_profiler.sh b/Framework/Foundation/3rdparty/x9/profiling/compile_profiler.sh
new file mode 100755
index 0000000000000..0f89b0e9acdc1
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/profiling/compile_profiler.sh
@@ -0,0 +1,3 @@
+#/bin/bash
+
+gcc -Wextra -Wall -Werror -pedantic -flto -O3 -march=native x9_profiler.c ../x9.c -o X9_PROF
diff --git a/Framework/Foundation/3rdparty/x9/profiling/x9_profiler.c b/Framework/Foundation/3rdparty/x9/profiling/x9_profiler.c
new file mode 100644
index 0000000000000..d596dd816791d
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/profiling/x9_profiler.c
@@ -0,0 +1,524 @@
+/* x9_profiler.c:
+ *
+ *  One producer
+ *  One consumer
+ *  One message type
+ *
+ *  ┌────────┐       ┏━━━━━━━━┓       ┌────────┐
+ *  │Producer│──────▷┃ inbox  ┃◁ ─ ─ ─│Consumer│
+ *  └────────┘       ┗━━━━━━━━┛       └────────┘
+ *
+ *  '--test 1' uses 'x9_write_to_inbox_spin' and 'x9_read_from_inbox_spin'
+ *  '--test 2' uses x9_read_from_inbox and 'x9_read_from_inbox'
+ *
+ *  The advantage of '--test 2' is that, given its non spinning nature,
+ *  it's possible to gather more performance metrics.
+ */
+
+#define _GNU_SOURCE  /* cpu_*, pthread_setaffinity_np */
+#include <assert.h>  /* assert */
+#include <getopt.h>  /* required_argument, getopt_long */
+#include <pthread.h> /* pthread_t, pthread functions */
+#include <stdint.h>  /* uint8_t, uint64_t, int64_t */
+#include <stdio.h>   /* printf */
+#include <stdlib.h>  /* qsort, rand, RAND_MAX */
+#include <string.h>  /* memset, strcmp */
+#include <unistd.h>  /* sysconf, _SC_NPROCESSORS_ONLN */
+
+#include "../x9.h"
+
+typedef enum { HEADER = 1, SEPARATOR } stdout_output;
+
+#define ARG(arg_name) (!strcmp(long_options[option_idx].name, arg_name))
+
+typedef struct vector {
+  uint64_t size;
+  uint64_t used;
+  int64_t* data;
+} vector;
+
+typedef struct {
+  vector* inboxes_sizes;
+  vector* msgs_sizes;
+  vector* run_in_cores;
+  int64_t n_messages;
+  int64_t n_iterations;
+  int64_t test;
+} perf_config;
+
+typedef struct {
+  double time_secs;
+  double writer_hit_ratio;
+  double reader_hit_ratio;
+} perf_results;
+
+typedef struct {
+  x9_inbox* inbox;
+  uint64_t  msg_sz;
+  uint64_t  n_msgs;
+  double    writer_hit_ratio;
+  double    reader_hit_ratio;
+} th_struct;
+
+typedef struct {
+  uint8_t* a;
+} msg;
+
+__attribute__((noreturn)) static void abort_test(char const* const msg) {
+  printf("%s\n", msg);
+  abort();
+}
+
+static vector* vector_init(uint64_t const sz) {
+  if ((sz % 2)) { abort_test("ERROR: vector sz must be % 2 == 0"); }
+  vector* const v = calloc(1, sizeof(vector));
+  if (NULL == v) { abort_test("ERROR: failed to allocate vector."); }
+  v->data = calloc(1, sz * sizeof(*v->data));
+  if (NULL == v->data) { abort_test("ERROR: failed to allocate v->data."); }
+  v->size = sz;
+  return v;
+}
+
+static void vector_insert(vector* const v, int64_t value) {
+  if (v->used == v->size) {
+    v->size *= 2;
+    v->data = realloc(v->data, v->size * sizeof(*v->data));
+    if (NULL == v->data) { abort_test("ERROR: realloc failed."); }
+  }
+  v->data[v->used++] = value;
+}
+
+static void vector_free(vector* const v) {
+  free(v->data);
+  free(v);
+}
+
+static int random_int(int const min, int const max) {
+  return min + rand() / (RAND_MAX / (max - min + 1) + 1);
+}
+
+static void* producer_fn_test_1(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {.a = calloc(data->msg_sz, sizeof(uint8_t))};
+  if (NULL == m.a) { abort_test("ERROR: failed to allocate msg buffer"); }
+
+  for (uint64_t k = 0; k != data->n_msgs; ++k) {
+    int32_t const random_val = random_int(1, 9);
+    memset(m.a, random_val, data->msg_sz);
+    x9_write_to_inbox_spin(data->inbox, data->msg_sz, m.a);
+  }
+  free(m.a);
+  return 0;
+}
+
+static void* consumer_fn_test_1(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {.a = calloc(data->msg_sz, sizeof(uint8_t))};
+  if (NULL == m.a) { abort_test("ERROR: failed to allocate msg buffer"); }
+
+  for (uint64_t k = 0; k != data->n_msgs; ++k) {
+    x9_read_from_inbox_spin(data->inbox, data->msg_sz, m.a);
+    assert(m.a[(data->msg_sz - 1)] == (m.a[0]));
+  }
+  free(m.a);
+  return 0;
+}
+
+static void* producer_fn_test_2(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {.a = calloc(data->msg_sz, sizeof(uint8_t))};
+  if (NULL == m.a) { abort_test("ERROR: failed to allocate msg buffer"); }
+
+  uint64_t write_attempts = 0;
+  uint64_t msgs_written   = 0;
+
+  for (;;) {
+    if (msgs_written == data->n_msgs) { break; }
+    int32_t const random_val = random_int(1, 9);
+    memset(m.a, random_val, data->msg_sz);
+    if (x9_write_to_inbox(data->inbox, data->msg_sz, m.a)) { ++msgs_written; }
+    ++write_attempts;
+  }
+
+  data->writer_hit_ratio = (double)msgs_written / (double)write_attempts;
+  free(m.a);
+  return 0;
+}
+
+static void* consumer_fn_test_2(void* args) {
+  th_struct* data = (th_struct*)args;
+
+  msg m = {.a = calloc(data->msg_sz, sizeof(uint8_t))};
+  if (NULL == m.a) { abort_test("ERROR: failed to allocate msg buffer"); }
+
+  uint64_t read_attempts = 0;
+  uint64_t msgs_read     = 0;
+
+  for (;;) {
+    if (msgs_read == data->n_msgs) { break; }
+    if (x9_read_from_inbox(data->inbox, data->msg_sz, m.a)) {
+      ++msgs_read;
+      assert(m.a[(data->msg_sz - 1)] == (m.a[0]));
+    }
+    ++read_attempts;
+  }
+
+  data->reader_hit_ratio = (double)msgs_read / (double)read_attempts;
+  free(m.a);
+  return 0;
+}
+
+static perf_results run_test(uint64_t const ibx_sz,
+                             uint64_t const msg_sz,
+                             uint64_t const n_msgs,
+                             uint64_t const first_core,
+                             uint64_t const second_core,
+                             uint64_t const test
+
+) {
+  /* Create inbox */
+  x9_inbox* const inbox = x9_create_inbox(ibx_sz, "ibx_1", msg_sz);
+
+  /* Confirm that it's valid */
+  if (!(x9_inbox_is_valid(inbox))) {
+    abort_test("ERROR: x9_inbox is invalid");
+  }
+
+  /* Producer */
+  pthread_t      producer_th   = {0};
+  pthread_attr_t producer_attr = {0};
+  pthread_attr_init(&producer_attr);
+  th_struct producer_struct = {
+      .inbox = inbox, .msg_sz = msg_sz, .n_msgs = n_msgs};
+
+  /* Consumer */
+  pthread_t      consumer_th   = {0};
+  pthread_attr_t consumer_attr = {0};
+  pthread_attr_init(&consumer_attr);
+  th_struct consumer_struct = {
+      .inbox = inbox, .msg_sz = msg_sz, .n_msgs = n_msgs};
+
+  /* Set affinity */
+  cpu_set_t f_core = {0};
+  CPU_ZERO(&f_core);
+  CPU_SET(first_core, &f_core);
+
+  cpu_set_t s_core = {0};
+  CPU_ZERO(&s_core);
+  CPU_SET(second_core, &s_core);
+
+  pthread_attr_setaffinity_np(&producer_attr, sizeof(cpu_set_t), &f_core);
+  pthread_attr_setaffinity_np(&consumer_attr, sizeof(cpu_set_t), &s_core);
+
+  /* Start timer */
+  struct timespec tic = {0};
+  clock_gettime(CLOCK_MONOTONIC, &tic);
+
+  /* Launch threads */
+  if (1 == test) {
+    pthread_create(&consumer_th, &consumer_attr, consumer_fn_test_1,
+                   &consumer_struct);
+    pthread_create(&producer_th, &producer_attr, producer_fn_test_1,
+                   &producer_struct);
+
+  } else {
+    pthread_create(&consumer_th, &consumer_attr, consumer_fn_test_2,
+                   &consumer_struct);
+    pthread_create(&producer_th, &producer_attr, producer_fn_test_2,
+                   &producer_struct);
+  }
+
+  /* Join them */
+  pthread_join(consumer_th, NULL);
+  pthread_join(producer_th, NULL);
+
+  /* Stop timer */
+  struct timespec toc = {0};
+  clock_gettime(CLOCK_MONOTONIC, &toc);
+
+  uint64_t const before =
+      ((uint64_t)tic.tv_sec * 1000000000UL) + (uint64_t)tic.tv_nsec;
+  uint64_t const after =
+      ((uint64_t)toc.tv_sec * 1000000000UL) + (uint64_t)toc.tv_nsec;
+
+  /* Cleanup */
+  pthread_attr_destroy(&producer_attr);
+  pthread_attr_destroy(&consumer_attr);
+  x9_free_inbox(inbox);
+
+  return (perf_results){.time_secs        = (double)(after - before) / 1e9,
+                        .writer_hit_ratio = producer_struct.writer_hit_ratio,
+                        .reader_hit_ratio = consumer_struct.reader_hit_ratio};
+}
+
+static void parse_array_arguments(char* restrict const args,
+                                  vector* const write_to) {
+  char*             args_start = args;
+  char const* const args_end   = args + strlen(args);
+
+  for (; args_start < args_end;) {
+    char* begin = args_start;
+    char* end   = args_start;
+    for (; end != args_end; ++end) {
+      if (',' == *end) { break; }
+    }
+    args_start      = end + 1;
+    int64_t const n = strtoll(begin, &end, 10);
+    vector_insert(write_to, n);
+  }
+}
+
+static perf_config* parse_command_line_args(int argc, char** argv) {
+  perf_config* config = calloc(1, sizeof(perf_config));
+  if (NULL == config) {
+    abort_test("ERROR: failed to allocate 'perf_config'");
+  }
+
+  for (;;) {
+    int                  option_idx     = 0;
+    static struct option long_options[] = {
+        {"inboxes_szs", required_argument, 0, 0},
+        {"msgs_szs", required_argument, 0, 0},
+        {"n_msgs", required_argument, 0, 0},
+        {"n_its", required_argument, 0, 0},
+        {"run_in_cores", required_argument, 0, 0},
+        {"test", required_argument, 0, 0},
+        {0, 0, 0, 0}
+
+    };
+
+    int const c = getopt_long(argc, argv, "", long_options, &option_idx);
+    if (-1 == c) { break; }
+
+    switch (c) {
+      case 0:
+        if (ARG("inboxes_szs")) {
+          config->inboxes_sizes = vector_init(8);
+
+          parse_array_arguments(optarg, config->inboxes_sizes);
+
+          if (!config->inboxes_sizes->data[0]) {
+            abort_test(
+                "ERROR: test requires at least one value for "
+                "'--inboxes_sizes'");
+          }
+
+          for (uint64_t k = 0; k != config->inboxes_sizes->used; ++k) {
+            int64_t const n = config->inboxes_sizes->data[k];
+            if (!((n > 0) && ((n % 2) == 0))) {
+              abort_test(
+                  "ERROR: '--inboxes_sizes' values must be > 0 and % 2 == "
+                  "0");
+            }
+          }
+        }
+        if (ARG("msgs_szs")) {
+          config->msgs_sizes = vector_init(8);
+          parse_array_arguments(optarg, config->msgs_sizes);
+
+          if (!config->msgs_sizes->data[0]) {
+            abort_test(
+                "ERROR: test requires at least one value for "
+                "'--msgs_sizes'");
+          }
+
+          for (uint64_t k = 0; k != config->msgs_sizes->used; ++k) {
+            int64_t const n = config->msgs_sizes->data[k];
+            if (!(n > 0)) {
+              abort_test("ERROR: '--msgs_sizes' values must be > 0");
+            }
+          }
+        }
+
+        if (ARG("run_in_cores")) {
+          config->run_in_cores = vector_init(8);
+          parse_array_arguments(optarg, config->run_in_cores);
+
+          if (config->run_in_cores->used != 2) {
+            abort_test("ERROR: --run_in_cores requires two values.");
+          }
+
+          int64_t const n_cores = sysconf(_SC_NPROCESSORS_ONLN);
+
+          if ((config->run_in_cores->data[0] < 0) ||
+              config->run_in_cores->data[0] > n_cores ||
+              (config->run_in_cores->data[1] < 0) ||
+              config->run_in_cores->data[1] > n_cores) {
+            char abort_msg[128] = {0};
+            sprintf(abort_msg,
+                    "ERROR: '--run_in_cores' values must be between 0 "
+                    "and %ld",
+                    n_cores);
+            abort_test(abort_msg);
+          }
+        }
+
+        if (ARG("n_msgs")) {
+          int64_t const n = atoll(optarg);
+          if (!(n > 0)) { abort_test("ERROR: '--n_msgs' value must be > 0"); }
+          config->n_messages = n;
+        }
+
+        if (ARG("n_its")) {
+          int64_t const n = atoll(optarg);
+          if (!(n > 0)) { abort_test("ERROR: '--n_its' value must be > 0"); }
+          config->n_iterations = n;
+        }
+
+        if (ARG("test")) {
+          int64_t n = atoll(optarg);
+          if (!((n > 0) && (n < 3))) {
+            abort_test("ERROR: '--test' value must be either '1' or '2'");
+          }
+          config->test = n;
+        }
+    }
+  }
+
+  if (!config->test || !config->inboxes_sizes || !config->msgs_sizes ||
+      !config->n_messages || !config->n_iterations || !config->run_in_cores) {
+    abort_test("ERROR: missing command line arguments.");
+  }
+
+  if (1 == config->test) {
+    if (config->run_in_cores->data[0] == config->run_in_cores->data[1]) {
+      abort_test(
+          "ERROR: for '--test 1' the values of '--run_in_cores' can not be "
+          "equal because there's no sched_yield())'");
+    }
+  }
+  return config;
+}
+
+static void free_perf_config(perf_config* config) {
+  vector_free(config->inboxes_sizes);
+  vector_free(config->msgs_sizes);
+  vector_free(config->run_in_cores);
+  free(config);
+}
+
+static void print_to_stdout(perf_config const* const config,
+                            stdout_output const      what_to_print) {
+  char const* const i_sz     = "Inbox size";
+  char const* const sep      = " | ";
+  char const* const m_sz     = "Msg size";
+  char const* const time     = "Time (secs)";
+  char const* const m_sec    = "Msgs/second";
+  char const* const prod_hit = "Writer hit ratio";
+  char const* const cons_hit = "Reader hit ratio";
+
+  uint64_t const test_1_sep_len = strlen(i_sz) + strlen(m_sz) + strlen(time) +
+                                  strlen(m_sec) + (3 * strlen(sep));
+
+  uint64_t const test_2_sep_len = strlen(i_sz) + strlen(m_sz) + strlen(time) +
+                                  strlen(m_sec) + strlen(prod_hit) +
+                                  strlen(cons_hit) + (5 * strlen(sep));
+
+  if (HEADER == what_to_print) {
+    if (1 == config->test) {
+      printf("\n%s%s%s%s%s%s%s\n", i_sz, sep, m_sz, sep, time, sep, m_sec);
+
+    } else {
+      printf("\n%s%s%s%s%s%s%s%s%s%s%s\n", i_sz, sep, m_sz, sep, time, sep,
+             m_sec, sep, prod_hit, sep, cons_hit);
+    }
+  }
+  if (1 == config->test) {
+    for (uint64_t k = 0; k != test_1_sep_len; ++k) { fputs("-", stdout); }
+  } else {
+    for (uint64_t k = 0; k != test_2_sep_len; ++k) { fputs("-", stdout); }
+  }
+  puts("");
+}
+
+static int cmp(const void* a, const void* b) {
+  return (*(const double*)a > *(const double*)b)   ? 1
+         : (*(const double*)a < *(const double*)b) ? -1
+                                                   : 0;
+}
+
+static double calculate_median(uint64_t const sz, double* const arr) {
+  qsort(arr, sz, sizeof(double), cmp);
+  return ((sz % 2) == 0) ? ((arr[sz / 2 - 1] + arr[sz / 2]) / 2) : arr[sz / 2];
+}
+
+int main(int argc, char** argv) {
+  /* Seed random generator */
+  srand((uint32_t)time(0));
+
+  perf_config* config = parse_command_line_args(argc, argv);
+
+  print_to_stdout(config, HEADER);
+
+  double* time_secs = calloc((uint64_t)config->n_iterations, sizeof(double));
+  if (NULL == time_secs) {
+    abort_test("ERROR: failed to allocate 'time_secs'");
+  }
+
+  double* writer_hit_ratio =
+      calloc((uint64_t)config->n_iterations, sizeof(double));
+  if (NULL == writer_hit_ratio) {
+    abort_test("ERROR: failed to allocate 'writer_hit_ratio'");
+  }
+
+  double* reader_hit_ratio =
+      calloc((uint64_t)config->n_iterations, sizeof(double));
+  if (NULL == reader_hit_ratio) {
+    abort_test("ERROR: failed to allocate 'reader_hit_ratio'");
+  }
+
+  for (uint64_t k = 0; k != config->inboxes_sizes->used; ++k) {
+    if (config->inboxes_sizes->data[k]) {
+      for (uint64_t j = 0; j != config->msgs_sizes->used; ++j) {
+        if (config->msgs_sizes->data[j]) {
+          for (uint64_t it = 0; it != (uint64_t)config->n_iterations; ++it) {
+            perf_results results =
+                run_test((uint64_t)config->inboxes_sizes->data[k],
+                         (uint64_t)config->msgs_sizes->data[j],
+                         (uint64_t)config->n_messages,
+                         (uint64_t)config->run_in_cores->data[0],
+                         (uint64_t)config->run_in_cores->data[1],
+                         (uint64_t)config->test);
+
+            time_secs[it]        = results.time_secs;
+            writer_hit_ratio[it] = results.writer_hit_ratio;
+            reader_hit_ratio[it] = results.reader_hit_ratio;
+          }
+
+          double const median_secs =
+              calculate_median((uint64_t)config->n_iterations, time_secs);
+
+          printf("%10ld | ", config->inboxes_sizes->data[k]);
+          printf("%8ld | ", config->msgs_sizes->data[j]);
+          median_secs > 1 ? printf("%*.2f | ", 11, median_secs)
+                          : printf("%*.4f | ", 11, median_secs);
+          printf("%*.2fM", 10,
+                 ((double)config->n_messages / median_secs) / 1e6);
+
+          if (2 == config->test) {
+            double const median_writer_hit = calculate_median(
+                (uint64_t)config->n_iterations, writer_hit_ratio);
+
+            double const median_reader_hit = calculate_median(
+                (uint64_t)config->n_iterations, reader_hit_ratio);
+
+            printf(" |%*.2f%% | ", 16, median_writer_hit * 100);
+            printf("%*.2f%%", 15, median_reader_hit * 100);
+          }
+        }
+        puts("");
+      }
+      print_to_stdout(config, SEPARATOR);
+    }
+  }
+
+  puts("");
+  free(time_secs);
+  free(writer_hit_ratio);
+  free(reader_hit_ratio);
+  free_perf_config(config);
+  return EXIT_SUCCESS;
+}
diff --git a/Framework/Foundation/3rdparty/x9/x9.c b/Framework/Foundation/3rdparty/x9/x9.c
new file mode 100644
index 0000000000000..46d3d126de9f4
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/x9.c
@@ -0,0 +1,424 @@
+/*
+   X9 - high performance message passing library.
+   Copyright (c) 2023, Diogo Flores
+
+   BSD 2-Clause License (https://opensource.org/license/bsd-2-clause/)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at: diogoxflores@gmail.com
+*/
+
+#pragma GCC diagnostic ignored "-Wpadded"
+
+#include "x9.h"
+
+#include <assert.h>    /* assert */
+#include <immintrin.h> /* _mm_pause */
+#include <stdarg.h>    /* va_* */
+#include <stdatomic.h> /* atomic_* */
+#include <stdbool.h>   /* bool */
+#include <stdio.h>     /* printf */
+#include <stdlib.h>    /* aligned_alloc, calloc */
+#include <string.h>    /* strcmp */
+#include <string.h>    /* memcpy */
+
+/* CPU cache line size */
+#define X9_CL_SIZE       64
+#define X9_ALIGN_TO_CL() __attribute__((__aligned__(X9_CL_SIZE)))
+
+#ifdef X9_DEBUG
+static void x9_print_error_msg(char const* const error_msg) {
+  printf("X9_ERROR: %s\n", error_msg);
+  fflush(stdout);
+  return;
+}
+#endif
+
+/* --- Internal types --- */
+
+typedef struct {
+  _Atomic(bool) slot_has_data;
+  _Atomic(bool) msg_written;
+  _Atomic(bool) shared;
+  char const    pad[5];
+} x9_msg_header;
+
+typedef struct x9_inbox_internal {
+  _Atomic(uint64_t) read_idx  X9_ALIGN_TO_CL();
+  _Atomic(uint64_t) write_idx X9_ALIGN_TO_CL();
+  uint64_t sz                 X9_ALIGN_TO_CL();
+  uint64_t                    msg_sz;
+  uint64_t                    constant;
+  void*                       msgs;
+  char*                       name;
+  char                        pad[24];
+} x9_inbox;
+
+typedef struct x9_node_internal {
+  x9_inbox** inboxes;
+  uint64_t   n_inboxes;
+  char*      name;
+} x9_node;
+
+/* --- Internal functions --- */
+
+static inline uint64_t x9_load_idx(x9_inbox* const inbox,
+                                   bool const      read_idx) {
+  /* From paper: Faster Remainder by Direct Computation, Lemire et al */
+  register uint64_t const low_bits =
+      inbox->constant *
+      (read_idx ? atomic_load_explicit(&inbox->read_idx, __ATOMIC_RELAXED)
+                : atomic_load_explicit(&inbox->write_idx, __ATOMIC_RELAXED));
+  return ((__uint128_t)low_bits * inbox->sz) >> 64;
+}
+
+static inline uint64_t x9_increment_idx(x9_inbox* const inbox,
+                                        bool const      read_idx) {
+  /* From paper: Faster Remainder by Direct Computation, Lemire et al */
+  register uint64_t const low_bits =
+      inbox->constant *
+      (read_idx
+           ? atomic_fetch_add_explicit(&inbox->read_idx, 1, __ATOMIC_RELAXED)
+           : atomic_fetch_add_explicit(&inbox->write_idx, 1,
+                                       __ATOMIC_RELAXED));
+  return ((__uint128_t)low_bits * inbox->sz) >> 64;
+}
+
+static inline void* x9_header_ptr(x9_inbox const* const inbox,
+                                  uint64_t const        idx) {
+  return &((char*)inbox->msgs)[idx * (inbox->msg_sz + sizeof(x9_msg_header))];
+}
+
+/* --- Public functions --- */
+
+x9_inbox* x9_create_inbox(uint64_t const sz,
+                          char const* restrict const name,
+                          uint64_t const msg_sz) {
+  if (!((sz > 0) && !(sz % 2))) { goto inbox_incorrect_size; }
+
+  x9_inbox* inbox = aligned_alloc(X9_CL_SIZE, sizeof(x9_inbox));
+  if (NULL == inbox) { goto inbox_allocation_failed; }
+  memset(inbox, 0, sizeof(x9_inbox));
+
+  uint64_t const name_len = strlen(name);
+  char*          ibx_name = calloc(name_len + 1, sizeof(char));
+  if (NULL == ibx_name) { goto inbox_name_allocation_failed; }
+  memcpy(ibx_name, name, name_len);
+
+  void* msgs = calloc(sz, msg_sz + sizeof(x9_msg_header));
+  if (NULL == msgs) { goto inbox_msgs_allocation_failed; }
+
+  inbox->constant = UINT64_C(0xFFFFFFFFFFFFFFFF) / sz + 1;
+  inbox->name     = ibx_name;
+  inbox->msgs     = msgs;
+  inbox->sz       = sz;
+  inbox->msg_sz   = msg_sz;
+  return inbox;
+
+inbox_incorrect_size:
+#ifdef X9_DEBUG
+  x9_print_error_msg("INBOX_INCORRECT_SIZE");
+#endif
+  return NULL;
+
+inbox_allocation_failed:
+#ifdef X9_DEBUG
+  x9_print_error_msg("INBOX_ALLOCATION_FAILED");
+#endif
+  return NULL;
+
+inbox_name_allocation_failed:
+#ifdef X9_DEBUG
+  x9_print_error_msg("INBOX_NAME_ALLOCATION_FAILED");
+#endif
+  free(inbox);
+  return NULL;
+
+inbox_msgs_allocation_failed:
+#ifdef X9_DEBUG
+  x9_print_error_msg("INBOX_MSGS_ALLOCATION_FAILED");
+#endif
+  free(ibx_name);
+  free(inbox);
+  return NULL;
+}
+
+bool x9_inbox_is_valid(x9_inbox const* const inbox) {
+  return !(NULL == inbox);
+}
+
+bool x9_inbox_name_is(x9_inbox const* const inbox,
+                      char const* restrict const cmp) {
+  return !strcmp(inbox->name, cmp) ? true : false;
+}
+
+void x9_free_inbox(x9_inbox* const inbox) {
+  free(inbox->msgs);
+  free(inbox->name);
+  free(inbox);
+}
+
+x9_inbox* x9_select_inbox_from_node(x9_node const* const node,
+                                    char const* restrict const name) {
+  for (uint64_t k = 0; k != node->n_inboxes; ++k) {
+    if (x9_inbox_name_is(node->inboxes[k], name)) { return node->inboxes[k]; }
+  }
+#ifdef X9_DEBUG
+  x9_print_error_msg("NODE_DOES_NOT_CONTAIN_INBOX");
+#endif
+  return NULL;
+}
+
+x9_node* x9_create_node(char* restrict const name,
+                        uint64_t const n_inboxes,
+                        ...) {
+  if (!(n_inboxes > 0)) { goto node_incorrect_definition; }
+
+  x9_node* node = calloc(1, sizeof(x9_node));
+  if (NULL == node) { goto node_allocation_failed; }
+
+  uint64_t const name_len  = strlen(name);
+  char*          node_name = calloc(name_len + 1, sizeof(char));
+  if (NULL == node_name) { goto node_name_allocation_failed; }
+  memcpy(node_name, name, name_len);
+  node->name = node_name;
+
+  va_list argp = {0};
+  va_start(argp, n_inboxes);
+
+  x9_inbox** inboxes = calloc(n_inboxes, sizeof(x9_inbox));
+  if (NULL == inboxes) { goto node_inboxes_allocation_failed; }
+  node->inboxes   = inboxes;
+  node->n_inboxes = n_inboxes;
+
+  for (uint64_t k = 0; k != n_inboxes; ++k) {
+    x9_inbox* inbox = va_arg(argp, x9_inbox*);
+    if (k) {
+      for (uint64_t j = 0; j != k; ++j) {
+        if (inbox == node->inboxes[j]) {
+          goto node_multiple_equal_inboxes;
+        } else {
+          node->inboxes[k] = inbox;
+        }
+      }
+    } else {
+      node->inboxes[k] = inbox;
+    }
+  }
+  va_end(argp);
+
+  return node;
+
+node_incorrect_definition:
+#ifdef X9_DEBUG
+  x9_print_error_msg("NODE_INCORRECT_DEFINITION");
+#endif
+  return NULL;
+
+node_allocation_failed:
+#ifdef X9_DEBUG
+  x9_print_error_msg("NODE_ALLOCATION_FAILED");
+#endif
+  return NULL;
+
+node_name_allocation_failed:
+#ifdef X9_DEBUG
+  x9_print_error_msg("NODE_NAME_ALLOCATION_FAILED");
+#endif
+  free(node);
+  return NULL;
+
+node_inboxes_allocation_failed:
+#ifdef X9_DEBUG
+  x9_print_error_msg("NODE_INBOXES_ALLOCATION_FAILED");
+#endif
+  va_end(argp);
+  free(node_name);
+  free(node);
+  return NULL;
+
+node_multiple_equal_inboxes:
+#ifdef X9_DEBUG
+  x9_print_error_msg("NODE_MULTIPLE_EQUAL_INBOXES");
+#endif
+  va_end(argp);
+  free(node_name);
+  free(node->inboxes);
+  free(node);
+  return NULL;
+}
+
+bool x9_node_is_valid(x9_node const* const node) { return !(NULL == node); }
+
+bool x9_node_name_is(x9_node const* const node,
+                     char const* restrict const cmp) {
+  return !strcmp(node->name, cmp) ? true : false;
+}
+
+void x9_free_node(x9_node* const node) {
+  free(node->name);
+  free(node->inboxes);
+  free(node);
+}
+
+void x9_free_node_and_attached_inboxes(x9_node* const node) {
+  for (uint64_t k = 0; k != node->n_inboxes; ++k) {
+    if (NULL != node->inboxes[k]) {
+      x9_free_inbox(node->inboxes[k]);
+      node->inboxes[k] = NULL;
+    }
+  }
+  x9_free_node(node);
+}
+
+bool x9_write_to_inbox(x9_inbox* const inbox,
+                       uint64_t const  msg_sz,
+                       void const* restrict const msg) {
+  bool                          f      = false;
+  register uint64_t const       idx    = x9_load_idx(inbox, false);
+  register x9_msg_header* const header = x9_header_ptr(inbox, idx);
+
+  if (atomic_compare_exchange_strong_explicit(&header->slot_has_data, &f, true,
+                                              __ATOMIC_ACQUIRE,
+                                              __ATOMIC_RELAXED)) {
+    memcpy((char*)header + sizeof(x9_msg_header), msg, msg_sz);
+    atomic_fetch_add_explicit(&inbox->write_idx, 1, __ATOMIC_RELEASE);
+    atomic_store_explicit(&header->msg_written, true, __ATOMIC_RELEASE);
+    return true;
+  }
+  return false;
+}
+
+void x9_write_to_inbox_spin(x9_inbox* const inbox,
+                            uint64_t const  msg_sz,
+                            void const* restrict const msg) {
+  for (;;) {
+    bool                          f      = false;
+    register uint64_t const       idx    = x9_increment_idx(inbox, false);
+    register x9_msg_header* const header = x9_header_ptr(inbox, idx);
+    if (atomic_compare_exchange_weak_explicit(&header->slot_has_data, &f, true,
+                                              __ATOMIC_ACQUIRE,
+                                              __ATOMIC_RELAXED)) {
+      memcpy((char*)header + sizeof(x9_msg_header), msg, msg_sz);
+      atomic_store_explicit(&header->msg_written, true, __ATOMIC_RELEASE);
+      break;
+    }
+  }
+}
+
+void x9_broadcast_msg_to_all_node_inboxes(x9_node const* const node,
+                                          uint64_t const       msg_sz,
+                                          void const* restrict const msg) {
+  for (uint64_t k = 0; k != node->n_inboxes; ++k) {
+    x9_write_to_inbox_spin(node->inboxes[k], msg_sz, msg);
+  }
+}
+
+bool x9_read_from_inbox(x9_inbox* const inbox,
+                        uint64_t const  msg_sz,
+                        void* restrict const outparam) {
+  register uint64_t const       idx    = x9_load_idx(inbox, true);
+  register x9_msg_header* const header = x9_header_ptr(inbox, idx);
+
+  if (atomic_load_explicit(&header->slot_has_data, __ATOMIC_RELAXED)) {
+    if (atomic_load_explicit(&header->msg_written, __ATOMIC_ACQUIRE)) {
+      memcpy(outparam, (char*)header + sizeof(x9_msg_header), msg_sz);
+      atomic_store_explicit(&header->msg_written, false, __ATOMIC_RELAXED);
+      atomic_store_explicit(&header->slot_has_data, false, __ATOMIC_RELEASE);
+      atomic_fetch_add_explicit(&inbox->read_idx, 1, __ATOMIC_RELEASE);
+      return true;
+    }
+  }
+  return false;
+}
+
+void x9_read_from_inbox_spin(x9_inbox* const inbox,
+                             uint64_t const  msg_sz,
+                             void* restrict const outparam) {
+  register uint64_t const       idx    = x9_increment_idx(inbox, true);
+  register x9_msg_header* const header = x9_header_ptr(inbox, idx);
+
+  for (;;) {
+    _mm_pause();
+    if (atomic_load_explicit(&header->slot_has_data, __ATOMIC_RELAXED)) {
+      if (atomic_load_explicit(&header->msg_written, __ATOMIC_ACQUIRE)) {
+        memcpy(outparam, (char*)header + sizeof(x9_msg_header), msg_sz);
+        atomic_store_explicit(&header->msg_written, false, __ATOMIC_RELAXED);
+        atomic_store_explicit(&header->slot_has_data, false, __ATOMIC_RELEASE);
+        return;
+      }
+    }
+  }
+}
+
+bool x9_read_from_shared_inbox(x9_inbox* const inbox,
+                               uint64_t const  msg_sz,
+                               void* restrict const outparam) {
+  bool                          f      = false;
+  register uint64_t const       idx    = x9_load_idx(inbox, true);
+  register x9_msg_header* const header = x9_header_ptr(inbox, idx);
+
+  if (atomic_compare_exchange_strong_explicit(
+          &header->shared, &f, true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+    if (atomic_load_explicit(&header->slot_has_data, __ATOMIC_RELAXED)) {
+      if (atomic_load_explicit(&header->msg_written, __ATOMIC_ACQUIRE)) {
+        memcpy(outparam, (char*)header + sizeof(x9_msg_header), msg_sz);
+        atomic_fetch_add_explicit(&inbox->read_idx, 1, __ATOMIC_RELEASE);
+        atomic_store_explicit(&header->msg_written, false, __ATOMIC_RELAXED);
+        atomic_store_explicit(&header->slot_has_data, false, __ATOMIC_RELEASE);
+        atomic_store_explicit(&header->shared, false, __ATOMIC_RELEASE);
+        return true;
+      }
+    }
+    atomic_store_explicit(&header->shared, false, __ATOMIC_RELEASE);
+  }
+  return false;
+}
+
+void x9_read_from_shared_inbox_spin(x9_inbox* const inbox,
+                                    uint64_t const  msg_sz,
+                                    void* restrict const outparam) {
+  for (;;) {
+    bool                          f      = false;
+    register uint64_t const       idx    = x9_increment_idx(inbox, true);
+    register x9_msg_header* const header = x9_header_ptr(inbox, idx);
+
+    if (atomic_compare_exchange_strong_explicit(
+            &header->shared, &f, true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+      if (atomic_load_explicit(&header->slot_has_data, __ATOMIC_RELAXED)) {
+        if (atomic_load_explicit(&header->msg_written, __ATOMIC_ACQUIRE)) {
+          memcpy(outparam, (char*)header + sizeof(x9_msg_header), msg_sz);
+          atomic_store_explicit(&header->msg_written, false, __ATOMIC_RELAXED);
+          atomic_store_explicit(&header->slot_has_data, false,
+                                __ATOMIC_RELEASE);
+          atomic_store_explicit(&header->shared, false, __ATOMIC_RELEASE);
+          return;
+        }
+      }
+      atomic_store_explicit(&header->shared, false, __ATOMIC_RELEASE);
+    }
+  }
+}
+
diff --git a/Framework/Foundation/3rdparty/x9/x9.h b/Framework/Foundation/3rdparty/x9/x9.h
new file mode 100644
index 0000000000000..bd8717916ddc3
--- /dev/null
+++ b/Framework/Foundation/3rdparty/x9/x9.h
@@ -0,0 +1,166 @@
+/*
+   X9 - high performance message passing library.
+   Copyright (c) 2023, Diogo Flores
+
+   BSD 2-Clause License (https://opensource.org/license/bsd-2-clause/)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at: diogoxflores@gmail.com
+*/
+
+#pragma once
+
+#include <stdbool.h> /* bool */
+#include <stdint.h>  /* uint64_t */
+
+/* --- Opaque types --- */
+
+typedef struct x9_node_internal  x9_node;
+typedef struct x9_inbox_internal x9_inbox;
+
+/* --- Public API --- */
+
+/* Creates a x9_inbox with a buffer of size 'sz', which must be positive and
+ * mod 2 == 0, a 'name', which can be used for comparison by calling
+ * 'x9_inbox_name_is' or for selecting a specific x9_inbox from a x9_node by
+ * calling 'x9_select_inbox_from_node', and a 'msg_sz' which the inbox is
+ * expected to receive.
+ *
+ * Example:
+ *   x9_inbox* inbox = x9_create_inbox(512, "ibx", sizeof(<some struct>));*/
+__attribute__((nonnull)) x9_inbox* x9_create_inbox(
+    uint64_t const sz, char const* restrict const name, uint64_t const msg_sz);
+
+/* Variadic function that creates a 'x9_node', which is an abstraction that
+ * unifies x9_inbox(es).
+ * 'name' can be used for comparison by calling 'x9_node_name_is',
+ * 'n_inboxes' must be > 0 and equal the number of inboxes passed to the
+ * function to be attached to the x9_node.
+ *
+ * Example:
+ *   x9_node* node = x9_create_node("my_node", 3, inbox_1, inbox_2, inbox_3);*/
+__attribute__((nonnull)) x9_node* x9_create_node(char* restrict const name,
+                                                 uint64_t const n_inboxes,
+                                                 ...);
+
+/* Returns 'true' if the 'inbox' is valid, 'false' otherwise.
+ * Should always be called after 'x9_create_inbox' to validate the correct
+ * initialization of the underlying data structure, and after calling
+ * 'x9_select_inbox_from_node' to ensure that the user selected a valid inbox.
+ * If the return value is 'false', and it's unclear why it is so, the user can
+ * enable the 'X9_DEBUG' macro at compile time for precise error information.*/
+bool x9_inbox_is_valid(x9_inbox const* const inbox);
+
+/* Returns 'true' if the 'node' is valid, 'false' otherwise.
+ * Should always be called after 'x9_create_node' to validate the correct
+ * initialization of the underlying data structure.
+ * If the return value is 'false', and it's unclear why it is so, the user can
+ * enable the 'X9_DEBUG' macro at compile time for precise error information.*/
+bool x9_node_is_valid(x9_node const* const node);
+
+/* Selects a x9_inbox from a 'node' by its 'name'.
+ * Returns NULL if the 'node' does not contain an x9_inbox with such 'name'.*/
+__attribute__((nonnull)) x9_inbox* x9_select_inbox_from_node(
+    x9_node const* const node, char const* restrict const name);
+
+/* Returns 'true' if the 'inbox' name == 'cmp', 'false' otherwise.*/
+__attribute__((nonnull)) bool x9_inbox_name_is(x9_inbox const* const inbox,
+                                               char const* restrict const cmp);
+
+/* Returns 'true' if the 'node' name == 'cmp', 'false' otherwise.*/
+__attribute__((nonnull)) bool x9_node_name_is(x9_node const* const node,
+                                              char const* restrict const cmp);
+
+/* Frees the 'inbox' data structure and its internal components. */
+__attribute__((nonnull)) void x9_free_inbox(x9_inbox* const inbox);
+
+/* Frees the 'node' data structure and its internal components. */
+__attribute__((nonnull)) void x9_free_node(x9_node* const node);
+
+/* Frees the 'node' and all the x9_inbox(es) attached to it.
+ * IMPORTANT: should only be used when the attached x9_inbox(es) are not shared
+ * with other nodes.*/
+__attribute__((nonnull)) void x9_free_node_and_attached_inboxes(
+    x9_node* const node);
+
+/* Returns 'true' if a message was read, 'false' otherwise.
+ * If 'true', the message will be written to 'outparam'.
+ * IMPORTANT: Can only be used to read from inboxes where the thread calling
+ * this function is the only thread reading from said 'inbox'.*/
+__attribute__((nonnull)) bool x9_read_from_inbox(
+    x9_inbox* const inbox,
+    uint64_t const  msg_sz,
+    void* restrict const outparam);
+
+/* Reads the next unread message in the 'inbox' to 'outparam'.
+ * Uses spinning, that is, it wil not return until it has read a message, and
+ * it will keep checking if a message was written and try to read it. */
+__attribute__((nonnull)) void x9_read_from_inbox_spin(
+    x9_inbox* const inbox,
+    uint64_t const  msg_sz,
+    void* restrict const outparam);
+
+/* Returns 'true' if a message was read, 'false' otherwise.
+ * If 'true', the msg contents will be written to the 'outparam'.
+ * Use this function when multiple threads read from the same inbox. */
+__attribute__((nonnull)) bool x9_read_from_shared_inbox(
+    x9_inbox* const inbox,
+    uint64_t const  msg_sz,
+    void* restrict const outparam);
+
+/* Reads the next unread message in the 'inbox' to 'outparam'.
+ * Use this function when multiple threads read from the same inbox.
+ * Uses spinning, that is, it wil not return until it has read a message, and
+ * it will keep checking if a message was written and try to read it. */
+__attribute__((nonnull)) void x9_read_from_shared_inbox_spin(
+    x9_inbox* const inbox,
+    uint64_t const  msg_sz,
+    void* restrict const outparam);
+
+/* Returns 'true' if the message was written to the 'inbox', 'false'
+ * otherwise. */
+__attribute__((nonnull)) bool x9_write_to_inbox(
+    x9_inbox* const inbox,
+    uint64_t const  msg_sz,
+    void const* restrict const msg);
+
+/* Writes the 'msg' to the 'inbox'.
+ * Uses spinning, that is, it wil not not return until it has written the
+ * 'msg', and it will keep checking if the destination inbox has a free
+ * slot that it can write the 'msg' to. */
+__attribute__((nonnull)) void x9_write_to_inbox_spin(
+    x9_inbox* const inbox,
+    uint64_t const  msg_sz,
+    void const* restrict const msg);
+
+/* Writes the same 'msg' to all 'node' inboxes.
+ * Calls 'x9_write_to_inbox_spin' in the background.
+ * Users must guarantee that all 'node' inboxes accept messages of the
+ * same type (or at least of the same 'msg_sz') */
+__attribute__((nonnull)) void x9_broadcast_msg_to_all_node_inboxes(
+    x9_node const* const node,
+    uint64_t const       msg_sz,
+    void const* restrict const msg);
+

From 0b11e3f323b1bdf932d52b64b348835d19a066fe Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 19 Sep 2024 14:21:51 +0200
Subject: [PATCH 637/726] Platform independent yielding

---
 Framework/Foundation/3rdparty/x9/x9.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/Framework/Foundation/3rdparty/x9/x9.c b/Framework/Foundation/3rdparty/x9/x9.c
index 46d3d126de9f4..2ca4bb80237b3 100644
--- a/Framework/Foundation/3rdparty/x9/x9.c
+++ b/Framework/Foundation/3rdparty/x9/x9.c
@@ -35,7 +35,12 @@
 #include "x9.h"
 
 #include <assert.h>    /* assert */
+#if defined(__x86_64__) || defined(__i386__)
 #include <immintrin.h> /* _mm_pause */
+#elif defined(__aarch64__)
+#else
+#error Not supported architecture
+#endif
 #include <stdarg.h>    /* va_* */
 #include <stdatomic.h> /* atomic_* */
 #include <stdbool.h>   /* bool */
@@ -361,7 +366,13 @@ void x9_read_from_inbox_spin(x9_inbox* const inbox,
   register x9_msg_header* const header = x9_header_ptr(inbox, idx);
 
   for (;;) {
+#if defined(__x86_64__) || defined(__i386__)
     _mm_pause();
+#elif defined(__aarch64__)
+    __asm__ __volatile__ ("yield");
+#else
+#error Not supported architecture
+#endif
     if (atomic_load_explicit(&header->slot_has_data, __ATOMIC_RELAXED)) {
       if (atomic_load_explicit(&header->msg_written, __ATOMIC_ACQUIRE)) {
         memcpy(outparam, (char*)header + sizeof(x9_msg_header), msg_sz);

From 279e0fb4fdd991797fb60b55adbf54a421012e33 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 19 Sep 2024 14:21:51 +0200
Subject: [PATCH 638/726] Support using X9 from C++

---
 Framework/Foundation/3rdparty/x9/x9.h | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Framework/Foundation/3rdparty/x9/x9.h b/Framework/Foundation/3rdparty/x9/x9.h
index bd8717916ddc3..229d7adf7dba0 100644
--- a/Framework/Foundation/3rdparty/x9/x9.h
+++ b/Framework/Foundation/3rdparty/x9/x9.h
@@ -32,6 +32,11 @@
 
 #pragma once
 
+#ifdef __cplusplus
+extern "C" {
+#define restrict __restrict__
+#endif
+
 #include <stdbool.h> /* bool */
 #include <stdint.h>  /* uint64_t */
 
@@ -164,3 +169,6 @@ __attribute__((nonnull)) void x9_broadcast_msg_to_all_node_inboxes(
     uint64_t const       msg_sz,
     void const* restrict const msg);
 
+#ifdef __cplusplus
+}
+#endif

From a09eecf0cc22d2998be192887b202b7df94a82da Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 19 Sep 2024 14:21:51 +0200
Subject: [PATCH 639/726] DPL: Use X9 to make AsyncQueue atomic

---
 Framework/Core/CMakeLists.txt                 |  1 +
 Framework/Core/include/Framework/AsyncQueue.h | 12 +++++++
 Framework/Core/src/AsyncQueue.cxx             | 33 ++++++++++++++++++-
 3 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index 8c2d9650776c0..c01ef534eb212 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -163,6 +163,7 @@ o2_add_library(Framework
                                      O2::Headers
                                      O2::MemoryResources
                                      O2::PCG
+                                     O2::X9
                                      RapidJSON::RapidJSON
                                      Arrow::arrow_shared
                                      Microsoft.GSL::GSL
diff --git a/Framework/Core/include/Framework/AsyncQueue.h b/Framework/Core/include/Framework/AsyncQueue.h
index d214b9bfc09ba..0f25410179c1f 100644
--- a/Framework/Core/include/Framework/AsyncQueue.h
+++ b/Framework/Core/include/Framework/AsyncQueue.h
@@ -14,6 +14,10 @@
 #include "Framework/TimesliceSlot.h"
 #include <string>
 #include <vector>
+#include <atomic>
+
+typedef struct x9_inbox_internal x9_inbox;
+typedef struct x9_node_internal x9_node;
 
 namespace o2::framework
 {
@@ -89,6 +93,12 @@ struct AsyncQueue {
   std::vector<AsyncTaskSpec> prototypes;
   std::vector<AsyncTask> tasks;
   size_t iteration = 0;
+
+  std::atomic<bool> first = true;
+
+  // Inbox for the message queue used to append
+  // tasks to this queue.
+  x9_inbox* inbox = nullptr;
   AsyncQueue();
 };
 
@@ -104,6 +114,8 @@ struct AsyncQueueHelpers {
   /// 3. only execute the highest (timeslice, debounce) value
   static void run(AsyncQueue& queue, TimesliceId oldestPossibleTimeslice);
 
+  // Flush tasks which were posted but not yet committed to the queue
+  static void flushPending(AsyncQueue& queue);
   /// Reset the queue to its initial state
   static void reset(AsyncQueue& queue);
 };
diff --git a/Framework/Core/src/AsyncQueue.cxx b/Framework/Core/src/AsyncQueue.cxx
index e0e44da9cd19c..c9be88f096c0d 100644
--- a/Framework/Core/src/AsyncQueue.cxx
+++ b/Framework/Core/src/AsyncQueue.cxx
@@ -11,6 +11,7 @@
 
 #include "Framework/AsyncQueue.h"
 #include "Framework/Signpost.h"
+#include "x9.h"
 #include <numeric>
 
 O2_DECLARE_DYNAMIC_LOG(async_queue);
@@ -18,7 +19,9 @@ O2_DECLARE_DYNAMIC_LOG(async_queue);
 namespace o2::framework
 {
 AsyncQueue::AsyncQueue()
+  : inbox(x9_create_inbox(16, "async_queue", sizeof(AsyncTask)))
 {
+  this->inbox = x9_create_inbox(16, "async_queue", sizeof(AsyncTask));
 }
 
 auto AsyncQueueHelpers::create(AsyncQueue& queue, AsyncTaskSpec spec) -> AsyncTaskId
@@ -31,11 +34,39 @@ auto AsyncQueueHelpers::create(AsyncQueue& queue, AsyncTaskSpec spec) -> AsyncTa
 
 auto AsyncQueueHelpers::post(AsyncQueue& queue, AsyncTask const& task) -> void
 {
-  queue.tasks.push_back(task);
+  // Until we do not manage to write to the inbox, keep removing
+  // items from the queue if you are the first one which fails to
+  // write.
+  while (!x9_write_to_inbox(queue.inbox, sizeof(AsyncTask), &task)) {
+    AsyncQueueHelpers::flushPending(queue);
+  }
+}
+
+auto AsyncQueueHelpers::flushPending(AsyncQueue& queue) -> void
+{
+  bool isFirst = true;
+  if (!std::atomic_compare_exchange_strong(&queue.first, &isFirst, false)) {
+    // Not the first, try again.
+    return;
+  }
+  // First thread which does not manage to write to the queue.
+  // Flush it a bit before we try again.
+  AsyncTask toFlush;
+  // This potentially stalls if the inserting tasks are faster to insert
+  // than we are to retrieve. We should probably have a cut-off
+  while (x9_read_from_inbox(queue.inbox, sizeof(AsyncTask), &toFlush)) {
+    queue.tasks.push_back(toFlush);
+  }
+  queue.first = true;
 }
 
 auto AsyncQueueHelpers::run(AsyncQueue& queue, TimesliceId oldestPossible) -> void
 {
+  // We synchronize right before we run to get as many
+  // tasks as possible. Notice we might still miss some
+  // which will have to handled on a subsequent iteration.
+  AsyncQueueHelpers::flushPending(queue);
+
   if (queue.tasks.empty()) {
     return;
   }

From 5f34c0dc5f1bf6bb931c9417823aa17a9327d485 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 19 Sep 2024 23:46:56 +0200
Subject: [PATCH 640/726] DPL: avoid pragma once

---
 Framework/Foundation/3rdparty/x9/x9.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Framework/Foundation/3rdparty/x9/x9.h b/Framework/Foundation/3rdparty/x9/x9.h
index 229d7adf7dba0..587e0f3116642 100644
--- a/Framework/Foundation/3rdparty/x9/x9.h
+++ b/Framework/Foundation/3rdparty/x9/x9.h
@@ -30,7 +30,8 @@
    You can contact the author at: diogoxflores@gmail.com
 */
 
-#pragma once
+#ifndef O2_FRAMEWORK_FOUDATION_X9_H_
+#define O2_FRAMEWORK_FOUDATION_X9_H_
 
 #ifdef __cplusplus
 extern "C" {
@@ -172,3 +173,4 @@ __attribute__((nonnull)) void x9_broadcast_msg_to_all_node_inboxes(
 #ifdef __cplusplus
 }
 #endif
+#endif // O2_FRAMEWORK_FOUDATION_X9_H_

From a9e420f12d038d55c09fe926c6043baa5f439e7c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 20 Sep 2024 00:48:57 +0200
Subject: [PATCH 641/726] DPL Analysis: make sure VLAs use only one basket

Otherwise it seems to create issues with ROOT bulk reading if the baskets
are split across multiple clusters.

In particular, each row needs 4 bytes for the size of the elements in the row
and the _size branch needs to also have its size correctly set.
---
 .../Core/include/Framework/TableTreeHelpers.h    |  5 +++--
 Framework/Core/src/TableTreeHelpers.cxx          | 16 +++++++++++++++-
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/include/Framework/TableTreeHelpers.h b/Framework/Core/include/Framework/TableTreeHelpers.h
index e12f4a38a973b..ccc7035ba3435 100644
--- a/Framework/Core/include/Framework/TableTreeHelpers.h
+++ b/Framework/Core/include/Framework/TableTreeHelpers.h
@@ -83,8 +83,9 @@ class ColumnToBranch
   ColumnToBranch(ColumnToBranch const& other) = delete;
   ColumnToBranch(ColumnToBranch&& other) = delete;
   void at(const int64_t* pos);
-  int fieldSize() const { return mFieldSize; }
-  char const* branchName() const { return mBranchName.c_str(); }
+  [[nodiscard]] int fieldSize() const { return mFieldSize; }
+  [[nodiscard]] int columnEntries() const { return mColumn->length(); }
+  [[nodiscard]] char const* branchName() const { return mBranchName.c_str(); }
 
  private:
   void accessChunk();
diff --git a/Framework/Core/src/TableTreeHelpers.cxx b/Framework/Core/src/TableTreeHelpers.cxx
index e6e6a1349e619..14e4a43ea6cba 100644
--- a/Framework/Core/src/TableTreeHelpers.cxx
+++ b/Framework/Core/src/TableTreeHelpers.cxx
@@ -454,9 +454,23 @@ std::shared_ptr<TTree> TableToTree::process()
   }
 
   for (auto& reader : mColumnReaders) {
-    int idealBasketSize = 1024 + reader->fieldSize() * mRows; // minimal additional size needed, otherwise we get 2 baskets
+    int idealBasketSize = 1024 + reader->fieldSize() * reader->columnEntries(); // minimal additional size needed, otherwise we get 2 baskets
     int basketSize = std::max(32000, idealBasketSize);        // keep a minimum value
+    // std::cout << "Setting baskets size for " << reader->branchName() << " to " << basketSize << " =  1024 + "
+    //           << reader->fieldSize() << " * " << reader->columnEntries() << ". mRows was " << mRows << std::endl;
     mTree->SetBasketSize(reader->branchName(), basketSize);
+    // If it starts with fIndexArray, also set the size branch basket size
+    if (strncmp(reader->branchName(), "fIndexArray", strlen("fIndexArray")) == 0) {
+      std::string sizeBranch = reader->branchName();
+      sizeBranch += "_size";
+      //  std::cout << "Setting baskets size for " << sizeBranch << " to " << basketSize << " =  1024 + "
+      //            << reader->fieldSize() << " * " << reader->columnEntries() << ". mRows was " << mRows << std::endl;
+      // One int per array to keep track of the size
+      int idealBasketSize = 4 * mRows + 1024 + reader->fieldSize() * reader->columnEntries(); // minimal additional size needed, otherwise we get 2 baskets
+      int basketSize = std::max(32000, idealBasketSize);                                      // keep a minimum value
+      mTree->SetBasketSize(sizeBranch.c_str(), basketSize);
+      mTree->SetBasketSize(reader->branchName(), basketSize);
+    }
   }
 
   while (row < mRows) {

From 0a97f2eeea76801d060c35a8dbb54ca4b02f465a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sun, 22 Sep 2024 09:38:27 +0200
Subject: [PATCH 642/726] DPL: thread safe activity detection

This makes sure that the detection of active data processors in
a device is threadsafe by recording the last active one.

The main loop will then short circuit libuv in case the last active
dataprocessor was reported and it will reset the last active pointer
to nullptr if it has not changed in the meanwhile (meaning that another
data processor was actually able to process something).
---
 .../include/Framework/DataProcessingContext.h |  4 +-
 .../include/Framework/DataProcessingDevice.h  |  1 -
 .../Core/include/Framework/DeviceState.h      |  7 +++
 Framework/Core/src/DataProcessingDevice.cxx   | 60 ++++++++++++-------
 4 files changed, 46 insertions(+), 26 deletions(-)

diff --git a/Framework/Core/include/Framework/DataProcessingContext.h b/Framework/Core/include/Framework/DataProcessingContext.h
index d71ad203b1580..9b7cbc238c942 100644
--- a/Framework/Core/include/Framework/DataProcessingContext.h
+++ b/Framework/Core/include/Framework/DataProcessingContext.h
@@ -26,9 +26,7 @@ struct DataProcessorSpec;
 struct DataProcessorContext {
   DataProcessorContext(DataProcessorContext const&) = delete;
   DataProcessorContext() = default;
-  // These are specific of a given context and therefore
-  // not shared by threads.
-  bool* wasActive = nullptr;
+
   bool allDone = false;
   /// Latest run number we processed globally for this DataProcessor.
   int64_t lastRunNumberProcessed = -1;
diff --git a/Framework/Core/include/Framework/DataProcessingDevice.h b/Framework/Core/include/Framework/DataProcessingDevice.h
index d9565ebef84a2..67edaa99e532b 100644
--- a/Framework/Core/include/Framework/DataProcessingDevice.h
+++ b/Framework/Core/include/Framework/DataProcessingDevice.h
@@ -113,7 +113,6 @@ class DataProcessingDevice : public fair::mq::Device
   std::vector<fair::mq::RegionInfo> mPendingRegionInfos; /// A list of the region infos not yet notified.
   std::mutex mRegionInfoMutex;
   ProcessingPolicies mProcessingPolicies; /// User policies related to data processing
-  bool mWasActive = false;                /// Whether or not the device was active at last iteration.
   std::vector<uv_work_t> mHandles;        /// Handles to use to schedule work.
   std::vector<TaskStreamInfo> mStreams;   /// Information about the task running in the associated mHandle.
   /// Handle to wake up the main loop from other threads
diff --git a/Framework/Core/include/Framework/DeviceState.h b/Framework/Core/include/Framework/DeviceState.h
index 89961b3e92dc7..f6d863064ee66 100644
--- a/Framework/Core/include/Framework/DeviceState.h
+++ b/Framework/Core/include/Framework/DeviceState.h
@@ -30,6 +30,8 @@ typedef struct uv_async_s uv_async_t;
 namespace o2::framework
 {
 
+struct DataProcessorContext;
+
 /// Running state information of a given device
 struct DeviceState {
   /// Motivation for the loop being triggered.
@@ -108,6 +110,11 @@ struct DeviceState {
   /// the bits we are interested in.
   std::vector<int> severityStack;
   TransitionHandlingState transitionHandling = TransitionHandlingState::NoTransition;
+
+  // The DataProcessorContext which was most recently active.
+  // We use this to determine if we should trigger the loop without
+  // waiting for some events.
+  std::atomic<DataProcessorContext*> lastActiveDataProcessor = nullptr;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index f3fe328e78a06..169613b18a2ee 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -1061,7 +1061,11 @@ void DataProcessingDevice::InitTask()
   // Whenever we InitTask, we consider as if the previous iteration
   // was successful, so that even if there is no timer or receiving
   // channel, we can still start an enumeration.
-  mWasActive = true;
+  DataProcessorContext* initialContext = nullptr;
+  bool idle = state.lastActiveDataProcessor.compare_exchange_strong(initialContext, (DataProcessorContext*)-1);
+  if (!idle) {
+    LOG(error) << "DataProcessor " << state.lastActiveDataProcessor.load()->spec->name << " was unexpectedly active";
+  }
 
   // We should be ready to run here. Therefore we copy all the
   // required parts in the DataProcessorContext. Eventually we should
@@ -1093,8 +1097,6 @@ void DataProcessingDevice::InitTask()
 
 void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceContext& deviceContext)
 {
-  context.wasActive = &mWasActive;
-
   context.isSink = false;
   // If nothing is a sink, the rate limiting simply does not trigger.
   bool enableRateLimiting = std::stoi(fConfig->GetValue<std::string>("timeframes-rate-limit"));
@@ -1308,14 +1310,19 @@ void DataProcessingDevice::Run()
     {
       ServiceRegistryRef ref{mServiceRegistry};
       ref.get<DriverClient>().flushPending(mServiceRegistry);
-      auto shouldNotWait = (mWasActive &&
+      DataProcessorContext* lastActive = state.lastActiveDataProcessor.load();
+      // Reset to zero unless some other DataPorcessorContext completed in the meanwhile.
+      // In such case we will take care of it at next iteration.
+      state.lastActiveDataProcessor.compare_exchange_strong(lastActive, nullptr);
+
+      auto shouldNotWait = (lastActive != nullptr &&
                             (state.streaming != StreamingState::Idle) && (state.activeSignals.empty())) ||
                            (state.streaming == StreamingState::EndOfStreaming);
       if (firstLoop) {
         shouldNotWait = true;
         firstLoop = false;
       }
-      if (mWasActive) {
+      if (lastActive != nullptr) {
         state.loopReason |= DeviceState::LoopReason::PREVIOUSLY_ACTIVE;
       }
       if (NewStatePending()) {
@@ -1485,10 +1492,7 @@ void DataProcessingDevice::Run()
       } else {
         auto ref = ServiceRegistryRef{mServiceRegistry};
         ref.get<ComputingQuotaEvaluator>().handleExpired(reportExpiredOffer);
-        mWasActive = false;
       }
-    } else {
-      mWasActive = false;
     }
   }
 
@@ -1510,7 +1514,6 @@ void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
   O2_SIGNPOST_ID_FROM_POINTER(dpid, device, &context);
   O2_SIGNPOST_START(device, dpid, "do_prepare", "Starting DataProcessorContext::doPrepare.");
 
-  *context.wasActive = false;
   {
     ref.get<CallbackService>().call<CallbackService::Id::ClockTick>();
   }
@@ -1669,7 +1672,10 @@ void DataProcessingDevice::doPrepare(ServiceRegistryRef ref)
     socket.Events(&info.hasPendingEvents);
     if (info.hasPendingEvents) {
       info.readPolled = false;
-      *context.wasActive |= newMessages;
+      // In case there were messages, we consider it as activity
+      if (newMessages) {
+        state.lastActiveDataProcessor.store(&context);
+      }
     }
     O2_SIGNPOST_END(device, cid, "channels", "Done processing channel %{public}s (%d).",
                     channelSpec.name.c_str(), info.id.value);
@@ -1693,24 +1699,29 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
   auto& spec = ref.get<DeviceSpec const>();
 
   if (state.streaming == StreamingState::Idle) {
-    *context.wasActive = false;
     return;
   }
 
   context.completed.clear();
   context.completed.reserve(16);
-  *context.wasActive |= DataProcessingDevice::tryDispatchComputation(ref, context.completed);
+  if (DataProcessingDevice::tryDispatchComputation(ref, context.completed)) {
+    state.lastActiveDataProcessor.store(&context);
+  }
   DanglingContext danglingContext{*context.registry};
 
   context.preDanglingCallbacks(danglingContext);
-  if (*context.wasActive == false) {
+  if (state.lastActiveDataProcessor.load() == nullptr) {
     ref.get<CallbackService>().call<CallbackService::Id::Idle>();
   }
   auto activity = ref.get<DataRelayer>().processDanglingInputs(context.expirationHandlers, *context.registry, true);
-  *context.wasActive |= activity.expiredSlots > 0;
+  if (activity.expiredSlots > 0) {
+    state.lastActiveDataProcessor = &context;
+  }
 
   context.completed.clear();
-  *context.wasActive |= DataProcessingDevice::tryDispatchComputation(ref, context.completed);
+  if (DataProcessingDevice::tryDispatchComputation(ref, context.completed)) {
+    state.lastActiveDataProcessor = &context;
+  }
 
   context.postDanglingCallbacks(danglingContext);
 
@@ -1720,7 +1731,7 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
   // framework itself.
   if (context.allDone == true && state.streaming == StreamingState::Streaming) {
     switchState(StreamingState::EndOfStreaming);
-    *context.wasActive = true;
+    state.lastActiveDataProcessor = &context;
   }
 
   if (state.streaming == StreamingState::EndOfStreaming) {
@@ -1766,7 +1777,10 @@ void DataProcessingDevice::doRun(ServiceRegistryRef ref)
     // This is needed because the transport is deleted before the device.
     relayer.clear();
     switchState(StreamingState::Idle);
-    *context.wasActive = shouldProcess;
+    // In case  we should process, note the data processor responsible for it
+    if (shouldProcess) {
+      state.lastActiveDataProcessor = &context;
+    }
     // On end of stream we shut down all output pollers.
     O2_SIGNPOST_EVENT_EMIT(device, dpid, "state", "Shutting down output pollers.");
     for (auto& poller : state.activeOutputPollers) {
@@ -1834,6 +1848,7 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
     O2_SIGNPOST_ID_FROM_POINTER(cid, device, &info);
     auto ref = ServiceRegistryRef{*context.registry};
     auto& stats = ref.get<DataProcessingStats>();
+    auto& state = ref.get<DeviceState>();
     auto& parts = info.parts;
     stats.updateStats({(int)ProcessingStatsId::TOTAL_INPUTS, DataProcessingStats::Op::Set, (int64_t)parts.Size()});
 
@@ -1856,14 +1871,14 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
         O2_SIGNPOST_EVENT_EMIT(device, cid, "handle_data", "Got SourceInfoHeader with state %d", (int)sih->state);
         info.state = sih->state;
         insertInputInfo(pi, 2, InputType::SourceInfo, info.id);
-        *context.wasActive = true;
+        state.lastActiveDataProcessor = &context;
         continue;
       }
       auto dih = o2::header::get<DomainInfoHeader*>(headerData);
       if (dih) {
         O2_SIGNPOST_EVENT_EMIT(device, cid, "handle_data", "Got DomainInfoHeader with oldestPossibleTimeslice %d", (int)dih->oldestPossibleTimeslice);
         insertInputInfo(pi, 2, InputType::DomainInfo, info.id);
-        *context.wasActive = true;
+        state.lastActiveDataProcessor = &context;
         continue;
       }
       auto dh = o2::header::get<DataHeader*>(headerData);
@@ -1925,6 +1940,7 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
 
   auto handleValidMessages = [&info, ref, &reportError](std::vector<InputInfo> const& inputInfos) {
     auto& relayer = ref.get<DataRelayer>();
+    auto& state = ref.get<DeviceState>();
     static WaitBackpressurePolicy policy;
     auto& parts = info.parts;
     // We relay execution to make sure we have a complete set of parts
@@ -2012,7 +2028,7 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
         case InputType::SourceInfo: {
           LOGP(detail, "Received SourceInfo");
           auto& context = ref.get<DataProcessorContext>();
-          *context.wasActive = true;
+          state.lastActiveDataProcessor = &context;
           auto headerIndex = input.position;
           auto payloadIndex = input.position + 1;
           assert(payloadIndex < parts.Size());
@@ -2030,7 +2046,7 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
           /// We have back pressure, therefore we do not process DomainInfo anymore.
           /// until the previous message are processed.
           auto& context = ref.get<DataProcessorContext>();
-          *context.wasActive = true;
+          state.lastActiveDataProcessor = &context;
           auto headerIndex = input.position;
           auto payloadIndex = input.position + 1;
           assert(payloadIndex < parts.Size());
@@ -2058,7 +2074,7 @@ void DataProcessingDevice::handleData(ServiceRegistryRef ref, InputChannelInfo&
       auto& context = ref.get<DataProcessorContext>();
       context.domainInfoUpdatedCallback(*context.registry, oldestPossibleTimeslice, info.id);
       ref.get<CallbackService>().call<CallbackService::Id::DomainInfoUpdated>((ServiceRegistryRef)*context.registry, (size_t)oldestPossibleTimeslice, (ChannelIndex)info.id);
-      *context.wasActive = true;
+      state.lastActiveDataProcessor = &context;
     }
     auto it = std::remove_if(parts.fParts.begin(), parts.fParts.end(), [](auto& msg) -> bool { return msg.get() == nullptr; });
     parts.fParts.erase(it, parts.end());

From f51a114942282eb4c6a0b0fab0a90e1f57b42f81 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 26 Sep 2024 12:19:30 +0200
Subject: [PATCH 643/726] DPL: introduce a --data-processing-timeout

Used to force stop the data processing and start the calibration.
In case it is larger than the exit-transition-timeout, the two will be coalesced
into a single transition.
---
 .../Core/include/Framework/DeviceContext.h    |  2 ++
 .../Core/include/Framework/DeviceState.h      | 10 ++++++
 Framework/Core/src/DataProcessingDevice.cxx   | 34 ++++++++++++++++++-
 Framework/Core/src/DeviceSpecHelpers.cxx      |  2 ++
 Framework/Core/src/O2ControlHelpers.cxx       |  9 ++++-
 Framework/Core/src/runDataProcessing.cxx      | 20 ++++++-----
 .../test_FrameworkDataFlowToO2Control.cxx     | 10 ++++--
 7 files changed, 74 insertions(+), 13 deletions(-)

diff --git a/Framework/Core/include/Framework/DeviceContext.h b/Framework/Core/include/Framework/DeviceContext.h
index 3777e7f608b75..4593e5e819ccf 100644
--- a/Framework/Core/include/Framework/DeviceContext.h
+++ b/Framework/Core/include/Framework/DeviceContext.h
@@ -28,9 +28,11 @@ struct ComputingQuotaStats;
 struct DeviceContext {
   ComputingQuotaStats* quotaStats = nullptr;
   uv_timer_t* gracePeriodTimer = nullptr;
+  uv_timer_t* dataProcessingGracePeriodTimer = nullptr;
   uv_signal_t* sigusr1Handle = nullptr;
   int expectedRegionCallbacks = 0;
   int exitTransitionTimeout = 0;
+  int dataProcessingTimeout = 0;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/include/Framework/DeviceState.h b/Framework/Core/include/Framework/DeviceState.h
index f6d863064ee66..5644d85a904d4 100644
--- a/Framework/Core/include/Framework/DeviceState.h
+++ b/Framework/Core/include/Framework/DeviceState.h
@@ -68,8 +68,18 @@ struct DeviceState {
     STREAM_CONTEXT_LOG = 1 << 4,         // Log for the StreamContext callbacks
   };
 
+  enum ProcessingType : int {
+    Any,             // Any kind of processing is allowed
+    CalibrationOnly, // Only calibrations are allowed to be processed / produced
+  };
+
   std::vector<InputChannelInfo> inputChannelInfos;
   StreamingState streaming = StreamingState::Streaming;
+  // What kind of processing is allowed. By default we allow any.
+  // If we are past the data processing timeout, this will be
+  // CalibrationOnly. We need to reset it at every start.
+  ProcessingType allowedProcessing = ProcessingType::Any;
+
   bool quitRequested = false;
   std::atomic<int64_t> cleanupCount = -1;
 
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 169613b18a2ee..564466f7a1e3a 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -142,11 +142,28 @@ void on_transition_requested_expired(uv_timer_t* handle)
   if (hasOnlyGenerated(spec)) {
     O2_SIGNPOST_EVENT_EMIT_INFO(calibration, cid, "callback", "Grace period for source expired. Exiting.");
   } else {
-    O2_SIGNPOST_EVENT_EMIT_WARN(calibration, cid, "callback", "Grace period for data / calibration expired. Exiting.");
+    O2_SIGNPOST_EVENT_EMIT_INFO(calibration, cid, "callback", "Grace period for %{public}s expired. Exiting.",
+                                state.allowedProcessing == DeviceState::CalibrationOnly ? "calibration" : "data & calibration");
   }
   state.transitionHandling = TransitionHandlingState::Expired;
 }
 
+void on_data_processing_expired(uv_timer_t* handle)
+{
+  auto* ref = (ServiceRegistryRef*)handle->data;
+  auto& state = ref->get<DeviceState>();
+  state.loopReason |= DeviceState::TIMER_EXPIRED;
+
+  // Check if this is a source device
+  O2_SIGNPOST_ID_FROM_POINTER(cid, device, handle);
+
+  // Source devices should never end up in this callback, since the exitTransitionTimeout should
+  // be reset to the dataProcessingTimeout and the timers cohalesced.
+  assert(hasOnlyGenerated(ref->get<DeviceSpec const>()) == false);
+  O2_SIGNPOST_EVENT_EMIT_INFO(calibration, cid, "callback", "Grace period for data processing expired. Only calibrations from this point onwards.");
+  state.allowedProcessing = DeviceState::CalibrationOnly;
+}
+
 void on_communication_requested(uv_async_t* s)
 {
   auto* state = (DeviceState*)s->data;
@@ -1015,6 +1032,7 @@ void DataProcessingDevice::InitTask()
 
   deviceContext.expectedRegionCallbacks = std::stoi(fConfig->GetValue<std::string>("expected-region-callbacks"));
   deviceContext.exitTransitionTimeout = std::stoi(fConfig->GetValue<std::string>("exit-transition-timeout"));
+  deviceContext.dataProcessingTimeout = std::stoi(fConfig->GetValue<std::string>("data-processing-timeout"));
 
   for (auto& channel : GetChannels()) {
     channel.second.at(0).Transport()->SubscribeToRegionEvents([&context = deviceContext,
@@ -1209,6 +1227,7 @@ void DataProcessingDevice::PreRun()
   O2_SIGNPOST_START(device, cid, "PreRun", "Entering PreRun callback.");
   state.quitRequested = false;
   state.streaming = StreamingState::Streaming;
+  state.allowedProcessing = DeviceState::Any;
   for (auto& info : state.inputChannelInfos) {
     if (info.state != InputChannelState::Pull) {
       info.state = InputChannelState::Running;
@@ -1339,6 +1358,19 @@ void DataProcessingDevice::Run()
           state.streaming = StreamingState::EndOfStreaming;
         }
 
+        // If this is a source device, dataTransitionTimeout and dataProcessingTimeout are effectively
+        // the same (because source devices are not allowed to produce any calibration).
+        // should be the same.
+        if (hasOnlyGenerated(spec) && deviceContext.dataProcessingTimeout > 0) {
+          deviceContext.exitTransitionTimeout = deviceContext.dataProcessingTimeout;
+        }
+
+        // We do not do anything in particular if the data processing timeout would go past the exitTransitionTimeout
+        if (deviceContext.dataProcessingTimeout > 0 && deviceContext.dataProcessingTimeout < deviceContext.exitTransitionTimeout) {
+          uv_update_time(state.loop);
+          O2_SIGNPOST_EVENT_EMIT(calibration, lid, "timer_setup", "Starting %d s timer for dataProcessingTimeout.", deviceContext.dataProcessingTimeout);
+          uv_timer_start(deviceContext.dataProcessingGracePeriodTimer, on_data_processing_expired, deviceContext.dataProcessingTimeout * 1000, 0);
+        }
         if (deviceContext.exitTransitionTimeout != 0 && state.streaming != StreamingState::Idle) {
           state.transitionHandling = TransitionHandlingState::Requested;
           ref.get<CallbackService>().call<CallbackService::Id::ExitRequested>(ServiceRegistryRef{ref});
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 8076242bd274e..d1fad2bb66f8b 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1537,6 +1537,7 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
         realOdesc.add_options()("child-driver", bpo::value<std::string>());
         realOdesc.add_options()("rate", bpo::value<std::string>());
         realOdesc.add_options()("exit-transition-timeout", bpo::value<std::string>());
+        realOdesc.add_options()("data-processing-timeout", bpo::value<std::string>());
         realOdesc.add_options()("expected-region-callbacks", bpo::value<std::string>());
         realOdesc.add_options()("timeframes-rate-limit", bpo::value<std::string>());
         realOdesc.add_options()("environment", bpo::value<std::string>());
@@ -1723,6 +1724,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
     ("control-port", bpo::value<std::string>(), "Utility port to be used by O2 Control")                                                                             //
     ("rate", bpo::value<std::string>(), "rate for a data source device (Hz)")                                                                                        //
     ("exit-transition-timeout", bpo::value<std::string>(), "timeout before switching to READY state")                                                                //
+    ("data-processing-timeout", bpo::value<std::string>(), "timeout after which only calibration can happen")                                                        //
     ("expected-region-callbacks", bpo::value<std::string>(), "region callbacks to expect before starting")                                                           //
     ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframes can be in fly")                                                    //
     ("shm-monitor", bpo::value<std::string>(), "whether to use the shared memory monitor")                                                                           //
diff --git a/Framework/Core/src/O2ControlHelpers.cxx b/Framework/Core/src/O2ControlHelpers.cxx
index ad539dc4af03f..46b1eb12ea26a 100644
--- a/Framework/Core/src/O2ControlHelpers.cxx
+++ b/Framework/Core/src/O2ControlHelpers.cxx
@@ -262,6 +262,8 @@ void dumpCommand(std::ostream& dumpOut, const DeviceExecution& execution, std::s
   dumpOut << indLevel << indScheme << "- \"-b\"\n";
   dumpOut << indLevel << indScheme << "- \"--exit-transition-timeout\"\n";
   dumpOut << indLevel << indScheme << "- \"'{{ exit_transition_timeout }}'\"\n";
+  dumpOut << indLevel << indScheme << "- \"--data-processing-timeout\"\n";
+  dumpOut << indLevel << indScheme << "- \"'{{ data_processing_timeout }}'\"\n";
   dumpOut << indLevel << indScheme << "- \"--monitoring-backend\"\n";
   dumpOut << indLevel << indScheme << "- \"'{{ monitoring_dpl_url }}'\"\n";
   dumpOut << indLevel << indScheme << "- \"--session\"\n";
@@ -393,15 +395,20 @@ void dumpTask(std::ostream& dumpOut, const DeviceSpec& spec, const DeviceExecuti
   dumpOut << indLevel << "defaults:\n";
   dumpOut << indLevel << indScheme << "log_task_stdout: none\n";
   dumpOut << indLevel << indScheme << "log_task_stderr: none\n";
-  std::string exitTransitionTimeout = "15";
+  std::string exitTransitionTimeout = "15"; // Allow 15 seconds to finish processing and calibrations
+  std::string dataProcessingTimeout = "10"; // Allow only ten seconds to finish processing
   if (execution.args.size() > 2) {
     for (size_t i = 0; i < execution.args.size() - 1; ++i) {
       if (strcmp(execution.args[i], "--exit-transition-timeout") == 0) {
         exitTransitionTimeout = execution.args[i + 1];
       }
+      if (strcmp(execution.args[i], "--data-processing-timeout") == 0) {
+        dataProcessingTimeout = execution.args[i + 1];
+      }
     }
   }
   dumpOut << indLevel << indScheme << "exit_transition_timeout: " << exitTransitionTimeout << "\n";
+  dumpOut << indLevel << indScheme << "data_processing_timeout: " << dataProcessingTimeout << "\n";
 
   if (bfs::path(execution.args[0]).filename().string() != execution.args[0]) {
     LOG(warning) << "The workflow template generation was started with absolute or relative executables paths."
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 7d298998d0563..0cf775a33de18 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1036,6 +1036,7 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
   // declared in the workflow definition are allowed.
   runner.AddHook<fair::mq::hooks::SetCustomCmdLineOptions>([&spec, driverConfig, defaultDriverClient](fair::mq::DeviceRunner& r) {
     std::string defaultExitTransitionTimeout = "0";
+    std::string defaultDataProcessingTimeout = "0";
     std::string defaultInfologgerMode = "";
     o2::framework::DeploymentMode deploymentMode = o2::framework::DefaultsHelpers::deploymentMode();
     if (deploymentMode == o2::framework::DeploymentMode::OnlineDDS) {
@@ -1047,15 +1048,16 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
     boost::program_options::options_description optsDesc;
     ConfigParamsHelper::populateBoostProgramOptions(optsDesc, spec.options, gHiddenDeviceOptions);
     char const* defaultSignposts = getenv("DPL_SIGNPOSTS");
-    optsDesc.add_options()("monitoring-backend", bpo::value<std::string>()->default_value("default"), "monitoring backend info")                                                           //
-      ("driver-client-backend", bpo::value<std::string>()->default_value(defaultDriverClient), "backend for device -> driver communicataon: stdout://: use stdout, ws://: use websockets") //
-      ("infologger-severity", bpo::value<std::string>()->default_value(""), "minimum FairLogger severity to send to InfoLogger")                                                           //
-      ("dpl-tracing-flags", bpo::value<std::string>()->default_value(""), "pipe `|` separate list of events to be traced")                                                                 //
-      ("signposts", bpo::value<std::string>()->default_value(defaultSignposts ? defaultSignposts : ""), "comma separated list of signposts to enable")                                     //
-      ("expected-region-callbacks", bpo::value<std::string>()->default_value("0"), "how many region callbacks we are expecting")                                                           //
-      ("exit-transition-timeout", bpo::value<std::string>()->default_value(defaultExitTransitionTimeout), "how many second to wait before switching from RUN to READY")                    //
-      ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframe can be in fly at the same moment (0 disables)")                                         //
-      ("configuration,cfg", bpo::value<std::string>()->default_value("command-line"), "configuration backend")                                                                             //
+    optsDesc.add_options()("monitoring-backend", bpo::value<std::string>()->default_value("default"), "monitoring backend info")                                                                   //
+      ("driver-client-backend", bpo::value<std::string>()->default_value(defaultDriverClient), "backend for device -> driver communicataon: stdout://: use stdout, ws://: use websockets")         //
+      ("infologger-severity", bpo::value<std::string>()->default_value(""), "minimum FairLogger severity to send to InfoLogger")                                                                   //
+      ("dpl-tracing-flags", bpo::value<std::string>()->default_value(""), "pipe `|` separate list of events to be traced")                                                                         //
+      ("signposts", bpo::value<std::string>()->default_value(defaultSignposts ? defaultSignposts : ""), "comma separated list of signposts to enable")                                             //
+      ("expected-region-callbacks", bpo::value<std::string>()->default_value("0"), "how many region callbacks we are expecting")                                                                   //
+      ("exit-transition-timeout", bpo::value<std::string>()->default_value(defaultExitTransitionTimeout), "how many second to wait before switching from RUN to READY")                            //
+      ("data-processing-timeout", bpo::value<std::string>()->default_value(defaultDataProcessingTimeout), "how many second to wait before stopping data processing and allowing data calibration") //
+      ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframe can be in fly at the same moment (0 disables)")                                                 //
+      ("configuration,cfg", bpo::value<std::string>()->default_value("command-line"), "configuration backend")                                                                                     //
       ("infologger-mode", bpo::value<std::string>()->default_value(defaultInfologgerMode), "O2_INFOLOGGER_MODE override");
     r.fConfig.AddToCmdLineOptions(optsDesc, true);
   });
diff --git a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
index 4cda5fd76336a..982f8e05b19ec 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
@@ -15,10 +15,8 @@
 #include "../src/DeviceSpecHelpers.h"
 #include "../src/SimpleResourceManager.h"
 #include "../src/ComputingResourceHelpers.h"
-#include "Framework/DataAllocator.h"
 #include "Framework/DeviceControl.h"
 #include "Framework/DeviceSpec.h"
-#include "Framework/ProcessingContext.h"
 #include "Framework/WorkflowSpec.h"
 #include "Framework/DriverConfig.h"
 #include "Framework/O2ControlParameters.h"
@@ -173,6 +171,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"
@@ -270,6 +270,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"
@@ -367,6 +369,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"
@@ -461,6 +465,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"

From 37ec795ba36f4f7e30c76287f0c520d955ff71cb Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 26 Sep 2024 12:23:55 +0200
Subject: [PATCH 644/726] DPL: introduce a --data-processing-timeout

Used to force stop the data processing and start the calibration.
In case it is larger than the exit-transition-timeout, the two will be coalesced
into a single transition.

For the time being the only side effect is to set the DeviceState::allowedProcessing and
to terminate the source devices earlier than the data processing ones.

Next PR will make sure that only calibrations are treated when DeviceState::allowedProcessing is set to ProcessingType::Calibration.
---
 .../Core/include/Framework/DeviceContext.h    |  2 ++
 .../Core/include/Framework/DeviceState.h      | 10 ++++++
 Framework/Core/src/DataProcessingDevice.cxx   | 34 ++++++++++++++++++-
 Framework/Core/src/DeviceSpecHelpers.cxx      |  2 ++
 Framework/Core/src/O2ControlHelpers.cxx       |  9 ++++-
 Framework/Core/src/runDataProcessing.cxx      | 20 ++++++-----
 .../test_FrameworkDataFlowToO2Control.cxx     | 10 ++++--
 7 files changed, 74 insertions(+), 13 deletions(-)

diff --git a/Framework/Core/include/Framework/DeviceContext.h b/Framework/Core/include/Framework/DeviceContext.h
index 3777e7f608b75..4593e5e819ccf 100644
--- a/Framework/Core/include/Framework/DeviceContext.h
+++ b/Framework/Core/include/Framework/DeviceContext.h
@@ -28,9 +28,11 @@ struct ComputingQuotaStats;
 struct DeviceContext {
   ComputingQuotaStats* quotaStats = nullptr;
   uv_timer_t* gracePeriodTimer = nullptr;
+  uv_timer_t* dataProcessingGracePeriodTimer = nullptr;
   uv_signal_t* sigusr1Handle = nullptr;
   int expectedRegionCallbacks = 0;
   int exitTransitionTimeout = 0;
+  int dataProcessingTimeout = 0;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/include/Framework/DeviceState.h b/Framework/Core/include/Framework/DeviceState.h
index f6d863064ee66..5644d85a904d4 100644
--- a/Framework/Core/include/Framework/DeviceState.h
+++ b/Framework/Core/include/Framework/DeviceState.h
@@ -68,8 +68,18 @@ struct DeviceState {
     STREAM_CONTEXT_LOG = 1 << 4,         // Log for the StreamContext callbacks
   };
 
+  enum ProcessingType : int {
+    Any,             // Any kind of processing is allowed
+    CalibrationOnly, // Only calibrations are allowed to be processed / produced
+  };
+
   std::vector<InputChannelInfo> inputChannelInfos;
   StreamingState streaming = StreamingState::Streaming;
+  // What kind of processing is allowed. By default we allow any.
+  // If we are past the data processing timeout, this will be
+  // CalibrationOnly. We need to reset it at every start.
+  ProcessingType allowedProcessing = ProcessingType::Any;
+
   bool quitRequested = false;
   std::atomic<int64_t> cleanupCount = -1;
 
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 169613b18a2ee..564466f7a1e3a 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -142,11 +142,28 @@ void on_transition_requested_expired(uv_timer_t* handle)
   if (hasOnlyGenerated(spec)) {
     O2_SIGNPOST_EVENT_EMIT_INFO(calibration, cid, "callback", "Grace period for source expired. Exiting.");
   } else {
-    O2_SIGNPOST_EVENT_EMIT_WARN(calibration, cid, "callback", "Grace period for data / calibration expired. Exiting.");
+    O2_SIGNPOST_EVENT_EMIT_INFO(calibration, cid, "callback", "Grace period for %{public}s expired. Exiting.",
+                                state.allowedProcessing == DeviceState::CalibrationOnly ? "calibration" : "data & calibration");
   }
   state.transitionHandling = TransitionHandlingState::Expired;
 }
 
+void on_data_processing_expired(uv_timer_t* handle)
+{
+  auto* ref = (ServiceRegistryRef*)handle->data;
+  auto& state = ref->get<DeviceState>();
+  state.loopReason |= DeviceState::TIMER_EXPIRED;
+
+  // Check if this is a source device
+  O2_SIGNPOST_ID_FROM_POINTER(cid, device, handle);
+
+  // Source devices should never end up in this callback, since the exitTransitionTimeout should
+  // be reset to the dataProcessingTimeout and the timers cohalesced.
+  assert(hasOnlyGenerated(ref->get<DeviceSpec const>()) == false);
+  O2_SIGNPOST_EVENT_EMIT_INFO(calibration, cid, "callback", "Grace period for data processing expired. Only calibrations from this point onwards.");
+  state.allowedProcessing = DeviceState::CalibrationOnly;
+}
+
 void on_communication_requested(uv_async_t* s)
 {
   auto* state = (DeviceState*)s->data;
@@ -1015,6 +1032,7 @@ void DataProcessingDevice::InitTask()
 
   deviceContext.expectedRegionCallbacks = std::stoi(fConfig->GetValue<std::string>("expected-region-callbacks"));
   deviceContext.exitTransitionTimeout = std::stoi(fConfig->GetValue<std::string>("exit-transition-timeout"));
+  deviceContext.dataProcessingTimeout = std::stoi(fConfig->GetValue<std::string>("data-processing-timeout"));
 
   for (auto& channel : GetChannels()) {
     channel.second.at(0).Transport()->SubscribeToRegionEvents([&context = deviceContext,
@@ -1209,6 +1227,7 @@ void DataProcessingDevice::PreRun()
   O2_SIGNPOST_START(device, cid, "PreRun", "Entering PreRun callback.");
   state.quitRequested = false;
   state.streaming = StreamingState::Streaming;
+  state.allowedProcessing = DeviceState::Any;
   for (auto& info : state.inputChannelInfos) {
     if (info.state != InputChannelState::Pull) {
       info.state = InputChannelState::Running;
@@ -1339,6 +1358,19 @@ void DataProcessingDevice::Run()
           state.streaming = StreamingState::EndOfStreaming;
         }
 
+        // If this is a source device, dataTransitionTimeout and dataProcessingTimeout are effectively
+        // the same (because source devices are not allowed to produce any calibration).
+        // should be the same.
+        if (hasOnlyGenerated(spec) && deviceContext.dataProcessingTimeout > 0) {
+          deviceContext.exitTransitionTimeout = deviceContext.dataProcessingTimeout;
+        }
+
+        // We do not do anything in particular if the data processing timeout would go past the exitTransitionTimeout
+        if (deviceContext.dataProcessingTimeout > 0 && deviceContext.dataProcessingTimeout < deviceContext.exitTransitionTimeout) {
+          uv_update_time(state.loop);
+          O2_SIGNPOST_EVENT_EMIT(calibration, lid, "timer_setup", "Starting %d s timer for dataProcessingTimeout.", deviceContext.dataProcessingTimeout);
+          uv_timer_start(deviceContext.dataProcessingGracePeriodTimer, on_data_processing_expired, deviceContext.dataProcessingTimeout * 1000, 0);
+        }
         if (deviceContext.exitTransitionTimeout != 0 && state.streaming != StreamingState::Idle) {
           state.transitionHandling = TransitionHandlingState::Requested;
           ref.get<CallbackService>().call<CallbackService::Id::ExitRequested>(ServiceRegistryRef{ref});
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 8076242bd274e..d1fad2bb66f8b 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1537,6 +1537,7 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
         realOdesc.add_options()("child-driver", bpo::value<std::string>());
         realOdesc.add_options()("rate", bpo::value<std::string>());
         realOdesc.add_options()("exit-transition-timeout", bpo::value<std::string>());
+        realOdesc.add_options()("data-processing-timeout", bpo::value<std::string>());
         realOdesc.add_options()("expected-region-callbacks", bpo::value<std::string>());
         realOdesc.add_options()("timeframes-rate-limit", bpo::value<std::string>());
         realOdesc.add_options()("environment", bpo::value<std::string>());
@@ -1723,6 +1724,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
     ("control-port", bpo::value<std::string>(), "Utility port to be used by O2 Control")                                                                             //
     ("rate", bpo::value<std::string>(), "rate for a data source device (Hz)")                                                                                        //
     ("exit-transition-timeout", bpo::value<std::string>(), "timeout before switching to READY state")                                                                //
+    ("data-processing-timeout", bpo::value<std::string>(), "timeout after which only calibration can happen")                                                        //
     ("expected-region-callbacks", bpo::value<std::string>(), "region callbacks to expect before starting")                                                           //
     ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframes can be in fly")                                                    //
     ("shm-monitor", bpo::value<std::string>(), "whether to use the shared memory monitor")                                                                           //
diff --git a/Framework/Core/src/O2ControlHelpers.cxx b/Framework/Core/src/O2ControlHelpers.cxx
index ad539dc4af03f..46b1eb12ea26a 100644
--- a/Framework/Core/src/O2ControlHelpers.cxx
+++ b/Framework/Core/src/O2ControlHelpers.cxx
@@ -262,6 +262,8 @@ void dumpCommand(std::ostream& dumpOut, const DeviceExecution& execution, std::s
   dumpOut << indLevel << indScheme << "- \"-b\"\n";
   dumpOut << indLevel << indScheme << "- \"--exit-transition-timeout\"\n";
   dumpOut << indLevel << indScheme << "- \"'{{ exit_transition_timeout }}'\"\n";
+  dumpOut << indLevel << indScheme << "- \"--data-processing-timeout\"\n";
+  dumpOut << indLevel << indScheme << "- \"'{{ data_processing_timeout }}'\"\n";
   dumpOut << indLevel << indScheme << "- \"--monitoring-backend\"\n";
   dumpOut << indLevel << indScheme << "- \"'{{ monitoring_dpl_url }}'\"\n";
   dumpOut << indLevel << indScheme << "- \"--session\"\n";
@@ -393,15 +395,20 @@ void dumpTask(std::ostream& dumpOut, const DeviceSpec& spec, const DeviceExecuti
   dumpOut << indLevel << "defaults:\n";
   dumpOut << indLevel << indScheme << "log_task_stdout: none\n";
   dumpOut << indLevel << indScheme << "log_task_stderr: none\n";
-  std::string exitTransitionTimeout = "15";
+  std::string exitTransitionTimeout = "15"; // Allow 15 seconds to finish processing and calibrations
+  std::string dataProcessingTimeout = "10"; // Allow only ten seconds to finish processing
   if (execution.args.size() > 2) {
     for (size_t i = 0; i < execution.args.size() - 1; ++i) {
       if (strcmp(execution.args[i], "--exit-transition-timeout") == 0) {
         exitTransitionTimeout = execution.args[i + 1];
       }
+      if (strcmp(execution.args[i], "--data-processing-timeout") == 0) {
+        dataProcessingTimeout = execution.args[i + 1];
+      }
     }
   }
   dumpOut << indLevel << indScheme << "exit_transition_timeout: " << exitTransitionTimeout << "\n";
+  dumpOut << indLevel << indScheme << "data_processing_timeout: " << dataProcessingTimeout << "\n";
 
   if (bfs::path(execution.args[0]).filename().string() != execution.args[0]) {
     LOG(warning) << "The workflow template generation was started with absolute or relative executables paths."
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 7d298998d0563..0cf775a33de18 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1036,6 +1036,7 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
   // declared in the workflow definition are allowed.
   runner.AddHook<fair::mq::hooks::SetCustomCmdLineOptions>([&spec, driverConfig, defaultDriverClient](fair::mq::DeviceRunner& r) {
     std::string defaultExitTransitionTimeout = "0";
+    std::string defaultDataProcessingTimeout = "0";
     std::string defaultInfologgerMode = "";
     o2::framework::DeploymentMode deploymentMode = o2::framework::DefaultsHelpers::deploymentMode();
     if (deploymentMode == o2::framework::DeploymentMode::OnlineDDS) {
@@ -1047,15 +1048,16 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
     boost::program_options::options_description optsDesc;
     ConfigParamsHelper::populateBoostProgramOptions(optsDesc, spec.options, gHiddenDeviceOptions);
     char const* defaultSignposts = getenv("DPL_SIGNPOSTS");
-    optsDesc.add_options()("monitoring-backend", bpo::value<std::string>()->default_value("default"), "monitoring backend info")                                                           //
-      ("driver-client-backend", bpo::value<std::string>()->default_value(defaultDriverClient), "backend for device -> driver communicataon: stdout://: use stdout, ws://: use websockets") //
-      ("infologger-severity", bpo::value<std::string>()->default_value(""), "minimum FairLogger severity to send to InfoLogger")                                                           //
-      ("dpl-tracing-flags", bpo::value<std::string>()->default_value(""), "pipe `|` separate list of events to be traced")                                                                 //
-      ("signposts", bpo::value<std::string>()->default_value(defaultSignposts ? defaultSignposts : ""), "comma separated list of signposts to enable")                                     //
-      ("expected-region-callbacks", bpo::value<std::string>()->default_value("0"), "how many region callbacks we are expecting")                                                           //
-      ("exit-transition-timeout", bpo::value<std::string>()->default_value(defaultExitTransitionTimeout), "how many second to wait before switching from RUN to READY")                    //
-      ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframe can be in fly at the same moment (0 disables)")                                         //
-      ("configuration,cfg", bpo::value<std::string>()->default_value("command-line"), "configuration backend")                                                                             //
+    optsDesc.add_options()("monitoring-backend", bpo::value<std::string>()->default_value("default"), "monitoring backend info")                                                                   //
+      ("driver-client-backend", bpo::value<std::string>()->default_value(defaultDriverClient), "backend for device -> driver communicataon: stdout://: use stdout, ws://: use websockets")         //
+      ("infologger-severity", bpo::value<std::string>()->default_value(""), "minimum FairLogger severity to send to InfoLogger")                                                                   //
+      ("dpl-tracing-flags", bpo::value<std::string>()->default_value(""), "pipe `|` separate list of events to be traced")                                                                         //
+      ("signposts", bpo::value<std::string>()->default_value(defaultSignposts ? defaultSignposts : ""), "comma separated list of signposts to enable")                                             //
+      ("expected-region-callbacks", bpo::value<std::string>()->default_value("0"), "how many region callbacks we are expecting")                                                                   //
+      ("exit-transition-timeout", bpo::value<std::string>()->default_value(defaultExitTransitionTimeout), "how many second to wait before switching from RUN to READY")                            //
+      ("data-processing-timeout", bpo::value<std::string>()->default_value(defaultDataProcessingTimeout), "how many second to wait before stopping data processing and allowing data calibration") //
+      ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframe can be in fly at the same moment (0 disables)")                                                 //
+      ("configuration,cfg", bpo::value<std::string>()->default_value("command-line"), "configuration backend")                                                                                     //
       ("infologger-mode", bpo::value<std::string>()->default_value(defaultInfologgerMode), "O2_INFOLOGGER_MODE override");
     r.fConfig.AddToCmdLineOptions(optsDesc, true);
   });
diff --git a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
index 4cda5fd76336a..982f8e05b19ec 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
@@ -15,10 +15,8 @@
 #include "../src/DeviceSpecHelpers.h"
 #include "../src/SimpleResourceManager.h"
 #include "../src/ComputingResourceHelpers.h"
-#include "Framework/DataAllocator.h"
 #include "Framework/DeviceControl.h"
 #include "Framework/DeviceSpec.h"
-#include "Framework/ProcessingContext.h"
 #include "Framework/WorkflowSpec.h"
 #include "Framework/DriverConfig.h"
 #include "Framework/O2ControlParameters.h"
@@ -173,6 +171,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"
@@ -270,6 +270,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"
@@ -367,6 +369,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"
@@ -461,6 +465,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"

From 15af17c7985c1662a7f23544fc49b9dbd1a8e0e2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 26 Sep 2024 12:56:07 +0200
Subject: [PATCH 645/726] DPL: cleanup unneeded headers

---
 Framework/Core/src/DataProcessingDevice.cxx | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 564466f7a1e3a..c74904c880408 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -10,7 +10,6 @@
 // or submit itself to any jurisdiction.
 #include "Framework/AsyncQueue.h"
 #include "Framework/DataProcessingDevice.h"
-#include "Framework/ChannelMatching.h"
 #include "Framework/ControlService.h"
 #include "Framework/ComputingQuotaEvaluator.h"
 #include "Framework/DataProcessingHeader.h"
@@ -28,7 +27,6 @@
 #include "ConfigurationOptionsRetriever.h"
 #include "Framework/FairMQDeviceProxy.h"
 #include "Framework/CallbackService.h"
-#include "Framework/TMessageSerializer.h"
 #include "Framework/InputRecord.h"
 #include "Framework/InputSpan.h"
 #if defined(__APPLE__) || defined(NDEBUG)
@@ -37,23 +35,20 @@
 #include "Framework/Signpost.h"
 #include "Framework/TimingHelpers.h"
 #include "Framework/SourceInfoHeader.h"
-#include "Framework/Logger.h"
 #include "Framework/DriverClient.h"
-#include "Framework/Monitoring.h"
 #include "Framework/TimesliceIndex.h"
 #include "Framework/VariableContextHelpers.h"
 #include "Framework/DataProcessingContext.h"
+#include "Framework/DataProcessingHeader.h"
 #include "Framework/DeviceContext.h"
 #include "Framework/RawDeviceService.h"
 #include "Framework/StreamContext.h"
 #include "Framework/DefaultsHelpers.h"
+#include "Framework/ServiceRegistryRef.h"
 
-#include "PropertyTreeHelpers.h"
-#include "DataProcessingStatus.h"
 #include "DecongestionService.h"
 #include "Framework/DataProcessingHelpers.h"
 #include "DataRelayerHelpers.h"
-#include "ProcessingPoliciesHelpers.h"
 #include "Headers/DataHeader.h"
 #include "Headers/DataHeaderHelpers.h"
 
@@ -66,6 +61,7 @@
 #include <fairmq/ProgOptions.h>
 #include <Configuration/ConfigurationInterface.h>
 #include <Configuration/ConfigurationFactory.h>
+#include <Monitoring/Monitoring.h>
 #include <TMessage.h>
 #include <TClonesArray.h>
 
@@ -74,7 +70,6 @@
 #include <vector>
 #include <numeric>
 #include <memory>
-#include <unordered_map>
 #include <uv.h>
 #include <execinfo.h>
 #include <sstream>

From 654875ad6687f85ed2b086a3736a8742812b1d80 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 26 Sep 2024 15:09:27 +0200
Subject: [PATCH 646/726] DPL: drop incoming non-calibration data when we
 process calibration only

---
 Framework/Core/src/DataRelayer.cxx | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index ad085afac0b03..68f109dca4dad 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -402,6 +402,12 @@ void DataRelayer::pruneCache(TimesliceSlot slot, OnDropCallback onDrop)
   pruneCache(slot);
 }
 
+bool isCalibrationData(std::unique_ptr<fair::mq::Message>& first)
+{
+  auto* dh = o2::header::get<DataHeader*>(first->GetData());
+  return dh->flagsDerivedHeader & DataProcessingHeader::KEEP_AT_EOS_FLAG;
+}
+
 DataRelayer::RelayChoice
   DataRelayer::relay(void const* rawHeader,
                      std::unique_ptr<fair::mq::Message>* messages,
@@ -456,7 +462,7 @@ DataRelayer::RelayChoice
                      &nPayloads,
                      &cache = mCache,
                      &services = mContext,
-                     numInputTypes = mDistinctRoutesIndex.size()](TimesliceId timeslice, int input, TimesliceSlot slot, InputInfo const& info) {
+                     numInputTypes = mDistinctRoutesIndex.size()](TimesliceId timeslice, int input, TimesliceSlot slot, InputInfo const& info) -> size_t {
     O2_SIGNPOST_ID_GENERATE(aid, data_relayer);
     O2_SIGNPOST_EVENT_EMIT(data_relayer, aid, "saveInSlot", "saving %{public}s@%zu in slot %zu from %{public}s",
                            fmt::format("{:x}", *o2::header::get<DataHeader*>(messages[0]->GetData())).c_str(),
@@ -468,11 +474,20 @@ DataRelayer::RelayChoice
     // TODO: make sure that multiple parts can only be added within the same call of
     // DataRelayer::relay
     assert(nPayloads > 0);
+    size_t saved = 0;
     for (size_t mi = 0; mi < nMessages; ++mi) {
       assert(mi + nPayloads < nMessages);
+      // We are in calibration mode and the data does not have the calibration bit set.
+      // We do not store it.
+      if (services.get<DeviceState>().allowedProcessing == DeviceState::ProcessingType::Any || isCalibrationData(messages[mi])) {
+        mi += nPayloads;
+        continue;
+      }
       target.add([&messages, &mi](size_t i) -> fair::mq::MessagePtr& { return messages[mi + i]; }, nPayloads + 1);
       mi += nPayloads;
+      saved += nPayloads;
     }
+    return saved;
   };
 
   auto updateStatistics = [ref = mContext](TimesliceIndex::ActionTaken action) {
@@ -551,7 +566,10 @@ DataRelayer::RelayChoice
       this->pruneCache(slot, onDrop);
       mPruneOps.erase(std::remove_if(mPruneOps.begin(), mPruneOps.end(), [slot](const auto& x) { return x.slot == slot; }), mPruneOps.end());
     }
-    saveInSlot(timeslice, input, slot, info);
+    size_t saved = saveInSlot(timeslice, input, slot, info);
+    if (saved == 0) {
+      return RelayChoice{.type = RelayChoice::Type::Dropped, .timeslice = timeslice};
+    }
     index.publishSlot(slot);
     index.markAsDirty(slot, true);
     stats.updateStats({static_cast<short>(ProcessingStatsId::RELAYED_MESSAGES), DataProcessingStats::Op::Add, (int)1});
@@ -633,7 +651,10 @@ DataRelayer::RelayChoice
       // cache still holds the old data, so we prune it.
       this->pruneCache(slot, onDrop);
       mPruneOps.erase(std::remove_if(mPruneOps.begin(), mPruneOps.end(), [slot](const auto& x) { return x.slot == slot; }), mPruneOps.end());
-      saveInSlot(timeslice, input, slot, info);
+      size_t saved = saveInSlot(timeslice, input, slot, info);
+      if (saved == 0) {
+        return RelayChoice{.type = RelayChoice::Type::Dropped, .timeslice = timeslice};
+      }
       index.publishSlot(slot);
       index.markAsDirty(slot, true);
       return RelayChoice{.type = RelayChoice::Type::WillRelay};

From 80a0d943067977986e9c5205ce80de40b52a6b53 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 26 Sep 2024 19:44:38 +0200
Subject: [PATCH 647/726] DPL: introduce a --data-processing-timeout

Used to force stop the data processing and start the calibration.
In case it is larger than the exit-transition-timeout, the two will be coalesced
into a single transition.

For the time being the only side effect is to set the DeviceState::allowedProcessing and
to terminate the source devices earlier than the data processing ones.

Next PR will make sure that only calibrations are treated when DeviceState::allowedProcessing is set to ProcessingType::Calibration.
---
 .../Core/include/Framework/DeviceContext.h    |  2 +
 .../Core/include/Framework/DeviceState.h      | 10 +++++
 Framework/Core/src/DataProcessingDevice.cxx   | 43 ++++++++++++++++++-
 Framework/Core/src/DeviceSpecHelpers.cxx      |  2 +
 Framework/Core/src/O2ControlHelpers.cxx       |  9 +++-
 Framework/Core/src/runDataProcessing.cxx      | 20 +++++----
 .../test_FrameworkDataFlowToO2Control.cxx     | 10 ++++-
 7 files changed, 83 insertions(+), 13 deletions(-)

diff --git a/Framework/Core/include/Framework/DeviceContext.h b/Framework/Core/include/Framework/DeviceContext.h
index 3777e7f608b75..4593e5e819ccf 100644
--- a/Framework/Core/include/Framework/DeviceContext.h
+++ b/Framework/Core/include/Framework/DeviceContext.h
@@ -28,9 +28,11 @@ struct ComputingQuotaStats;
 struct DeviceContext {
   ComputingQuotaStats* quotaStats = nullptr;
   uv_timer_t* gracePeriodTimer = nullptr;
+  uv_timer_t* dataProcessingGracePeriodTimer = nullptr;
   uv_signal_t* sigusr1Handle = nullptr;
   int expectedRegionCallbacks = 0;
   int exitTransitionTimeout = 0;
+  int dataProcessingTimeout = 0;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/include/Framework/DeviceState.h b/Framework/Core/include/Framework/DeviceState.h
index f6d863064ee66..5644d85a904d4 100644
--- a/Framework/Core/include/Framework/DeviceState.h
+++ b/Framework/Core/include/Framework/DeviceState.h
@@ -68,8 +68,18 @@ struct DeviceState {
     STREAM_CONTEXT_LOG = 1 << 4,         // Log for the StreamContext callbacks
   };
 
+  enum ProcessingType : int {
+    Any,             // Any kind of processing is allowed
+    CalibrationOnly, // Only calibrations are allowed to be processed / produced
+  };
+
   std::vector<InputChannelInfo> inputChannelInfos;
   StreamingState streaming = StreamingState::Streaming;
+  // What kind of processing is allowed. By default we allow any.
+  // If we are past the data processing timeout, this will be
+  // CalibrationOnly. We need to reset it at every start.
+  ProcessingType allowedProcessing = ProcessingType::Any;
+
   bool quitRequested = false;
   std::atomic<int64_t> cleanupCount = -1;
 
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 169613b18a2ee..fe594bdeb7ed1 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -142,11 +142,28 @@ void on_transition_requested_expired(uv_timer_t* handle)
   if (hasOnlyGenerated(spec)) {
     O2_SIGNPOST_EVENT_EMIT_INFO(calibration, cid, "callback", "Grace period for source expired. Exiting.");
   } else {
-    O2_SIGNPOST_EVENT_EMIT_WARN(calibration, cid, "callback", "Grace period for data / calibration expired. Exiting.");
+    O2_SIGNPOST_EVENT_EMIT_INFO(calibration, cid, "callback", "Grace period for %{public}s expired. Exiting.",
+                                state.allowedProcessing == DeviceState::CalibrationOnly ? "calibration" : "data & calibration");
   }
   state.transitionHandling = TransitionHandlingState::Expired;
 }
 
+void on_data_processing_expired(uv_timer_t* handle)
+{
+  auto* ref = (ServiceRegistryRef*)handle->data;
+  auto& state = ref->get<DeviceState>();
+  state.loopReason |= DeviceState::TIMER_EXPIRED;
+
+  // Check if this is a source device
+  O2_SIGNPOST_ID_FROM_POINTER(cid, device, handle);
+
+  // Source devices should never end up in this callback, since the exitTransitionTimeout should
+  // be reset to the dataProcessingTimeout and the timers cohalesced.
+  assert(hasOnlyGenerated(ref->get<DeviceSpec const>()) == false);
+  O2_SIGNPOST_EVENT_EMIT_INFO(calibration, cid, "callback", "Grace period for data processing expired. Only calibrations from this point onwards.");
+  state.allowedProcessing = DeviceState::CalibrationOnly;
+}
+
 void on_communication_requested(uv_async_t* s)
 {
   auto* state = (DeviceState*)s->data;
@@ -949,6 +966,10 @@ void DataProcessingDevice::startPollers()
   deviceContext.gracePeriodTimer = (uv_timer_t*)malloc(sizeof(uv_timer_t));
   deviceContext.gracePeriodTimer->data = new ServiceRegistryRef(mServiceRegistry);
   uv_timer_init(state.loop, deviceContext.gracePeriodTimer);
+
+  deviceContext.dataProcessingGracePeriodTimer = (uv_timer_t*)malloc(sizeof(uv_timer_t));
+  deviceContext.dataProcessingGracePeriodTimer->data = new ServiceRegistryRef(mServiceRegistry);
+  uv_timer_init(state.loop, deviceContext.dataProcessingGracePeriodTimer);
 }
 
 void DataProcessingDevice::stopPollers()
@@ -980,6 +1001,11 @@ void DataProcessingDevice::stopPollers()
   delete (ServiceRegistryRef*)deviceContext.gracePeriodTimer->data;
   free(deviceContext.gracePeriodTimer);
   deviceContext.gracePeriodTimer = nullptr;
+
+  uv_timer_stop(deviceContext.dataProcessingGracePeriodTimer);
+  delete (ServiceRegistryRef*)deviceContext.dataProcessingGracePeriodTimer->data;
+  free(deviceContext.dataProcessingGracePeriodTimer);
+  deviceContext.dataProcessingGracePeriodTimer = nullptr;
 }
 
 void DataProcessingDevice::InitTask()
@@ -1015,6 +1041,7 @@ void DataProcessingDevice::InitTask()
 
   deviceContext.expectedRegionCallbacks = std::stoi(fConfig->GetValue<std::string>("expected-region-callbacks"));
   deviceContext.exitTransitionTimeout = std::stoi(fConfig->GetValue<std::string>("exit-transition-timeout"));
+  deviceContext.dataProcessingTimeout = std::stoi(fConfig->GetValue<std::string>("data-processing-timeout"));
 
   for (auto& channel : GetChannels()) {
     channel.second.at(0).Transport()->SubscribeToRegionEvents([&context = deviceContext,
@@ -1209,6 +1236,7 @@ void DataProcessingDevice::PreRun()
   O2_SIGNPOST_START(device, cid, "PreRun", "Entering PreRun callback.");
   state.quitRequested = false;
   state.streaming = StreamingState::Streaming;
+  state.allowedProcessing = DeviceState::Any;
   for (auto& info : state.inputChannelInfos) {
     if (info.state != InputChannelState::Pull) {
       info.state = InputChannelState::Running;
@@ -1339,6 +1367,19 @@ void DataProcessingDevice::Run()
           state.streaming = StreamingState::EndOfStreaming;
         }
 
+        // If this is a source device, dataTransitionTimeout and dataProcessingTimeout are effectively
+        // the same (because source devices are not allowed to produce any calibration).
+        // should be the same.
+        if (hasOnlyGenerated(spec) && deviceContext.dataProcessingTimeout > 0) {
+          deviceContext.exitTransitionTimeout = deviceContext.dataProcessingTimeout;
+        }
+
+        // We do not do anything in particular if the data processing timeout would go past the exitTransitionTimeout
+        if (deviceContext.dataProcessingTimeout > 0 && deviceContext.dataProcessingTimeout < deviceContext.exitTransitionTimeout) {
+          uv_update_time(state.loop);
+          O2_SIGNPOST_EVENT_EMIT(calibration, lid, "timer_setup", "Starting %d s timer for dataProcessingTimeout.", deviceContext.dataProcessingTimeout);
+          uv_timer_start(deviceContext.dataProcessingGracePeriodTimer, on_data_processing_expired, deviceContext.dataProcessingTimeout * 1000, 0);
+        }
         if (deviceContext.exitTransitionTimeout != 0 && state.streaming != StreamingState::Idle) {
           state.transitionHandling = TransitionHandlingState::Requested;
           ref.get<CallbackService>().call<CallbackService::Id::ExitRequested>(ServiceRegistryRef{ref});
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 8076242bd274e..d1fad2bb66f8b 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1537,6 +1537,7 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
         realOdesc.add_options()("child-driver", bpo::value<std::string>());
         realOdesc.add_options()("rate", bpo::value<std::string>());
         realOdesc.add_options()("exit-transition-timeout", bpo::value<std::string>());
+        realOdesc.add_options()("data-processing-timeout", bpo::value<std::string>());
         realOdesc.add_options()("expected-region-callbacks", bpo::value<std::string>());
         realOdesc.add_options()("timeframes-rate-limit", bpo::value<std::string>());
         realOdesc.add_options()("environment", bpo::value<std::string>());
@@ -1723,6 +1724,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
     ("control-port", bpo::value<std::string>(), "Utility port to be used by O2 Control")                                                                             //
     ("rate", bpo::value<std::string>(), "rate for a data source device (Hz)")                                                                                        //
     ("exit-transition-timeout", bpo::value<std::string>(), "timeout before switching to READY state")                                                                //
+    ("data-processing-timeout", bpo::value<std::string>(), "timeout after which only calibration can happen")                                                        //
     ("expected-region-callbacks", bpo::value<std::string>(), "region callbacks to expect before starting")                                                           //
     ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframes can be in fly")                                                    //
     ("shm-monitor", bpo::value<std::string>(), "whether to use the shared memory monitor")                                                                           //
diff --git a/Framework/Core/src/O2ControlHelpers.cxx b/Framework/Core/src/O2ControlHelpers.cxx
index ad539dc4af03f..46b1eb12ea26a 100644
--- a/Framework/Core/src/O2ControlHelpers.cxx
+++ b/Framework/Core/src/O2ControlHelpers.cxx
@@ -262,6 +262,8 @@ void dumpCommand(std::ostream& dumpOut, const DeviceExecution& execution, std::s
   dumpOut << indLevel << indScheme << "- \"-b\"\n";
   dumpOut << indLevel << indScheme << "- \"--exit-transition-timeout\"\n";
   dumpOut << indLevel << indScheme << "- \"'{{ exit_transition_timeout }}'\"\n";
+  dumpOut << indLevel << indScheme << "- \"--data-processing-timeout\"\n";
+  dumpOut << indLevel << indScheme << "- \"'{{ data_processing_timeout }}'\"\n";
   dumpOut << indLevel << indScheme << "- \"--monitoring-backend\"\n";
   dumpOut << indLevel << indScheme << "- \"'{{ monitoring_dpl_url }}'\"\n";
   dumpOut << indLevel << indScheme << "- \"--session\"\n";
@@ -393,15 +395,20 @@ void dumpTask(std::ostream& dumpOut, const DeviceSpec& spec, const DeviceExecuti
   dumpOut << indLevel << "defaults:\n";
   dumpOut << indLevel << indScheme << "log_task_stdout: none\n";
   dumpOut << indLevel << indScheme << "log_task_stderr: none\n";
-  std::string exitTransitionTimeout = "15";
+  std::string exitTransitionTimeout = "15"; // Allow 15 seconds to finish processing and calibrations
+  std::string dataProcessingTimeout = "10"; // Allow only ten seconds to finish processing
   if (execution.args.size() > 2) {
     for (size_t i = 0; i < execution.args.size() - 1; ++i) {
       if (strcmp(execution.args[i], "--exit-transition-timeout") == 0) {
         exitTransitionTimeout = execution.args[i + 1];
       }
+      if (strcmp(execution.args[i], "--data-processing-timeout") == 0) {
+        dataProcessingTimeout = execution.args[i + 1];
+      }
     }
   }
   dumpOut << indLevel << indScheme << "exit_transition_timeout: " << exitTransitionTimeout << "\n";
+  dumpOut << indLevel << indScheme << "data_processing_timeout: " << dataProcessingTimeout << "\n";
 
   if (bfs::path(execution.args[0]).filename().string() != execution.args[0]) {
     LOG(warning) << "The workflow template generation was started with absolute or relative executables paths."
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 7d298998d0563..0cf775a33de18 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1036,6 +1036,7 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
   // declared in the workflow definition are allowed.
   runner.AddHook<fair::mq::hooks::SetCustomCmdLineOptions>([&spec, driverConfig, defaultDriverClient](fair::mq::DeviceRunner& r) {
     std::string defaultExitTransitionTimeout = "0";
+    std::string defaultDataProcessingTimeout = "0";
     std::string defaultInfologgerMode = "";
     o2::framework::DeploymentMode deploymentMode = o2::framework::DefaultsHelpers::deploymentMode();
     if (deploymentMode == o2::framework::DeploymentMode::OnlineDDS) {
@@ -1047,15 +1048,16 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
     boost::program_options::options_description optsDesc;
     ConfigParamsHelper::populateBoostProgramOptions(optsDesc, spec.options, gHiddenDeviceOptions);
     char const* defaultSignposts = getenv("DPL_SIGNPOSTS");
-    optsDesc.add_options()("monitoring-backend", bpo::value<std::string>()->default_value("default"), "monitoring backend info")                                                           //
-      ("driver-client-backend", bpo::value<std::string>()->default_value(defaultDriverClient), "backend for device -> driver communicataon: stdout://: use stdout, ws://: use websockets") //
-      ("infologger-severity", bpo::value<std::string>()->default_value(""), "minimum FairLogger severity to send to InfoLogger")                                                           //
-      ("dpl-tracing-flags", bpo::value<std::string>()->default_value(""), "pipe `|` separate list of events to be traced")                                                                 //
-      ("signposts", bpo::value<std::string>()->default_value(defaultSignposts ? defaultSignposts : ""), "comma separated list of signposts to enable")                                     //
-      ("expected-region-callbacks", bpo::value<std::string>()->default_value("0"), "how many region callbacks we are expecting")                                                           //
-      ("exit-transition-timeout", bpo::value<std::string>()->default_value(defaultExitTransitionTimeout), "how many second to wait before switching from RUN to READY")                    //
-      ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframe can be in fly at the same moment (0 disables)")                                         //
-      ("configuration,cfg", bpo::value<std::string>()->default_value("command-line"), "configuration backend")                                                                             //
+    optsDesc.add_options()("monitoring-backend", bpo::value<std::string>()->default_value("default"), "monitoring backend info")                                                                   //
+      ("driver-client-backend", bpo::value<std::string>()->default_value(defaultDriverClient), "backend for device -> driver communicataon: stdout://: use stdout, ws://: use websockets")         //
+      ("infologger-severity", bpo::value<std::string>()->default_value(""), "minimum FairLogger severity to send to InfoLogger")                                                                   //
+      ("dpl-tracing-flags", bpo::value<std::string>()->default_value(""), "pipe `|` separate list of events to be traced")                                                                         //
+      ("signposts", bpo::value<std::string>()->default_value(defaultSignposts ? defaultSignposts : ""), "comma separated list of signposts to enable")                                             //
+      ("expected-region-callbacks", bpo::value<std::string>()->default_value("0"), "how many region callbacks we are expecting")                                                                   //
+      ("exit-transition-timeout", bpo::value<std::string>()->default_value(defaultExitTransitionTimeout), "how many second to wait before switching from RUN to READY")                            //
+      ("data-processing-timeout", bpo::value<std::string>()->default_value(defaultDataProcessingTimeout), "how many second to wait before stopping data processing and allowing data calibration") //
+      ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframe can be in fly at the same moment (0 disables)")                                                 //
+      ("configuration,cfg", bpo::value<std::string>()->default_value("command-line"), "configuration backend")                                                                                     //
       ("infologger-mode", bpo::value<std::string>()->default_value(defaultInfologgerMode), "O2_INFOLOGGER_MODE override");
     r.fConfig.AddToCmdLineOptions(optsDesc, true);
   });
diff --git a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
index 4cda5fd76336a..982f8e05b19ec 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
@@ -15,10 +15,8 @@
 #include "../src/DeviceSpecHelpers.h"
 #include "../src/SimpleResourceManager.h"
 #include "../src/ComputingResourceHelpers.h"
-#include "Framework/DataAllocator.h"
 #include "Framework/DeviceControl.h"
 #include "Framework/DeviceSpec.h"
-#include "Framework/ProcessingContext.h"
 #include "Framework/WorkflowSpec.h"
 #include "Framework/DriverConfig.h"
 #include "Framework/O2ControlParameters.h"
@@ -173,6 +171,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"
@@ -270,6 +270,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"
@@ -367,6 +369,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"
@@ -461,6 +465,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"

From e76d45ba0138a4860599935ce1a748fe7cbe75a4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 26 Sep 2024 19:44:38 +0200
Subject: [PATCH 648/726] DPL: introduce a --data-processing-timeout

Used to force stop the data processing and start the calibration.
In case it is larger than the exit-transition-timeout, the two will be coalesced
into a single transition.

For the time being the only side effect is to set the DeviceState::allowedProcessing and
to terminate the source devices earlier than the data processing ones.

Next PR will make sure that only calibrations are treated when DeviceState::allowedProcessing is set to ProcessingType::Calibration.
---
 .../Core/include/Framework/DeviceContext.h    |  2 +
 .../Core/include/Framework/DeviceState.h      | 10 +++++
 Framework/Core/src/DataProcessingDevice.cxx   | 43 ++++++++++++++++++-
 Framework/Core/src/DeviceSpecHelpers.cxx      |  2 +
 Framework/Core/src/O2ControlHelpers.cxx       |  9 +++-
 Framework/Core/src/runDataProcessing.cxx      | 20 +++++----
 .../test_FrameworkDataFlowToO2Control.cxx     | 14 +++++-
 7 files changed, 87 insertions(+), 13 deletions(-)

diff --git a/Framework/Core/include/Framework/DeviceContext.h b/Framework/Core/include/Framework/DeviceContext.h
index 3777e7f608b75..4593e5e819ccf 100644
--- a/Framework/Core/include/Framework/DeviceContext.h
+++ b/Framework/Core/include/Framework/DeviceContext.h
@@ -28,9 +28,11 @@ struct ComputingQuotaStats;
 struct DeviceContext {
   ComputingQuotaStats* quotaStats = nullptr;
   uv_timer_t* gracePeriodTimer = nullptr;
+  uv_timer_t* dataProcessingGracePeriodTimer = nullptr;
   uv_signal_t* sigusr1Handle = nullptr;
   int expectedRegionCallbacks = 0;
   int exitTransitionTimeout = 0;
+  int dataProcessingTimeout = 0;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/include/Framework/DeviceState.h b/Framework/Core/include/Framework/DeviceState.h
index f6d863064ee66..5644d85a904d4 100644
--- a/Framework/Core/include/Framework/DeviceState.h
+++ b/Framework/Core/include/Framework/DeviceState.h
@@ -68,8 +68,18 @@ struct DeviceState {
     STREAM_CONTEXT_LOG = 1 << 4,         // Log for the StreamContext callbacks
   };
 
+  enum ProcessingType : int {
+    Any,             // Any kind of processing is allowed
+    CalibrationOnly, // Only calibrations are allowed to be processed / produced
+  };
+
   std::vector<InputChannelInfo> inputChannelInfos;
   StreamingState streaming = StreamingState::Streaming;
+  // What kind of processing is allowed. By default we allow any.
+  // If we are past the data processing timeout, this will be
+  // CalibrationOnly. We need to reset it at every start.
+  ProcessingType allowedProcessing = ProcessingType::Any;
+
   bool quitRequested = false;
   std::atomic<int64_t> cleanupCount = -1;
 
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index 169613b18a2ee..fe594bdeb7ed1 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -142,11 +142,28 @@ void on_transition_requested_expired(uv_timer_t* handle)
   if (hasOnlyGenerated(spec)) {
     O2_SIGNPOST_EVENT_EMIT_INFO(calibration, cid, "callback", "Grace period for source expired. Exiting.");
   } else {
-    O2_SIGNPOST_EVENT_EMIT_WARN(calibration, cid, "callback", "Grace period for data / calibration expired. Exiting.");
+    O2_SIGNPOST_EVENT_EMIT_INFO(calibration, cid, "callback", "Grace period for %{public}s expired. Exiting.",
+                                state.allowedProcessing == DeviceState::CalibrationOnly ? "calibration" : "data & calibration");
   }
   state.transitionHandling = TransitionHandlingState::Expired;
 }
 
+void on_data_processing_expired(uv_timer_t* handle)
+{
+  auto* ref = (ServiceRegistryRef*)handle->data;
+  auto& state = ref->get<DeviceState>();
+  state.loopReason |= DeviceState::TIMER_EXPIRED;
+
+  // Check if this is a source device
+  O2_SIGNPOST_ID_FROM_POINTER(cid, device, handle);
+
+  // Source devices should never end up in this callback, since the exitTransitionTimeout should
+  // be reset to the dataProcessingTimeout and the timers cohalesced.
+  assert(hasOnlyGenerated(ref->get<DeviceSpec const>()) == false);
+  O2_SIGNPOST_EVENT_EMIT_INFO(calibration, cid, "callback", "Grace period for data processing expired. Only calibrations from this point onwards.");
+  state.allowedProcessing = DeviceState::CalibrationOnly;
+}
+
 void on_communication_requested(uv_async_t* s)
 {
   auto* state = (DeviceState*)s->data;
@@ -949,6 +966,10 @@ void DataProcessingDevice::startPollers()
   deviceContext.gracePeriodTimer = (uv_timer_t*)malloc(sizeof(uv_timer_t));
   deviceContext.gracePeriodTimer->data = new ServiceRegistryRef(mServiceRegistry);
   uv_timer_init(state.loop, deviceContext.gracePeriodTimer);
+
+  deviceContext.dataProcessingGracePeriodTimer = (uv_timer_t*)malloc(sizeof(uv_timer_t));
+  deviceContext.dataProcessingGracePeriodTimer->data = new ServiceRegistryRef(mServiceRegistry);
+  uv_timer_init(state.loop, deviceContext.dataProcessingGracePeriodTimer);
 }
 
 void DataProcessingDevice::stopPollers()
@@ -980,6 +1001,11 @@ void DataProcessingDevice::stopPollers()
   delete (ServiceRegistryRef*)deviceContext.gracePeriodTimer->data;
   free(deviceContext.gracePeriodTimer);
   deviceContext.gracePeriodTimer = nullptr;
+
+  uv_timer_stop(deviceContext.dataProcessingGracePeriodTimer);
+  delete (ServiceRegistryRef*)deviceContext.dataProcessingGracePeriodTimer->data;
+  free(deviceContext.dataProcessingGracePeriodTimer);
+  deviceContext.dataProcessingGracePeriodTimer = nullptr;
 }
 
 void DataProcessingDevice::InitTask()
@@ -1015,6 +1041,7 @@ void DataProcessingDevice::InitTask()
 
   deviceContext.expectedRegionCallbacks = std::stoi(fConfig->GetValue<std::string>("expected-region-callbacks"));
   deviceContext.exitTransitionTimeout = std::stoi(fConfig->GetValue<std::string>("exit-transition-timeout"));
+  deviceContext.dataProcessingTimeout = std::stoi(fConfig->GetValue<std::string>("data-processing-timeout"));
 
   for (auto& channel : GetChannels()) {
     channel.second.at(0).Transport()->SubscribeToRegionEvents([&context = deviceContext,
@@ -1209,6 +1236,7 @@ void DataProcessingDevice::PreRun()
   O2_SIGNPOST_START(device, cid, "PreRun", "Entering PreRun callback.");
   state.quitRequested = false;
   state.streaming = StreamingState::Streaming;
+  state.allowedProcessing = DeviceState::Any;
   for (auto& info : state.inputChannelInfos) {
     if (info.state != InputChannelState::Pull) {
       info.state = InputChannelState::Running;
@@ -1339,6 +1367,19 @@ void DataProcessingDevice::Run()
           state.streaming = StreamingState::EndOfStreaming;
         }
 
+        // If this is a source device, dataTransitionTimeout and dataProcessingTimeout are effectively
+        // the same (because source devices are not allowed to produce any calibration).
+        // should be the same.
+        if (hasOnlyGenerated(spec) && deviceContext.dataProcessingTimeout > 0) {
+          deviceContext.exitTransitionTimeout = deviceContext.dataProcessingTimeout;
+        }
+
+        // We do not do anything in particular if the data processing timeout would go past the exitTransitionTimeout
+        if (deviceContext.dataProcessingTimeout > 0 && deviceContext.dataProcessingTimeout < deviceContext.exitTransitionTimeout) {
+          uv_update_time(state.loop);
+          O2_SIGNPOST_EVENT_EMIT(calibration, lid, "timer_setup", "Starting %d s timer for dataProcessingTimeout.", deviceContext.dataProcessingTimeout);
+          uv_timer_start(deviceContext.dataProcessingGracePeriodTimer, on_data_processing_expired, deviceContext.dataProcessingTimeout * 1000, 0);
+        }
         if (deviceContext.exitTransitionTimeout != 0 && state.streaming != StreamingState::Idle) {
           state.transitionHandling = TransitionHandlingState::Requested;
           ref.get<CallbackService>().call<CallbackService::Id::ExitRequested>(ServiceRegistryRef{ref});
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index 8076242bd274e..d1fad2bb66f8b 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1537,6 +1537,7 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped,
         realOdesc.add_options()("child-driver", bpo::value<std::string>());
         realOdesc.add_options()("rate", bpo::value<std::string>());
         realOdesc.add_options()("exit-transition-timeout", bpo::value<std::string>());
+        realOdesc.add_options()("data-processing-timeout", bpo::value<std::string>());
         realOdesc.add_options()("expected-region-callbacks", bpo::value<std::string>());
         realOdesc.add_options()("timeframes-rate-limit", bpo::value<std::string>());
         realOdesc.add_options()("environment", bpo::value<std::string>());
@@ -1723,6 +1724,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
     ("control-port", bpo::value<std::string>(), "Utility port to be used by O2 Control")                                                                             //
     ("rate", bpo::value<std::string>(), "rate for a data source device (Hz)")                                                                                        //
     ("exit-transition-timeout", bpo::value<std::string>(), "timeout before switching to READY state")                                                                //
+    ("data-processing-timeout", bpo::value<std::string>(), "timeout after which only calibration can happen")                                                        //
     ("expected-region-callbacks", bpo::value<std::string>(), "region callbacks to expect before starting")                                                           //
     ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframes can be in fly")                                                    //
     ("shm-monitor", bpo::value<std::string>(), "whether to use the shared memory monitor")                                                                           //
diff --git a/Framework/Core/src/O2ControlHelpers.cxx b/Framework/Core/src/O2ControlHelpers.cxx
index ad539dc4af03f..46b1eb12ea26a 100644
--- a/Framework/Core/src/O2ControlHelpers.cxx
+++ b/Framework/Core/src/O2ControlHelpers.cxx
@@ -262,6 +262,8 @@ void dumpCommand(std::ostream& dumpOut, const DeviceExecution& execution, std::s
   dumpOut << indLevel << indScheme << "- \"-b\"\n";
   dumpOut << indLevel << indScheme << "- \"--exit-transition-timeout\"\n";
   dumpOut << indLevel << indScheme << "- \"'{{ exit_transition_timeout }}'\"\n";
+  dumpOut << indLevel << indScheme << "- \"--data-processing-timeout\"\n";
+  dumpOut << indLevel << indScheme << "- \"'{{ data_processing_timeout }}'\"\n";
   dumpOut << indLevel << indScheme << "- \"--monitoring-backend\"\n";
   dumpOut << indLevel << indScheme << "- \"'{{ monitoring_dpl_url }}'\"\n";
   dumpOut << indLevel << indScheme << "- \"--session\"\n";
@@ -393,15 +395,20 @@ void dumpTask(std::ostream& dumpOut, const DeviceSpec& spec, const DeviceExecuti
   dumpOut << indLevel << "defaults:\n";
   dumpOut << indLevel << indScheme << "log_task_stdout: none\n";
   dumpOut << indLevel << indScheme << "log_task_stderr: none\n";
-  std::string exitTransitionTimeout = "15";
+  std::string exitTransitionTimeout = "15"; // Allow 15 seconds to finish processing and calibrations
+  std::string dataProcessingTimeout = "10"; // Allow only ten seconds to finish processing
   if (execution.args.size() > 2) {
     for (size_t i = 0; i < execution.args.size() - 1; ++i) {
       if (strcmp(execution.args[i], "--exit-transition-timeout") == 0) {
         exitTransitionTimeout = execution.args[i + 1];
       }
+      if (strcmp(execution.args[i], "--data-processing-timeout") == 0) {
+        dataProcessingTimeout = execution.args[i + 1];
+      }
     }
   }
   dumpOut << indLevel << indScheme << "exit_transition_timeout: " << exitTransitionTimeout << "\n";
+  dumpOut << indLevel << indScheme << "data_processing_timeout: " << dataProcessingTimeout << "\n";
 
   if (bfs::path(execution.args[0]).filename().string() != execution.args[0]) {
     LOG(warning) << "The workflow template generation was started with absolute or relative executables paths."
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 7d298998d0563..0cf775a33de18 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1036,6 +1036,7 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
   // declared in the workflow definition are allowed.
   runner.AddHook<fair::mq::hooks::SetCustomCmdLineOptions>([&spec, driverConfig, defaultDriverClient](fair::mq::DeviceRunner& r) {
     std::string defaultExitTransitionTimeout = "0";
+    std::string defaultDataProcessingTimeout = "0";
     std::string defaultInfologgerMode = "";
     o2::framework::DeploymentMode deploymentMode = o2::framework::DefaultsHelpers::deploymentMode();
     if (deploymentMode == o2::framework::DeploymentMode::OnlineDDS) {
@@ -1047,15 +1048,16 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
     boost::program_options::options_description optsDesc;
     ConfigParamsHelper::populateBoostProgramOptions(optsDesc, spec.options, gHiddenDeviceOptions);
     char const* defaultSignposts = getenv("DPL_SIGNPOSTS");
-    optsDesc.add_options()("monitoring-backend", bpo::value<std::string>()->default_value("default"), "monitoring backend info")                                                           //
-      ("driver-client-backend", bpo::value<std::string>()->default_value(defaultDriverClient), "backend for device -> driver communicataon: stdout://: use stdout, ws://: use websockets") //
-      ("infologger-severity", bpo::value<std::string>()->default_value(""), "minimum FairLogger severity to send to InfoLogger")                                                           //
-      ("dpl-tracing-flags", bpo::value<std::string>()->default_value(""), "pipe `|` separate list of events to be traced")                                                                 //
-      ("signposts", bpo::value<std::string>()->default_value(defaultSignposts ? defaultSignposts : ""), "comma separated list of signposts to enable")                                     //
-      ("expected-region-callbacks", bpo::value<std::string>()->default_value("0"), "how many region callbacks we are expecting")                                                           //
-      ("exit-transition-timeout", bpo::value<std::string>()->default_value(defaultExitTransitionTimeout), "how many second to wait before switching from RUN to READY")                    //
-      ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframe can be in fly at the same moment (0 disables)")                                         //
-      ("configuration,cfg", bpo::value<std::string>()->default_value("command-line"), "configuration backend")                                                                             //
+    optsDesc.add_options()("monitoring-backend", bpo::value<std::string>()->default_value("default"), "monitoring backend info")                                                                   //
+      ("driver-client-backend", bpo::value<std::string>()->default_value(defaultDriverClient), "backend for device -> driver communicataon: stdout://: use stdout, ws://: use websockets")         //
+      ("infologger-severity", bpo::value<std::string>()->default_value(""), "minimum FairLogger severity to send to InfoLogger")                                                                   //
+      ("dpl-tracing-flags", bpo::value<std::string>()->default_value(""), "pipe `|` separate list of events to be traced")                                                                         //
+      ("signposts", bpo::value<std::string>()->default_value(defaultSignposts ? defaultSignposts : ""), "comma separated list of signposts to enable")                                             //
+      ("expected-region-callbacks", bpo::value<std::string>()->default_value("0"), "how many region callbacks we are expecting")                                                                   //
+      ("exit-transition-timeout", bpo::value<std::string>()->default_value(defaultExitTransitionTimeout), "how many second to wait before switching from RUN to READY")                            //
+      ("data-processing-timeout", bpo::value<std::string>()->default_value(defaultDataProcessingTimeout), "how many second to wait before stopping data processing and allowing data calibration") //
+      ("timeframes-rate-limit", bpo::value<std::string>()->default_value("0"), "how many timeframe can be in fly at the same moment (0 disables)")                                                 //
+      ("configuration,cfg", bpo::value<std::string>()->default_value("command-line"), "configuration backend")                                                                                     //
       ("infologger-mode", bpo::value<std::string>()->default_value(defaultInfologgerMode), "O2_INFOLOGGER_MODE override");
     r.fConfig.AddToCmdLineOptions(optsDesc, true);
   });
diff --git a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
index 4cda5fd76336a..d5f402aa16caa 100644
--- a/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
+++ b/Framework/Core/test/test_FrameworkDataFlowToO2Control.cxx
@@ -15,10 +15,8 @@
 #include "../src/DeviceSpecHelpers.h"
 #include "../src/SimpleResourceManager.h"
 #include "../src/ComputingResourceHelpers.h"
-#include "Framework/DataAllocator.h"
 #include "Framework/DeviceControl.h"
 #include "Framework/DeviceSpec.h"
-#include "Framework/ProcessingContext.h"
 #include "Framework/WorkflowSpec.h"
 #include "Framework/DriverConfig.h"
 #include "Framework/O2ControlParameters.h"
@@ -141,6 +139,7 @@ const std::vector expectedTasks{
   log_task_stdout: none
   log_task_stderr: none
   exit_transition_timeout: 15
+  data_processing_timeout: 10
   _module_cmdline: >-
     source /etc/profile.d/modules.sh && MODULEPATH={{ modulepath }} module load O2 QualityControl Control-OCCPlugin &&
     {{ dpl_command }} | bcsadc/foo
@@ -173,6 +172,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"
@@ -236,6 +237,7 @@ const std::vector expectedTasks{
   log_task_stdout: none
   log_task_stderr: none
   exit_transition_timeout: 15
+  data_processing_timeout: 10
   _module_cmdline: >-
     source /etc/profile.d/modules.sh && MODULEPATH={{ modulepath }} module load O2 QualityControl Control-OCCPlugin &&
     {{ dpl_command }} | foo
@@ -270,6 +272,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"
@@ -333,6 +337,7 @@ const std::vector expectedTasks{
   log_task_stdout: none
   log_task_stderr: none
   exit_transition_timeout: 15
+  data_processing_timeout: 10
   _module_cmdline: >-
     source /etc/profile.d/modules.sh && MODULEPATH={{ modulepath }} module load O2 QualityControl Control-OCCPlugin &&
     {{ dpl_command }} | foo
@@ -367,6 +372,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"
@@ -430,6 +437,7 @@ const std::vector expectedTasks{
   log_task_stdout: none
   log_task_stderr: none
   exit_transition_timeout: 15
+  data_processing_timeout: 10
   _module_cmdline: >-
     source /etc/profile.d/modules.sh && MODULEPATH={{ modulepath }} module load O2 QualityControl Control-OCCPlugin &&
     {{ dpl_command }} | foo
@@ -461,6 +469,8 @@ const std::vector expectedTasks{
     - "-b"
     - "--exit-transition-timeout"
     - "'{{ exit_transition_timeout }}'"
+    - "--data-processing-timeout"
+    - "'{{ data_processing_timeout }}'"
     - "--monitoring-backend"
     - "'{{ monitoring_dpl_url }}'"
     - "--session"

From 5fde66b84337acf589ddc00bb141b3e069d03ae2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 28 Sep 2024 12:36:52 +0200
Subject: [PATCH 649/726] DPL: cleanup unneeded headers

---
 Framework/Core/include/Framework/DeviceState.h |  2 --
 Framework/Core/src/DataProcessingDevice.cxx    | 11 +++--------
 2 files changed, 3 insertions(+), 10 deletions(-)

diff --git a/Framework/Core/include/Framework/DeviceState.h b/Framework/Core/include/Framework/DeviceState.h
index 5644d85a904d4..4a080e72907f6 100644
--- a/Framework/Core/include/Framework/DeviceState.h
+++ b/Framework/Core/include/Framework/DeviceState.h
@@ -17,8 +17,6 @@
 
 #include <vector>
 #include <string>
-#include <map>
-#include <utility>
 #include <atomic>
 
 typedef struct uv_loop_s uv_loop_t;
diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index fe594bdeb7ed1..f4123de250e79 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -10,7 +10,6 @@
 // or submit itself to any jurisdiction.
 #include "Framework/AsyncQueue.h"
 #include "Framework/DataProcessingDevice.h"
-#include "Framework/ChannelMatching.h"
 #include "Framework/ControlService.h"
 #include "Framework/ComputingQuotaEvaluator.h"
 #include "Framework/DataProcessingHeader.h"
@@ -28,7 +27,6 @@
 #include "ConfigurationOptionsRetriever.h"
 #include "Framework/FairMQDeviceProxy.h"
 #include "Framework/CallbackService.h"
-#include "Framework/TMessageSerializer.h"
 #include "Framework/InputRecord.h"
 #include "Framework/InputSpan.h"
 #if defined(__APPLE__) || defined(NDEBUG)
@@ -37,23 +35,20 @@
 #include "Framework/Signpost.h"
 #include "Framework/TimingHelpers.h"
 #include "Framework/SourceInfoHeader.h"
-#include "Framework/Logger.h"
 #include "Framework/DriverClient.h"
-#include "Framework/Monitoring.h"
 #include "Framework/TimesliceIndex.h"
 #include "Framework/VariableContextHelpers.h"
 #include "Framework/DataProcessingContext.h"
+#include "Framework/DataProcessingHeader.h"
 #include "Framework/DeviceContext.h"
 #include "Framework/RawDeviceService.h"
 #include "Framework/StreamContext.h"
 #include "Framework/DefaultsHelpers.h"
+#include "Framework/ServiceRegistryRef.h"
 
-#include "PropertyTreeHelpers.h"
-#include "DataProcessingStatus.h"
 #include "DecongestionService.h"
 #include "Framework/DataProcessingHelpers.h"
 #include "DataRelayerHelpers.h"
-#include "ProcessingPoliciesHelpers.h"
 #include "Headers/DataHeader.h"
 #include "Headers/DataHeaderHelpers.h"
 
@@ -66,6 +61,7 @@
 #include <fairmq/ProgOptions.h>
 #include <Configuration/ConfigurationInterface.h>
 #include <Configuration/ConfigurationFactory.h>
+#include <Monitoring/Monitoring.h>
 #include <TMessage.h>
 #include <TClonesArray.h>
 
@@ -74,7 +70,6 @@
 #include <vector>
 #include <numeric>
 #include <memory>
-#include <unordered_map>
 #include <uv.h>
 #include <execinfo.h>
 #include <sstream>

From bec156687cbcea07fc8439e5cc8dfe50462ef2d6 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 28 Sep 2024 12:37:45 +0200
Subject: [PATCH 650/726] DPL: drop incoming non-calibration data when we
 process calibration only

---
 Framework/Core/src/DataRelayer.cxx       | 27 +++++++++++++++++++++---
 Framework/Core/test/test_DataRelayer.cxx |  3 +++
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx
index ad085afac0b03..c2ae459aace38 100644
--- a/Framework/Core/src/DataRelayer.cxx
+++ b/Framework/Core/src/DataRelayer.cxx
@@ -402,6 +402,12 @@ void DataRelayer::pruneCache(TimesliceSlot slot, OnDropCallback onDrop)
   pruneCache(slot);
 }
 
+bool isCalibrationData(std::unique_ptr<fair::mq::Message>& first)
+{
+  auto* dh = o2::header::get<DataHeader*>(first->GetData());
+  return dh->flagsDerivedHeader & DataProcessingHeader::KEEP_AT_EOS_FLAG;
+}
+
 DataRelayer::RelayChoice
   DataRelayer::relay(void const* rawHeader,
                      std::unique_ptr<fair::mq::Message>* messages,
@@ -456,7 +462,7 @@ DataRelayer::RelayChoice
                      &nPayloads,
                      &cache = mCache,
                      &services = mContext,
-                     numInputTypes = mDistinctRoutesIndex.size()](TimesliceId timeslice, int input, TimesliceSlot slot, InputInfo const& info) {
+                     numInputTypes = mDistinctRoutesIndex.size()](TimesliceId timeslice, int input, TimesliceSlot slot, InputInfo const& info) -> size_t {
     O2_SIGNPOST_ID_GENERATE(aid, data_relayer);
     O2_SIGNPOST_EVENT_EMIT(data_relayer, aid, "saveInSlot", "saving %{public}s@%zu in slot %zu from %{public}s",
                            fmt::format("{:x}", *o2::header::get<DataHeader*>(messages[0]->GetData())).c_str(),
@@ -468,11 +474,20 @@ DataRelayer::RelayChoice
     // TODO: make sure that multiple parts can only be added within the same call of
     // DataRelayer::relay
     assert(nPayloads > 0);
+    size_t saved = 0;
     for (size_t mi = 0; mi < nMessages; ++mi) {
       assert(mi + nPayloads < nMessages);
+      // We are in calibration mode and the data does not have the calibration bit set.
+      // We do not store it.
+      if (services.get<DeviceState>().allowedProcessing == DeviceState::ProcessingType::CalibrationOnly && !isCalibrationData(messages[mi])) {
+        mi += nPayloads;
+        continue;
+      }
       target.add([&messages, &mi](size_t i) -> fair::mq::MessagePtr& { return messages[mi + i]; }, nPayloads + 1);
       mi += nPayloads;
+      saved += nPayloads;
     }
+    return saved;
   };
 
   auto updateStatistics = [ref = mContext](TimesliceIndex::ActionTaken action) {
@@ -551,7 +566,10 @@ DataRelayer::RelayChoice
       this->pruneCache(slot, onDrop);
       mPruneOps.erase(std::remove_if(mPruneOps.begin(), mPruneOps.end(), [slot](const auto& x) { return x.slot == slot; }), mPruneOps.end());
     }
-    saveInSlot(timeslice, input, slot, info);
+    size_t saved = saveInSlot(timeslice, input, slot, info);
+    if (saved == 0) {
+      return RelayChoice{.type = RelayChoice::Type::Dropped, .timeslice = timeslice};
+    }
     index.publishSlot(slot);
     index.markAsDirty(slot, true);
     stats.updateStats({static_cast<short>(ProcessingStatsId::RELAYED_MESSAGES), DataProcessingStats::Op::Add, (int)1});
@@ -633,7 +651,10 @@ DataRelayer::RelayChoice
       // cache still holds the old data, so we prune it.
       this->pruneCache(slot, onDrop);
       mPruneOps.erase(std::remove_if(mPruneOps.begin(), mPruneOps.end(), [slot](const auto& x) { return x.slot == slot; }), mPruneOps.end());
-      saveInSlot(timeslice, input, slot, info);
+      size_t saved = saveInSlot(timeslice, input, slot, info);
+      if (saved == 0) {
+        return RelayChoice{.type = RelayChoice::Type::Dropped, .timeslice = timeslice};
+      }
       index.publishSlot(slot);
       index.markAsDirty(slot, true);
       return RelayChoice{.type = RelayChoice::Type::WillRelay};
diff --git a/Framework/Core/test/test_DataRelayer.cxx b/Framework/Core/test/test_DataRelayer.cxx
index d112e8736399a..ec5d48fbd78da 100644
--- a/Framework/Core/test/test_DataRelayer.cxx
+++ b/Framework/Core/test/test_DataRelayer.cxx
@@ -10,6 +10,7 @@
 // or submit itself to any jurisdiction.
 
 #include <catch_amalgamated.hpp>
+#include "Framework/DeviceState.h"
 #include "Headers/DataHeader.h"
 #include "Headers/Stack.h"
 #include "MemoryResources/MemoryResources.h"
@@ -60,10 +61,12 @@ TEST_CASE("DataRelayer")
     stats.registerMetric(spec);
   }
 
+  DeviceState state;
   ref.registerService(ServiceRegistryHelpers::handleForService<Monitoring>(&monitoring));
   ref.registerService(ServiceRegistryHelpers::handleForService<DataProcessingStats>(&stats));
   ref.registerService(ServiceRegistryHelpers::handleForService<DataProcessingStates>(&states));
   ref.registerService(ServiceRegistryHelpers::handleForService<DriverConfig const>(&driverConfig));
+  ref.registerService(ServiceRegistryHelpers::handleForService<DeviceState>(&state));
   // A simple test where an input is provided
   // and the subsequent InputRecord is immediately requested.
   SECTION("TestNoWait")

From c528be7bc4a67319e191020103284ec2004a09d4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 30 Sep 2024 13:11:56 +0200
Subject: [PATCH 651/726] DPL Analysis: add option to artificially slow down
 AOD reader

---
 .../AnalysisSupport/src/AODJAlienReaderHelpers.cxx  | 13 +++++++++++--
 Framework/Core/src/WorkflowCustomizationHelpers.cxx |  1 +
 Framework/Core/src/WorkflowHelpers.cxx              |  1 -
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 5ce2eca49c07b..da69f0e29cbc5 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -41,8 +41,6 @@
 #include <arrow/table.h>
 #include <arrow/util/key_value_metadata.h>
 
-#include <thread>
-
 using namespace o2;
 using namespace o2::aod;
 
@@ -140,6 +138,8 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
 
     auto filename = options.get<std::string>("aod-file-private");
 
+    auto maxRate = options.get<float>("aod-max-io-rate");
+
     std::string parentFileReplacement;
     if (options.isSet("aod-parent-base-path-replacement")) {
       parentFileReplacement = options.get<std::string>("aod-parent-base-path-replacement");
@@ -192,6 +192,7 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
                            fileCounter,
                            numTF,
                            watchdog,
+                           maxRate,
                            didir, reportTFN, reportTFFileName](Monitoring& monitoring, DataAllocator& outputs, ControlService& control, DeviceSpec const& device) {
       // Each parallel reader device.inputTimesliceId reads the files fileCounter*device.maxInputTimeslices+device.inputTimesliceId
       // the TF to read is numTF
@@ -222,6 +223,7 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
         return;
       }
 
+      int64_t startTime = uv_hrtime();
       for (auto& route : requestedTables) {
         if ((device.inputTimesliceId % route.maxTimeslices) != route.timeslice) {
           continue;
@@ -278,6 +280,13 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
         }
         first = false;
       }
+      int64_t stopTime = uv_hrtime();
+      int64_t currentDelta = (stopTime - startTime) / 1000000000; // in s
+      float extraTime = (totalSizeUncompressed / 1000000 - currentDelta * maxRate) / maxRate;
+      // We only sleep if we read faster than the max-read-rate.
+      if (extraTime > 0.) {
+        uv_sleep(extraTime * 1000); // in milliseconds
+      }
       totalDFSent++;
       monitoring.send(Metric{(uint64_t)totalDFSent, "df-sent"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL));
       monitoring.send(Metric{(uint64_t)totalSizeUncompressed / 1000, "aod-bytes-read-uncompressed"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL));
diff --git a/Framework/Core/src/WorkflowCustomizationHelpers.cxx b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
index 05abb5dab98cd..6236526c665e5 100644
--- a/Framework/Core/src/WorkflowCustomizationHelpers.cxx
+++ b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
@@ -68,6 +68,7 @@ std::vector<ConfigParamSpec> WorkflowCustomizationHelpers::requiredWorkflowOptio
            {"aod-writer-keep", VariantType::String, "", {"Comma separated list of ORIGIN/DESCRIPTION/SUBSPECIFICATION:treename:col1/col2/..:filename"}},
 
            {"fairmq-rate-logging", VariantType::Int, 0, {"Rate logging for FairMQ channels"}},
+           {"aod-max-io-rate", VariantType::Float, 0.0f, {"Maximum I/O throughput in MB/s"}},
            {"fairmq-recv-buffer-size", VariantType::Int, 4, {"recvBufferSize option for FairMQ channels"}},
            {"fairmq-send-buffer-size", VariantType::Int, 4, {"sendBufferSize option for FairMQ channels"}},
            /// Find out a place where we can write the sockets
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 30246492b8b65..450bd1c9a51e7 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -34,7 +34,6 @@
 #include <utility>
 #include <vector>
 #include <climits>
-#include <thread>
 
 O2_DECLARE_DYNAMIC_LOG(workflow_helpers);
 

From 56d0945c6c5c48d0f207252a261b06ce47acad9b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 30 Sep 2024 13:11:56 +0200
Subject: [PATCH 652/726] DPL Analysis: add option to artificially slow down
 AOD reader

---
 .../AnalysisSupport/src/AODJAlienReaderHelpers.cxx  | 13 +++++++++++--
 Framework/Core/src/WorkflowCustomizationHelpers.cxx |  1 +
 Framework/Core/src/WorkflowHelpers.cxx              |  1 -
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 5ce2eca49c07b..da69f0e29cbc5 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -41,8 +41,6 @@
 #include <arrow/table.h>
 #include <arrow/util/key_value_metadata.h>
 
-#include <thread>
-
 using namespace o2;
 using namespace o2::aod;
 
@@ -140,6 +138,8 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
 
     auto filename = options.get<std::string>("aod-file-private");
 
+    auto maxRate = options.get<float>("aod-max-io-rate");
+
     std::string parentFileReplacement;
     if (options.isSet("aod-parent-base-path-replacement")) {
       parentFileReplacement = options.get<std::string>("aod-parent-base-path-replacement");
@@ -192,6 +192,7 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
                            fileCounter,
                            numTF,
                            watchdog,
+                           maxRate,
                            didir, reportTFN, reportTFFileName](Monitoring& monitoring, DataAllocator& outputs, ControlService& control, DeviceSpec const& device) {
       // Each parallel reader device.inputTimesliceId reads the files fileCounter*device.maxInputTimeslices+device.inputTimesliceId
       // the TF to read is numTF
@@ -222,6 +223,7 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
         return;
       }
 
+      int64_t startTime = uv_hrtime();
       for (auto& route : requestedTables) {
         if ((device.inputTimesliceId % route.maxTimeslices) != route.timeslice) {
           continue;
@@ -278,6 +280,13 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
         }
         first = false;
       }
+      int64_t stopTime = uv_hrtime();
+      int64_t currentDelta = (stopTime - startTime) / 1000000000; // in s
+      float extraTime = (totalSizeUncompressed / 1000000 - currentDelta * maxRate) / maxRate;
+      // We only sleep if we read faster than the max-read-rate.
+      if (extraTime > 0.) {
+        uv_sleep(extraTime * 1000); // in milliseconds
+      }
       totalDFSent++;
       monitoring.send(Metric{(uint64_t)totalDFSent, "df-sent"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL));
       monitoring.send(Metric{(uint64_t)totalSizeUncompressed / 1000, "aod-bytes-read-uncompressed"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL));
diff --git a/Framework/Core/src/WorkflowCustomizationHelpers.cxx b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
index 05abb5dab98cd..6236526c665e5 100644
--- a/Framework/Core/src/WorkflowCustomizationHelpers.cxx
+++ b/Framework/Core/src/WorkflowCustomizationHelpers.cxx
@@ -68,6 +68,7 @@ std::vector<ConfigParamSpec> WorkflowCustomizationHelpers::requiredWorkflowOptio
            {"aod-writer-keep", VariantType::String, "", {"Comma separated list of ORIGIN/DESCRIPTION/SUBSPECIFICATION:treename:col1/col2/..:filename"}},
 
            {"fairmq-rate-logging", VariantType::Int, 0, {"Rate logging for FairMQ channels"}},
+           {"aod-max-io-rate", VariantType::Float, 0.0f, {"Maximum I/O throughput in MB/s"}},
            {"fairmq-recv-buffer-size", VariantType::Int, 4, {"recvBufferSize option for FairMQ channels"}},
            {"fairmq-send-buffer-size", VariantType::Int, 4, {"sendBufferSize option for FairMQ channels"}},
            /// Find out a place where we can write the sockets
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 30246492b8b65..450bd1c9a51e7 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -34,7 +34,6 @@
 #include <utility>
 #include <vector>
 #include <climits>
-#include <thread>
 
 O2_DECLARE_DYNAMIC_LOG(workflow_helpers);
 

From e55a6286f4337a0135adf8b8ec4851857ae24c5a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 1 Oct 2024 15:08:27 +0200
Subject: [PATCH 653/726] DPL Analysis: add option to artificially slow down
 AOD reader

---
 .../src/AODJAlienReaderHelpers.cxx            | 20 +++++++++++++++++--
 Framework/Core/src/WorkflowHelpers.cxx        |  4 +++-
 Framework/Core/src/runDataProcessing.cxx      |  1 +
 3 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 5ce2eca49c07b..353c9b1abe9d0 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -41,8 +41,6 @@
 #include <arrow/table.h>
 #include <arrow/util/key_value_metadata.h>
 
-#include <thread>
-
 using namespace o2;
 using namespace o2::aod;
 
@@ -140,6 +138,8 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
 
     auto filename = options.get<std::string>("aod-file-private");
 
+    auto maxRate = options.get<float>("aod-max-io-rate");
+
     std::string parentFileReplacement;
     if (options.isSet("aod-parent-base-path-replacement")) {
       parentFileReplacement = options.get<std::string>("aod-parent-base-path-replacement");
@@ -192,6 +192,7 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
                            fileCounter,
                            numTF,
                            watchdog,
+                           maxRate,
                            didir, reportTFN, reportTFFileName](Monitoring& monitoring, DataAllocator& outputs, ControlService& control, DeviceSpec const& device) {
       // Each parallel reader device.inputTimesliceId reads the files fileCounter*device.maxInputTimeslices+device.inputTimesliceId
       // the TF to read is numTF
@@ -222,6 +223,8 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
         return;
       }
 
+      int64_t startTime = uv_hrtime();
+      int64_t startSize = totalSizeCompressed;
       for (auto& route : requestedTables) {
         if ((device.inputTimesliceId % route.maxTimeslices) != route.timeslice) {
           continue;
@@ -278,6 +281,19 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
         }
         first = false;
       }
+      int64_t stopSize = totalSizeCompressed;
+      int64_t bytesDelta = stopSize - startSize;
+      int64_t stopTime = uv_hrtime();
+      float currentDelta = float(stopTime - startTime) / 1000000000; // in s
+      if (ceil(maxRate) > 0.) {
+        float extraTime = (bytesDelta / 1000000 - currentDelta * maxRate) / maxRate;
+        // We only sleep if we read faster than the max-read-rate.
+        LOGP(info, "Read {} MB in {} s. Sleeping for {} seconds to stay within {} MB/s limit.", bytesDelta / 1000000, currentDelta, extraTime, maxRate);
+
+        if (extraTime > 0.) {
+          uv_sleep(extraTime * 1000); // in milliseconds
+        }
+      }
       totalDFSent++;
       monitoring.send(Metric{(uint64_t)totalDFSent, "df-sent"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL));
       monitoring.send(Metric{(uint64_t)totalSizeUncompressed / 1000, "aod-bytes-read-uncompressed"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL));
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 30246492b8b65..6349bd5889eba 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -12,6 +12,7 @@
 #include "AnalysisSupportHelpers.h"
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/AODReaderHelpers.h"
+#include "Framework/ConfigParamSpec.h"
 #include "Framework/ConfigParamsHelper.h"
 #include "Framework/CommonDataProcessors.h"
 #include "Framework/ConfigContext.h"
@@ -27,6 +28,7 @@
 #include "Framework/DefaultsHelpers.h"
 #include "Framework/Signpost.h"
 
+#include "Framework/Variant.h"
 #include "Headers/DataHeader.h"
 #include <algorithm>
 #include <list>
@@ -34,7 +36,6 @@
 #include <utility>
 #include <vector>
 #include <climits>
-#include <thread>
 
 O2_DECLARE_DYNAMIC_LOG(workflow_helpers);
 
@@ -213,6 +214,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
                          aodLifetime}},
     .algorithm = AlgorithmSpec::dummyAlgorithm(),
     .options = {ConfigParamSpec{"aod-file-private", VariantType::String, ctx.options().get<std::string>("aod-file"), {"AOD file"}},
+                ConfigParamSpec{"aod-max-io-rate", VariantType::Float, 0.f, {"Maximum I/O rate in MB/s"}},
                 ConfigParamSpec{"aod-reader-json", VariantType::String, {"json configuration file"}},
                 ConfigParamSpec{"aod-parent-access-level", VariantType::String, {"Allow parent file access up to specified level. Default: no (0)"}},
                 ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}},
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 0cf775a33de18..d4d0c836aa925 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -2026,6 +2026,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
             "--aod-writer-resmode",
             "--aod-writer-maxfilesize",
             "--aod-writer-keep",
+            "--aod-max-io-rate",
             "--aod-parent-access-level",
             "--aod-parent-base-path-replacement",
             "--driver-client-backend",

From 46d8ccf9f4b7709f63959e7ed3c678e732e69b85 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 1 Oct 2024 16:18:18 +0200
Subject: [PATCH 654/726] DPL Analysis: add option to artificially slow down
 AOD reader

---
 .../src/AODJAlienReaderHelpers.cxx            | 19 +++++++++++++++++--
 Framework/Core/src/WorkflowHelpers.cxx        |  4 +++-
 Framework/Core/src/runDataProcessing.cxx      |  1 +
 3 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 5ce2eca49c07b..a8b708668dae1 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -41,8 +41,6 @@
 #include <arrow/table.h>
 #include <arrow/util/key_value_metadata.h>
 
-#include <thread>
-
 using namespace o2;
 using namespace o2::aod;
 
@@ -140,6 +138,8 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
 
     auto filename = options.get<std::string>("aod-file-private");
 
+    auto maxRate = options.get<float>("aod-max-io-rate");
+
     std::string parentFileReplacement;
     if (options.isSet("aod-parent-base-path-replacement")) {
       parentFileReplacement = options.get<std::string>("aod-parent-base-path-replacement");
@@ -192,6 +192,7 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
                            fileCounter,
                            numTF,
                            watchdog,
+                           maxRate,
                            didir, reportTFN, reportTFFileName](Monitoring& monitoring, DataAllocator& outputs, ControlService& control, DeviceSpec const& device) {
       // Each parallel reader device.inputTimesliceId reads the files fileCounter*device.maxInputTimeslices+device.inputTimesliceId
       // the TF to read is numTF
@@ -222,6 +223,8 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
         return;
       }
 
+      int64_t startTime = uv_hrtime();
+      int64_t startSize = totalSizeCompressed;
       for (auto& route : requestedTables) {
         if ((device.inputTimesliceId % route.maxTimeslices) != route.timeslice) {
           continue;
@@ -278,6 +281,18 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
         }
         first = false;
       }
+      int64_t stopSize = totalSizeCompressed;
+      int64_t bytesDelta = stopSize - startSize;
+      int64_t stopTime = uv_hrtime();
+      float currentDelta = float(stopTime - startTime) / 1000000000; // in s
+      if (ceil(maxRate) > 0.) {
+        float extraTime = (bytesDelta / 1000000 - currentDelta * maxRate) / maxRate;
+        // We only sleep if we read faster than the max-read-rate.
+        if (extraTime > 0.) {
+          LOGP(info, "Read {} MB in {} s. Sleeping for {} seconds to stay within {} MB/s limit.", bytesDelta / 1000000, currentDelta, extraTime, maxRate);
+          uv_sleep(extraTime * 1000); // in milliseconds
+        }
+      }
       totalDFSent++;
       monitoring.send(Metric{(uint64_t)totalDFSent, "df-sent"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL));
       monitoring.send(Metric{(uint64_t)totalSizeUncompressed / 1000, "aod-bytes-read-uncompressed"}.addTag(Key::Subsystem, monitoring::tags::Value::DPL));
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 30246492b8b65..6349bd5889eba 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -12,6 +12,7 @@
 #include "AnalysisSupportHelpers.h"
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/AODReaderHelpers.h"
+#include "Framework/ConfigParamSpec.h"
 #include "Framework/ConfigParamsHelper.h"
 #include "Framework/CommonDataProcessors.h"
 #include "Framework/ConfigContext.h"
@@ -27,6 +28,7 @@
 #include "Framework/DefaultsHelpers.h"
 #include "Framework/Signpost.h"
 
+#include "Framework/Variant.h"
 #include "Headers/DataHeader.h"
 #include <algorithm>
 #include <list>
@@ -34,7 +36,6 @@
 #include <utility>
 #include <vector>
 #include <climits>
-#include <thread>
 
 O2_DECLARE_DYNAMIC_LOG(workflow_helpers);
 
@@ -213,6 +214,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
                          aodLifetime}},
     .algorithm = AlgorithmSpec::dummyAlgorithm(),
     .options = {ConfigParamSpec{"aod-file-private", VariantType::String, ctx.options().get<std::string>("aod-file"), {"AOD file"}},
+                ConfigParamSpec{"aod-max-io-rate", VariantType::Float, 0.f, {"Maximum I/O rate in MB/s"}},
                 ConfigParamSpec{"aod-reader-json", VariantType::String, {"json configuration file"}},
                 ConfigParamSpec{"aod-parent-access-level", VariantType::String, {"Allow parent file access up to specified level. Default: no (0)"}},
                 ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}},
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 0cf775a33de18..d4d0c836aa925 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -2026,6 +2026,7 @@ int runStateMachine(DataProcessorSpecs const& workflow,
             "--aod-writer-resmode",
             "--aod-writer-maxfilesize",
             "--aod-writer-keep",
+            "--aod-max-io-rate",
             "--aod-parent-access-level",
             "--aod-parent-base-path-replacement",
             "--driver-client-backend",

From a129ab1f9166f7b956644410670b78f30ce50141 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 3 Oct 2024 10:54:01 +0200
Subject: [PATCH 655/726] DPL Analysis: re-enable prefetching

Now that ROOT supports prefetching with bulk reading, we can reenable this
which should cut in half the number of IOPS for the general case.
---
 Framework/Core/src/TableTreeHelpers.cxx | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/Framework/Core/src/TableTreeHelpers.cxx b/Framework/Core/src/TableTreeHelpers.cxx
index 14e4a43ea6cba..780126c89dd85 100644
--- a/Framework/Core/src/TableTreeHelpers.cxx
+++ b/Framework/Core/src/TableTreeHelpers.cxx
@@ -547,14 +547,14 @@ void TreeToTable::addAllColumns(TTree* tree, std::vector<std::string>&& names)
   if (mBranchReaders.empty()) {
     throw runtime_error("No columns will be read");
   }
-  //tree->SetCacheSize(50000000);
-  // FIXME: see https://github.com/root-project/root/issues/8962 and enable
-  // again once fixed.
-  //tree->SetClusterPrefetch(true);
-  //for (auto& reader : mBranchReaders) {
-  //  tree->AddBranchToCache(reader->branch());
-  //}
-  //tree->StopCacheLearningPhase();
+  // Was affected by https://github.com/root-project/root/issues/8962
+  // Re-enabling this seems to cut the number of IOPS in half
+  tree->SetCacheSize(25000000);
+  tree->SetClusterPrefetch(true);
+  for (auto& reader : mBranchReaders) {
+    tree->AddBranchToCache(reader->branch());
+  }
+  tree->StopCacheLearningPhase();
 }
 
 void TreeToTable::setLabel(const char* label)

From f8a68d8a42e676a8ff8d890edbc48f3a569e4a17 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 4 Oct 2024 10:04:30 +0200
Subject: [PATCH 656/726] DPL: enable prefetching only for branches with one
 basket

---
 Framework/Core/src/TableTreeHelpers.cxx | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/TableTreeHelpers.cxx b/Framework/Core/src/TableTreeHelpers.cxx
index 780126c89dd85..37bcfbd02139b 100644
--- a/Framework/Core/src/TableTreeHelpers.cxx
+++ b/Framework/Core/src/TableTreeHelpers.cxx
@@ -552,7 +552,11 @@ void TreeToTable::addAllColumns(TTree* tree, std::vector<std::string>&& names)
   tree->SetCacheSize(25000000);
   tree->SetClusterPrefetch(true);
   for (auto& reader : mBranchReaders) {
-    tree->AddBranchToCache(reader->branch());
+    // Unfortunately ROOT 6-32-06 is still buggy. Adding a branch to
+    // the cache only if the number of baskets allows it.
+    if (reader->branch()->GetListOfBaskets()->GetEntriesFast() == 1) {
+      tree->AddBranchToCache(reader->branch());
+    }
   }
   tree->StopCacheLearningPhase();
 }

From 0ee3c7ecbd2edf99d2c2d1b5896411022c23f0fe Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 4 Oct 2024 10:04:30 +0200
Subject: [PATCH 657/726] DPL: enable prefetching only for branches with one
 basket

---
 Framework/Core/src/TableTreeHelpers.cxx | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/Framework/Core/src/TableTreeHelpers.cxx b/Framework/Core/src/TableTreeHelpers.cxx
index 780126c89dd85..2ae438ee916a8 100644
--- a/Framework/Core/src/TableTreeHelpers.cxx
+++ b/Framework/Core/src/TableTreeHelpers.cxx
@@ -549,12 +549,25 @@ void TreeToTable::addAllColumns(TTree* tree, std::vector<std::string>&& names)
   }
   // Was affected by https://github.com/root-project/root/issues/8962
   // Re-enabling this seems to cut the number of IOPS in half
-  tree->SetCacheSize(25000000);
-  tree->SetClusterPrefetch(true);
+  bool hasMultibasket = false;
+  // Unfortunately ROOT 6-32-06 is still buggy. Adding a branch to
+  // the cache only if the number of baskets allows it.
   for (auto& reader : mBranchReaders) {
-    tree->AddBranchToCache(reader->branch());
+    if (reader->branch()->GetListOfBaskets()->GetEntriesFast() != 1) {
+      LOGP(warning, "Branch {} has more than one basket. Disabling prefetching for tree {}.", reader->branch()->GetName(), tree->GetName());
+      hasMultibasket = true;
+      break;
+    }
+  }
+  if (hasMultibasket == false) {
+    tree->SetCacheSize(25000000);
+    tree->SetClusterPrefetch(true);
+    for (auto& reader : mBranchReaders) {
+      std::cout << "Branch added to cache " << reader->branch()->GetName() << std::endl;
+      tree->AddBranchToCache(reader->branch());
+    }
+    tree->StopCacheLearningPhase();
   }
-  tree->StopCacheLearningPhase();
 }
 
 void TreeToTable::setLabel(const char* label)

From 7442ab6b3105d09aeed34c0b9153eff89278744a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 7 Oct 2024 16:53:29 +0200
Subject: [PATCH 658/726] DPL: add separate getter for DataRef

Will simplify splitting of the large get<> method using concepts.
---
 .../Core/include/Framework/InputRecord.h      | 35 +++++++++++--------
 1 file changed, 21 insertions(+), 14 deletions(-)

diff --git a/Framework/Core/include/Framework/InputRecord.h b/Framework/Core/include/Framework/InputRecord.h
index 3e7efb83efd25..5f33d018e5950 100644
--- a/Framework/Core/include/Framework/InputRecord.h
+++ b/Framework/Core/include/Framework/InputRecord.h
@@ -33,6 +33,7 @@
 #include <cassert>
 #include <memory>
 #include <type_traits>
+#include <concepts>
 
 #include <fairmq/FwdDecls.h>
 
@@ -206,6 +207,25 @@ class InputRecord
     }
     return this->getByPos(pos, part);
   }
+
+  template <typename R>
+    requires std::is_convertible_v<R, char const*>
+  DataRef getRef(R binding, int part = 0) const {
+    return getDataRefByString(binding, part);
+  }
+
+  template <typename R>
+    requires requires (R r) {r.c_str();}
+  DataRef getRef(R binding, int part = 0) const {
+    return getDataRefByString(binding.c_str(), part);
+  }
+
+  template <typename R>
+    requires std::is_convertible_v<R, DataRef>
+  DataRef getRef(R ref, int part = 0) const {
+    return ref;
+  }
+
   /// Get the object of specified type T for the binding R.
   /// If R is a string like object, we look up by name the InputSpec and
   /// return the data associated to the given label.
@@ -220,20 +240,7 @@ class InputRecord
   template <typename T = DataRef, typename R>
   decltype(auto) get(R binding, int part = 0) const
   {
-    DataRef ref{nullptr, nullptr};
-    using decayed = std::decay_t<R>;
-
-    // Get the actual dataref
-    if constexpr (std::is_same_v<decayed, char const*> ||
-                  std::is_same_v<decayed, char*>) {
-      ref = getDataRefByString(binding, part);
-    } else if constexpr (std::is_same_v<decayed, std::string>) {
-      ref = getDataRefByString(binding.c_str(), part);
-    } else if constexpr (std::is_same_v<decayed, DataRef>) {
-      ref = binding;
-    } else {
-      static_assert(always_static_assert_v<R>, "Unknown binding type");
-    }
+    DataRef ref = getRef(binding, part);
 
     using PointerLessValueT = std::remove_pointer_t<T>;
 

From 05455b2ef8e0e91d3117fb62e6f77ccde32db1b5 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 7 Oct 2024 16:53:29 +0200
Subject: [PATCH 659/726] DPL: prototype for getting CCDB Metadata

---
 Framework/Core/include/Framework/InputRecord.h | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/Framework/Core/include/Framework/InputRecord.h b/Framework/Core/include/Framework/InputRecord.h
index 5f33d018e5950..825a724c91039 100644
--- a/Framework/Core/include/Framework/InputRecord.h
+++ b/Framework/Core/include/Framework/InputRecord.h
@@ -40,6 +40,9 @@
 namespace o2::framework
 {
 
+// Wrapper class to get CCDB metadata
+struct CCDBMetadataExtractor{};
+
 struct InputSpec;
 class InputSpan;
 class CallbackService;
@@ -461,6 +464,14 @@ class InputRecord
     }
   }
 
+  template <typename T = DataRef, typename R>
+  std::map<std::string, std::string> &get(R binding, int part = 0) const
+    requires std::same_as<T, CCDBMetadataExtractor>
+  {
+    // FIXME: needs proper implementation by Ruben
+    return getRef(binding, part);
+  }
+
   /// Helper method to be used to check if a given part of the InputRecord is present.
   [[nodiscard]] bool isValid(std::string const& s) const
   {

From 888fe18fcb51423809df50c823e05b24ecc9aa4c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 9 Oct 2024 14:01:22 +0200
Subject: [PATCH 660/726] DPL: add separate getter for DataRef

Will simplify splitting of the large get<> method using concepts.
---
 .../Core/include/Framework/InputRecord.h      | 38 ++++++++++++-------
 1 file changed, 24 insertions(+), 14 deletions(-)

diff --git a/Framework/Core/include/Framework/InputRecord.h b/Framework/Core/include/Framework/InputRecord.h
index 3e7efb83efd25..449e55fc7fcbc 100644
--- a/Framework/Core/include/Framework/InputRecord.h
+++ b/Framework/Core/include/Framework/InputRecord.h
@@ -33,6 +33,7 @@
 #include <cassert>
 #include <memory>
 #include <type_traits>
+#include <concepts>
 
 #include <fairmq/FwdDecls.h>
 
@@ -206,6 +207,28 @@ class InputRecord
     }
     return this->getByPos(pos, part);
   }
+
+  template <typename R>
+    requires std::is_convertible_v<R, char const*>
+  DataRef getRef(R binding, int part = 0) const
+  {
+    return getDataRefByString(binding, part);
+  }
+
+  template <typename R>
+    requires requires(R r) { r.c_str(); }
+  DataRef getRef(R binding, int part = 0) const
+  {
+    return getDataRefByString(binding.c_str(), part);
+  }
+
+  template <typename R>
+    requires std::is_convertible_v<R, DataRef>
+  DataRef getRef(R ref, int part = 0) const
+  {
+    return ref;
+  }
+
   /// Get the object of specified type T for the binding R.
   /// If R is a string like object, we look up by name the InputSpec and
   /// return the data associated to the given label.
@@ -220,20 +243,7 @@ class InputRecord
   template <typename T = DataRef, typename R>
   decltype(auto) get(R binding, int part = 0) const
   {
-    DataRef ref{nullptr, nullptr};
-    using decayed = std::decay_t<R>;
-
-    // Get the actual dataref
-    if constexpr (std::is_same_v<decayed, char const*> ||
-                  std::is_same_v<decayed, char*>) {
-      ref = getDataRefByString(binding, part);
-    } else if constexpr (std::is_same_v<decayed, std::string>) {
-      ref = getDataRefByString(binding.c_str(), part);
-    } else if constexpr (std::is_same_v<decayed, DataRef>) {
-      ref = binding;
-    } else {
-      static_assert(always_static_assert_v<R>, "Unknown binding type");
-    }
+    DataRef ref = getRef(binding, part);
 
     using PointerLessValueT = std::remove_pointer_t<T>;
 

From 044260538f105e4b825f10fa7402ab228eb93774 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 9 Oct 2024 15:41:42 +0200
Subject: [PATCH 661/726] DPL: do not deserialise metadata unless requested

---
 Framework/Core/src/DataRefUtils.cxx | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Framework/Core/src/DataRefUtils.cxx b/Framework/Core/src/DataRefUtils.cxx
index b32c47387e69a..d3710a6234642 100644
--- a/Framework/Core/src/DataRefUtils.cxx
+++ b/Framework/Core/src/DataRefUtils.cxx
@@ -102,8 +102,6 @@ void* DataRefUtils::decodeCCDB(DataRef const& ref, std::type_info const& tinfo)
     return nullptr;
   }
 
-  extractCCDBHeaders(ref);
-
   TClass* tcl = TClass::GetClass(tinfo);
   result = extractFromTFile(memFile, tcl, "ccdb_object");
   if (!result) {

From 717ffb8a4ecf64d60a8ebe948c57b1bdb43b4161 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 9 Oct 2024 15:42:38 +0200
Subject: [PATCH 662/726] DPL: add helper method to retrieve and cache CCDB
 metadata

---
 .../Core/include/Framework/InputRecord.h      | 47 +++++++++++++++++++
 .../Core/include/Framework/ObjectCache.h      |  7 +++
 2 files changed, 54 insertions(+)

diff --git a/Framework/Core/include/Framework/InputRecord.h b/Framework/Core/include/Framework/InputRecord.h
index 449e55fc7fcbc..4de7256e0cc95 100644
--- a/Framework/Core/include/Framework/InputRecord.h
+++ b/Framework/Core/include/Framework/InputRecord.h
@@ -40,6 +40,10 @@
 namespace o2::framework
 {
 
+// Wrapper class to get CCDB metadata
+struct CCDBMetadataExtractor {
+};
+
 struct InputSpec;
 class InputSpan;
 class CallbackService;
@@ -464,6 +468,49 @@ class InputRecord
     }
   }
 
+  template <typename T = DataRef, typename R>
+  std::map<std::string, std::string>& get(R binding, int part = 0) const
+    requires std::same_as<T, CCDBMetadataExtractor>
+  {
+    auto ref = getRef(binding, part);
+    auto header = DataRefUtils::getHeader<header::DataHeader*>(ref);
+    auto payloadSize = DataRefUtils::getPayloadSize(ref);
+    auto method = header->payloadSerializationMethod;
+    if (method != header::gSerializationMethodCCDB) {
+      throw runtime_error("Attempt to extract metadata from a non-CCDB serialised message");
+    }
+    // This is to support deserialising objects from CCDB. Contrary to what happens for
+    // other objects, those objects are most likely long lived, so we
+    // keep around an instance of the associated object and deserialise it only when
+    // it's updated.
+    auto id = ObjectCache::Id::fromRef(ref);
+    ConcreteDataMatcher matcher{header->dataOrigin, header->dataDescription, header->subSpecification};
+    // If the matcher does not have an entry in the cache, deserialise it
+    // and cache the deserialised object at the given id.
+    auto path = fmt::format("{}", DataSpecUtils::describe(matcher));
+    LOGP(debug, "{}", path);
+    auto& cache = mRegistry.get<ObjectCache>();
+    auto cacheEntry = cache.matcherToMetadataId.find(path);
+    if (cacheEntry == cache.matcherToMetadataId.end()) {
+      cache.matcherToMetadataId.insert(std::make_pair(path, id));
+      cache.idToMetadata[id] = extracCCDBMetadata(ref);
+      LOGP(info, "Caching CCDB metadata {}: {}", id.value, path);
+      return cache.idToMetadata[id];
+    }
+    auto& oldId = cacheEntry->second;
+    // The id in the cache is the same, let's simply return it.
+    if (oldId.value == id.value) {
+      LOGP(debug, "Returning cached CCDB metatada {}: {}", id.value, path);
+      return cache.idToMetadata[id];
+    }
+    // The id in the cache is different. Let's destroy the old cached entry
+    // and create a new one.
+    LOGP(info, "Replacing cached entry {} with {} for {}", oldId.value, id.value, path);
+    cache.idToObject[id] = extracCCDBMetadata(ref);
+    oldId.value = id.value;
+    return cache.idToObject[id];
+  }
+
   /// Helper method to be used to check if a given part of the InputRecord is present.
   [[nodiscard]] bool isValid(std::string const& s) const
   {
diff --git a/Framework/Core/include/Framework/ObjectCache.h b/Framework/Core/include/Framework/ObjectCache.h
index 22120a216092f..a6873aec8a1ac 100644
--- a/Framework/Core/include/Framework/ObjectCache.h
+++ b/Framework/Core/include/Framework/ObjectCache.h
@@ -13,6 +13,7 @@
 
 #include "Framework/DataRef.h"
 #include <unordered_map>
+#include <map>
 
 namespace o2::framework
 {
@@ -46,6 +47,12 @@ struct ObjectCache {
   /// A map from a CacheId (which is the void* ptr of the previous map).
   /// to an actual (type erased) pointer to the deserialised object.
   std::unordered_map<Id, void*, Id::hash_fn> idToObject;
+
+  /// A cache to the deserialised metadata
+  /// We keep it separate because we want to avoid that looking up
+  /// the metadata also pollutes the object cache.
+  std::unordered_map<std::string, Id> matcherToMetadataId;
+  std::unordered_map<Id, std::map<std::string, std::string>, Id::hash_fn> idToMetadata;
 };
 
 } // namespace o2::framework

From eaabbab0d934184605ccb9a6c097d2e0de53319c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 9 Oct 2024 21:45:15 +0200
Subject: [PATCH 663/726] DPL: add helper method to retrieve and cache CCDB
 metadata

---
 .../Core/include/Framework/InputRecord.h      | 47 +++++++++++++++++++
 .../Core/include/Framework/ObjectCache.h      |  7 +++
 2 files changed, 54 insertions(+)

diff --git a/Framework/Core/include/Framework/InputRecord.h b/Framework/Core/include/Framework/InputRecord.h
index 449e55fc7fcbc..49b8ad70438a7 100644
--- a/Framework/Core/include/Framework/InputRecord.h
+++ b/Framework/Core/include/Framework/InputRecord.h
@@ -40,6 +40,10 @@
 namespace o2::framework
 {
 
+// Wrapper class to get CCDB metadata
+struct CCDBMetadataExtractor {
+};
+
 struct InputSpec;
 class InputSpan;
 class CallbackService;
@@ -464,6 +468,49 @@ class InputRecord
     }
   }
 
+  template <typename T = DataRef, typename R>
+  std::map<std::string, std::string>& get(R binding, int part = 0) const
+    requires std::same_as<T, CCDBMetadataExtractor>
+  {
+    auto ref = getRef(binding, part);
+    auto header = DataRefUtils::getHeader<header::DataHeader*>(ref);
+    auto payloadSize = DataRefUtils::getPayloadSize(ref);
+    auto method = header->payloadSerializationMethod;
+    if (method != header::gSerializationMethodCCDB) {
+      throw runtime_error("Attempt to extract metadata from a non-CCDB serialised message");
+    }
+    // This is to support deserialising objects from CCDB. Contrary to what happens for
+    // other objects, those objects are most likely long lived, so we
+    // keep around an instance of the associated object and deserialise it only when
+    // it's updated.
+    auto id = ObjectCache::Id::fromRef(ref);
+    ConcreteDataMatcher matcher{header->dataOrigin, header->dataDescription, header->subSpecification};
+    // If the matcher does not have an entry in the cache, deserialise it
+    // and cache the deserialised object at the given id.
+    auto path = fmt::format("{}", DataSpecUtils::describe(matcher));
+    LOGP(debug, "{}", path);
+    auto& cache = mRegistry.get<ObjectCache>();
+    auto cacheEntry = cache.matcherToMetadataId.find(path);
+    if (cacheEntry == cache.matcherToMetadataId.end()) {
+      cache.matcherToMetadataId.insert(std::make_pair(path, id));
+      cache.idToMetadata[id] = extractCCDBHeaders(ref);
+      LOGP(info, "Caching CCDB metadata {}: {}", id.value, path);
+      return cache.idToMetadata[id];
+    }
+    auto& oldId = cacheEntry->second;
+    // The id in the cache is the same, let's simply return it.
+    if (oldId.value == id.value) {
+      LOGP(debug, "Returning cached CCDB metatada {}: {}", id.value, path);
+      return cache.idToMetadata[id];
+    }
+    // The id in the cache is different. Let's destroy the old cached entry
+    // and create a new one.
+    LOGP(info, "Replacing cached entry {} with {} for {}", oldId.value, id.value, path);
+    cache.idToObject[id] = extracCCDBMetadata(ref);
+    oldId.value = id.value;
+    return cache.idToObject[id];
+  }
+
   /// Helper method to be used to check if a given part of the InputRecord is present.
   [[nodiscard]] bool isValid(std::string const& s) const
   {
diff --git a/Framework/Core/include/Framework/ObjectCache.h b/Framework/Core/include/Framework/ObjectCache.h
index 22120a216092f..a6873aec8a1ac 100644
--- a/Framework/Core/include/Framework/ObjectCache.h
+++ b/Framework/Core/include/Framework/ObjectCache.h
@@ -13,6 +13,7 @@
 
 #include "Framework/DataRef.h"
 #include <unordered_map>
+#include <map>
 
 namespace o2::framework
 {
@@ -46,6 +47,12 @@ struct ObjectCache {
   /// A map from a CacheId (which is the void* ptr of the previous map).
   /// to an actual (type erased) pointer to the deserialised object.
   std::unordered_map<Id, void*, Id::hash_fn> idToObject;
+
+  /// A cache to the deserialised metadata
+  /// We keep it separate because we want to avoid that looking up
+  /// the metadata also pollutes the object cache.
+  std::unordered_map<std::string, Id> matcherToMetadataId;
+  std::unordered_map<Id, std::map<std::string, std::string>, Id::hash_fn> idToMetadata;
 };
 
 } // namespace o2::framework

From b3870af3f65dd75639e7a209fec21e592ff3b36f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 10 Oct 2024 10:53:10 +0200
Subject: [PATCH 664/726] DPL: allow customising DataProcessingStats intervals

In particular, this allows setting a minimum value for online metrics.
---
 .../Core/include/Framework/DataProcessingStats.h   | 11 ++++++++++-
 Framework/Core/src/CommonServices.cxx              |  5 ++++-
 Framework/Core/src/DataProcessingStats.cxx         | 14 ++++++++------
 Framework/Core/src/DeviceSpecHelpers.cxx           |  1 +
 Framework/Core/src/runDataProcessing.cxx           |  1 +
 .../Core/test/test_ComputingQuotaEvaluator.cxx     |  2 +-
 Framework/Core/test/test_DataProcessingStats.cxx   | 12 ++++++------
 Framework/Core/test/test_DataRelayer.cxx           |  2 +-
 8 files changed, 32 insertions(+), 16 deletions(-)

diff --git a/Framework/Core/include/Framework/DataProcessingStats.h b/Framework/Core/include/Framework/DataProcessingStats.h
index 286df61ba1464..ec96bf8e9973c 100644
--- a/Framework/Core/include/Framework/DataProcessingStats.h
+++ b/Framework/Core/include/Framework/DataProcessingStats.h
@@ -16,6 +16,7 @@
 #include <atomic>
 #include <cstdint>
 #include <array>
+#include <memory>
 #include <numeric>
 #include <mutex>
 #include <utility>
@@ -69,8 +70,16 @@ enum struct ProcessingStatsId : short {
 
 /// Helper struct to hold statistics about the data processing happening.
 struct DataProcessingStats {
+  // Parameters for the default behaviour
+  struct DefaultConfig {
+    int64_t minOnlinePublishInterval = 0;
+  };
+
+  DefaultConfig config = {};
+
   DataProcessingStats(std::function<void(int64_t& base, int64_t& offset)> getRealtimeBase,
-                      std::function<int64_t(int64_t base, int64_t offset)> getTimestamp);
+                      std::function<int64_t(int64_t base, int64_t offset)> getTimestamp,
+                      DefaultConfig config);
 
   constexpr static ServiceKind service_kind = ServiceKind::Global;
   constexpr static unsigned short MAX_METRICS = 1 << 15;
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 9192cbd0c965d..1c6f192cfb723 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -867,8 +867,11 @@ o2::framework::ServiceSpec CommonServices::dataProcessingStats()
       clock_gettime(CLOCK_REALTIME, &now);
       uv_update_time(state.loop);
       uint64_t offset = now.tv_sec * 1000 - uv_now(state.loop);
+      DataProcessingStats::DefaultConfig config = {
+        .minOnlinePublishInterval = std::stoi(options.GetProperty<std::string>("dpl-stats-min-online-publishing-interval").c_str())};
       auto* stats = new DataProcessingStats(TimingHelpers::defaultRealtimeBaseConfigurator(offset, state.loop),
-                                            TimingHelpers::defaultCPUTimeConfigurator(state.loop));
+                                            TimingHelpers::defaultCPUTimeConfigurator(state.loop),
+                                            config);
       auto& runningWorkflow = services.get<RunningWorkflowInfo const>();
 
       // It makes no sense to update the stats more often than every 5s
diff --git a/Framework/Core/src/DataProcessingStats.cxx b/Framework/Core/src/DataProcessingStats.cxx
index 920f4d7c7addb..3b02a0aacdd70 100644
--- a/Framework/Core/src/DataProcessingStats.cxx
+++ b/Framework/Core/src/DataProcessingStats.cxx
@@ -11,21 +11,20 @@
 
 #include "Framework/DataProcessingStats.h"
 #include "Framework/RuntimeError.h"
-#include "Framework/ServiceRegistryRef.h"
-#include "Framework/DeviceState.h"
 #include "Framework/Logger.h"
 #include <uv.h>
-#include <iostream>
 #include <atomic>
-#include <utility>
+#include <thread>
 
 namespace o2::framework
 {
 
 DataProcessingStats::DataProcessingStats(std::function<void(int64_t& base, int64_t& offset)> getRealtimeBase_,
-                                         std::function<int64_t(int64_t base, int64_t offset)> getTimestamp_)
+                                         std::function<int64_t(int64_t base, int64_t offset)> getTimestamp_,
+                                         DefaultConfig config_)
   : getTimestamp(getTimestamp_),
-    getRealtimeBase(getRealtimeBase_)
+    getRealtimeBase(getRealtimeBase_),
+    config(config_)
 {
   getRealtimeBase(realTimeBase, initialTimeOffset);
 }
@@ -269,6 +268,9 @@ void DataProcessingStats::registerMetric(MetricSpec const& spec)
   metricSpecs[spec.metricId] = spec;
   metricsNames[spec.metricId] = spec.name;
   metrics[spec.metricId] = spec.defaultValue;
+  if (metricSpecs[spec.metricId].scope == Scope::Online) {
+    metricSpecs[spec.metricId].minPublishInterval = std::max(metricSpecs[spec.metricId].minPublishInterval, config.minOnlinePublishInterval);
+  }
   int64_t currentTime = getTimestamp(realTimeBase, initialTimeOffset);
   updateInfos[spec.metricId] = UpdateInfo{currentTime, currentTime};
   updated[spec.metricId] = spec.sendInitialValue;
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index d1fad2bb66f8b..ffafe4dc4fd66 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1753,6 +1753,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
     ("configuration,cfg", bpo::value<std::string>(), "configuration connection string")                                                                              //
     ("driver-client-backend", bpo::value<std::string>(), "driver connection string")                                                                                 //
     ("monitoring-backend", bpo::value<std::string>(), "monitoring connection string")                                                                                //
+    ("dpl-stats-min-online-publishing-interval", bpo::value<std::string>(), "minimum flushing interval for online metrics (in ms)")                                  //
     ("infologger-mode", bpo::value<std::string>(), "O2_INFOLOGGER_MODE override")                                                                                    //
     ("infologger-severity", bpo::value<std::string>(), "minimun FairLogger severity which goes to info logger")                                                      //
     ("dpl-tracing-flags", bpo::value<std::string>(), "pipe separated list of events to trace")                                                                       //
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index d4d0c836aa925..54550f04d0a8d 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1049,6 +1049,7 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
     ConfigParamsHelper::populateBoostProgramOptions(optsDesc, spec.options, gHiddenDeviceOptions);
     char const* defaultSignposts = getenv("DPL_SIGNPOSTS");
     optsDesc.add_options()("monitoring-backend", bpo::value<std::string>()->default_value("default"), "monitoring backend info")                                                                   //
+      ("dpl-stats-min-online-publishing-interval", bpo::value<std::string>()->default_value("0"), "minimum flushing interval for online metrics (in ms)")                                          //
       ("driver-client-backend", bpo::value<std::string>()->default_value(defaultDriverClient), "backend for device -> driver communicataon: stdout://: use stdout, ws://: use websockets")         //
       ("infologger-severity", bpo::value<std::string>()->default_value(""), "minimum FairLogger severity to send to InfoLogger")                                                                   //
       ("dpl-tracing-flags", bpo::value<std::string>()->default_value(""), "pipe `|` separate list of events to be traced")                                                                         //
diff --git a/Framework/Core/test/test_ComputingQuotaEvaluator.cxx b/Framework/Core/test/test_ComputingQuotaEvaluator.cxx
index 26b41a3791485..92fedcfe78614 100644
--- a/Framework/Core/test/test_ComputingQuotaEvaluator.cxx
+++ b/Framework/Core/test/test_ComputingQuotaEvaluator.cxx
@@ -75,7 +75,7 @@ TEST_CASE("TestComputingQuotaEvaluator")
   };
 
   DataProcessingStats stats(TimingHelpers::defaultRealtimeBaseConfigurator(0, uv_default_loop()),
-                            TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()));
+                            TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()), {});
 
   ServiceRegistry registry;
   ServiceRegistryRef ref(registry);
diff --git a/Framework/Core/test/test_DataProcessingStats.cxx b/Framework/Core/test/test_DataProcessingStats.cxx
index c23b7fe0436d5..5f14ead966dc0 100644
--- a/Framework/Core/test/test_DataProcessingStats.cxx
+++ b/Framework/Core/test/test_DataProcessingStats.cxx
@@ -30,7 +30,7 @@ using namespace o2::framework;
 TEST_CASE("DataProcessingStats")
 {
   DataProcessingStats stats(TimingHelpers::defaultRealtimeBaseConfigurator(0, uv_default_loop()),
-                            TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()));
+                            TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()), {});
 
   o2::framework::clean_all_runtime_errors();
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric});
@@ -190,7 +190,7 @@ TEST_CASE("DataProcessingStatsOutOfOrder")
     int64_t value[] = {0, 1000, 999, 998};
     return base + value[count++] - offset;
   };
-  DataProcessingStats stats(realtimeTime, cpuTime);
+  DataProcessingStats stats(realtimeTime, cpuTime, {});
   // Notice this will consume one value in the cpuTime.
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric});
   stats.updateStats({DummyMetric, DataProcessingStats::Op::Set, 2});
@@ -222,7 +222,7 @@ TEST_CASE("DataProcessingStatsInstantaneousRate")
 
   // I want to push deltas since the last update and have the immediate time
   // averaged being stored.
-  DataProcessingStats stats(realtimeConfigurator, cpuTimeConfigurator);
+  DataProcessingStats stats(realtimeConfigurator, cpuTimeConfigurator, {});
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric, .kind = DataProcessingStats::Kind::Rate});
   REQUIRE(stats.updateInfos[DummyMetric].timestamp == 0);
   REQUIRE(stats.updateInfos[DummyMetric].lastPublished == 0);
@@ -265,7 +265,7 @@ TEST_CASE("DataProcessingStatsCumulativeRate")
 
   // I want to push deltas since the last update and have the immediate time
   // averaged being stored.
-  DataProcessingStats stats(realtimeConfigurator, cpuTimeConfigurator);
+  DataProcessingStats stats(realtimeConfigurator, cpuTimeConfigurator, {});
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric, .kind = DataProcessingStats::Kind::Rate});
   REQUIRE(stats.updateInfos[DummyMetric].timestamp == 1000);
   REQUIRE(stats.updateInfos[DummyMetric].lastPublished == 1000);
@@ -310,7 +310,7 @@ TEST_CASE("DataProcessingStatsPublishing")
 
   // I want to push deltas since the last update and have the immediate time
   // averaged being stored.
-  DataProcessingStats stats(realtimeTimestamp, cpuTimeTimestamp);
+  DataProcessingStats stats(realtimeTimestamp, cpuTimeTimestamp, {});
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric, .minPublishInterval = 5000});
   stats.registerMetric({.name = "dummy_metric2", .metricId = DummyMetric2, .minPublishInterval = 2000});
   REQUIRE(stats.updateInfos[DummyMetric].timestamp == 1000);
@@ -355,7 +355,7 @@ TEST_CASE("DataProcessingStatsPublishingRepeated")
 
   // I want to push deltas since the last update and have the immediate time
   // averaged being stored.
-  DataProcessingStats stats(realtimeTimestamp, cpuTimeTimestamp);
+  DataProcessingStats stats(realtimeTimestamp, cpuTimeTimestamp, {});
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric, .minPublishInterval = 3000, .maxRefreshLatency = 9000});
   REQUIRE(stats.updateInfos[DummyMetric].timestamp == 1000);
   REQUIRE(stats.updateInfos[DummyMetric].lastPublished == 1000);
diff --git a/Framework/Core/test/test_DataRelayer.cxx b/Framework/Core/test/test_DataRelayer.cxx
index ec5d48fbd78da..64a1827820638 100644
--- a/Framework/Core/test/test_DataRelayer.cxx
+++ b/Framework/Core/test/test_DataRelayer.cxx
@@ -48,7 +48,7 @@ TEST_CASE("DataRelayer")
     TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()));
   DataProcessingStats stats(
     TimingHelpers::defaultRealtimeBaseConfigurator(0, uv_default_loop()),
-    TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()));
+    TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()), {});
   int quickUpdateInterval = 1;
   using MetricSpec = DataProcessingStats::MetricSpec;
   std::vector<MetricSpec> specs{

From d9a9b14299e5596d23c88bd02ad423a1c0592e40 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 10 Oct 2024 10:53:10 +0200
Subject: [PATCH 665/726] DPL: allow customising DataProcessingStats intervals

In particular, this allows setting a minimum value for online metrics.
---
 .../Core/include/Framework/DataProcessingStats.h   | 11 ++++++++++-
 Framework/Core/src/CommonServices.cxx              | 11 +++++++----
 Framework/Core/src/DataProcessingStats.cxx         | 14 ++++++++------
 Framework/Core/src/DeviceSpecHelpers.cxx           |  1 +
 Framework/Core/src/runDataProcessing.cxx           |  1 +
 .../Core/test/test_ComputingQuotaEvaluator.cxx     |  2 +-
 Framework/Core/test/test_DataProcessingStats.cxx   | 12 ++++++------
 Framework/Core/test/test_DataRelayer.cxx           |  2 +-
 8 files changed, 35 insertions(+), 19 deletions(-)

diff --git a/Framework/Core/include/Framework/DataProcessingStats.h b/Framework/Core/include/Framework/DataProcessingStats.h
index 286df61ba1464..ec96bf8e9973c 100644
--- a/Framework/Core/include/Framework/DataProcessingStats.h
+++ b/Framework/Core/include/Framework/DataProcessingStats.h
@@ -16,6 +16,7 @@
 #include <atomic>
 #include <cstdint>
 #include <array>
+#include <memory>
 #include <numeric>
 #include <mutex>
 #include <utility>
@@ -69,8 +70,16 @@ enum struct ProcessingStatsId : short {
 
 /// Helper struct to hold statistics about the data processing happening.
 struct DataProcessingStats {
+  // Parameters for the default behaviour
+  struct DefaultConfig {
+    int64_t minOnlinePublishInterval = 0;
+  };
+
+  DefaultConfig config = {};
+
   DataProcessingStats(std::function<void(int64_t& base, int64_t& offset)> getRealtimeBase,
-                      std::function<int64_t(int64_t base, int64_t offset)> getTimestamp);
+                      std::function<int64_t(int64_t base, int64_t offset)> getTimestamp,
+                      DefaultConfig config);
 
   constexpr static ServiceKind service_kind = ServiceKind::Global;
   constexpr static unsigned short MAX_METRICS = 1 << 15;
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 9192cbd0c965d..0ee3d639c1b61 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -715,14 +715,14 @@ o2::framework::ServiceSpec
       O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Queueing oldest possible timeslice %" PRIu64 " propagation for execution.",
                              (uint64_t)oldestPossibleOutput.timeslice.value);
       AsyncQueueHelpers::post(
-        queue, AsyncTask{ .timeslice = TimesliceId{oldestPossibleTimeslice}, 
-                .id = decongestion.oldestPossibleTimesliceTask, 
+        queue, AsyncTask{ .timeslice = TimesliceId{oldestPossibleTimeslice},
+                .id = decongestion.oldestPossibleTimesliceTask,
                 .debounce = -1, .callback = decongestionCallback}
             .user<DecongestionContext>(DecongestionContext{.ref = services, .oldestPossibleOutput = oldestPossibleOutput}));
 
       if (decongestion.orderedCompletionPolicyActive) {
         AsyncQueueHelpers::post(
-          queue, AsyncTask{.timeslice = TimesliceId{oldestPossibleOutput.timeslice.value},.id = decongestion.oldestPossibleTimesliceTask,  .debounce = -1, 
+          queue, AsyncTask{.timeslice = TimesliceId{oldestPossibleOutput.timeslice.value},.id = decongestion.oldestPossibleTimesliceTask,  .debounce = -1,
           .callback = decongestionCallbackOrdered}
           .user<DecongestionContext>({.ref = services, .oldestPossibleOutput = oldestPossibleOutput}));
       } },
@@ -867,8 +867,11 @@ o2::framework::ServiceSpec CommonServices::dataProcessingStats()
       clock_gettime(CLOCK_REALTIME, &now);
       uv_update_time(state.loop);
       uint64_t offset = now.tv_sec * 1000 - uv_now(state.loop);
+      DataProcessingStats::DefaultConfig config = {
+        .minOnlinePublishInterval = std::stoi(options.GetProperty<std::string>("dpl-stats-min-online-publishing-interval").c_str())};
       auto* stats = new DataProcessingStats(TimingHelpers::defaultRealtimeBaseConfigurator(offset, state.loop),
-                                            TimingHelpers::defaultCPUTimeConfigurator(state.loop));
+                                            TimingHelpers::defaultCPUTimeConfigurator(state.loop),
+                                            config);
       auto& runningWorkflow = services.get<RunningWorkflowInfo const>();
 
       // It makes no sense to update the stats more often than every 5s
diff --git a/Framework/Core/src/DataProcessingStats.cxx b/Framework/Core/src/DataProcessingStats.cxx
index 920f4d7c7addb..3b02a0aacdd70 100644
--- a/Framework/Core/src/DataProcessingStats.cxx
+++ b/Framework/Core/src/DataProcessingStats.cxx
@@ -11,21 +11,20 @@
 
 #include "Framework/DataProcessingStats.h"
 #include "Framework/RuntimeError.h"
-#include "Framework/ServiceRegistryRef.h"
-#include "Framework/DeviceState.h"
 #include "Framework/Logger.h"
 #include <uv.h>
-#include <iostream>
 #include <atomic>
-#include <utility>
+#include <thread>
 
 namespace o2::framework
 {
 
 DataProcessingStats::DataProcessingStats(std::function<void(int64_t& base, int64_t& offset)> getRealtimeBase_,
-                                         std::function<int64_t(int64_t base, int64_t offset)> getTimestamp_)
+                                         std::function<int64_t(int64_t base, int64_t offset)> getTimestamp_,
+                                         DefaultConfig config_)
   : getTimestamp(getTimestamp_),
-    getRealtimeBase(getRealtimeBase_)
+    getRealtimeBase(getRealtimeBase_),
+    config(config_)
 {
   getRealtimeBase(realTimeBase, initialTimeOffset);
 }
@@ -269,6 +268,9 @@ void DataProcessingStats::registerMetric(MetricSpec const& spec)
   metricSpecs[spec.metricId] = spec;
   metricsNames[spec.metricId] = spec.name;
   metrics[spec.metricId] = spec.defaultValue;
+  if (metricSpecs[spec.metricId].scope == Scope::Online) {
+    metricSpecs[spec.metricId].minPublishInterval = std::max(metricSpecs[spec.metricId].minPublishInterval, config.minOnlinePublishInterval);
+  }
   int64_t currentTime = getTimestamp(realTimeBase, initialTimeOffset);
   updateInfos[spec.metricId] = UpdateInfo{currentTime, currentTime};
   updated[spec.metricId] = spec.sendInitialValue;
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index d1fad2bb66f8b..ffafe4dc4fd66 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1753,6 +1753,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
     ("configuration,cfg", bpo::value<std::string>(), "configuration connection string")                                                                              //
     ("driver-client-backend", bpo::value<std::string>(), "driver connection string")                                                                                 //
     ("monitoring-backend", bpo::value<std::string>(), "monitoring connection string")                                                                                //
+    ("dpl-stats-min-online-publishing-interval", bpo::value<std::string>(), "minimum flushing interval for online metrics (in ms)")                                  //
     ("infologger-mode", bpo::value<std::string>(), "O2_INFOLOGGER_MODE override")                                                                                    //
     ("infologger-severity", bpo::value<std::string>(), "minimun FairLogger severity which goes to info logger")                                                      //
     ("dpl-tracing-flags", bpo::value<std::string>(), "pipe separated list of events to trace")                                                                       //
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index d4d0c836aa925..54550f04d0a8d 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1049,6 +1049,7 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
     ConfigParamsHelper::populateBoostProgramOptions(optsDesc, spec.options, gHiddenDeviceOptions);
     char const* defaultSignposts = getenv("DPL_SIGNPOSTS");
     optsDesc.add_options()("monitoring-backend", bpo::value<std::string>()->default_value("default"), "monitoring backend info")                                                                   //
+      ("dpl-stats-min-online-publishing-interval", bpo::value<std::string>()->default_value("0"), "minimum flushing interval for online metrics (in ms)")                                          //
       ("driver-client-backend", bpo::value<std::string>()->default_value(defaultDriverClient), "backend for device -> driver communicataon: stdout://: use stdout, ws://: use websockets")         //
       ("infologger-severity", bpo::value<std::string>()->default_value(""), "minimum FairLogger severity to send to InfoLogger")                                                                   //
       ("dpl-tracing-flags", bpo::value<std::string>()->default_value(""), "pipe `|` separate list of events to be traced")                                                                         //
diff --git a/Framework/Core/test/test_ComputingQuotaEvaluator.cxx b/Framework/Core/test/test_ComputingQuotaEvaluator.cxx
index 26b41a3791485..92fedcfe78614 100644
--- a/Framework/Core/test/test_ComputingQuotaEvaluator.cxx
+++ b/Framework/Core/test/test_ComputingQuotaEvaluator.cxx
@@ -75,7 +75,7 @@ TEST_CASE("TestComputingQuotaEvaluator")
   };
 
   DataProcessingStats stats(TimingHelpers::defaultRealtimeBaseConfigurator(0, uv_default_loop()),
-                            TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()));
+                            TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()), {});
 
   ServiceRegistry registry;
   ServiceRegistryRef ref(registry);
diff --git a/Framework/Core/test/test_DataProcessingStats.cxx b/Framework/Core/test/test_DataProcessingStats.cxx
index c23b7fe0436d5..5f14ead966dc0 100644
--- a/Framework/Core/test/test_DataProcessingStats.cxx
+++ b/Framework/Core/test/test_DataProcessingStats.cxx
@@ -30,7 +30,7 @@ using namespace o2::framework;
 TEST_CASE("DataProcessingStats")
 {
   DataProcessingStats stats(TimingHelpers::defaultRealtimeBaseConfigurator(0, uv_default_loop()),
-                            TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()));
+                            TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()), {});
 
   o2::framework::clean_all_runtime_errors();
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric});
@@ -190,7 +190,7 @@ TEST_CASE("DataProcessingStatsOutOfOrder")
     int64_t value[] = {0, 1000, 999, 998};
     return base + value[count++] - offset;
   };
-  DataProcessingStats stats(realtimeTime, cpuTime);
+  DataProcessingStats stats(realtimeTime, cpuTime, {});
   // Notice this will consume one value in the cpuTime.
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric});
   stats.updateStats({DummyMetric, DataProcessingStats::Op::Set, 2});
@@ -222,7 +222,7 @@ TEST_CASE("DataProcessingStatsInstantaneousRate")
 
   // I want to push deltas since the last update and have the immediate time
   // averaged being stored.
-  DataProcessingStats stats(realtimeConfigurator, cpuTimeConfigurator);
+  DataProcessingStats stats(realtimeConfigurator, cpuTimeConfigurator, {});
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric, .kind = DataProcessingStats::Kind::Rate});
   REQUIRE(stats.updateInfos[DummyMetric].timestamp == 0);
   REQUIRE(stats.updateInfos[DummyMetric].lastPublished == 0);
@@ -265,7 +265,7 @@ TEST_CASE("DataProcessingStatsCumulativeRate")
 
   // I want to push deltas since the last update and have the immediate time
   // averaged being stored.
-  DataProcessingStats stats(realtimeConfigurator, cpuTimeConfigurator);
+  DataProcessingStats stats(realtimeConfigurator, cpuTimeConfigurator, {});
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric, .kind = DataProcessingStats::Kind::Rate});
   REQUIRE(stats.updateInfos[DummyMetric].timestamp == 1000);
   REQUIRE(stats.updateInfos[DummyMetric].lastPublished == 1000);
@@ -310,7 +310,7 @@ TEST_CASE("DataProcessingStatsPublishing")
 
   // I want to push deltas since the last update and have the immediate time
   // averaged being stored.
-  DataProcessingStats stats(realtimeTimestamp, cpuTimeTimestamp);
+  DataProcessingStats stats(realtimeTimestamp, cpuTimeTimestamp, {});
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric, .minPublishInterval = 5000});
   stats.registerMetric({.name = "dummy_metric2", .metricId = DummyMetric2, .minPublishInterval = 2000});
   REQUIRE(stats.updateInfos[DummyMetric].timestamp == 1000);
@@ -355,7 +355,7 @@ TEST_CASE("DataProcessingStatsPublishingRepeated")
 
   // I want to push deltas since the last update and have the immediate time
   // averaged being stored.
-  DataProcessingStats stats(realtimeTimestamp, cpuTimeTimestamp);
+  DataProcessingStats stats(realtimeTimestamp, cpuTimeTimestamp, {});
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric, .minPublishInterval = 3000, .maxRefreshLatency = 9000});
   REQUIRE(stats.updateInfos[DummyMetric].timestamp == 1000);
   REQUIRE(stats.updateInfos[DummyMetric].lastPublished == 1000);
diff --git a/Framework/Core/test/test_DataRelayer.cxx b/Framework/Core/test/test_DataRelayer.cxx
index ec5d48fbd78da..64a1827820638 100644
--- a/Framework/Core/test/test_DataRelayer.cxx
+++ b/Framework/Core/test/test_DataRelayer.cxx
@@ -48,7 +48,7 @@ TEST_CASE("DataRelayer")
     TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()));
   DataProcessingStats stats(
     TimingHelpers::defaultRealtimeBaseConfigurator(0, uv_default_loop()),
-    TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()));
+    TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()), {});
   int quickUpdateInterval = 1;
   using MetricSpec = DataProcessingStats::MetricSpec;
   std::vector<MetricSpec> specs{

From 9c822996fedb928578f9424ef044ec2f93b2415b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 10 Oct 2024 15:30:14 +0200
Subject: [PATCH 666/726] DPL: allow customising DataProcessingStats intervals

In particular, this allows setting a minimum value for online metrics.
---
 .../Core/include/Framework/DataProcessingStats.h   | 11 ++++++++++-
 Framework/Core/src/CommonServices.cxx              | 11 +++++++----
 Framework/Core/src/DataProcessingStats.cxx         | 14 ++++++++------
 Framework/Core/src/DeviceSpecHelpers.cxx           |  1 +
 Framework/Core/src/runDataProcessing.cxx           |  1 +
 .../Core/test/test_ComputingQuotaEvaluator.cxx     |  2 +-
 Framework/Core/test/test_DataProcessingStats.cxx   | 12 ++++++------
 Framework/Core/test/test_DataRelayer.cxx           |  2 +-
 8 files changed, 35 insertions(+), 19 deletions(-)

diff --git a/Framework/Core/include/Framework/DataProcessingStats.h b/Framework/Core/include/Framework/DataProcessingStats.h
index 286df61ba1464..ec96bf8e9973c 100644
--- a/Framework/Core/include/Framework/DataProcessingStats.h
+++ b/Framework/Core/include/Framework/DataProcessingStats.h
@@ -16,6 +16,7 @@
 #include <atomic>
 #include <cstdint>
 #include <array>
+#include <memory>
 #include <numeric>
 #include <mutex>
 #include <utility>
@@ -69,8 +70,16 @@ enum struct ProcessingStatsId : short {
 
 /// Helper struct to hold statistics about the data processing happening.
 struct DataProcessingStats {
+  // Parameters for the default behaviour
+  struct DefaultConfig {
+    int64_t minOnlinePublishInterval = 0;
+  };
+
+  DefaultConfig config = {};
+
   DataProcessingStats(std::function<void(int64_t& base, int64_t& offset)> getRealtimeBase,
-                      std::function<int64_t(int64_t base, int64_t offset)> getTimestamp);
+                      std::function<int64_t(int64_t base, int64_t offset)> getTimestamp,
+                      DefaultConfig config);
 
   constexpr static ServiceKind service_kind = ServiceKind::Global;
   constexpr static unsigned short MAX_METRICS = 1 << 15;
diff --git a/Framework/Core/src/CommonServices.cxx b/Framework/Core/src/CommonServices.cxx
index 9192cbd0c965d..bc750181d54e2 100644
--- a/Framework/Core/src/CommonServices.cxx
+++ b/Framework/Core/src/CommonServices.cxx
@@ -715,14 +715,14 @@ o2::framework::ServiceSpec
       O2_SIGNPOST_EVENT_EMIT(data_processor_context, cid, "oldest_possible_timeslice", "Queueing oldest possible timeslice %" PRIu64 " propagation for execution.",
                              (uint64_t)oldestPossibleOutput.timeslice.value);
       AsyncQueueHelpers::post(
-        queue, AsyncTask{ .timeslice = TimesliceId{oldestPossibleTimeslice}, 
-                .id = decongestion.oldestPossibleTimesliceTask, 
+        queue, AsyncTask{ .timeslice = TimesliceId{oldestPossibleTimeslice},
+                .id = decongestion.oldestPossibleTimesliceTask,
                 .debounce = -1, .callback = decongestionCallback}
             .user<DecongestionContext>(DecongestionContext{.ref = services, .oldestPossibleOutput = oldestPossibleOutput}));
 
       if (decongestion.orderedCompletionPolicyActive) {
         AsyncQueueHelpers::post(
-          queue, AsyncTask{.timeslice = TimesliceId{oldestPossibleOutput.timeslice.value},.id = decongestion.oldestPossibleTimesliceTask,  .debounce = -1, 
+          queue, AsyncTask{.timeslice = TimesliceId{oldestPossibleOutput.timeslice.value},.id = decongestion.oldestPossibleTimesliceTask,  .debounce = -1,
           .callback = decongestionCallbackOrdered}
           .user<DecongestionContext>({.ref = services, .oldestPossibleOutput = oldestPossibleOutput}));
       } },
@@ -867,8 +867,11 @@ o2::framework::ServiceSpec CommonServices::dataProcessingStats()
       clock_gettime(CLOCK_REALTIME, &now);
       uv_update_time(state.loop);
       uint64_t offset = now.tv_sec * 1000 - uv_now(state.loop);
+      DataProcessingStats::DefaultConfig config = {
+        .minOnlinePublishInterval = std::stoi(options.GetProperty<std::string>("dpl-stats-min-online-publishing-interval").c_str()) * 1000};
       auto* stats = new DataProcessingStats(TimingHelpers::defaultRealtimeBaseConfigurator(offset, state.loop),
-                                            TimingHelpers::defaultCPUTimeConfigurator(state.loop));
+                                            TimingHelpers::defaultCPUTimeConfigurator(state.loop),
+                                            config);
       auto& runningWorkflow = services.get<RunningWorkflowInfo const>();
 
       // It makes no sense to update the stats more often than every 5s
diff --git a/Framework/Core/src/DataProcessingStats.cxx b/Framework/Core/src/DataProcessingStats.cxx
index 920f4d7c7addb..3b02a0aacdd70 100644
--- a/Framework/Core/src/DataProcessingStats.cxx
+++ b/Framework/Core/src/DataProcessingStats.cxx
@@ -11,21 +11,20 @@
 
 #include "Framework/DataProcessingStats.h"
 #include "Framework/RuntimeError.h"
-#include "Framework/ServiceRegistryRef.h"
-#include "Framework/DeviceState.h"
 #include "Framework/Logger.h"
 #include <uv.h>
-#include <iostream>
 #include <atomic>
-#include <utility>
+#include <thread>
 
 namespace o2::framework
 {
 
 DataProcessingStats::DataProcessingStats(std::function<void(int64_t& base, int64_t& offset)> getRealtimeBase_,
-                                         std::function<int64_t(int64_t base, int64_t offset)> getTimestamp_)
+                                         std::function<int64_t(int64_t base, int64_t offset)> getTimestamp_,
+                                         DefaultConfig config_)
   : getTimestamp(getTimestamp_),
-    getRealtimeBase(getRealtimeBase_)
+    getRealtimeBase(getRealtimeBase_),
+    config(config_)
 {
   getRealtimeBase(realTimeBase, initialTimeOffset);
 }
@@ -269,6 +268,9 @@ void DataProcessingStats::registerMetric(MetricSpec const& spec)
   metricSpecs[spec.metricId] = spec;
   metricsNames[spec.metricId] = spec.name;
   metrics[spec.metricId] = spec.defaultValue;
+  if (metricSpecs[spec.metricId].scope == Scope::Online) {
+    metricSpecs[spec.metricId].minPublishInterval = std::max(metricSpecs[spec.metricId].minPublishInterval, config.minOnlinePublishInterval);
+  }
   int64_t currentTime = getTimestamp(realTimeBase, initialTimeOffset);
   updateInfos[spec.metricId] = UpdateInfo{currentTime, currentTime};
   updated[spec.metricId] = spec.sendInitialValue;
diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx
index d1fad2bb66f8b..f2644ed66ba08 100644
--- a/Framework/Core/src/DeviceSpecHelpers.cxx
+++ b/Framework/Core/src/DeviceSpecHelpers.cxx
@@ -1753,6 +1753,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic
     ("configuration,cfg", bpo::value<std::string>(), "configuration connection string")                                                                              //
     ("driver-client-backend", bpo::value<std::string>(), "driver connection string")                                                                                 //
     ("monitoring-backend", bpo::value<std::string>(), "monitoring connection string")                                                                                //
+    ("dpl-stats-min-online-publishing-interval", bpo::value<std::string>(), "minimum flushing interval for online metrics (in s)")                                   //
     ("infologger-mode", bpo::value<std::string>(), "O2_INFOLOGGER_MODE override")                                                                                    //
     ("infologger-severity", bpo::value<std::string>(), "minimun FairLogger severity which goes to info logger")                                                      //
     ("dpl-tracing-flags", bpo::value<std::string>(), "pipe separated list of events to trace")                                                                       //
diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index d4d0c836aa925..58e669e127f03 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -1049,6 +1049,7 @@ int doChild(int argc, char** argv, ServiceRegistry& serviceRegistry,
     ConfigParamsHelper::populateBoostProgramOptions(optsDesc, spec.options, gHiddenDeviceOptions);
     char const* defaultSignposts = getenv("DPL_SIGNPOSTS");
     optsDesc.add_options()("monitoring-backend", bpo::value<std::string>()->default_value("default"), "monitoring backend info")                                                                   //
+      ("dpl-stats-min-online-publishing-interval", bpo::value<std::string>()->default_value("0"), "minimum flushing interval for online metrics (in s)")                                           //
       ("driver-client-backend", bpo::value<std::string>()->default_value(defaultDriverClient), "backend for device -> driver communicataon: stdout://: use stdout, ws://: use websockets")         //
       ("infologger-severity", bpo::value<std::string>()->default_value(""), "minimum FairLogger severity to send to InfoLogger")                                                                   //
       ("dpl-tracing-flags", bpo::value<std::string>()->default_value(""), "pipe `|` separate list of events to be traced")                                                                         //
diff --git a/Framework/Core/test/test_ComputingQuotaEvaluator.cxx b/Framework/Core/test/test_ComputingQuotaEvaluator.cxx
index 26b41a3791485..92fedcfe78614 100644
--- a/Framework/Core/test/test_ComputingQuotaEvaluator.cxx
+++ b/Framework/Core/test/test_ComputingQuotaEvaluator.cxx
@@ -75,7 +75,7 @@ TEST_CASE("TestComputingQuotaEvaluator")
   };
 
   DataProcessingStats stats(TimingHelpers::defaultRealtimeBaseConfigurator(0, uv_default_loop()),
-                            TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()));
+                            TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()), {});
 
   ServiceRegistry registry;
   ServiceRegistryRef ref(registry);
diff --git a/Framework/Core/test/test_DataProcessingStats.cxx b/Framework/Core/test/test_DataProcessingStats.cxx
index c23b7fe0436d5..5f14ead966dc0 100644
--- a/Framework/Core/test/test_DataProcessingStats.cxx
+++ b/Framework/Core/test/test_DataProcessingStats.cxx
@@ -30,7 +30,7 @@ using namespace o2::framework;
 TEST_CASE("DataProcessingStats")
 {
   DataProcessingStats stats(TimingHelpers::defaultRealtimeBaseConfigurator(0, uv_default_loop()),
-                            TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()));
+                            TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()), {});
 
   o2::framework::clean_all_runtime_errors();
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric});
@@ -190,7 +190,7 @@ TEST_CASE("DataProcessingStatsOutOfOrder")
     int64_t value[] = {0, 1000, 999, 998};
     return base + value[count++] - offset;
   };
-  DataProcessingStats stats(realtimeTime, cpuTime);
+  DataProcessingStats stats(realtimeTime, cpuTime, {});
   // Notice this will consume one value in the cpuTime.
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric});
   stats.updateStats({DummyMetric, DataProcessingStats::Op::Set, 2});
@@ -222,7 +222,7 @@ TEST_CASE("DataProcessingStatsInstantaneousRate")
 
   // I want to push deltas since the last update and have the immediate time
   // averaged being stored.
-  DataProcessingStats stats(realtimeConfigurator, cpuTimeConfigurator);
+  DataProcessingStats stats(realtimeConfigurator, cpuTimeConfigurator, {});
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric, .kind = DataProcessingStats::Kind::Rate});
   REQUIRE(stats.updateInfos[DummyMetric].timestamp == 0);
   REQUIRE(stats.updateInfos[DummyMetric].lastPublished == 0);
@@ -265,7 +265,7 @@ TEST_CASE("DataProcessingStatsCumulativeRate")
 
   // I want to push deltas since the last update and have the immediate time
   // averaged being stored.
-  DataProcessingStats stats(realtimeConfigurator, cpuTimeConfigurator);
+  DataProcessingStats stats(realtimeConfigurator, cpuTimeConfigurator, {});
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric, .kind = DataProcessingStats::Kind::Rate});
   REQUIRE(stats.updateInfos[DummyMetric].timestamp == 1000);
   REQUIRE(stats.updateInfos[DummyMetric].lastPublished == 1000);
@@ -310,7 +310,7 @@ TEST_CASE("DataProcessingStatsPublishing")
 
   // I want to push deltas since the last update and have the immediate time
   // averaged being stored.
-  DataProcessingStats stats(realtimeTimestamp, cpuTimeTimestamp);
+  DataProcessingStats stats(realtimeTimestamp, cpuTimeTimestamp, {});
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric, .minPublishInterval = 5000});
   stats.registerMetric({.name = "dummy_metric2", .metricId = DummyMetric2, .minPublishInterval = 2000});
   REQUIRE(stats.updateInfos[DummyMetric].timestamp == 1000);
@@ -355,7 +355,7 @@ TEST_CASE("DataProcessingStatsPublishingRepeated")
 
   // I want to push deltas since the last update and have the immediate time
   // averaged being stored.
-  DataProcessingStats stats(realtimeTimestamp, cpuTimeTimestamp);
+  DataProcessingStats stats(realtimeTimestamp, cpuTimeTimestamp, {});
   stats.registerMetric({.name = "dummy_metric", .metricId = DummyMetric, .minPublishInterval = 3000, .maxRefreshLatency = 9000});
   REQUIRE(stats.updateInfos[DummyMetric].timestamp == 1000);
   REQUIRE(stats.updateInfos[DummyMetric].lastPublished == 1000);
diff --git a/Framework/Core/test/test_DataRelayer.cxx b/Framework/Core/test/test_DataRelayer.cxx
index ec5d48fbd78da..64a1827820638 100644
--- a/Framework/Core/test/test_DataRelayer.cxx
+++ b/Framework/Core/test/test_DataRelayer.cxx
@@ -48,7 +48,7 @@ TEST_CASE("DataRelayer")
     TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()));
   DataProcessingStats stats(
     TimingHelpers::defaultRealtimeBaseConfigurator(0, uv_default_loop()),
-    TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()));
+    TimingHelpers::defaultCPUTimeConfigurator(uv_default_loop()), {});
   int quickUpdateInterval = 1;
   using MetricSpec = DataProcessingStats::MetricSpec;
   std::vector<MetricSpec> specs{

From 3f4e271d87ec8beec10f7efce3ba92606e4ec853 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 11 Oct 2024 10:37:46 +0200
Subject: [PATCH 667/726] DPL Analysis: enable cache without prefetching

This give the same results in terms of reduced IOPs, without
the problems with the multiple basket branches.

It also adds explicitly to the cache all the _size branches.

Not yet clear if we need to have ROOT v6.32.06-alice2 as well for
this to work correctly.
---
 Framework/Core/src/TableTreeHelpers.cxx | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/src/TableTreeHelpers.cxx b/Framework/Core/src/TableTreeHelpers.cxx
index 780126c89dd85..795155870c847 100644
--- a/Framework/Core/src/TableTreeHelpers.cxx
+++ b/Framework/Core/src/TableTreeHelpers.cxx
@@ -550,9 +550,17 @@ void TreeToTable::addAllColumns(TTree* tree, std::vector<std::string>&& names)
   // Was affected by https://github.com/root-project/root/issues/8962
   // Re-enabling this seems to cut the number of IOPS in half
   tree->SetCacheSize(25000000);
-  tree->SetClusterPrefetch(true);
+  // tree->SetClusterPrefetch(true);
   for (auto& reader : mBranchReaders) {
     tree->AddBranchToCache(reader->branch());
+    if (strncmp(reader->branch()->GetName(), "fIndexArray", strlen("fIndexArray")) == 0) {
+      std::string sizeBranchName = reader->branch()->GetName();
+      sizeBranchName += "_size";
+      TBranch* sizeBranch = (TBranch*)tree->GetBranch(sizeBranchName.c_str());
+      if (sizeBranch) {
+        tree->AddBranchToCache(sizeBranch);
+      }
+    }
   }
   tree->StopCacheLearningPhase();
 }

From 4f9d0f93f696afeb202fd78c0162a6bf6b7ba433 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 11 Oct 2024 10:52:55 +0200
Subject: [PATCH 668/726] DPL Analysis: enable cache without prefetching

This give the same results in terms of reduced IOPs, without
the problems with the multiple basket branches.

It also adds explicitly to the cache all the _size branches.

Not yet clear if we need to have ROOT v6.32.06-alice2 as well for
this to work correctly.
---
 Framework/Core/src/TableTreeHelpers.cxx | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/Framework/Core/src/TableTreeHelpers.cxx b/Framework/Core/src/TableTreeHelpers.cxx
index 14e4a43ea6cba..795155870c847 100644
--- a/Framework/Core/src/TableTreeHelpers.cxx
+++ b/Framework/Core/src/TableTreeHelpers.cxx
@@ -547,14 +547,22 @@ void TreeToTable::addAllColumns(TTree* tree, std::vector<std::string>&& names)
   if (mBranchReaders.empty()) {
     throw runtime_error("No columns will be read");
   }
-  //tree->SetCacheSize(50000000);
-  // FIXME: see https://github.com/root-project/root/issues/8962 and enable
-  // again once fixed.
-  //tree->SetClusterPrefetch(true);
-  //for (auto& reader : mBranchReaders) {
-  //  tree->AddBranchToCache(reader->branch());
-  //}
-  //tree->StopCacheLearningPhase();
+  // Was affected by https://github.com/root-project/root/issues/8962
+  // Re-enabling this seems to cut the number of IOPS in half
+  tree->SetCacheSize(25000000);
+  // tree->SetClusterPrefetch(true);
+  for (auto& reader : mBranchReaders) {
+    tree->AddBranchToCache(reader->branch());
+    if (strncmp(reader->branch()->GetName(), "fIndexArray", strlen("fIndexArray")) == 0) {
+      std::string sizeBranchName = reader->branch()->GetName();
+      sizeBranchName += "_size";
+      TBranch* sizeBranch = (TBranch*)tree->GetBranch(sizeBranchName.c_str());
+      if (sizeBranch) {
+        tree->AddBranchToCache(sizeBranch);
+      }
+    }
+  }
+  tree->StopCacheLearningPhase();
 }
 
 void TreeToTable::setLabel(const char* label)

From 7b10347d8db7b19626a426baac697ed41ff6d601 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 16 Oct 2024 13:09:31 +0200
Subject: [PATCH 669/726] DPL: speedup homogeneous_apply_ref for less than 20
 elements

Adding the requires statement should reduce greatly work done by the compiler
to build and then optimise out the if constexprs.
---
 .../include/Framework/StructToTuple.h         | 26 +++++++++++++++----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/Framework/Foundation/include/Framework/StructToTuple.h b/Framework/Foundation/include/Framework/StructToTuple.h
index becf22d01b8fe..dde40c2e9dfa9 100644
--- a/Framework/Foundation/include/Framework/StructToTuple.h
+++ b/Framework/Foundation/include/Framework/StructToTuple.h
@@ -163,13 +163,19 @@ consteval auto brace_constructible_size(auto... Members)
     return std::array<decltype(l(p0)), d##0>{DPL_FENUM_##d##0(l, p, )}; \
   }
 
-template <bool B = false, typename L, class T>
-auto homogeneous_apply_refs(L l, T&& object)
+template <bool B, typename T>
+consteval int nested_brace_constructible_size()
 {
   using type = std::decay_t<T>;
   constexpr int nesting = B ? 1 : 0;
-  constexpr unsigned long numElements = brace_constructible_size<type>() - nesting;
-  static_assert(numElements < 99, "Too many elements in the struct");
+  return brace_constructible_size<type>() - nesting;
+}
+
+template <bool B = false, typename L, class T>
+  requires(nested_brace_constructible_size<B, T>() < 99 && nested_brace_constructible_size<B, T>() >= 20)
+auto homogeneous_apply_refs(L l, T&& object)
+{
+  constexpr unsigned long numElements = nested_brace_constructible_size<B, T>();
   // clang-format off
   if DPL_HOMOGENEOUS_APPLY_ENTRY (9, 9)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (9, 8)
@@ -251,7 +257,17 @@ auto homogeneous_apply_refs(L l, T&& object)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (2, 2)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (2, 1)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY_TENS (2)
-  else if DPL_HOMOGENEOUS_APPLY_ENTRY (1, 9)
+  else { return std::array<bool,0>(); }
+  // clang-format on
+}
+
+template <bool B = false, typename L, class T>
+  requires(nested_brace_constructible_size<B, T>() < 20 && nested_brace_constructible_size<B, T>() >= 0)
+auto homogeneous_apply_refs(L l, T&& object)
+{
+  constexpr unsigned long numElements = nested_brace_constructible_size<B, T>();
+  // clang-format off
+  if DPL_HOMOGENEOUS_APPLY_ENTRY (1, 9)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (1, 8)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (1, 7)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (1, 6)

From de41e1b8a179a6d6407ce769408aa988dc1b322c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 17 Oct 2024 11:01:17 +0200
Subject: [PATCH 670/726] DPL: speedup homogeneous_apply_ref

Adding the requires statement should reduce greatly work done by the compiler
to build and then optimise out the if constexprs.
---
 .../include/Framework/StructToTuple.h         | 122 ++++++++++++++++--
 1 file changed, 109 insertions(+), 13 deletions(-)

diff --git a/Framework/Foundation/include/Framework/StructToTuple.h b/Framework/Foundation/include/Framework/StructToTuple.h
index becf22d01b8fe..c600e33883a74 100644
--- a/Framework/Foundation/include/Framework/StructToTuple.h
+++ b/Framework/Foundation/include/Framework/StructToTuple.h
@@ -163,13 +163,19 @@ consteval auto brace_constructible_size(auto... Members)
     return std::array<decltype(l(p0)), d##0>{DPL_FENUM_##d##0(l, p, )}; \
   }
 
-template <bool B = false, typename L, class T>
-auto homogeneous_apply_refs(L l, T&& object)
+template <bool B, typename T>
+consteval int nested_brace_constructible_size()
 {
   using type = std::decay_t<T>;
   constexpr int nesting = B ? 1 : 0;
-  constexpr unsigned long numElements = brace_constructible_size<type>() - nesting;
-  static_assert(numElements < 99, "Too many elements in the struct");
+  return brace_constructible_size<type>() - nesting;
+}
+
+template <bool B = false, typename L, class T, int D = nested_brace_constructible_size<B, T>() / 10>
+  requires(D == 9)
+auto homogeneous_apply_refs(L l, T&& object)
+{
+  constexpr int numElements = nested_brace_constructible_size<B, T>();
   // clang-format off
   if DPL_HOMOGENEOUS_APPLY_ENTRY (9, 9)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (9, 8)
@@ -181,7 +187,17 @@ auto homogeneous_apply_refs(L l, T&& object)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (9, 2)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (9, 1)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY_TENS (9)
-  else if DPL_HOMOGENEOUS_APPLY_ENTRY (8, 9)
+  else { return std::array<bool,0>(); }
+  // clang-format on
+}
+
+template <bool B = false, typename L, class T, int D = nested_brace_constructible_size<B, T>() / 10>
+  requires(D == 8)
+auto homogeneous_apply_refs(L l, T&& object)
+{
+  constexpr int numElements = nested_brace_constructible_size<B, T>();
+  // clang-format off
+  if DPL_HOMOGENEOUS_APPLY_ENTRY (8, 9)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (8, 8)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (8, 7)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (8, 6)
@@ -191,7 +207,17 @@ auto homogeneous_apply_refs(L l, T&& object)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (8, 2)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (8, 1)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY_TENS (8)
-  else if DPL_HOMOGENEOUS_APPLY_ENTRY (7, 9)
+  else { return std::array<bool,0>(); }
+  // clang-format on
+}
+
+template <bool B = false, typename L, class T, int D = nested_brace_constructible_size<B, T>() / 10>
+  requires(D == 7)
+auto homogeneous_apply_refs(L l, T&& object)
+{
+  constexpr int numElements = nested_brace_constructible_size<B, T>();
+  // clang-format off
+  if DPL_HOMOGENEOUS_APPLY_ENTRY (7, 9)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (7, 8)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (7, 7)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (7, 6)
@@ -201,7 +227,17 @@ auto homogeneous_apply_refs(L l, T&& object)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (7, 2)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (7, 1)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY_TENS (7)
-  else if DPL_HOMOGENEOUS_APPLY_ENTRY (6, 9)
+  else { return std::array<bool,0>(); }
+  // clang-format on
+}
+
+template <bool B = false, typename L, class T, int D = nested_brace_constructible_size<B, T>() / 10>
+  requires(D == 6)
+auto homogeneous_apply_refs(L l, T&& object)
+{
+  constexpr int numElements = nested_brace_constructible_size<B, T>();
+  // clang-format off
+  if DPL_HOMOGENEOUS_APPLY_ENTRY (6, 9)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (6, 8)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (6, 7)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (6, 6)
@@ -211,7 +247,17 @@ auto homogeneous_apply_refs(L l, T&& object)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (6, 2)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (6, 1)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY_TENS (6)
-  else if DPL_HOMOGENEOUS_APPLY_ENTRY (5, 9)
+  else { return std::array<bool,0>(); }
+  // clang-format on
+}
+
+template <bool B = false, typename L, class T, int D = nested_brace_constructible_size<B, T>() / 10>
+  requires(D == 5)
+auto homogeneous_apply_refs(L l, T&& object)
+{
+  constexpr int numElements = nested_brace_constructible_size<B, T>();
+  // clang-format off
+  if DPL_HOMOGENEOUS_APPLY_ENTRY (5, 9)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (5, 8)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (5, 7)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (5, 6)
@@ -221,7 +267,17 @@ auto homogeneous_apply_refs(L l, T&& object)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (5, 2)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (5, 1)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY_TENS (5)
-  else if DPL_HOMOGENEOUS_APPLY_ENTRY (4, 9)
+  else { return std::array<bool,0>(); }
+  // clang-format on
+}
+
+template <bool B = false, typename L, class T, int D = nested_brace_constructible_size<B, T>() / 10>
+  requires(D == 4)
+auto homogeneous_apply_refs(L l, T&& object)
+{
+  constexpr int numElements = nested_brace_constructible_size<B, T>();
+  // clang-format off
+  if DPL_HOMOGENEOUS_APPLY_ENTRY (4, 9)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (4, 8)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (4, 7)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (4, 6)
@@ -231,7 +287,17 @@ auto homogeneous_apply_refs(L l, T&& object)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (4, 2)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (4, 1)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY_TENS (4)
-  else if DPL_HOMOGENEOUS_APPLY_ENTRY (3, 9)
+  else { return std::array<bool,0>(); }
+  // clang-format on
+}
+
+template <bool B = false, typename L, class T, int D = nested_brace_constructible_size<B, T>() / 10>
+  requires(D == 3)
+auto homogeneous_apply_refs(L l, T&& object)
+{
+  constexpr int numElements = nested_brace_constructible_size<B, T>();
+  // clang-format off
+  if DPL_HOMOGENEOUS_APPLY_ENTRY (3, 9)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (3, 8)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (3, 7)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (3, 6)
@@ -241,7 +307,17 @@ auto homogeneous_apply_refs(L l, T&& object)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (3, 2)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (3, 1)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY_TENS (3)
-  else if DPL_HOMOGENEOUS_APPLY_ENTRY (2, 9)
+  else { return std::array<bool,0>(); }
+  // clang-format on
+}
+
+template <bool B = false, typename L, class T, int D = nested_brace_constructible_size<B, T>() / 10>
+  requires(D == 2)
+auto homogeneous_apply_refs(L l, T&& object)
+{
+  constexpr int numElements = nested_brace_constructible_size<B, T>();
+  // clang-format off
+  if DPL_HOMOGENEOUS_APPLY_ENTRY (2, 9)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (2, 8)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (2, 7)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (2, 6)
@@ -251,7 +327,17 @@ auto homogeneous_apply_refs(L l, T&& object)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (2, 2)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (2, 1)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY_TENS (2)
-  else if DPL_HOMOGENEOUS_APPLY_ENTRY (1, 9)
+  else { return std::array<bool,0>(); }
+  // clang-format on
+}
+
+template <bool B = false, typename L, class T, int D = nested_brace_constructible_size<B, T>() / 10>
+  requires(D == 1)
+auto homogeneous_apply_refs(L l, T&& object)
+{
+  constexpr int numElements = nested_brace_constructible_size<B, T>();
+  // clang-format off
+  if DPL_HOMOGENEOUS_APPLY_ENTRY (1, 9)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (1, 8)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (1, 7)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (1, 6)
@@ -261,7 +347,17 @@ auto homogeneous_apply_refs(L l, T&& object)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (1, 2)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY (1, 1)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY_TENS (1)
-  else if DPL_HOMOGENEOUS_APPLY_ENTRY_LOW (9)
+  else { return std::array<bool,0>(); }
+  // clang-format on
+}
+
+template <bool B = false, typename L, class T, int D = nested_brace_constructible_size<B, T>() / 10>
+  requires(D == 0)
+auto homogeneous_apply_refs(L l, T&& object)
+{
+  constexpr int numElements = nested_brace_constructible_size<B, T>();
+  // clang-format off
+  if DPL_HOMOGENEOUS_APPLY_ENTRY_LOW (9)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY_LOW (8)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY_LOW (7)
   else if DPL_HOMOGENEOUS_APPLY_ENTRY_LOW (6)

From b5b3522d6e7075664414e8e68ce6ead2264e167c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 17 Oct 2024 22:38:07 +0200
Subject: [PATCH 671/726] DPL analysis: hide internal symbols

---
 .../Core/include/Framework/AnalysisTask.h     | 47 ++++++++-----------
 1 file changed, 20 insertions(+), 27 deletions(-)

diff --git a/Framework/Core/include/Framework/AnalysisTask.h b/Framework/Core/include/Framework/AnalysisTask.h
index 9d623adcf2bd3..ea57b643fadb2 100644
--- a/Framework/Core/include/Framework/AnalysisTask.h
+++ b/Framework/Core/include/Framework/AnalysisTask.h
@@ -60,6 +60,7 @@ static constexpr bool is_enumeration_v<Enumeration<BEGIN, END, STEP>> = true;
 
 // Helper struct which builds a DataProcessorSpec from
 // the contents of an AnalysisTask...
+namespace {
 struct AnalysisDataProcessorBuilder {
   template <typename T>
   static ConfigParamSpec getSpec()
@@ -374,6 +375,7 @@ struct AnalysisDataProcessorBuilder {
     std::invoke(processingFunction, task, g, std::get<A>(at)...);
   }
 };
+}
 
 struct SetDefaultProcesses {
   std::vector<std::pair<std::string, bool>> map;
@@ -385,8 +387,9 @@ struct TaskName {
   std::string value;
 };
 
+namespace {
 template <typename T, typename... A>
-auto getTaskNameSetProcesses(TaskName first, SetDefaultProcesses second, A... args)
+auto getTaskNameSetProcesses(std::string& outputName, TaskName first, SetDefaultProcesses second, A... args)
 {
   auto task = std::make_shared<T>(std::forward<A>(args)...);
   for (auto& setting : second.map) {
@@ -396,11 +399,12 @@ auto getTaskNameSetProcesses(TaskName first, SetDefaultProcesses second, A... ar
       },
       *task.get());
   }
-  return std::make_tuple(first.value, task);
+  outputName = first.value;
+  return task;
 }
 
 template <typename T, typename... A>
-auto getTaskNameSetProcesses(SetDefaultProcesses first, TaskName second, A... args)
+auto getTaskNameSetProcesses(std::string& outputName, SetDefaultProcesses first, TaskName second, A... args)
 {
   auto task = std::make_shared<T>(std::forward<A>(args)...);
   for (auto& setting : first.map) {
@@ -410,11 +414,12 @@ auto getTaskNameSetProcesses(SetDefaultProcesses first, TaskName second, A... ar
       },
       *task.get());
   }
-  return std::make_tuple(second.value, task);
+  outputName = second.value;
+  return task;
 }
 
 template <typename T, typename... A>
-auto getTaskNameSetProcesses(SetDefaultProcesses first, A... args)
+auto getTaskNameSetProcesses(std::string& outputName, SetDefaultProcesses first, A... args)
 {
   auto task = std::make_shared<T>(std::forward<A>(args)...);
   for (auto& setting : first.map) {
@@ -425,40 +430,27 @@ auto getTaskNameSetProcesses(SetDefaultProcesses first, A... args)
       *task.get());
   }
   auto type_name_str = type_name<T>();
-  std::string name = type_to_task_name(type_name_str);
-  return std::make_tuple(name, task);
+  outputName = type_to_task_name(type_name_str);
+  return task;
 }
 
 template <typename T, typename... A>
-auto getTaskNameSetProcesses(TaskName first, A... args)
+auto getTaskNameSetProcesses(std::string& outputName, TaskName first, A... args)
 {
   auto task = std::make_shared<T>(std::forward<A>(args)...);
-  return std::make_tuple(first.value, task);
+  outputName = first.value;
+  return task;
 }
 
 template <typename T, typename... A>
-auto getTaskNameSetProcesses(A... args)
+auto getTaskNameSetProcesses(std::string& outputName, A... args)
 {
   auto task = std::make_shared<T>(std::forward<A>(args)...);
   auto type_name_str = type_name<T>();
-  std::string name = type_to_task_name(type_name_str);
-  return std::make_tuple(name, task);
-}
-
-template <typename T, typename... A>
-auto getTaskName(TaskName first, A... args)
-{
-  auto task = std::make_shared<T>(std::forward<A>(args)...);
-  return std::make_tuple(first.value, task);
+  outputName = type_to_task_name(type_name_str);
+  return task;
 }
 
-template <typename T, typename... A>
-auto getTaskName(A... args)
-{
-  auto task = std::make_shared<T>(std::forward<A>(args)...);
-  auto type_name_str = type_name<T>();
-  std::string name = type_to_task_name(type_name_str);
-  return std::make_tuple(name, task);
 }
 
 /// Adaptor to make an AlgorithmSpec from a o2::framework::Task
@@ -468,7 +460,8 @@ DataProcessorSpec adaptAnalysisTask(ConfigContext const& ctx, Args&&... args)
 {
   TH1::AddDirectory(false);
 
-  auto [name_str, task] = getTaskNameSetProcesses<T>(args...);
+  std::string name_str;
+  auto task = getTaskNameSetProcesses<T>(name_str, args...);
 
   auto suffix = ctx.options().get<std::string>("workflow-suffix");
   if (!suffix.empty()) {

From b47179b0152febed8c72936b455e7eedfc7bc5af Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 18 Oct 2024 08:48:18 +0200
Subject: [PATCH 672/726] DPL Analysis: cleanup unused code

Are we really reading the whole table every time?
---
 .../AnalysisSupport/src/DataInputDirector.cxx | 25 +++----------------
 1 file changed, 3 insertions(+), 22 deletions(-)

diff --git a/Framework/AnalysisSupport/src/DataInputDirector.cxx b/Framework/AnalysisSupport/src/DataInputDirector.cxx
index 6e072d47e523f..172ecd66c0e64 100644
--- a/Framework/AnalysisSupport/src/DataInputDirector.cxx
+++ b/Framework/AnalysisSupport/src/DataInputDirector.cxx
@@ -35,15 +35,6 @@
 #include <utility>
 #endif
 
-std::vector<std::string> getColumnNames(o2::header::DataHeader dh)
-{
-  auto description = std::string(dh.dataDescription.str);
-  auto origin = std::string(dh.dataOrigin.str);
-
-  // default: column names = {}
-  return {};
-}
-
 namespace o2::framework
 {
 using namespace rapidjson;
@@ -395,20 +386,10 @@ bool DataInputDescriptor::readTree(DataAllocator& outputs, header::DataHeader dh
 
   // add branches to read
   // fill the table
-  auto colnames = getColumnNames(dh);
   t2t->setLabel(tree->GetName());
-  if (colnames.size() == 0) {
-    totalSizeCompressed += tree->GetZipBytes();
-    totalSizeUncompressed += tree->GetTotBytes();
-    t2t->addAllColumns(tree);
-  } else {
-    for (auto& colname : colnames) {
-      TBranch* branch = tree->GetBranch(colname.c_str());
-      totalSizeCompressed += branch->GetZipBytes("*");
-      totalSizeUncompressed += branch->GetTotBytes("*");
-    }
-    t2t->addAllColumns(tree, std::move(colnames));
-  }
+  totalSizeCompressed += tree->GetZipBytes();
+  totalSizeUncompressed += tree->GetTotBytes();
+  t2t->addAllColumns(tree);
   t2t->fill(tree);
   delete tree;
 

From 113c9472ecbf2917f256a349d8056d102ca15ec4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 18 Oct 2024 09:56:01 +0200
Subject: [PATCH 673/726] DPL: move byteswapping helpers to Endian.h

---
 Framework/Core/src/TableTreeHelpers.cxx       | 48 ------------------
 .../Foundation/include/Framework/Endian.h     | 49 +++++++++++++++++++
 2 files changed, 49 insertions(+), 48 deletions(-)

diff --git a/Framework/Core/src/TableTreeHelpers.cxx b/Framework/Core/src/TableTreeHelpers.cxx
index 14e4a43ea6cba..ebddcf300461a 100644
--- a/Framework/Core/src/TableTreeHelpers.cxx
+++ b/Framework/Core/src/TableTreeHelpers.cxx
@@ -128,54 +128,6 @@ BranchToColumn::BranchToColumn(TBranch* branch, bool VLA, std::string name, EDat
   }
 }
 
-template <typename T>
-inline T doSwap(T)
-{
-  static_assert(always_static_assert_v<T>, "Unsupported type");
-}
-
-template <>
-inline uint16_t doSwap(uint16_t x)
-{
-  return swap16_(x);
-}
-
-template <>
-inline uint32_t doSwap(uint32_t x)
-{
-  return swap32_(x);
-}
-
-template <>
-inline uint64_t doSwap(uint64_t x)
-{
-  return swap64_(x);
-}
-
-template <typename T>
-void doSwapCopy_(void* dest, void* source, int size) noexcept
-{
-  auto tdest = static_cast<T*>(dest);
-  auto tsrc = static_cast<T*>(source);
-  for (auto i = 0; i < size; ++i) {
-    tdest[i] = doSwap<T>(tsrc[i]);
-  }
-}
-
-void swapCopy(unsigned char* dest, char* source, int size, int typeSize) noexcept
-{
-  switch (typeSize) {
-    case 1:
-      return (void)std::memcpy(dest, source, size);
-    case 2:
-      return doSwapCopy_<uint16_t>(dest, source, size);
-    case 4:
-      return doSwapCopy_<uint32_t>(dest, source, size);
-    case 8:
-      return doSwapCopy_<uint64_t>(dest, source, size);
-  }
-}
-
 std::pair<std::shared_ptr<arrow::ChunkedArray>, std::shared_ptr<arrow::Field>> BranchToColumn::read(TBuffer* buffer)
 {
   auto totalEntries = mBranch->GetEntries();
diff --git a/Framework/Foundation/include/Framework/Endian.h b/Framework/Foundation/include/Framework/Endian.h
index 9749eaf084c6f..a8fee915ee340 100644
--- a/Framework/Foundation/include/Framework/Endian.h
+++ b/Framework/Foundation/include/Framework/Endian.h
@@ -12,6 +12,9 @@
 #ifndef O2_FRAMEWORK_ENDIAN_H_
 #define O2_FRAMEWORK_ENDIAN_H_
 
+#include <cstdint>
+#include <concepts>
+#include <cstring>
 // Lookup file for __BYTE_ORDER
 #ifdef __APPLE__
 #include <machine/endian.h>
@@ -29,4 +32,50 @@
 #define O2_HOST_BYTE_ORDER __BYTE_ORDER
 #define O2_BIG_ENDIAN __BIG_ENDIAN
 #define O2_LITTLE_ENDIAN __LITTLE_ENDIAN
+
+
+template <typename T>
+  requires std::same_as<T, uint16_t>
+inline uint16_t doSwap(uint16_t x)
+{
+  return swap16_(x);
+}
+
+template <typename T>
+  requires std::same_as<T, uint32_t>
+inline uint32_t doSwap(uint32_t x)
+{
+  return swap32_(x);
+}
+
+template <typename T>
+  requires std::same_as<T, uint64_t>
+inline uint64_t doSwap(uint64_t x)
+{
+  return swap64_(x);
+}
+
+template <typename T>
+inline void doSwapCopy_(void* dest, void* source, int size) noexcept
+{
+  auto tdest = static_cast<T*>(dest);
+  auto tsrc = static_cast<T*>(source);
+  for (auto i = 0; i < size; ++i) {
+    tdest[i] = doSwap<T>(tsrc[i]);
+  }
+}
+
+inline void swapCopy(unsigned char* dest, char* source, int size, int typeSize) noexcept
+{
+  switch (typeSize) {
+    case 1:
+      return (void)std::memcpy(dest, source, size);
+    case 2:
+      return doSwapCopy_<uint16_t>(dest, source, size);
+    case 4:
+      return doSwapCopy_<uint32_t>(dest, source, size);
+    case 8:
+      return doSwapCopy_<uint64_t>(dest, source, size);
+  }
+}
 #endif // O2_FRAMEWORK_ENDIAN_H_

From 41c21d70d08284492b7ef87a469c5fe2fd0bfe35 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 18 Oct 2024 11:12:31 +0200
Subject: [PATCH 674/726] DPL: finally use concepts to separate the make method

In the past, substituting SFINAE with constexpr made sense because
understanding what was going on in case of failure was extremely complicated.

Now that we have constrains and cocepts, it makes sense to split again the
method since the diagnostics are good and the compiler is limited to optimise
the code in question, without actually having to instanciate all the templates.
---
 .../Core/include/Framework/DataAllocator.h    | 188 ++++++++++--------
 1 file changed, 103 insertions(+), 85 deletions(-)

diff --git a/Framework/Core/include/Framework/DataAllocator.h b/Framework/Core/include/Framework/DataAllocator.h
index d746031ad58c5..f321fa7660dc8 100644
--- a/Framework/Core/include/Framework/DataAllocator.h
+++ b/Framework/Core/include/Framework/DataAllocator.h
@@ -57,14 +57,6 @@ namespace o2::framework
 {
 struct ServiceRegistry;
 
-#define ERROR_STRING                                    \
-  "data type T not supported by API, "                  \
-  "\n specializations available for"                    \
-  "\n - trivially copyable, non-polymorphic structures" \
-  "\n - arrays of those"                                \
-  "\n - TObject with additional constructor arguments"  \
-  "\n - std containers of those"
-
 /// Helper to allow framework managed objecs to have a callback
 /// when they go out of scope. For example, this could
 /// be used to serialize a message into a buffer before the
@@ -130,6 +122,10 @@ struct LifetimeHolder {
   }
 };
 
+template <typename T>
+concept VectorOfMessageableTypes = is_specialization_v<T, std::vector> &&
+                                   is_messageable<typename T::value_type>::value;
+
 /// This allocator is responsible to make sure that the messages created match
 /// the provided spec and that depending on how many pipelined reader we
 /// have, messages get created on the channel for the reader of the current
@@ -163,93 +159,115 @@ class DataAllocator
   // and with subspecification 0xdeadbeef.
   void cookDeadBeef(const Output& spec);
 
-  /// Generic helper to create an object which is owned by the framework and
-  /// returned as a reference to the own object.
-  /// Note: decltype(auto) will deduce the return type from the expression and it
-  /// will be lvalue reference for the framework-owned objects. Instances of local
-  /// variables like shared_ptr will be returned by value/move/return value optimization.
-  /// Objects created this way will be sent to the channel specified by @spec
   template <typename T, typename... Args>
+    requires(is_specialization_v<T, UninitializedVector>)
   decltype(auto) make(const Output& spec, Args... args)
   {
     auto& timingInfo = mRegistry.get<TimingInfo>();
     auto& context = mRegistry.get<MessageContext>();
 
-    if constexpr (is_specialization_v<T, UninitializedVector>) {
-      auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
-      // plain buffer as polymorphic spectator std::vector, which does not run constructors / destructors
-      using ValueType = typename T::value_type;
-
-      // Note: initial payload size is 0 and will be set by the context before sending
-      fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, 0);
-      return context.add<MessageContext::VectorObject<ValueType, MessageContext::ContainerRefObject<std::vector<ValueType, o2::pmr::NoConstructAllocator<ValueType>>>>>(
-                      std::move(headerMessage), routeIndex, 0, std::forward<Args>(args)...)
-        .get();
-    } else if constexpr (is_specialization_v<T, std::vector> && has_messageable_value_type<T>::value) {
-      auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
-      // this catches all std::vector objects with messageable value type before checking if is also
-      // has a root dictionary, so non-serialized transmission is preferred
-      using ValueType = typename T::value_type;
-
-      // Note: initial payload size is 0 and will be set by the context before sending
-      fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, 0);
-      return context.add<MessageContext::VectorObject<ValueType>>(std::move(headerMessage), routeIndex, 0, std::forward<Args>(args)...).get();
-    } else if constexpr (has_root_dictionary<T>::value == true && is_messageable<T>::value == false) {
-      auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
-      // Extended support for types implementing the Root ClassDef interface, both TObject
-      // derived types and others
-      if constexpr (enable_root_serialization<T>::value) {
-        fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodROOT, 0);
-
-        return context.add<typename enable_root_serialization<T>::object_type>(std::move(headerMessage), routeIndex, std::forward<Args>(args)...).get();
-      } else {
-        static_assert(enable_root_serialization<T>::value, "Please make sure you include RootMessageContext.h");
-      }
-      // Note: initial payload size is 0 and will be set by the context before sending
-    } else if constexpr (std::is_base_of_v<std::string, T>) {
-      auto* s = new std::string(args...);
-      adopt(spec, s);
-      return *s;
-    } else if constexpr (requires { static_cast<struct TableBuilder>(std::declval<std::decay_t<T>>()); }) {
-      auto tb = std::move(LifetimeHolder<TableBuilder>(new std::decay_t<T>(args...)));
-      adopt(spec, tb);
-      return tb;
-    } else if constexpr (requires { static_cast<struct TreeToTable>(std::declval<std::decay_t<T>>()); }) {
-      auto t2t = std::move(LifetimeHolder<TreeToTable>(new std::decay_t<T>(args...)));
-      adopt(spec, t2t);
-      return t2t;
-    } else if constexpr (sizeof...(Args) == 0) {
-      if constexpr (is_messageable<T>::value == true) {
-        return *reinterpret_cast<T*>(newChunk(spec, sizeof(T)).data());
-      } else {
-        static_assert(always_static_assert_v<T>, ERROR_STRING);
-      }
-    } else if constexpr (sizeof...(Args) == 1) {
-      using FirstArg = typename std::tuple_element<0, std::tuple<Args...>>::type;
-      if constexpr (std::is_integral_v<FirstArg>) {
-        if constexpr (is_messageable<T>::value == true) {
-          auto [nElements] = std::make_tuple(args...);
-          auto size = nElements * sizeof(T);
-          auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
-
-          fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, size);
-          return context.add<MessageContext::SpanObject<T>>(std::move(headerMessage), routeIndex, 0, nElements).get();
-        }
-      } else if constexpr (std::is_same_v<FirstArg, std::shared_ptr<arrow::Schema>>) {
-        if constexpr (std::is_base_of_v<arrow::ipc::RecordBatchWriter, T>) {
-          auto [schema] = std::make_tuple(args...);
-          std::shared_ptr<arrow::ipc::RecordBatchWriter> writer;
-          create(spec, &writer, schema);
-          return writer;
-        }
-      } else {
-        static_assert(always_static_assert_v<T>, ERROR_STRING);
-      }
+    auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
+    // plain buffer as polymorphic spectator std::vector, which does not run constructors / destructors
+    using ValueType = typename T::value_type;
+
+    // Note: initial payload size is 0 and will be set by the context before sending
+    fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, 0);
+    return context.add<MessageContext::VectorObject<ValueType, MessageContext::ContainerRefObject<std::vector<ValueType, o2::pmr::NoConstructAllocator<ValueType>>>>>(
+                    std::move(headerMessage), routeIndex, 0, std::forward<Args>(args)...)
+      .get();
+  }
+
+  template <typename T, typename... Args>
+    requires VectorOfMessageableTypes<T>
+  decltype(auto) make(const Output& spec, Args... args)
+  {
+    auto& timingInfo = mRegistry.get<TimingInfo>();
+    auto& context = mRegistry.get<MessageContext>();
+
+    auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
+    // this catches all std::vector objects with messageable value type before checking if is also
+    // has a root dictionary, so non-serialized transmission is preferred
+    using ValueType = typename T::value_type;
+
+    // Note: initial payload size is 0 and will be set by the context before sending
+    fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, 0);
+    return context.add<MessageContext::VectorObject<ValueType>>(std::move(headerMessage), routeIndex, 0, std::forward<Args>(args)...).get();
+  }
+
+  template <typename T, typename... Args>
+    requires(!VectorOfMessageableTypes<T> && has_root_dictionary<T>::value == true && is_messageable<T>::value == false)
+  decltype(auto) make(const Output& spec, Args... args)
+  {
+    auto& timingInfo = mRegistry.get<TimingInfo>();
+    auto& context = mRegistry.get<MessageContext>();
+
+    auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
+    // Extended support for types implementing the Root ClassDef interface, both TObject
+    // derived types and others
+    if constexpr (enable_root_serialization<T>::value) {
+      fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodROOT, 0);
+
+      return context.add<typename enable_root_serialization<T>::object_type>(std::move(headerMessage), routeIndex, std::forward<Args>(args)...).get();
     } else {
-      static_assert(always_static_assert_v<T>, ERROR_STRING);
+      static_assert(enable_root_serialization<T>::value, "Please make sure you include RootMessageContext.h");
     }
   }
 
+  template <typename T, typename... Args>
+    requires(std::is_base_of_v<std::string, T>)
+  decltype(auto) make(const Output& spec, Args... args)
+  {
+    auto* s = new std::string(args...);
+    adopt(spec, s);
+    return *s;
+  }
+
+  template <typename T, typename... Args>
+    requires(requires { static_cast<struct TableBuilder>(std::declval<std::decay_t<T>>()); })
+  decltype(auto) make(const Output& spec, Args... args)
+  {
+    auto tb = std::move(LifetimeHolder<TableBuilder>(new std::decay_t<T>(args...)));
+    adopt(spec, tb);
+    return tb;
+  }
+
+  template <typename T, typename... Args>
+    requires(requires { static_cast<struct TreeToTable>(std::declval<std::decay_t<T>>()); })
+  decltype(auto) make(const Output& spec, Args... args)
+  {
+    auto t2t = std::move(LifetimeHolder<TreeToTable>(new std::decay_t<T>(args...)));
+    adopt(spec, t2t);
+    return t2t;
+  }
+
+  template <typename T, typename... Args>
+    requires(sizeof...(Args) == 0 && is_messageable<T>::value == true)
+  decltype(auto) make(const Output& spec, Args... args)
+  {
+    return *reinterpret_cast<T*>(newChunk(spec, sizeof(T)).data());
+  }
+
+  template <typename T, typename Arg>
+    requires(std::is_integral_v<Arg> && is_messageable<T>::value)
+  decltype(auto) make(const Output& spec, Arg args)
+  {
+    auto& timingInfo = mRegistry.get<TimingInfo>();
+    auto& context = mRegistry.get<MessageContext>();
+    auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
+
+    fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, sizeof(T));
+    return context.add<MessageContext::SpanObject<T>>(std::move(headerMessage), routeIndex, 0, 1).get();
+  }
+
+  template <typename T, typename Arg>
+    requires(std::is_same_v<Arg, std::shared_ptr<arrow::Schema>>)
+  decltype(auto) make(const Output& spec, Arg schema)
+  {
+    std::shared_ptr<arrow::ipc::RecordBatchWriter> writer;
+    create(spec, &writer, schema);
+    return writer;
+  }
+
   /// Adopt a string in the framework and serialize / send
   /// it to the consumers of @a spec once done.
   void

From 96537e5b8b3eac1967d4f377b1c8903a3bf1edde Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 18 Oct 2024 12:33:49 +0200
Subject: [PATCH 675/726] DPL: finally use concepts to separate the make method

In the past, substituting SFINAE with constexpr made sense because
understanding what was going on in case of failure was extremely complicated.

Now that we have constrains and cocepts, it makes sense to split again the
method since the diagnostics are good and the compiler is limited to optimise
the code in question, without actually having to instanciate all the templates.
---
 .../Core/include/Framework/DataAllocator.h    | 188 ++++++++++--------
 1 file changed, 103 insertions(+), 85 deletions(-)

diff --git a/Framework/Core/include/Framework/DataAllocator.h b/Framework/Core/include/Framework/DataAllocator.h
index d746031ad58c5..dbfed7179e898 100644
--- a/Framework/Core/include/Framework/DataAllocator.h
+++ b/Framework/Core/include/Framework/DataAllocator.h
@@ -57,14 +57,6 @@ namespace o2::framework
 {
 struct ServiceRegistry;
 
-#define ERROR_STRING                                    \
-  "data type T not supported by API, "                  \
-  "\n specializations available for"                    \
-  "\n - trivially copyable, non-polymorphic structures" \
-  "\n - arrays of those"                                \
-  "\n - TObject with additional constructor arguments"  \
-  "\n - std containers of those"
-
 /// Helper to allow framework managed objecs to have a callback
 /// when they go out of scope. For example, this could
 /// be used to serialize a message into a buffer before the
@@ -130,6 +122,10 @@ struct LifetimeHolder {
   }
 };
 
+template <typename T>
+concept VectorOfMessageableTypes = is_specialization_v<T, std::vector> &&
+                                   is_messageable<typename T::value_type>::value;
+
 /// This allocator is responsible to make sure that the messages created match
 /// the provided spec and that depending on how many pipelined reader we
 /// have, messages get created on the channel for the reader of the current
@@ -163,93 +159,115 @@ class DataAllocator
   // and with subspecification 0xdeadbeef.
   void cookDeadBeef(const Output& spec);
 
-  /// Generic helper to create an object which is owned by the framework and
-  /// returned as a reference to the own object.
-  /// Note: decltype(auto) will deduce the return type from the expression and it
-  /// will be lvalue reference for the framework-owned objects. Instances of local
-  /// variables like shared_ptr will be returned by value/move/return value optimization.
-  /// Objects created this way will be sent to the channel specified by @spec
   template <typename T, typename... Args>
+    requires(is_specialization_v<T, UninitializedVector>)
   decltype(auto) make(const Output& spec, Args... args)
   {
     auto& timingInfo = mRegistry.get<TimingInfo>();
     auto& context = mRegistry.get<MessageContext>();
 
-    if constexpr (is_specialization_v<T, UninitializedVector>) {
-      auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
-      // plain buffer as polymorphic spectator std::vector, which does not run constructors / destructors
-      using ValueType = typename T::value_type;
-
-      // Note: initial payload size is 0 and will be set by the context before sending
-      fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, 0);
-      return context.add<MessageContext::VectorObject<ValueType, MessageContext::ContainerRefObject<std::vector<ValueType, o2::pmr::NoConstructAllocator<ValueType>>>>>(
-                      std::move(headerMessage), routeIndex, 0, std::forward<Args>(args)...)
-        .get();
-    } else if constexpr (is_specialization_v<T, std::vector> && has_messageable_value_type<T>::value) {
-      auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
-      // this catches all std::vector objects with messageable value type before checking if is also
-      // has a root dictionary, so non-serialized transmission is preferred
-      using ValueType = typename T::value_type;
-
-      // Note: initial payload size is 0 and will be set by the context before sending
-      fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, 0);
-      return context.add<MessageContext::VectorObject<ValueType>>(std::move(headerMessage), routeIndex, 0, std::forward<Args>(args)...).get();
-    } else if constexpr (has_root_dictionary<T>::value == true && is_messageable<T>::value == false) {
-      auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
-      // Extended support for types implementing the Root ClassDef interface, both TObject
-      // derived types and others
-      if constexpr (enable_root_serialization<T>::value) {
-        fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodROOT, 0);
-
-        return context.add<typename enable_root_serialization<T>::object_type>(std::move(headerMessage), routeIndex, std::forward<Args>(args)...).get();
-      } else {
-        static_assert(enable_root_serialization<T>::value, "Please make sure you include RootMessageContext.h");
-      }
-      // Note: initial payload size is 0 and will be set by the context before sending
-    } else if constexpr (std::is_base_of_v<std::string, T>) {
-      auto* s = new std::string(args...);
-      adopt(spec, s);
-      return *s;
-    } else if constexpr (requires { static_cast<struct TableBuilder>(std::declval<std::decay_t<T>>()); }) {
-      auto tb = std::move(LifetimeHolder<TableBuilder>(new std::decay_t<T>(args...)));
-      adopt(spec, tb);
-      return tb;
-    } else if constexpr (requires { static_cast<struct TreeToTable>(std::declval<std::decay_t<T>>()); }) {
-      auto t2t = std::move(LifetimeHolder<TreeToTable>(new std::decay_t<T>(args...)));
-      adopt(spec, t2t);
-      return t2t;
-    } else if constexpr (sizeof...(Args) == 0) {
-      if constexpr (is_messageable<T>::value == true) {
-        return *reinterpret_cast<T*>(newChunk(spec, sizeof(T)).data());
-      } else {
-        static_assert(always_static_assert_v<T>, ERROR_STRING);
-      }
-    } else if constexpr (sizeof...(Args) == 1) {
-      using FirstArg = typename std::tuple_element<0, std::tuple<Args...>>::type;
-      if constexpr (std::is_integral_v<FirstArg>) {
-        if constexpr (is_messageable<T>::value == true) {
-          auto [nElements] = std::make_tuple(args...);
-          auto size = nElements * sizeof(T);
-          auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
-
-          fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, size);
-          return context.add<MessageContext::SpanObject<T>>(std::move(headerMessage), routeIndex, 0, nElements).get();
-        }
-      } else if constexpr (std::is_same_v<FirstArg, std::shared_ptr<arrow::Schema>>) {
-        if constexpr (std::is_base_of_v<arrow::ipc::RecordBatchWriter, T>) {
-          auto [schema] = std::make_tuple(args...);
-          std::shared_ptr<arrow::ipc::RecordBatchWriter> writer;
-          create(spec, &writer, schema);
-          return writer;
-        }
-      } else {
-        static_assert(always_static_assert_v<T>, ERROR_STRING);
-      }
+    auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
+    // plain buffer as polymorphic spectator std::vector, which does not run constructors / destructors
+    using ValueType = typename T::value_type;
+
+    // Note: initial payload size is 0 and will be set by the context before sending
+    fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, 0);
+    return context.add<MessageContext::VectorObject<ValueType, MessageContext::ContainerRefObject<std::vector<ValueType, o2::pmr::NoConstructAllocator<ValueType>>>>>(
+                    std::move(headerMessage), routeIndex, 0, std::forward<Args>(args)...)
+      .get();
+  }
+
+  template <typename T, typename... Args>
+    requires VectorOfMessageableTypes<T>
+  decltype(auto) make(const Output& spec, Args... args)
+  {
+    auto& timingInfo = mRegistry.get<TimingInfo>();
+    auto& context = mRegistry.get<MessageContext>();
+
+    auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
+    // this catches all std::vector objects with messageable value type before checking if is also
+    // has a root dictionary, so non-serialized transmission is preferred
+    using ValueType = typename T::value_type;
+
+    // Note: initial payload size is 0 and will be set by the context before sending
+    fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, 0);
+    return context.add<MessageContext::VectorObject<ValueType>>(std::move(headerMessage), routeIndex, 0, std::forward<Args>(args)...).get();
+  }
+
+  template <typename T, typename... Args>
+    requires(!VectorOfMessageableTypes<T> && has_root_dictionary<T>::value == true && is_messageable<T>::value == false)
+  decltype(auto) make(const Output& spec, Args... args)
+  {
+    auto& timingInfo = mRegistry.get<TimingInfo>();
+    auto& context = mRegistry.get<MessageContext>();
+
+    auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
+    // Extended support for types implementing the Root ClassDef interface, both TObject
+    // derived types and others
+    if constexpr (enable_root_serialization<T>::value) {
+      fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodROOT, 0);
+
+      return context.add<typename enable_root_serialization<T>::object_type>(std::move(headerMessage), routeIndex, std::forward<Args>(args)...).get();
     } else {
-      static_assert(always_static_assert_v<T>, ERROR_STRING);
+      static_assert(enable_root_serialization<T>::value, "Please make sure you include RootMessageContext.h");
     }
   }
 
+  template <typename T, typename... Args>
+    requires(std::is_base_of_v<std::string, T>)
+  decltype(auto) make(const Output& spec, Args... args)
+  {
+    auto* s = new std::string(args...);
+    adopt(spec, s);
+    return *s;
+  }
+
+  template <typename T, typename... Args>
+    requires(requires { static_cast<struct TableBuilder>(std::declval<std::decay_t<T>>()); })
+  decltype(auto) make(const Output& spec, Args... args)
+  {
+    auto tb = std::move(LifetimeHolder<TableBuilder>(new std::decay_t<T>(args...)));
+    adopt(spec, tb);
+    return tb;
+  }
+
+  template <typename T, typename... Args>
+    requires(requires { static_cast<struct TreeToTable>(std::declval<std::decay_t<T>>()); })
+  decltype(auto) make(const Output& spec, Args... args)
+  {
+    auto t2t = std::move(LifetimeHolder<TreeToTable>(new std::decay_t<T>(args...)));
+    adopt(spec, t2t);
+    return t2t;
+  }
+
+  template <typename T>
+    requires(is_messageable<T>::value == true)
+  decltype(auto) make(const Output& spec)
+  {
+    return *reinterpret_cast<T*>(newChunk(spec, sizeof(T)).data());
+  }
+
+  template <typename T>
+    requires(is_messageable<T>::value)
+  decltype(auto) make(const Output& spec, std::integral auto nElements)
+  {
+    auto& timingInfo = mRegistry.get<TimingInfo>();
+    auto& context = mRegistry.get<MessageContext>();
+    auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
+
+    fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, nElements * sizeof(T));
+    return context.add<MessageContext::SpanObject<T>>(std::move(headerMessage), routeIndex, 0, nElements).get();
+  }
+
+  template <typename T, typename Arg>
+    requires(std::is_same_v<Arg, std::shared_ptr<arrow::Schema>>)
+  decltype(auto) make(const Output& spec, Arg schema)
+  {
+    std::shared_ptr<arrow::ipc::RecordBatchWriter> writer;
+    create(spec, &writer, schema);
+    return writer;
+  }
+
   /// Adopt a string in the framework and serialize / send
   /// it to the consumers of @a spec once done.
   void

From 431dc1a2e2dd4a1f475595e300561b79db952b45 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 19 Oct 2024 20:37:35 +0200
Subject: [PATCH 676/726] DPL: use constraint rather than static_assert

---
 Framework/Core/include/Framework/DataAllocator.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/include/Framework/DataAllocator.h b/Framework/Core/include/Framework/DataAllocator.h
index d746031ad58c5..0f1030c44ca86 100644
--- a/Framework/Core/include/Framework/DataAllocator.h
+++ b/Framework/Core/include/Framework/DataAllocator.h
@@ -144,8 +144,8 @@ class DataAllocator
   using DataDescription = o2::header::DataDescription;
   using SubSpecificationType = o2::header::DataHeader::SubSpecificationType;
   template <typename T>
+    requires std::is_fundamental_v<T>
   struct UninitializedVector {
-    static_assert(std::is_fundamental<T>::value, "UninitializedVector only allowed for fundamental types");
     using value_type = T;
   };
 

From 4a90351b7276743ced108f4086afd7d1566ab177 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 28 Oct 2024 11:35:06 +0100
Subject: [PATCH 677/726] DPL: finally use concepts to separate the make method

In the past, substituting SFINAE with constexpr made sense because
understanding what was going on in case of failure was extremely complicated.

Now that we have constrains and cocepts, it makes sense to split again the
method since the diagnostics are good and the compiler is limited to optimise
the code in question, without actually having to instanciate all the templates.
---
 .../Core/include/Framework/DataAllocator.h    | 189 ++++++++++--------
 1 file changed, 104 insertions(+), 85 deletions(-)

diff --git a/Framework/Core/include/Framework/DataAllocator.h b/Framework/Core/include/Framework/DataAllocator.h
index 0f1030c44ca86..9c563b06910b5 100644
--- a/Framework/Core/include/Framework/DataAllocator.h
+++ b/Framework/Core/include/Framework/DataAllocator.h
@@ -31,6 +31,7 @@
 #include <TClass.h>
 #include <gsl/span>
 
+#include <memory>
 #include <vector>
 #include <map>
 #include <string>
@@ -57,14 +58,6 @@ namespace o2::framework
 {
 struct ServiceRegistry;
 
-#define ERROR_STRING                                    \
-  "data type T not supported by API, "                  \
-  "\n specializations available for"                    \
-  "\n - trivially copyable, non-polymorphic structures" \
-  "\n - arrays of those"                                \
-  "\n - TObject with additional constructor arguments"  \
-  "\n - std containers of those"
-
 /// Helper to allow framework managed objecs to have a callback
 /// when they go out of scope. For example, this could
 /// be used to serialize a message into a buffer before the
@@ -130,6 +123,10 @@ struct LifetimeHolder {
   }
 };
 
+template <typename T>
+concept VectorOfMessageableTypes = is_specialization_v<T, std::vector> &&
+                                   is_messageable<typename T::value_type>::value;
+
 /// This allocator is responsible to make sure that the messages created match
 /// the provided spec and that depending on how many pipelined reader we
 /// have, messages get created on the channel for the reader of the current
@@ -143,6 +140,7 @@ class DataAllocator
   using DataOrigin = o2::header::DataOrigin;
   using DataDescription = o2::header::DataDescription;
   using SubSpecificationType = o2::header::DataHeader::SubSpecificationType;
+
   template <typename T>
     requires std::is_fundamental_v<T>
   struct UninitializedVector {
@@ -163,93 +161,114 @@ class DataAllocator
   // and with subspecification 0xdeadbeef.
   void cookDeadBeef(const Output& spec);
 
-  /// Generic helper to create an object which is owned by the framework and
-  /// returned as a reference to the own object.
-  /// Note: decltype(auto) will deduce the return type from the expression and it
-  /// will be lvalue reference for the framework-owned objects. Instances of local
-  /// variables like shared_ptr will be returned by value/move/return value optimization.
-  /// Objects created this way will be sent to the channel specified by @spec
   template <typename T, typename... Args>
+    requires is_specialization_v<T, o2::framework::DataAllocator::UninitializedVector>
   decltype(auto) make(const Output& spec, Args... args)
   {
     auto& timingInfo = mRegistry.get<TimingInfo>();
     auto& context = mRegistry.get<MessageContext>();
 
-    if constexpr (is_specialization_v<T, UninitializedVector>) {
-      auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
-      // plain buffer as polymorphic spectator std::vector, which does not run constructors / destructors
-      using ValueType = typename T::value_type;
-
-      // Note: initial payload size is 0 and will be set by the context before sending
-      fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, 0);
-      return context.add<MessageContext::VectorObject<ValueType, MessageContext::ContainerRefObject<std::vector<ValueType, o2::pmr::NoConstructAllocator<ValueType>>>>>(
-                      std::move(headerMessage), routeIndex, 0, std::forward<Args>(args)...)
-        .get();
-    } else if constexpr (is_specialization_v<T, std::vector> && has_messageable_value_type<T>::value) {
-      auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
-      // this catches all std::vector objects with messageable value type before checking if is also
-      // has a root dictionary, so non-serialized transmission is preferred
-      using ValueType = typename T::value_type;
-
-      // Note: initial payload size is 0 and will be set by the context before sending
-      fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, 0);
-      return context.add<MessageContext::VectorObject<ValueType>>(std::move(headerMessage), routeIndex, 0, std::forward<Args>(args)...).get();
-    } else if constexpr (has_root_dictionary<T>::value == true && is_messageable<T>::value == false) {
-      auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
-      // Extended support for types implementing the Root ClassDef interface, both TObject
-      // derived types and others
-      if constexpr (enable_root_serialization<T>::value) {
-        fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodROOT, 0);
-
-        return context.add<typename enable_root_serialization<T>::object_type>(std::move(headerMessage), routeIndex, std::forward<Args>(args)...).get();
-      } else {
-        static_assert(enable_root_serialization<T>::value, "Please make sure you include RootMessageContext.h");
-      }
-      // Note: initial payload size is 0 and will be set by the context before sending
-    } else if constexpr (std::is_base_of_v<std::string, T>) {
-      auto* s = new std::string(args...);
-      adopt(spec, s);
-      return *s;
-    } else if constexpr (requires { static_cast<struct TableBuilder>(std::declval<std::decay_t<T>>()); }) {
-      auto tb = std::move(LifetimeHolder<TableBuilder>(new std::decay_t<T>(args...)));
-      adopt(spec, tb);
-      return tb;
-    } else if constexpr (requires { static_cast<struct TreeToTable>(std::declval<std::decay_t<T>>()); }) {
-      auto t2t = std::move(LifetimeHolder<TreeToTable>(new std::decay_t<T>(args...)));
-      adopt(spec, t2t);
-      return t2t;
-    } else if constexpr (sizeof...(Args) == 0) {
-      if constexpr (is_messageable<T>::value == true) {
-        return *reinterpret_cast<T*>(newChunk(spec, sizeof(T)).data());
-      } else {
-        static_assert(always_static_assert_v<T>, ERROR_STRING);
-      }
-    } else if constexpr (sizeof...(Args) == 1) {
-      using FirstArg = typename std::tuple_element<0, std::tuple<Args...>>::type;
-      if constexpr (std::is_integral_v<FirstArg>) {
-        if constexpr (is_messageable<T>::value == true) {
-          auto [nElements] = std::make_tuple(args...);
-          auto size = nElements * sizeof(T);
-          auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
-
-          fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, size);
-          return context.add<MessageContext::SpanObject<T>>(std::move(headerMessage), routeIndex, 0, nElements).get();
-        }
-      } else if constexpr (std::is_same_v<FirstArg, std::shared_ptr<arrow::Schema>>) {
-        if constexpr (std::is_base_of_v<arrow::ipc::RecordBatchWriter, T>) {
-          auto [schema] = std::make_tuple(args...);
-          std::shared_ptr<arrow::ipc::RecordBatchWriter> writer;
-          create(spec, &writer, schema);
-          return writer;
-        }
-      } else {
-        static_assert(always_static_assert_v<T>, ERROR_STRING);
-      }
+    auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
+    // plain buffer as polymorphic spectator std::vector, which does not run constructors / destructors
+    using ValueType = typename T::value_type;
+
+    // Note: initial payload size is 0 and will be set by the context before sending
+    fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, 0);
+    return context.add<MessageContext::VectorObject<ValueType, MessageContext::ContainerRefObject<std::vector<ValueType, o2::pmr::NoConstructAllocator<ValueType>>>>>(
+                    std::move(headerMessage), routeIndex, 0, std::forward<Args>(args)...)
+      .get();
+  }
+
+  template <typename T, typename... Args>
+    requires VectorOfMessageableTypes<T>
+  decltype(auto) make(const Output& spec, Args... args)
+  {
+    auto& timingInfo = mRegistry.get<TimingInfo>();
+    auto& context = mRegistry.get<MessageContext>();
+
+    auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
+    // this catches all std::vector objects with messageable value type before checking if is also
+    // has a root dictionary, so non-serialized transmission is preferred
+    using ValueType = typename T::value_type;
+
+    // Note: initial payload size is 0 and will be set by the context before sending
+    fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, 0);
+    return context.add<MessageContext::VectorObject<ValueType>>(std::move(headerMessage), routeIndex, 0, std::forward<Args>(args)...).get();
+  }
+
+  template <typename T, typename... Args>
+    requires(!VectorOfMessageableTypes<T> && has_root_dictionary<T>::value == true && is_messageable<T>::value == false)
+  decltype(auto) make(const Output& spec, Args... args)
+  {
+    auto& timingInfo = mRegistry.get<TimingInfo>();
+    auto& context = mRegistry.get<MessageContext>();
+
+    auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
+    // Extended support for types implementing the Root ClassDef interface, both TObject
+    // derived types and others
+    if constexpr (enable_root_serialization<T>::value) {
+      fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodROOT, 0);
+
+      return context.add<typename enable_root_serialization<T>::object_type>(std::move(headerMessage), routeIndex, std::forward<Args>(args)...).get();
     } else {
-      static_assert(always_static_assert_v<T>, ERROR_STRING);
+      static_assert(enable_root_serialization<T>::value, "Please make sure you include RootMessageContext.h");
     }
   }
 
+  template <typename T, typename... Args>
+    requires std::is_base_of_v<std::string, T>
+  decltype(auto) make(const Output& spec, Args... args)
+  {
+    auto* s = new std::string(args...);
+    adopt(spec, s);
+    return *s;
+  }
+
+  template <typename T, typename... Args>
+    requires(requires { static_cast<struct TableBuilder>(std::declval<std::decay_t<T>>()); })
+  decltype(auto) make(const Output& spec, Args... args)
+  {
+    auto tb = std::move(LifetimeHolder<TableBuilder>(new std::decay_t<T>(args...)));
+    adopt(spec, tb);
+    return tb;
+  }
+
+  template <typename T, typename... Args>
+    requires(requires { static_cast<struct TreeToTable>(std::declval<std::decay_t<T>>()); })
+  decltype(auto) make(const Output& spec, Args... args)
+  {
+    auto t2t = std::move(LifetimeHolder<TreeToTable>(new std::decay_t<T>(args...)));
+    adopt(spec, t2t);
+    return t2t;
+  }
+
+  template <typename T>
+    requires is_messageable<T>::value && (!is_specialization_v<T, UninitializedVector>)
+  decltype(auto) make(const Output& spec)
+  {
+    return *reinterpret_cast<T*>(newChunk(spec, sizeof(T)).data());
+  }
+
+  template <typename T>
+    requires is_messageable<T>::value && (!is_specialization_v<T, UninitializedVector>)
+  decltype(auto) make(const Output& spec, std::integral auto nElements)
+  {
+    auto& timingInfo = mRegistry.get<TimingInfo>();
+    auto& context = mRegistry.get<MessageContext>();
+    auto routeIndex = matchDataHeader(spec, timingInfo.timeslice);
+
+    fair::mq::MessagePtr headerMessage = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodNone, nElements * sizeof(T));
+    return context.add<MessageContext::SpanObject<T>>(std::move(headerMessage), routeIndex, 0, nElements).get();
+  }
+
+  template <typename T, typename Arg>
+  decltype(auto) make(const Output& spec, std::same_as<std::shared_ptr<arrow::Schema>> auto schema)
+  {
+    std::shared_ptr<arrow::ipc::RecordBatchWriter> writer;
+    create(spec, &writer, schema);
+    return writer;
+  }
+
   /// Adopt a string in the framework and serialize / send
   /// it to the consumers of @a spec once done.
   void

From 56e990d86bc64ba5794de88eaec2f2a988813f39 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 30 Oct 2024 00:23:02 +0100
Subject: [PATCH 678/726] DPL: use concepts rather than SFINAE

Notice that we do not impose the arguments, so that we still get a
compilation error if they are wrong.
---
 Framework/Core/include/Framework/Task.h | 67 +++----------------------
 1 file changed, 6 insertions(+), 61 deletions(-)

diff --git a/Framework/Core/include/Framework/Task.h b/Framework/Core/include/Framework/Task.h
index 3d80f1ae9f0f6..02272f463ed9f 100644
--- a/Framework/Core/include/Framework/Task.h
+++ b/Framework/Core/include/Framework/Task.h
@@ -14,66 +14,11 @@
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/CallbackService.h"
 #include "Framework/EndOfStreamContext.h"
-#include <utility>
 #include <memory>
 
 namespace o2::framework
 {
 
-/// Check if the class task has EndOfStream
-template <typename T>
-class has_endOfStream
-{
-  typedef char one;
-  struct two {
-    char x[2];
-  };
-
-  template <typename C>
-  static one test(decltype(&C::endOfStream));
-  template <typename C>
-  static two test(...);
-
- public:
-  enum { value = sizeof(test<T>(nullptr)) == sizeof(char) };
-};
-
-/// Check if the class task has EndOfStream
-template <typename T>
-class has_finaliseCCDB
-{
-  typedef char one;
-  struct two {
-    char x[2];
-  };
-
-  template <typename C>
-  static one test(decltype(&C::finaliseCCDB));
-  template <typename C>
-  static two test(...);
-
- public:
-  enum { value = sizeof(test<T>(nullptr)) == sizeof(char) };
-};
-
-/// Check if the class task has Stop
-template <typename T>
-class has_stop
-{
-  typedef char one;
-  struct two {
-    char x[2];
-  };
-
-  template <typename C>
-  static one test(decltype(&C::stop));
-  template <typename C>
-  static two test(...);
-
- public:
-  enum { value = sizeof(test<T>(nullptr)) == sizeof(char) };
-};
-
 /// A more familiar task API for the DPL.
 /// This allows you to define your own tasks as subclasses
 /// of o2::framework::Task and to pass them in the specification
@@ -115,19 +60,19 @@ AlgorithmSpec adaptFromTask(Args&&... args)
 {
   return AlgorithmSpec::InitCallback{[=](InitContext& ic) {
     auto task = std::make_shared<T>(args...);
-    if constexpr (has_endOfStream<T>::value) {
+    if constexpr (requires (T&) { &T::endOfStream; }) {
       auto& callbacks = ic.services().get<CallbackService>();
       callbacks.set<CallbackService::Id::EndOfStream>([task](EndOfStreamContext& eosContext) {
         task->endOfStream(eosContext);
       });
     }
-    if constexpr (has_finaliseCCDB<T>::value) {
+    if constexpr (requires (T&) { &T::finaliseCCDB; }) {
       auto& callbacks = ic.services().get<CallbackService>();
       callbacks.set<CallbackService::Id::CCDBDeserialised>([task](ConcreteDataMatcher& matcher, void* obj) {
         task->finaliseCCDB(matcher, obj);
       });
     }
-    if constexpr (has_stop<T>::value) {
+    if constexpr (requires (T&) { &T::stop; }) {
       auto& callbacks = ic.services().get<CallbackService>();
       callbacks.set<CallbackService::Id::Stop>([task]() {
         task->stop();
@@ -144,19 +89,19 @@ template <typename T>
 AlgorithmSpec adoptTask(std::shared_ptr<T> task)
 {
   return AlgorithmSpec::InitCallback{[task](InitContext& ic) {
-    if constexpr (has_endOfStream<T>::value) {
+    if constexpr (requires (T&) { &T::endOfStream; }) {
       auto& callbacks = ic.services().get<CallbackService>();
       callbacks.set<CallbackService::Id::EndOfStream>([task](EndOfStreamContext& eosContext) {
         task->endOfStream(eosContext);
       });
     }
-    if constexpr (has_finaliseCCDB<T>::value) {
+    if constexpr (requires (T&) { &T::finaliseCCDB; }) {
       auto& callbacks = ic.services().get<CallbackService>();
       callbacks.set<CallbackService::Id::CCDBDeserialised>([task](ConcreteDataMatcher& matcher, void* obj) {
         task->finaliseCCDB(matcher, obj);
       });
     }
-    if constexpr (has_stop<T>::value) {
+    if constexpr (requires (T&) { &T::stop; }) {
       auto& callbacks = ic.services().get<CallbackService>();
       callbacks.set<CallbackService::Id::Stop>([task]() {
         task->stop();

From b574ea1a9eb66f35a0a468a3caa2ce760155b50e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 30 Oct 2024 00:24:20 +0100
Subject: [PATCH 679/726] DPL: use concepts rather than SFINAE

Notice that we do not impose the arguments, so that we still get a
compilation error if they are wrong.
---
 Framework/Core/include/Framework/Task.h | 67 +++----------------------
 1 file changed, 6 insertions(+), 61 deletions(-)

diff --git a/Framework/Core/include/Framework/Task.h b/Framework/Core/include/Framework/Task.h
index 3d80f1ae9f0f6..edb343e898a33 100644
--- a/Framework/Core/include/Framework/Task.h
+++ b/Framework/Core/include/Framework/Task.h
@@ -14,66 +14,11 @@
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/CallbackService.h"
 #include "Framework/EndOfStreamContext.h"
-#include <utility>
 #include <memory>
 
 namespace o2::framework
 {
 
-/// Check if the class task has EndOfStream
-template <typename T>
-class has_endOfStream
-{
-  typedef char one;
-  struct two {
-    char x[2];
-  };
-
-  template <typename C>
-  static one test(decltype(&C::endOfStream));
-  template <typename C>
-  static two test(...);
-
- public:
-  enum { value = sizeof(test<T>(nullptr)) == sizeof(char) };
-};
-
-/// Check if the class task has EndOfStream
-template <typename T>
-class has_finaliseCCDB
-{
-  typedef char one;
-  struct two {
-    char x[2];
-  };
-
-  template <typename C>
-  static one test(decltype(&C::finaliseCCDB));
-  template <typename C>
-  static two test(...);
-
- public:
-  enum { value = sizeof(test<T>(nullptr)) == sizeof(char) };
-};
-
-/// Check if the class task has Stop
-template <typename T>
-class has_stop
-{
-  typedef char one;
-  struct two {
-    char x[2];
-  };
-
-  template <typename C>
-  static one test(decltype(&C::stop));
-  template <typename C>
-  static two test(...);
-
- public:
-  enum { value = sizeof(test<T>(nullptr)) == sizeof(char) };
-};
-
 /// A more familiar task API for the DPL.
 /// This allows you to define your own tasks as subclasses
 /// of o2::framework::Task and to pass them in the specification
@@ -115,19 +60,19 @@ AlgorithmSpec adaptFromTask(Args&&... args)
 {
   return AlgorithmSpec::InitCallback{[=](InitContext& ic) {
     auto task = std::make_shared<T>(args...);
-    if constexpr (has_endOfStream<T>::value) {
+    if constexpr (requires { &T::endOfStream; }) {
       auto& callbacks = ic.services().get<CallbackService>();
       callbacks.set<CallbackService::Id::EndOfStream>([task](EndOfStreamContext& eosContext) {
         task->endOfStream(eosContext);
       });
     }
-    if constexpr (has_finaliseCCDB<T>::value) {
+    if constexpr (requires { &T::finaliseCCDB; }) {
       auto& callbacks = ic.services().get<CallbackService>();
       callbacks.set<CallbackService::Id::CCDBDeserialised>([task](ConcreteDataMatcher& matcher, void* obj) {
         task->finaliseCCDB(matcher, obj);
       });
     }
-    if constexpr (has_stop<T>::value) {
+    if constexpr (requires { &T::stop; }) {
       auto& callbacks = ic.services().get<CallbackService>();
       callbacks.set<CallbackService::Id::Stop>([task]() {
         task->stop();
@@ -144,19 +89,19 @@ template <typename T>
 AlgorithmSpec adoptTask(std::shared_ptr<T> task)
 {
   return AlgorithmSpec::InitCallback{[task](InitContext& ic) {
-    if constexpr (has_endOfStream<T>::value) {
+    if constexpr (requires { &T::endOfStream; }) {
       auto& callbacks = ic.services().get<CallbackService>();
       callbacks.set<CallbackService::Id::EndOfStream>([task](EndOfStreamContext& eosContext) {
         task->endOfStream(eosContext);
       });
     }
-    if constexpr (has_finaliseCCDB<T>::value) {
+    if constexpr (requires { &T::finaliseCCDB; }) {
       auto& callbacks = ic.services().get<CallbackService>();
       callbacks.set<CallbackService::Id::CCDBDeserialised>([task](ConcreteDataMatcher& matcher, void* obj) {
         task->finaliseCCDB(matcher, obj);
       });
     }
-    if constexpr (has_stop<T>::value) {
+    if constexpr (requires { &T::stop; }) {
       auto& callbacks = ic.services().get<CallbackService>();
       callbacks.set<CallbackService::Id::Stop>([task]() {
         task->stop();

From b5b6da4959aaec95bd23d9da17372339756ba573 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 31 Oct 2024 09:49:06 +0100
Subject: [PATCH 680/726] DPL: hide helper in an anonymous namespace

---
 .../include/Framework/StructToTuple.h         | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/Framework/Foundation/include/Framework/StructToTuple.h b/Framework/Foundation/include/Framework/StructToTuple.h
index c600e33883a74..5748329f6a50d 100644
--- a/Framework/Foundation/include/Framework/StructToTuple.h
+++ b/Framework/Foundation/include/Framework/StructToTuple.h
@@ -14,23 +14,26 @@
 #include <Framework/Traits.h>
 #include <array>
 
-namespace o2::framework
+namespace
 {
-struct any_type {
-  template <class T>
-  constexpr operator T(); // non explicit
-};
-
 template <class T, typename... Args>
 decltype(void(T{std::declval<Args>()...}), std::true_type())
-  test(int);
+  brace_test(int);
 
 template <class T, typename... Args>
 std::false_type
-  test(...);
+  brace_test(...);
+} // namespace
+
+namespace o2::framework
+{
+struct any_type {
+  template <class T>
+  constexpr operator T(); // non explicit
+};
 
 template <class T, typename... Args>
-struct is_braces_constructible : decltype(test<T, Args...>(0)) {
+struct is_braces_constructible : decltype(brace_test<T, Args...>(0)) {
 };
 
 #define DPL_REPEAT_0(x)

From 215c14da9bb7261617a031ba33009192255a41b5 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 1 Nov 2024 17:27:37 +0100
Subject: [PATCH 681/726] DPL: example on how to use the Arrow Dataset API

---
 Framework/Core/CMakeLists.txt                 |   2 +
 .../include/Framework/RootArrowFilesystem.h   | 203 ++++++++
 Framework/Core/src/RootArrowFilesystem.cxx    | 454 ++++++++++++++++++
 Framework/Core/test/test_Root2ArrowTable.cxx  | 176 +++++++
 dependencies/O2Dependencies.cmake             |  27 ++
 5 files changed, 862 insertions(+)
 create mode 100644 Framework/Core/include/Framework/RootArrowFilesystem.h
 create mode 100644 Framework/Core/src/RootArrowFilesystem.cxx

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index c01ef534eb212..02367afdcc556 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -117,6 +117,7 @@ o2_add_library(Framework
                        src/ResourcesMonitoringHelper.cxx
                        src/ResourcePolicy.cxx
                        src/ResourcePolicyHelpers.cxx
+                       src/RootArrowFilesystem.cxx
                        src/SendingPolicy.cxx
                        src/ServiceRegistry.cxx
                        src/ServiceSpec.cxx
@@ -166,6 +167,7 @@ o2_add_library(Framework
                                      O2::X9
                                      RapidJSON::RapidJSON
                                      Arrow::arrow_shared
+                                     ArrowDataset::arrow_dataset_shared
                                      Microsoft.GSL::GSL
                                      O2::FrameworkLogger
                                      Gandiva::gandiva_shared
diff --git a/Framework/Core/include/Framework/RootArrowFilesystem.h b/Framework/Core/include/Framework/RootArrowFilesystem.h
new file mode 100644
index 0000000000000..e1787dae69f31
--- /dev/null
+++ b/Framework/Core/include/Framework/RootArrowFilesystem.h
@@ -0,0 +1,203 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef O2_FRAMEWORK_ROOT_ARROW_FILESYSTEM_H_
+#define O2_FRAMEWORK_ROOT_ARROW_FILESYSTEM_H_
+
+#include <arrow/dataset/type_fwd.h>
+#include <arrow/dataset/file_base.h>
+#include <arrow/filesystem/type_fwd.h>
+#include <arrow/type_fwd.h>
+
+class TTree;
+class TBufferFile;
+class TDirectoryFile;
+
+namespace o2::framework
+{
+
+class TTreeFileWriteOptions : public arrow::dataset::FileWriteOptions
+{
+ public:
+  TTreeFileWriteOptions(std::shared_ptr<arrow::dataset::FileFormat> format)
+    : FileWriteOptions(format)
+  {
+  }
+};
+
+// This is a virtual filesystem based on a ttree, where branches with the
+// same prefix get grouped into a fragment
+class TTreeFileSystem : public arrow::fs::FileSystem
+{
+ public:
+  arrow::Result<arrow::fs::FileInfo> GetFileInfo(const std::string& path) override;
+  arrow::Result<arrow::fs::FileInfoVector> GetFileInfo(const arrow::fs::FileSelector& select) override;
+
+  bool Equals(const FileSystem& other) const override
+  {
+    return this->type_name() == other.type_name();
+  }
+
+  arrow::Status CreateDir(const std::string& path, bool recursive) override;
+
+  arrow::Status DeleteDir(const std::string& path) override;
+
+  arrow::Status CopyFile(const std::string& src, const std::string& dest) override;
+
+  arrow::Status Move(const std::string& src, const std::string& dest) override;
+
+  arrow::Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override;
+
+  arrow::Status DeleteRootDirContents() override;
+
+  arrow::Status DeleteFile(const std::string& path) override;
+
+  arrow::Result<std::shared_ptr<arrow::io::InputStream>> OpenInputStream(const std::string& path) override;
+
+  arrow::Result<std::shared_ptr<arrow::io::RandomAccessFile>> OpenInputFile(const std::string& path) override;
+
+  arrow::Result<std::shared_ptr<arrow::io::OutputStream>> OpenOutputStream(
+    const std::string& path,
+    const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override;
+
+  arrow::Result<std::shared_ptr<arrow::io::OutputStream>> OpenAppendStream(
+    const std::string& path,
+    const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override;
+
+  virtual TTree* GetTree(arrow::dataset::FileSource) = 0;
+};
+
+class SingleTreeFileSystem : public TTreeFileSystem
+{
+ public:
+  SingleTreeFileSystem(TTree* tree)
+    : TTreeFileSystem(),
+      mTree(tree)
+  {
+  }
+
+  std::string type_name() const override
+  {
+    return "ttree";
+  }
+
+  TTree* GetTree(arrow::dataset::FileSource) override
+  {
+    // Simply return the only TTree we have
+    return mTree;
+  }
+
+ private:
+  TTree* mTree;
+};
+
+class TFileFileSystem : public TTreeFileSystem
+{
+ public:
+  TFileFileSystem(TDirectoryFile* f, size_t readahead);
+
+  std::string type_name() const override
+  {
+    return "TDirectoryFile";
+  }
+
+  TTree* GetTree(arrow::dataset::FileSource source) override;
+
+  // We can go back to the TFile in case this is needed.
+  TDirectoryFile* GetFile()
+  {
+    return mFile;
+  }
+
+ private:
+  TDirectoryFile* mFile;
+};
+
+class TBufferFileFS : public TTreeFileSystem
+{
+ public:
+  TBufferFileFS(TBufferFile* f);
+
+  std::string type_name() const override
+  {
+    return "tbufferfile";
+  }
+
+  TTree* GetTree(arrow::dataset::FileSource) override
+  {
+    // Simply return the only TTree we have
+    return mTree;
+  }
+
+ private:
+  TTree* mTree;
+};
+
+class TTreeFileFragment : public arrow::dataset::FileFragment
+{
+ public:
+  TTreeFileFragment(arrow::dataset::FileSource source,
+                    std::shared_ptr<arrow::dataset::FileFormat> format,
+                    arrow::compute::Expression partition_expression,
+                    std::shared_ptr<arrow::Schema> physical_schema)
+    : FileFragment(std::move(source), std::move(format), std::move(partition_expression), std::move(physical_schema))
+  {
+  }
+};
+
+class TTreeFileFormat : public arrow::dataset::FileFormat
+{
+  size_t& mTotCompressedSize;
+  size_t& mTotUncompressedSize;
+
+ public:
+  TTreeFileFormat(size_t& totalCompressedSize, size_t& totalUncompressedSize)
+    : FileFormat({}),
+      mTotCompressedSize(totalCompressedSize),
+      mTotUncompressedSize(totalUncompressedSize)
+  {
+  }
+
+  ~TTreeFileFormat() override = default;
+
+  std::string type_name() const override
+  {
+    return "ttree";
+  }
+
+  bool Equals(const FileFormat& other) const override
+  {
+    return other.type_name() == this->type_name();
+  }
+
+  arrow::Result<bool> IsSupported(const arrow::dataset::FileSource& source) const override
+  {
+    auto fs = std::dynamic_pointer_cast<TTreeFileSystem>(source.filesystem());
+    return fs != nullptr;
+  }
+
+  arrow::Result<std::shared_ptr<arrow::Schema>> Inspect(const arrow::dataset::FileSource& source) const override;
+  /// \brief Create a FileFragment for a FileSource.
+  arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> MakeFragment(
+    arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
+    std::shared_ptr<arrow::Schema> physical_schema) override;
+
+  arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options, arrow::fs::FileLocator destination_locator) const override;
+
+  std::shared_ptr<arrow::dataset::FileWriteOptions> DefaultWriteOptions() override;
+
+  arrow::Result<arrow::RecordBatchGenerator> ScanBatchesAsync(
+    const std::shared_ptr<arrow::dataset::ScanOptions>& options,
+    const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const override;
+};
+
+} // namespace o2::framework
+
+#endif // O2_FRAMEWORK_ROOT_ARROW_FILESYSTEM_H_
diff --git a/Framework/Core/src/RootArrowFilesystem.cxx b/Framework/Core/src/RootArrowFilesystem.cxx
new file mode 100644
index 0000000000000..a130f51f7e5a9
--- /dev/null
+++ b/Framework/Core/src/RootArrowFilesystem.cxx
@@ -0,0 +1,454 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/RootArrowFilesystem.h"
+#include "Framework/Endian.h"
+#include "Framework/RuntimeError.h"
+#include <arrow/array/builder_nested.h>
+#include <arrow/array/builder_primitive.h>
+#include <stdexcept>
+#include <TFile.h>
+#include <TLeaf.h>
+#include <TBufferFile.h>
+#include <TTree.h>
+#include <TDirectoryFile.h>
+
+namespace
+{
+struct BranchInfo {
+  std::string name;
+  TBranch* ptr;
+  bool mVLA;
+};
+} // namespace
+
+auto arrowTypeFromROOT(EDataType type, int size)
+{
+  auto typeGenerator = [](std::shared_ptr<arrow::DataType> const& type, int size) -> std::shared_ptr<arrow::DataType> {
+    switch (size) {
+      case -1:
+        return arrow::list(type);
+      case 1:
+        return std::move(type);
+      default:
+        return arrow::fixed_size_list(type, size);
+    }
+  };
+
+  switch (type) {
+    case EDataType::kBool_t:
+      return typeGenerator(arrow::boolean(), size);
+    case EDataType::kUChar_t:
+      return typeGenerator(arrow::uint8(), size);
+    case EDataType::kUShort_t:
+      return typeGenerator(arrow::uint16(), size);
+    case EDataType::kUInt_t:
+      return typeGenerator(arrow::uint32(), size);
+    case EDataType::kULong64_t:
+      return typeGenerator(arrow::uint64(), size);
+    case EDataType::kChar_t:
+      return typeGenerator(arrow::int8(), size);
+    case EDataType::kShort_t:
+      return typeGenerator(arrow::int16(), size);
+    case EDataType::kInt_t:
+      return typeGenerator(arrow::int32(), size);
+    case EDataType::kLong64_t:
+      return typeGenerator(arrow::int64(), size);
+    case EDataType::kFloat_t:
+      return typeGenerator(arrow::float32(), size);
+    case EDataType::kDouble_t:
+      return typeGenerator(arrow::float64(), size);
+    default:
+      throw o2::framework::runtime_error_f("Unsupported branch type: %d", static_cast<int>(type));
+  }
+}
+namespace o2::framework
+{
+
+TFileFileSystem::TFileFileSystem(TDirectoryFile* f, size_t readahead)
+  : TTreeFileSystem(),
+    mFile(f)
+{
+  ((TFile*)mFile)->SetReadaheadSize(50 * 1024 * 1024);
+}
+
+TTree* TFileFileSystem::GetTree(arrow::dataset::FileSource source)
+{
+  // Simply return the only TTree we have
+  return (TTree*)mFile->Get(source.path().c_str());
+}
+
+arrow::Result<arrow::fs::FileInfo> TTreeFileSystem::GetFileInfo(const std::string& path)
+{
+  arrow::fs::FileInfo result;
+  result.set_type(arrow::fs::FileType::NotFound);
+  result.set_path(path);
+  arrow::dataset::FileSource source(path, shared_from_this());
+
+  for (auto branch : *GetTree(source)->GetListOfBranches()) {
+    if (strncmp(branch->GetName(), result.path().c_str(), path.size()) == 0) {
+      result.set_type(arrow::fs::FileType::File);
+      return result;
+    }
+  }
+  return result;
+}
+
+arrow::Result<arrow::fs::FileInfoVector> TTreeFileSystem::GetFileInfo(const arrow::fs::FileSelector& select)
+{
+  arrow::fs::FileInfoVector results;
+  auto selected = this->GetFileInfo(select.base_dir);
+  if (selected.ok()) {
+    results.emplace_back(*selected);
+  }
+  return results;
+}
+
+arrow::Status TTreeFileSystem::CreateDir(const std::string& path, bool recursive)
+{
+  return arrow::Status::NotImplemented("Read only filesystem");
+}
+
+arrow::Status TTreeFileSystem::DeleteDir(const std::string& path)
+{
+  return arrow::Status::NotImplemented("Read only filesystem");
+}
+
+arrow::Status TTreeFileSystem::CopyFile(const std::string& src, const std::string& dest)
+{
+  return arrow::Status::NotImplemented("Read only filesystem");
+}
+
+arrow::Status TTreeFileSystem::Move(const std::string& src, const std::string& dest)
+{
+  return arrow::Status::NotImplemented("Read only filesystem");
+}
+
+arrow::Status TTreeFileSystem::DeleteDirContents(const std::string& path, bool missing_dir_ok)
+{
+  return arrow::Status::NotImplemented("Read only filesystem");
+}
+
+arrow::Status TTreeFileSystem::DeleteRootDirContents()
+{
+  return arrow::Status::NotImplemented("Read only filesystem");
+}
+
+arrow::Status TTreeFileSystem::DeleteFile(const std::string& path)
+{
+  return arrow::Status::NotImplemented("Read only filesystem");
+}
+
+arrow::Result<std::shared_ptr<arrow::io::InputStream>> TTreeFileSystem::OpenInputStream(const std::string& path)
+{
+  return arrow::Status::NotImplemented("Non streamable filesystem");
+}
+
+arrow::Result<std::shared_ptr<arrow::io::RandomAccessFile>> TTreeFileSystem::OpenInputFile(const std::string& path)
+{
+  return arrow::Status::NotImplemented("No random access file system");
+}
+
+arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenOutputStream(
+  const std::string& path,
+  const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
+{
+  return arrow::Status::NotImplemented("Non streamable filesystem");
+}
+
+arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenAppendStream(
+  const std::string& path,
+  const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
+{
+  return arrow::Status::NotImplemented("No random access file system");
+}
+
+arrow::Result<std::shared_ptr<arrow::Schema>> TTreeFileFormat::Inspect(const arrow::dataset::FileSource& source) const
+{
+  arrow::Schema schema{{}};
+  auto fs = std::dynamic_pointer_cast<TTreeFileSystem>(source.filesystem());
+  TTree* tree = fs->GetTree(source);
+
+  auto branches = tree->GetListOfBranches();
+  auto n = branches->GetEntries();
+
+  std::vector<BranchInfo> branchInfos;
+  for (auto i = 0; i < n; ++i) {
+    auto branch = static_cast<TBranch*>(branches->At(i));
+    auto name = std::string{branch->GetName()};
+    auto pos = name.find("_size");
+    if (pos != std::string::npos) {
+      name.erase(pos);
+      branchInfos.emplace_back(BranchInfo{name, (TBranch*)nullptr, true});
+    } else {
+      auto lookup = std::find_if(branchInfos.begin(), branchInfos.end(), [&](BranchInfo const& bi) {
+        return bi.name == name;
+      });
+      if (lookup == branchInfos.end()) {
+        branchInfos.emplace_back(BranchInfo{name, branch, false});
+      } else {
+        lookup->ptr = branch;
+      }
+    }
+  }
+
+  std::vector<std::shared_ptr<arrow::Field>> fields;
+  tree->SetCacheSize(25000000);
+  for (auto& bi : branchInfos) {
+    static TClass* cls;
+    EDataType type;
+    bi.ptr->GetExpectedType(cls, type);
+    auto listSize = -1;
+    if (!bi.mVLA) {
+      listSize = static_cast<TLeaf*>(bi.ptr->GetListOfLeaves()->At(0))->GetLenStatic();
+    }
+    auto field = std::make_shared<arrow::Field>(bi.ptr->GetName(), arrowTypeFromROOT(type, listSize));
+    fields.push_back(field);
+
+    tree->AddBranchToCache(bi.ptr);
+    if (strncmp(bi.ptr->GetName(), "fIndexArray", strlen("fIndexArray")) == 0) {
+      std::string sizeBranchName = bi.ptr->GetName();
+      sizeBranchName += "_size";
+      auto* sizeBranch = (TBranch*)tree->GetBranch(sizeBranchName.c_str());
+      if (sizeBranch) {
+        tree->AddBranchToCache(sizeBranch);
+      }
+    }
+  }
+  tree->StopCacheLearningPhase();
+
+  return std::make_shared<arrow::Schema>(fields);
+}
+
+/// \brief Create a FileFragment for a FileSource.
+arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> TTreeFileFormat::MakeFragment(
+  arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
+  std::shared_ptr<arrow::Schema> physical_schema)
+{
+  std::shared_ptr<arrow::dataset::FileFormat> format = std::make_shared<TTreeFileFormat>(mTotCompressedSize, mTotUncompressedSize);
+
+  auto fragment = std::make_shared<TTreeFileFragment>(std::move(source), std::move(format),
+                                                      std::move(partition_expression),
+                                                      std::move(physical_schema));
+  return std::dynamic_pointer_cast<arrow::dataset::FileFragment>(fragment);
+}
+
+arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> TTreeFileFormat::MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options, arrow::fs::FileLocator destination_locator) const
+{
+  throw std::runtime_error("Unsupported operation");
+}
+
+std::shared_ptr<arrow::dataset::FileWriteOptions> TTreeFileFormat::DefaultWriteOptions()
+{
+  std::shared_ptr<TTreeFileWriteOptions> options(
+    new TTreeFileWriteOptions(shared_from_this()));
+  return options;
+}
+
+arrow::Result<arrow::RecordBatchGenerator> TTreeFileFormat::ScanBatchesAsync(
+  const std::shared_ptr<arrow::dataset::ScanOptions>& options,
+  const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const
+{
+  // Get the fragment as a TTreeFragment. This might be PART of a TTree.
+  auto treeFragment = std::dynamic_pointer_cast<TTreeFileFragment>(fragment);
+  // This is the schema we want to read
+  auto dataset_schema = options->dataset_schema;
+
+  auto generator = [pool = options->pool, treeFragment, dataset_schema, &totalCompressedSize = mTotCompressedSize,
+                    &totalUncompressedSize = mTotUncompressedSize]() -> arrow::Future<std::shared_ptr<arrow::RecordBatch>> {
+    auto schema = treeFragment->format()->Inspect(treeFragment->source());
+
+    std::vector<std::shared_ptr<arrow::Array>> columns;
+    std::vector<std::shared_ptr<arrow::Field>> fields = dataset_schema->fields();
+    auto physical_schema = *treeFragment->ReadPhysicalSchema();
+
+    static TBufferFile buffer{TBuffer::EMode::kWrite, 4 * 1024 * 1024};
+    auto fs = std::dynamic_pointer_cast<TTreeFileSystem>(treeFragment->source().filesystem());
+    int64_t rows = -1;
+    TTree* tree = fs->GetTree(treeFragment->source());
+    for (auto& field : fields) {
+      // The field actually on disk
+      auto physicalField = physical_schema->GetFieldByName(field->name());
+      TBranch* branch = tree->GetBranch(physicalField->name().c_str());
+      assert(branch);
+      buffer.Reset();
+      auto totalEntries = branch->GetEntries();
+      if (rows == -1) {
+        rows = totalEntries;
+      }
+      if (rows != totalEntries) {
+        throw runtime_error_f("Unmatching number of rows for branch %s", branch->GetName());
+      }
+      arrow::Status status;
+      int readEntries = 0;
+      std::shared_ptr<arrow::Array> array;
+      auto listType = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type());
+      if (physicalField->type() == arrow::boolean() ||
+          (listType && physicalField->type()->field(0)->type() == arrow::boolean())) {
+        if (listType) {
+          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
+          auto status = arrow::MakeBuilder(pool, physicalField->type()->field(0)->type(), &builder);
+          if (!status.ok()) {
+            throw runtime_error("Cannot create value builder");
+          }
+          auto listBuilder = std::make_unique<arrow::FixedSizeListBuilder>(pool, std::move(builder), listType->list_size());
+          auto valueBuilder = listBuilder.get()->value_builder();
+          // boolean array special case: we need to use builder to create the bitmap
+          status = valueBuilder->Reserve(totalEntries * listType->list_size());
+          status &= listBuilder->Reserve(totalEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to reserve memory for array builder");
+          }
+          while (readEntries < totalEntries) {
+            auto readLast = branch->GetBulkRead().GetBulkEntries(readEntries, buffer);
+            readEntries += readLast;
+            status &= static_cast<arrow::BooleanBuilder*>(valueBuilder)->AppendValues(reinterpret_cast<uint8_t const*>(buffer.GetCurrent()), readLast * listType->list_size());
+          }
+          status &= static_cast<arrow::FixedSizeListBuilder*>(listBuilder.get())->AppendValues(readEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to append values to array");
+          }
+          status &= listBuilder->Finish(&array);
+          if (!status.ok()) {
+            throw runtime_error("Failed to create array");
+          }
+        } else if (listType == nullptr) {
+          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
+          auto status = arrow::MakeBuilder(pool, physicalField->type(), &builder);
+          if (!status.ok()) {
+            throw runtime_error("Cannot create builder");
+          }
+          auto valueBuilder = static_cast<arrow::BooleanBuilder*>(builder.get());
+          // boolean array special case: we need to use builder to create the bitmap
+          status = valueBuilder->Reserve(totalEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to reserve memory for array builder");
+          }
+          while (readEntries < totalEntries) {
+            auto readLast = branch->GetBulkRead().GetBulkEntries(readEntries, buffer);
+            readEntries += readLast;
+            status &= valueBuilder->AppendValues(reinterpret_cast<uint8_t const*>(buffer.GetCurrent()), readLast);
+          }
+          if (!status.ok()) {
+            throw runtime_error("Failed to append values to array");
+          }
+          status &= valueBuilder->Finish(&array);
+          if (!status.ok()) {
+            throw runtime_error("Failed to create array");
+          }
+        }
+      } else {
+        // other types: use serialized read to build arrays directly.
+        auto typeSize = physicalField->type()->byte_width();
+        // This is needed for branches which have not been persisted.
+        auto bytes = branch->GetTotBytes();
+        auto branchSize = bytes ? bytes : 1000000;
+        auto&& result = arrow::AllocateResizableBuffer(branchSize, pool);
+        if (!result.ok()) {
+          throw runtime_error("Cannot allocate values buffer");
+        }
+        std::shared_ptr<arrow::Buffer> arrowValuesBuffer = std::move(result).ValueUnsafe();
+        auto ptr = arrowValuesBuffer->mutable_data();
+        if (ptr == nullptr) {
+          throw runtime_error("Invalid buffer");
+        }
+
+        std::unique_ptr<TBufferFile> offsetBuffer = nullptr;
+
+        uint32_t offset = 0;
+        int count = 0;
+        std::shared_ptr<arrow::Buffer> arrowOffsetBuffer;
+        std::span<int> offsets;
+        int size = 0;
+        uint32_t totalSize = 0;
+        TBranch* mSizeBranch = nullptr;
+        int64_t listSize = 1;
+        if (auto fixedSizeList = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type())) {
+          listSize = fixedSizeList->list_size();
+        } else if (auto vlaListType = std::dynamic_pointer_cast<arrow::ListType>(physicalField->type())) {
+          listSize = -1;
+        }
+        if (listSize == -1) {
+          mSizeBranch = branch->GetTree()->GetBranch((std::string{branch->GetName()} + "_size").c_str());
+          offsetBuffer = std::make_unique<TBufferFile>(TBuffer::EMode::kWrite, 4 * 1024 * 1024);
+          result = arrow::AllocateResizableBuffer((totalEntries + 1) * (int64_t)sizeof(int), pool);
+          if (!result.ok()) {
+            throw runtime_error("Cannot allocate offset buffer");
+          }
+          arrowOffsetBuffer = std::move(result).ValueUnsafe();
+          unsigned char* ptrOffset = arrowOffsetBuffer->mutable_data();
+          auto* tPtrOffset = reinterpret_cast<int*>(ptrOffset);
+          offsets = std::span<int>{tPtrOffset, tPtrOffset + totalEntries + 1};
+
+          // read sizes first
+          while (readEntries < totalEntries) {
+            auto readLast = mSizeBranch->GetBulkRead().GetEntriesSerialized(readEntries, *offsetBuffer);
+            readEntries += readLast;
+            for (auto i = 0; i < readLast; ++i) {
+              offsets[count++] = (int)offset;
+              offset += swap32_(reinterpret_cast<uint32_t*>(offsetBuffer->GetCurrent())[i]);
+            }
+          }
+          offsets[count] = (int)offset;
+          totalSize = offset;
+          readEntries = 0;
+        }
+
+        while (readEntries < totalEntries) {
+          auto readLast = branch->GetBulkRead().GetEntriesSerialized(readEntries, buffer);
+          if (listSize == -1) {
+            size = offsets[readEntries + readLast] - offsets[readEntries];
+          } else {
+            size = readLast * listSize;
+          }
+          readEntries += readLast;
+          swapCopy(ptr, buffer.GetCurrent(), size, typeSize);
+          ptr += (ptrdiff_t)(size * typeSize);
+        }
+        if (listSize >= 1) {
+          totalSize = readEntries * listSize;
+        }
+        std::shared_ptr<arrow::PrimitiveArray> varray;
+        switch (listSize) {
+          case -1:
+            varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
+            array = std::make_shared<arrow::ListArray>(physicalField->type(), readEntries, arrowOffsetBuffer, varray);
+            break;
+          case 1:
+            array = std::make_shared<arrow::PrimitiveArray>(physicalField->type(), readEntries, arrowValuesBuffer);
+            break;
+          default:
+            varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
+            array = std::make_shared<arrow::FixedSizeListArray>(physicalField->type(), readEntries, varray);
+        }
+      }
+
+      branch->SetStatus(false);
+      branch->DropBaskets("all");
+      branch->Reset();
+      branch->GetTransientBuffer(0)->Expand(0);
+
+      columns.push_back(array);
+    }
+    auto batch = arrow::RecordBatch::Make(dataset_schema, rows, columns);
+    totalCompressedSize += tree->GetZipBytes();
+    totalUncompressedSize += tree->GetTotBytes();
+    return batch;
+  };
+  return generator;
+}
+
+TBufferFileFS::TBufferFileFS(TBufferFile* f)
+  : TTreeFileSystem(),
+    mTree((TTree*)f->ReadObject(TTree::Class()))
+{
+}
+
+} // namespace o2::framework
diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx
index 109d9718f5f09..69d39f193c7df 100644
--- a/Framework/Core/test/test_Root2ArrowTable.cxx
+++ b/Framework/Core/test/test_Root2ArrowTable.cxx
@@ -18,13 +18,20 @@
 
 #include <ROOT/RDataFrame.hxx>
 #include <ROOT/RArrowDS.hxx>
+#include <TBufferFile.h>
+#include <TMemFile.h>
+#include <TDirectory.h>
 #include <TTree.h>
 #include <TRandom.h>
+#include <TFile.h>
+
+#include <arrow/dataset/scanner.h>
 #include <arrow/table.h>
 #include <arrow/ipc/writer.h>
 #include <arrow/io/memory.h>
 #include <arrow/ipc/writer.h>
 #include <arrow/ipc/reader.h>
+#include "Framework/RootArrowFilesystem.h"
 
 using namespace o2::framework;
 
@@ -181,3 +188,172 @@ TEST_CASE("RootTree2TableViaASoA")
     REQUIRE(row.ij()[1] == row.ev());
   }
 }
+
+TEST_CASE("RootTree2Fragment")
+{
+  using namespace o2::framework;
+  /// A directory holding a tree
+
+  /// Create a simple TTree
+  TBufferFile* file = new TBufferFile(TBuffer::kWrite);
+
+  TTree t1("t1", "a simple Tree with simple variables");
+  Float_t xyz[3];
+  Int_t ij[2];
+  Float_t px = 0, py = 1, pz = 2;
+  Double_t random;
+  Int_t ev;
+  t1.Branch("px", &px, "px/F");
+  t1.Branch("py", &py, "py/F");
+  t1.Branch("pz", &pz, "pz/F");
+  t1.Branch("random", &random, "random/D");
+  t1.Branch("ev", &ev, "ev/I");
+  t1.Branch("xyz", xyz, "xyz[3]/F");
+  t1.Branch("ij", ij, "ij[2]/I");
+  // fill the tree
+  for (Int_t i = 0; i < 1000; i++) {
+    xyz[0] = 1;
+    xyz[1] = 2;
+    xyz[2] = 3;
+    gRandom->Rannor(px, py);
+    pz = px * px + py * py;
+    xyz[2] = i + 1;
+    ij[0] = i;
+    ij[1] = i + 1;
+    random = gRandom->Rndm();
+    ev = i + 1;
+    t1.Fill();
+  }
+  file->WriteObjectAny(&t1, t1.Class());
+  auto* fileRead = new TBufferFile(TBuffer::kRead, file->BufferSize(), file->Buffer(), false, nullptr);
+
+  size_t totalSizeCompressed = 0;
+  size_t totalSizeUncompressed = 0;
+  auto format = std::make_shared<TTreeFileFormat>(totalSizeCompressed, totalSizeUncompressed);
+  auto fs = std::make_shared<TBufferFileFS>(fileRead);
+  arrow::dataset::FileSource source("p", fs);
+  REQUIRE(format->IsSupported(source) == true);
+  auto schemaOpt = format->Inspect(source);
+  REQUIRE(schemaOpt.ok());
+  auto schema = *schemaOpt;
+  REQUIRE(schema->num_fields() == 7);
+  REQUIRE(schema->field(0)->type()->id() == arrow::float32()->id());
+  REQUIRE(schema->field(1)->type()->id() == arrow::float32()->id());
+  REQUIRE(schema->field(2)->type()->id() == arrow::float32()->id());
+  REQUIRE(schema->field(3)->type()->id() == arrow::float64()->id());
+  REQUIRE(schema->field(4)->type()->id() == arrow::int32()->id());
+  REQUIRE(schema->field(5)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id());
+  REQUIRE(schema->field(6)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id());
+  auto fragment = format->MakeFragment(source, {}, schema);
+  REQUIRE(fragment.ok());
+  auto options = std::make_shared<arrow::dataset::ScanOptions>();
+  options->dataset_schema = schema;
+  auto scanner = format->ScanBatchesAsync(options, *fragment);
+  REQUIRE(scanner.ok());
+  auto batches = (*scanner)();
+  auto result = batches.result();
+  REQUIRE(result.ok());
+  REQUIRE((*result)->columns().size() == 7);
+  REQUIRE((*result)->num_rows() == 1000);
+}
+
+TEST_CASE("RootTree2Dataset")
+{
+  using namespace o2::framework;
+  /// A directory holding a tree
+  // auto *f = new TFile("Foo.root", "RECREATE");
+  auto* f = new TMemFile("foo", "RECREATE");
+  f->mkdir("DF_1");
+  f->mkdir("DF_2");
+
+  f->cd("DF_1");
+  auto* t = new TTree("tracks", "a simple Tree with simple variables");
+  {
+    Float_t xyz[3];
+    Int_t ij[2];
+    Float_t px = 0, py = 1, pz = 2;
+    Double_t random;
+    Int_t ev;
+    t->Branch("px", &px, "px/F");
+    t->Branch("py", &py, "py/F");
+    t->Branch("pz", &pz, "pz/F");
+    t->Branch("random", &random, "random/D");
+    t->Branch("ev", &ev, "ev/I");
+    t->Branch("xyz", xyz, "xyz[3]/F");
+    t->Branch("ij", ij, "ij[2]/I");
+    // fill the tree
+    for (Int_t i = 0; i < 1000; i++) {
+      xyz[0] = 1;
+      xyz[1] = 2;
+      xyz[2] = 3;
+      gRandom->Rannor(px, py);
+      pz = px * px + py * py;
+      xyz[2] = i + 1;
+      ij[0] = i;
+      ij[1] = i + 1;
+      random = gRandom->Rndm();
+      ev = i + 1;
+      t->Fill();
+    }
+  }
+
+  f->cd("DF_2");
+  t = new TTree("tracks", "a simple Tree with simple variables");
+  {
+    Float_t xyz[3];
+    Int_t ij[2];
+    Float_t px = 0, py = 1, pz = 2;
+    Double_t random;
+    Int_t ev;
+    t->Branch("px", &px, "px/F");
+    t->Branch("py", &py, "py/F");
+    t->Branch("pz", &pz, "pz/F");
+    t->Branch("random", &random, "random/D");
+    t->Branch("ev", &ev, "ev/I");
+    t->Branch("xyz", xyz, "xyz[3]/F");
+    t->Branch("ij", ij, "ij[2]/I");
+    // fill the tree
+    for (Int_t i = 0; i < 100; i++) {
+      xyz[0] = 1;
+      xyz[1] = 2;
+      xyz[2] = 3;
+      gRandom->Rannor(px, py);
+      pz = px * px + py * py;
+      xyz[2] = i + 1;
+      ij[0] = i;
+      ij[1] = i + 1;
+      random = gRandom->Rndm();
+      ev = i + 1;
+      t->Fill();
+    }
+  }
+
+  size_t totalSizeCompressed = 0;
+  size_t totalSizeUncompressed = 0;
+  auto format = std::make_shared<TTreeFileFormat>(totalSizeCompressed, totalSizeUncompressed);
+  auto fs = std::make_shared<TFileFileSystem>(f, 50 * 1024 * 1024);
+  arrow::dataset::FileSource source("DF_2/tracks", fs);
+  REQUIRE(format->IsSupported(source) == true);
+  auto schemaOpt = format->Inspect(source);
+  REQUIRE(schemaOpt.ok());
+  auto schema = *schemaOpt;
+  REQUIRE(schema->num_fields() == 7);
+  REQUIRE(schema->field(0)->type()->id() == arrow::float32()->id());
+  REQUIRE(schema->field(1)->type()->id() == arrow::float32()->id());
+  REQUIRE(schema->field(2)->type()->id() == arrow::float32()->id());
+  REQUIRE(schema->field(3)->type()->id() == arrow::float64()->id());
+  REQUIRE(schema->field(4)->type()->id() == arrow::int32()->id());
+  REQUIRE(schema->field(5)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id());
+  REQUIRE(schema->field(6)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id());
+  auto fragment = format->MakeFragment(source, {}, schema);
+  REQUIRE(fragment.ok());
+  auto options = std::make_shared<arrow::dataset::ScanOptions>();
+  options->dataset_schema = schema;
+  auto scanner = format->ScanBatchesAsync(options, *fragment);
+  REQUIRE(scanner.ok());
+  auto batches = (*scanner)();
+  auto result = batches.result();
+  REQUIRE(result.ok());
+  REQUIRE((*result)->columns().size() == 7);
+  REQUIRE((*result)->num_rows() == 100);
+}
diff --git a/dependencies/O2Dependencies.cmake b/dependencies/O2Dependencies.cmake
index 48a5d57d17c96..c5c22b3a79097 100644
--- a/dependencies/O2Dependencies.cmake
+++ b/dependencies/O2Dependencies.cmake
@@ -38,6 +38,33 @@ if (NOT TARGET Arrow::arrow_shared)
  add_library(Arrow::arrow_shared ALIAS arrow_shared)
 endif()
 
+if(NOT TARGET ArrowDataset::arrow_dataset_shared)
+  # ArrowDataset::arrow_dataset_shared is linked for no reason to parquet
+  # so we cannot use it because we do not want to build parquet itself.
+  # For that reason at the moment we need to do the lookup by hand.
+  get_target_property(ARROW_SHARED_LOCATION Arrow::arrow_shared LOCATION)
+  get_filename_component(ARROW_SHARED_DIR ${ARROW_SHARED_LOCATION} DIRECTORY)
+
+  find_library(ARROW_DATASET_SHARED arrow_dataset
+      PATHS ${ARROW_SHARED_DIR}
+      NO_DEFAULT_PATH
+  )
+
+  if(ARROW_DATASET_SHARED)
+    message(STATUS
+            "Found arrow_dataset_shared library at: ${ARROW_DATASET_SHARED}")
+  else()
+    message(FATAL_ERROR
+            "arrow_dataset_shared library not found in ${ARROW_SHARED_DIR}")
+  endif()
+
+  # Step 3: Create a target for ArrowDataset::arrow_dataset_shared
+  add_library(ArrowDataset::arrow_dataset_shared SHARED IMPORTED)
+  set_target_properties(ArrowDataset::arrow_dataset_shared PROPERTIES
+      IMPORTED_LOCATION ${ARROW_DATASET_SHARED}
+  )
+endif()
+
 if (NOT TARGET Gandiva::gandiva_shared)
   add_library(Gandiva::gandiva_shared ALIAS gandiva_shared)
 endif()

From dce97e92cbc482cfa6ed5d920d4a80e9decb0b8d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 6 Nov 2024 08:49:17 +0100
Subject: [PATCH 682/726] Layered arrow::FileSystem

This allows distinguishing the navigation of a ROOT file from the actual
contents (TTree or eventually, RNTuple).
---
 .../include/Framework/RootArrowFilesystem.h   |  50 +++++---
 Framework/Core/src/RootArrowFilesystem.cxx    | 116 ++++++++++++++----
 Framework/Core/test/test_Root2ArrowTable.cxx  |   2 +
 3 files changed, 126 insertions(+), 42 deletions(-)

diff --git a/Framework/Core/include/Framework/RootArrowFilesystem.h b/Framework/Core/include/Framework/RootArrowFilesystem.h
index e1787dae69f31..df00ce4fa8a76 100644
--- a/Framework/Core/include/Framework/RootArrowFilesystem.h
+++ b/Framework/Core/include/Framework/RootArrowFilesystem.h
@@ -15,6 +15,7 @@
 #include <arrow/dataset/file_base.h>
 #include <arrow/filesystem/type_fwd.h>
 #include <arrow/type_fwd.h>
+#include <memory>
 
 class TTree;
 class TBufferFile;
@@ -32,11 +33,13 @@ class TTreeFileWriteOptions : public arrow::dataset::FileWriteOptions
   }
 };
 
-// This is a virtual filesystem based on a ttree, where branches with the
-// same prefix get grouped into a fragment
-class TTreeFileSystem : public arrow::fs::FileSystem
+// This is to avoid having to implement a bunch of unimplemented methods
+// for all the possible virtual filesystem we can invent on top of ROOT
+// data structures.
+class VirtualRootFileSystemBase : public arrow::fs::FileSystem
 {
  public:
+  // Dummy implementation to avoid
   arrow::Result<arrow::fs::FileInfo> GetFileInfo(const std::string& path) override;
   arrow::Result<arrow::fs::FileInfoVector> GetFileInfo(const arrow::fs::FileSelector& select) override;
 
@@ -45,6 +48,8 @@ class TTreeFileSystem : public arrow::fs::FileSystem
     return this->type_name() == other.type_name();
   }
 
+  virtual std::shared_ptr<VirtualRootFileSystemBase> GetSubFilesystem(arrow::dataset::FileSource source) = 0;
+
   arrow::Status CreateDir(const std::string& path, bool recursive) override;
 
   arrow::Status DeleteDir(const std::string& path) override;
@@ -70,8 +75,19 @@ class TTreeFileSystem : public arrow::fs::FileSystem
   arrow::Result<std::shared_ptr<arrow::io::OutputStream>> OpenAppendStream(
     const std::string& path,
     const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override;
+};
+
+// A filesystem which allows me to get a TTree
+class TTreeFileSystem : public VirtualRootFileSystemBase
+{
+ public:
+  ~TTreeFileSystem() override;
 
-  virtual TTree* GetTree(arrow::dataset::FileSource) = 0;
+  std::shared_ptr<VirtualRootFileSystemBase> GetSubFilesystem(arrow::dataset::FileSource source) override
+  {
+    return std::dynamic_pointer_cast<VirtualRootFileSystemBase>(shared_from_this());
+  };
+  virtual TTree* GetTree(arrow::dataset::FileSource source) = 0;
 };
 
 class SingleTreeFileSystem : public TTreeFileSystem
@@ -98,9 +114,11 @@ class SingleTreeFileSystem : public TTreeFileSystem
   TTree* mTree;
 };
 
-class TFileFileSystem : public TTreeFileSystem
+class TFileFileSystem : public VirtualRootFileSystemBase
 {
  public:
+  arrow::Result<arrow::fs::FileInfo> GetFileInfo(const std::string& path) override;
+
   TFileFileSystem(TDirectoryFile* f, size_t readahead);
 
   std::string type_name() const override
@@ -108,7 +126,7 @@ class TFileFileSystem : public TTreeFileSystem
     return "TDirectoryFile";
   }
 
-  TTree* GetTree(arrow::dataset::FileSource source) override;
+  std::shared_ptr<VirtualRootFileSystemBase> GetSubFilesystem(arrow::dataset::FileSource source) override;
 
   // We can go back to the TFile in case this is needed.
   TDirectoryFile* GetFile()
@@ -120,24 +138,22 @@ class TFileFileSystem : public TTreeFileSystem
   TDirectoryFile* mFile;
 };
 
-class TBufferFileFS : public TTreeFileSystem
+class TBufferFileFS : public VirtualRootFileSystemBase
 {
  public:
   TBufferFileFS(TBufferFile* f);
 
+  arrow::Result<arrow::fs::FileInfo> GetFileInfo(const std::string& path) override;
   std::string type_name() const override
   {
     return "tbufferfile";
   }
 
-  TTree* GetTree(arrow::dataset::FileSource) override
-  {
-    // Simply return the only TTree we have
-    return mTree;
-  }
+  std::shared_ptr<VirtualRootFileSystemBase> GetSubFilesystem(arrow::dataset::FileSource source) override;
 
  private:
-  TTree* mTree;
+  TBufferFile* mBuffer;
+  std::shared_ptr<VirtualRootFileSystemBase> mFilesystem;
 };
 
 class TTreeFileFragment : public arrow::dataset::FileFragment
@@ -179,8 +195,12 @@ class TTreeFileFormat : public arrow::dataset::FileFormat
 
   arrow::Result<bool> IsSupported(const arrow::dataset::FileSource& source) const override
   {
-    auto fs = std::dynamic_pointer_cast<TTreeFileSystem>(source.filesystem());
-    return fs != nullptr;
+    auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
+    auto subFs = fs->GetSubFilesystem(source);
+    if (std::dynamic_pointer_cast<TTreeFileSystem>(subFs)) {
+      return true;
+    }
+    return false;
   }
 
   arrow::Result<std::shared_ptr<arrow::Schema>> Inspect(const arrow::dataset::FileSource& source) const override;
diff --git a/Framework/Core/src/RootArrowFilesystem.cxx b/Framework/Core/src/RootArrowFilesystem.cxx
index a130f51f7e5a9..46489141c3173 100644
--- a/Framework/Core/src/RootArrowFilesystem.cxx
+++ b/Framework/Core/src/RootArrowFilesystem.cxx
@@ -11,14 +11,19 @@
 #include "Framework/RootArrowFilesystem.h"
 #include "Framework/Endian.h"
 #include "Framework/RuntimeError.h"
+#include <Rtypes.h>
+#include <arrow/array/array_primitive.h>
 #include <arrow/array/builder_nested.h>
 #include <arrow/array/builder_primitive.h>
+#include <memory>
 #include <stdexcept>
 #include <TFile.h>
 #include <TLeaf.h>
 #include <TBufferFile.h>
 #include <TTree.h>
 #include <TDirectoryFile.h>
+#include <arrow/type.h>
+#include <arrow/type_fwd.h>
 
 namespace
 {
@@ -73,35 +78,52 @@ namespace o2::framework
 {
 
 TFileFileSystem::TFileFileSystem(TDirectoryFile* f, size_t readahead)
-  : TTreeFileSystem(),
+  : VirtualRootFileSystemBase(),
     mFile(f)
 {
   ((TFile*)mFile)->SetReadaheadSize(50 * 1024 * 1024);
 }
 
-TTree* TFileFileSystem::GetTree(arrow::dataset::FileSource source)
+std::shared_ptr<VirtualRootFileSystemBase> TFileFileSystem::GetSubFilesystem(arrow::dataset::FileSource source)
 {
-  // Simply return the only TTree we have
-  return (TTree*)mFile->Get(source.path().c_str());
+  auto tree = (TTree*)mFile->GetObjectChecked(source.path().c_str(), TClass::GetClass<TTree>());
+  if (tree) {
+    return std::shared_ptr<VirtualRootFileSystemBase>(new SingleTreeFileSystem(tree));
+  }
+
+
+  auto directory = (TDirectoryFile*)mFile->GetObjectChecked(source.path().c_str(), TClass::GetClass<TDirectory>());
+  if (directory) {
+    return std::shared_ptr<VirtualRootFileSystemBase>(new TFileFileSystem(directory, 50 * 1024 * 1024));
+  }
+  throw runtime_error_f("Unsupported file layout");
 }
 
-arrow::Result<arrow::fs::FileInfo> TTreeFileSystem::GetFileInfo(const std::string& path)
+arrow::Result<arrow::fs::FileInfo> TFileFileSystem::GetFileInfo(const std::string& path)
 {
   arrow::fs::FileInfo result;
   result.set_type(arrow::fs::FileType::NotFound);
   result.set_path(path);
   arrow::dataset::FileSource source(path, shared_from_this());
 
-  for (auto branch : *GetTree(source)->GetListOfBranches()) {
-    if (strncmp(branch->GetName(), result.path().c_str(), path.size()) == 0) {
-      result.set_type(arrow::fs::FileType::File);
-      return result;
-    }
+  auto fs = GetSubFilesystem(source);
+
+  // For now we only support single trees.
+  if (std::dynamic_pointer_cast<SingleTreeFileSystem>(fs)) {
+    result.set_type(arrow::fs::FileType::File);
+    return result;
   }
   return result;
 }
 
-arrow::Result<arrow::fs::FileInfoVector> TTreeFileSystem::GetFileInfo(const arrow::fs::FileSelector& select)
+arrow::Result<arrow::fs::FileInfo> VirtualRootFileSystemBase::GetFileInfo(std::string const&)
+{
+  arrow::fs::FileInfo result;
+  result.set_type(arrow::fs::FileType::NotFound);
+  return result;
+}
+
+arrow::Result<arrow::fs::FileInfoVector> VirtualRootFileSystemBase::GetFileInfo(const arrow::fs::FileSelector& select)
 {
   arrow::fs::FileInfoVector results;
   auto selected = this->GetFileInfo(select.base_dir);
@@ -111,59 +133,59 @@ arrow::Result<arrow::fs::FileInfoVector> TTreeFileSystem::GetFileInfo(const arro
   return results;
 }
 
-arrow::Status TTreeFileSystem::CreateDir(const std::string& path, bool recursive)
+arrow::Status VirtualRootFileSystemBase::CreateDir(const std::string& path, bool recursive)
 {
   return arrow::Status::NotImplemented("Read only filesystem");
 }
 
-arrow::Status TTreeFileSystem::DeleteDir(const std::string& path)
+arrow::Status VirtualRootFileSystemBase::DeleteDir(const std::string& path)
 {
   return arrow::Status::NotImplemented("Read only filesystem");
 }
 
-arrow::Status TTreeFileSystem::CopyFile(const std::string& src, const std::string& dest)
+arrow::Status VirtualRootFileSystemBase::CopyFile(const std::string& src, const std::string& dest)
 {
   return arrow::Status::NotImplemented("Read only filesystem");
 }
 
-arrow::Status TTreeFileSystem::Move(const std::string& src, const std::string& dest)
+arrow::Status VirtualRootFileSystemBase::Move(const std::string& src, const std::string& dest)
 {
   return arrow::Status::NotImplemented("Read only filesystem");
 }
 
-arrow::Status TTreeFileSystem::DeleteDirContents(const std::string& path, bool missing_dir_ok)
+arrow::Status VirtualRootFileSystemBase::DeleteDirContents(const std::string& path, bool missing_dir_ok)
 {
   return arrow::Status::NotImplemented("Read only filesystem");
 }
 
-arrow::Status TTreeFileSystem::DeleteRootDirContents()
+arrow::Status VirtualRootFileSystemBase::DeleteRootDirContents()
 {
   return arrow::Status::NotImplemented("Read only filesystem");
 }
 
-arrow::Status TTreeFileSystem::DeleteFile(const std::string& path)
+arrow::Status VirtualRootFileSystemBase::DeleteFile(const std::string& path)
 {
   return arrow::Status::NotImplemented("Read only filesystem");
 }
 
-arrow::Result<std::shared_ptr<arrow::io::InputStream>> TTreeFileSystem::OpenInputStream(const std::string& path)
+arrow::Result<std::shared_ptr<arrow::io::InputStream>> VirtualRootFileSystemBase::OpenInputStream(const std::string& path)
 {
   return arrow::Status::NotImplemented("Non streamable filesystem");
 }
 
-arrow::Result<std::shared_ptr<arrow::io::RandomAccessFile>> TTreeFileSystem::OpenInputFile(const std::string& path)
+arrow::Result<std::shared_ptr<arrow::io::RandomAccessFile>> VirtualRootFileSystemBase::OpenInputFile(const std::string& path)
 {
   return arrow::Status::NotImplemented("No random access file system");
 }
 
-arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenOutputStream(
+arrow::Result<std::shared_ptr<arrow::io::OutputStream>> VirtualRootFileSystemBase::OpenOutputStream(
   const std::string& path,
   const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
 {
   return arrow::Status::NotImplemented("Non streamable filesystem");
 }
 
-arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenAppendStream(
+arrow::Result<std::shared_ptr<arrow::io::OutputStream>> VirtualRootFileSystemBase::OpenAppendStream(
   const std::string& path,
   const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
 {
@@ -173,8 +195,13 @@ arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenApp
 arrow::Result<std::shared_ptr<arrow::Schema>> TTreeFileFormat::Inspect(const arrow::dataset::FileSource& source) const
 {
   arrow::Schema schema{{}};
-  auto fs = std::dynamic_pointer_cast<TTreeFileSystem>(source.filesystem());
-  TTree* tree = fs->GetTree(source);
+  auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
+  // Actually get the TTree from the ROOT file.
+  auto treeFs = std::dynamic_pointer_cast<TTreeFileSystem>(fs->GetSubFilesystem(source));
+  if (!treeFs.get()) {
+    throw runtime_error_f("Unknown filesystem %s\n", source.filesystem()->type_name().c_str());
+  }
+  TTree* tree = treeFs->GetTree(source);
 
   auto branches = tree->GetListOfBranches();
   auto n = branches->GetEntries();
@@ -270,7 +297,9 @@ arrow::Result<arrow::RecordBatchGenerator> TTreeFileFormat::ScanBatchesAsync(
     auto physical_schema = *treeFragment->ReadPhysicalSchema();
 
     static TBufferFile buffer{TBuffer::EMode::kWrite, 4 * 1024 * 1024};
-    auto fs = std::dynamic_pointer_cast<TTreeFileSystem>(treeFragment->source().filesystem());
+    auto containerFS = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(treeFragment->source().filesystem());
+    auto fs = std::dynamic_pointer_cast<TTreeFileSystem>(containerFS->GetSubFilesystem(treeFragment->source()));
+
     int64_t rows = -1;
     TTree* tree = fs->GetTree(treeFragment->source());
     for (auto& field : fields) {
@@ -446,9 +475,42 @@ arrow::Result<arrow::RecordBatchGenerator> TTreeFileFormat::ScanBatchesAsync(
 }
 
 TBufferFileFS::TBufferFileFS(TBufferFile* f)
-  : TTreeFileSystem(),
-    mTree((TTree*)f->ReadObject(TTree::Class()))
+  : VirtualRootFileSystemBase(),
+    mBuffer(f),
+    mFilesystem(nullptr)
+{
+}
+
+TTreeFileSystem::~TTreeFileSystem() = default;
+
+
+arrow::Result<arrow::fs::FileInfo> TBufferFileFS::GetFileInfo(const std::string& path)
+{
+  arrow::fs::FileInfo result;
+  result.set_type(arrow::fs::FileType::NotFound);
+  result.set_path(path);
+  arrow::dataset::FileSource source(path, shared_from_this());
+
+  // Only once to avoid rereading the streamed tree.
+  if (!mFilesystem.get()) {
+    return result;
+  }
+
+  // For now we only support single trees.
+  if (std::dynamic_pointer_cast<SingleTreeFileSystem>(mFilesystem)) {
+    result.set_type(arrow::fs::FileType::File);
+    return result;
+  }
+  return result;
+}
+
+std::shared_ptr<VirtualRootFileSystemBase> TBufferFileFS::GetSubFilesystem(arrow::dataset::FileSource source)
 {
+  if (!mFilesystem.get()) {
+    auto tree = ((TTree*)mBuffer->ReadObject(TTree::Class()));
+    mFilesystem = std::make_shared<SingleTreeFileSystem>(tree);
+  }
+  return mFilesystem;
 }
 
 } // namespace o2::framework
diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx
index 69d39f193c7df..599f1062c63a0 100644
--- a/Framework/Core/test/test_Root2ArrowTable.cxx
+++ b/Framework/Core/test/test_Root2ArrowTable.cxx
@@ -19,6 +19,7 @@
 #include <ROOT/RDataFrame.hxx>
 #include <ROOT/RArrowDS.hxx>
 #include <TBufferFile.h>
+#include <TClass.h>
 #include <TMemFile.h>
 #include <TDirectory.h>
 #include <TTree.h>
@@ -231,6 +232,7 @@ TEST_CASE("RootTree2Fragment")
   size_t totalSizeUncompressed = 0;
   auto format = std::make_shared<TTreeFileFormat>(totalSizeCompressed, totalSizeUncompressed);
   auto fs = std::make_shared<TBufferFileFS>(fileRead);
+
   arrow::dataset::FileSource source("p", fs);
   REQUIRE(format->IsSupported(source) == true);
   auto schemaOpt = format->Inspect(source);

From 0e15090fb13706f9e990c91a183892a877dcbc7c Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 6 Nov 2024 10:37:49 +0100
Subject: [PATCH 683/726] Merger: add ability to configure compression level

---
 Framework/AODMerger/src/aodMerger.cxx   | 13 +++++++++----
 Framework/AODMerger/src/aodStrainer.cxx |  9 +++++++--
 Framework/AODMerger/src/aodThinner.cxx  |  8 +++++++-
 3 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/Framework/AODMerger/src/aodMerger.cxx b/Framework/AODMerger/src/aodMerger.cxx
index feb945e7176f3..aa27d4f617b3b 100644
--- a/Framework/AODMerger/src/aodMerger.cxx
+++ b/Framework/AODMerger/src/aodMerger.cxx
@@ -39,6 +39,7 @@ int main(int argc, char* argv[])
   bool skipParentFilesList = false;
   int verbosity = 2;
   int exitCode = 0; // 0: success, >0: failure
+  int compression = 505;
 
   int option_index = 0;
   static struct option long_options[] = {
@@ -47,8 +48,9 @@ int main(int argc, char* argv[])
     {"max-size", required_argument, nullptr, 2},
     {"skip-non-existing-files", no_argument, nullptr, 3},
     {"skip-parent-files-list", no_argument, nullptr, 4},
-    {"verbosity", required_argument, nullptr, 5},
-    {"help", no_argument, nullptr, 6},
+    {"compression", no_argument, nullptr, 5},
+    {"verbosity", required_argument, nullptr, 'v'},
+    {"help", no_argument, nullptr, 'h'},
     {nullptr, 0, nullptr, 0}};
 
   while (true) {
@@ -66,14 +68,17 @@ int main(int argc, char* argv[])
     } else if (c == 4) {
       skipParentFilesList = true;
     } else if (c == 5) {
+      compression = atoi(optarg);
+    } else if (c == 'v') {
       verbosity = atoi(optarg);
-    } else if (c == 6) {
+    } else if (c == 'h') {
       printf("AO2D merging tool. Options: \n");
       printf("  --input <inputfile.txt>      Contains path to files to be merged. Default: %s\n", inputCollection.c_str());
       printf("  --output <outputfile.root>   Target output ROOT file. Default: %s\n", outputFileName.c_str());
       printf("  --max-size <size in Bytes>   Target directory size. Default: %ld. Set to 0 if file is not self-contained.\n", maxDirSize);
       printf("  --skip-non-existing-files    Flag to allow skipping of non-existing files in the input list.\n");
       printf("  --skip-parent-files-list     Flag to allow skipping the merging of the parent files list.\n");
+      printf("  --compression <root compression id>  Compression algorithm / level to use (default: %d)\n", compression);
       printf("  --verbosity <flag>           Verbosity of output (default: %d).\n", verbosity);
       return -1;
     } else {
@@ -95,7 +100,7 @@ int main(int argc, char* argv[])
   std::map<std::string, int> offsets;
   std::map<std::string, int> unassignedIndexOffset;
 
-  auto outputFile = TFile::Open(outputFileName.c_str(), "RECREATE", "", 505);
+  auto outputFile = TFile::Open(outputFileName.c_str(), "RECREATE", "", compression);
   TDirectory* outputDir = nullptr;
   long currentDirSize = 0;
 
diff --git a/Framework/AODMerger/src/aodStrainer.cxx b/Framework/AODMerger/src/aodStrainer.cxx
index a8c83d23e7361..0ecb8d0d81659 100644
--- a/Framework/AODMerger/src/aodStrainer.cxx
+++ b/Framework/AODMerger/src/aodStrainer.cxx
@@ -39,6 +39,7 @@ int main(int argc, char* argv[])
   double downsampling = 1.0;
   int verbosity = 2;
   int exitCode = 0; // 0: success, >0: failure
+  int compression = 505;
 
   std::random_device rd;  // Seed generator
   std::mt19937 gen(rd()); // Mersenne Twister generator
@@ -51,7 +52,8 @@ int main(int argc, char* argv[])
     {"verbosity", required_argument, nullptr, 2},
     {"tables", required_argument, nullptr, 3},
     {"downsampling", required_argument, nullptr, 4},
-    {"help", no_argument, nullptr, 5},
+    {"compression", required_argument, nullptr, 5},
+    {"help", no_argument, nullptr, 'h'},
     {nullptr, 0, nullptr, 0}};
 
   while (true) {
@@ -69,12 +71,15 @@ int main(int argc, char* argv[])
     } else if (c == 4) {
       downsampling = atof(optarg);
     } else if (c == 5) {
+      compression = atof(optarg);
+    } else if (c == 'h') {
       printf("AO2D strainer tool. Options: \n");
       printf("  --input <%s>      Contains path to files to be merged. Default: %s\n", inputAO2D.c_str(), inputAO2D.c_str());
       printf("  --output <%s>   Target output ROOT file. Default: %s\n", outputFileName.c_str(), outputFileName.c_str());
       printf("  --verbosity <flag>           Verbosity of output (default: %d).\n", verbosity);
       printf("  --tables <list of tables>    Comma separated list of tables (default: %s).\n", tables.c_str());
       printf("  --downsampling <downsample>  Fraction of DF to be kept (default: %f)\n", downsampling);
+      printf("  --compression <root compression id>  Compression algorithm / level to use (default: %d)\n", compression);
       return -1;
     } else {
       return -2;
@@ -95,7 +100,7 @@ int main(int argc, char* argv[])
     listOfTables.push_back(token);
   }
 
-  auto outputFile = TFile::Open(outputFileName.c_str(), "RECREATE", "", 505);
+  auto outputFile = TFile::Open(outputFileName.c_str(), "RECREATE", "", compression);
   TDirectory* outputDir = nullptr;
   TString line(inputAO2D.c_str());
   if (line.BeginsWith("alien://") && !gGrid && !TGrid::Connect("alien:")) {
diff --git a/Framework/AODMerger/src/aodThinner.cxx b/Framework/AODMerger/src/aodThinner.cxx
index 8459167195ea8..f9fb31f9ad08d 100644
--- a/Framework/AODMerger/src/aodThinner.cxx
+++ b/Framework/AODMerger/src/aodThinner.cxx
@@ -42,6 +42,7 @@ int main(int argc, char* argv[])
   std::string outputFileName("AO2D_thinned.root");
   int exitCode = 0; // 0: success, !=0: failure
   bool bOverwrite = false;
+  int compression = 505;
 
   int option_index = 1;
 
@@ -50,6 +51,7 @@ int main(int argc, char* argv[])
     {"input", required_argument, nullptr, 'i'},
     {"output", required_argument, nullptr, 'o'},
     {"overwrite", no_argument, nullptr, 'O'},
+    {"compression", no_argument, nullptr, 'c'},
     {"help", no_argument, nullptr, 'h'},
     {nullptr, 0, nullptr, 0}};
 
@@ -69,12 +71,16 @@ int main(int argc, char* argv[])
         bOverwrite = true;
         printf("Overwriting existing output file if existing\n");
         break;
+      case 'c':
+        compression = atoi(optarg);
+        break;
       case 'h':
       case '?':
       default:
         printf("AO2D thinning tool. Options: \n");
         printf("  --input/-i <inputfile.root>     Contains input file path to the file to be thinned. Default: %s\n", inputFileName.c_str());
         printf("  --output/-o <outputfile.root>   Target output ROOT file. Default: %s\n", outputFileName.c_str());
+        printf("  --compression/-c <compression id>   ROOT compression algorithm / level. Default: %d\n", compression);
         printf("\n");
         printf("  Optional Arguments:\n");
         printf("  --overwrite/-O                  Overwrite existing output file\n");
@@ -89,7 +95,7 @@ int main(int argc, char* argv[])
   TStopwatch clock;
   clock.Start(kTRUE);
 
-  auto outputFile = TFile::Open(outputFileName.c_str(), (bOverwrite) ? "RECREATE" : "CREATE", "", 505);
+  auto outputFile = TFile::Open(outputFileName.c_str(), (bOverwrite) ? "RECREATE" : "CREATE", "", compression);
   if (outputFile == nullptr) {
     printf("Error: File %s exists or cannot be created!\n", outputFileName.c_str());
     return 1;

From c48c1216dd455070b0b3d5cac4be8dfd7f4bc932 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 6 Nov 2024 16:24:28 +0100
Subject: [PATCH 684/726] DPL: allow configuring compression for the AOD writer

---
 .../include/Framework/DataOutputDirector.h    |  2 +-
 Framework/Core/src/AnalysisSupportHelpers.cxx | 18 +++---
 Framework/Core/src/AnalysisSupportHelpers.h   |  2 +-
 Framework/Core/src/ArrowSupport.cxx           |  7 ++-
 Framework/Core/src/ConfigParamDiscovery.cxx   |  1 +
 Framework/Core/src/DataOutputDirector.cxx     |  4 +-
 Framework/Core/src/Plugin.cxx                 | 61 +++++++++++++++++++
 Framework/Core/src/WorkflowHelpers.cxx        |  6 +-
 .../TestWorkflows/src/o2TestHistograms.cxx    | 17 ++++++
 9 files changed, 103 insertions(+), 15 deletions(-)

diff --git a/Framework/Core/include/Framework/DataOutputDirector.h b/Framework/Core/include/Framework/DataOutputDirector.h
index bdcb8faf976c0..e2554c9730ba5 100644
--- a/Framework/Core/include/Framework/DataOutputDirector.h
+++ b/Framework/Core/include/Framework/DataOutputDirector.h
@@ -84,7 +84,7 @@ struct DataOutputDirector {
   std::vector<DataOutputDescriptor*> getDataOutputDescriptors(InputSpec spec);
 
   // get the matching TFile
-  FileAndFolder getFileFolder(DataOutputDescriptor* dodesc, uint64_t folderNumber, std::string parentFileName);
+  FileAndFolder getFileFolder(DataOutputDescriptor* dodesc, uint64_t folderNumber, std::string parentFileName, int compression);
 
   // check file sizes
   bool checkFileSizes();
diff --git a/Framework/Core/src/AnalysisSupportHelpers.cxx b/Framework/Core/src/AnalysisSupportHelpers.cxx
index 3613bfedb887a..e949f27a6eed6 100644
--- a/Framework/Core/src/AnalysisSupportHelpers.cxx
+++ b/Framework/Core/src/AnalysisSupportHelpers.cxx
@@ -318,10 +318,10 @@ DataProcessorSpec AnalysisSupportHelpers::getOutputObjHistSink(std::vector<Outpu
 // add sink for the AODs
 DataProcessorSpec
   AnalysisSupportHelpers::getGlobalAODSink(std::shared_ptr<DataOutputDirector> dod,
-                                           std::vector<InputSpec> const& outputInputs)
+                                           std::vector<InputSpec> const& outputInputs, int compressionLevel)
 {
 
-  auto writerFunction = [dod, outputInputs](InitContext& ic) -> std::function<void(ProcessingContext&)> {
+  auto writerFunction = [dod, outputInputs, compressionLevel](InitContext& ic) -> std::function<void(ProcessingContext&)> {
     LOGP(debug, "======== getGlobalAODSink::Init ==========");
 
     // find out if any table needs to be saved
@@ -363,7 +363,7 @@ DataProcessorSpec
     std::vector<TString> aodMetaDataVals;
 
     // this functor is called once per time frame
-    return [dod, tfNumbers, tfFilenames, aodMetaDataKeys, aodMetaDataVals](ProcessingContext& pc) mutable -> void {
+    return [dod, tfNumbers, tfFilenames, aodMetaDataKeys, aodMetaDataVals, compressionLevel](ProcessingContext& pc) mutable -> void {
       LOGP(debug, "======== getGlobalAODSink::processing ==========");
       LOGP(debug, " processing data set with {} entries", pc.inputs().size());
 
@@ -457,7 +457,7 @@ DataProcessorSpec
         // a table can be saved in multiple ways
         // e.g. different selections of columns to different files
         for (auto d : ds) {
-          auto fileAndFolder = dod->getFileFolder(d, tfNumber, aodInputFile);
+          auto fileAndFolder = dod->getFileFolder(d, tfNumber, aodInputFile, compressionLevel);
           auto treename = fileAndFolder.folderName + "/" + d->treename;
           TableToTree ta2tr(table,
                             fileAndFolder.file,
@@ -495,11 +495,11 @@ DataProcessorSpec
   // the command line options relevant for the writer are global
   // see runDataProcessing.h
   DataProcessorSpec spec{
-    "internal-dpl-aod-writer",
-    outputInputs,
-    Outputs{},
-    AlgorithmSpec(writerFunction),
-    {}};
+    .name = "internal-dpl-aod-writer",
+    .inputs = outputInputs,
+    .outputs = {},
+    .algorithm = AlgorithmSpec{writerFunction},
+  };
 
   return spec;
 }
diff --git a/Framework/Core/src/AnalysisSupportHelpers.h b/Framework/Core/src/AnalysisSupportHelpers.h
index 43ce7ab85b96d..ba5bcedb4bc67 100644
--- a/Framework/Core/src/AnalysisSupportHelpers.h
+++ b/Framework/Core/src/AnalysisSupportHelpers.h
@@ -78,7 +78,7 @@ struct AnalysisSupportHelpers {
                                                 std::vector<OutputTaskInfo> const& tskmap);
   /// writes inputs of kind AOD to file
   static DataProcessorSpec getGlobalAODSink(std::shared_ptr<DataOutputDirector> dod,
-                                            std::vector<InputSpec> const& outputInputs);
+                                            std::vector<InputSpec> const& outputInputs, int compression);
 };
 
 }; // namespace o2::framework
diff --git a/Framework/Core/src/ArrowSupport.cxx b/Framework/Core/src/ArrowSupport.cxx
index 5c1a9050c4e40..1a656e4d60080 100644
--- a/Framework/Core/src/ArrowSupport.cxx
+++ b/Framework/Core/src/ArrowSupport.cxx
@@ -37,6 +37,7 @@
 #include "Headers/DataHeader.h"
 #include "Headers/DataHeaderHelpers.h"
 
+#include <RtypesCore.h>
 #include <fairmq/ProgOptions.h>
 
 #include <uv.h>
@@ -536,7 +537,11 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
         // add TFNumber and TFFilename as input to the writer
         outputsInputsAOD.emplace_back("tfn", "TFN", "TFNumber");
         outputsInputsAOD.emplace_back("tff", "TFF", "TFFilename");
-        workflow.push_back(AnalysisSupportHelpers::getGlobalAODSink(dod, outputsInputsAOD));
+        int compression = 505; 
+        if (ctx.options().hasOption("aod-writer-compression")) {
+          compression = ctx.options().get<int>("aod-writer-compression");
+        }
+        workflow.push_back(AnalysisSupportHelpers::getGlobalAODSink(dod, outputsInputsAOD, compression));
       }
       // Move the dummy sink at the end, if needed
       for (size_t i = 0; i < workflow.size(); ++i) {
diff --git a/Framework/Core/src/ConfigParamDiscovery.cxx b/Framework/Core/src/ConfigParamDiscovery.cxx
index 6c24a44bac566..9673f77ed0e42 100644
--- a/Framework/Core/src/ConfigParamDiscovery.cxx
+++ b/Framework/Core/src/ConfigParamDiscovery.cxx
@@ -27,6 +27,7 @@ std::vector<ConfigParamSpec> ConfigParamDiscovery::discover(ConfigParamRegistry&
   std::vector<char const*> capabilitiesSpecs = {
     "O2Framework:DiscoverMetadataInAODCapability",
     "O2Framework:DiscoverMetadataInCommandLineCapability",
+    "O2Framework:DiscoverAODOptionsInCommandLineCapability",
   };
 
   // Load all the requested plugins and discover what we can do.
diff --git a/Framework/Core/src/DataOutputDirector.cxx b/Framework/Core/src/DataOutputDirector.cxx
index 4b803e1050817..cfee1a4b8e8a9 100644
--- a/Framework/Core/src/DataOutputDirector.cxx
+++ b/Framework/Core/src/DataOutputDirector.cxx
@@ -455,7 +455,7 @@ std::vector<DataOutputDescriptor*> DataOutputDirector::getDataOutputDescriptors(
   return result;
 }
 
-FileAndFolder DataOutputDirector::getFileFolder(DataOutputDescriptor* dodesc, uint64_t folderNumber, std::string parentFileName)
+FileAndFolder DataOutputDirector::getFileFolder(DataOutputDescriptor* dodesc, uint64_t folderNumber, std::string parentFileName, int compression)
 {
   // initialisation
   FileAndFolder fileAndFolder;
@@ -488,7 +488,7 @@ FileAndFolder DataOutputDirector::getFileFolder(DataOutputDescriptor* dodesc, ui
       auto fn = resdirname + "/" + mfilenameBases[ind] + ".root";
       delete mfilePtrs[ind];
       mParentMaps[ind]->Clear();
-      mfilePtrs[ind] = TFile::Open(fn.c_str(), mfileMode.c_str(), "", 505);
+      mfilePtrs[ind] = TFile::Open(fn.c_str(), mfileMode.c_str(), "", compression);
     }
     fileAndFolder.file = mfilePtrs[ind];
 
diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx
index 2edf2e62a3633..a98771a913d01 100644
--- a/Framework/Core/src/Plugin.cxx
+++ b/Framework/Core/src/Plugin.cxx
@@ -15,6 +15,7 @@
 #include "Framework/Capability.h"
 #include "Framework/Signpost.h"
 #include "Framework/VariantJSONHelpers.h"
+#include <cstddef>
 #include <string_view>
 
 O2_DECLARE_DYNAMIC_LOG(capabilities);
@@ -47,6 +48,19 @@ auto lookForCommandLineOptions = [](ConfigParamRegistry& registry, int argc, cha
   return false;
 };
 
+auto lookForCommandLineAODOptions = [](ConfigParamRegistry& registry, int argc, char** argv) -> bool {
+  O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+  // If one of the options for aod-writer is specified, we should allow configuring compression.
+  for (size_t i = 0; i < argc; i++) {
+    std::string_view arg = argv[i];
+    if (arg.starts_with("--aod-writer-")) {
+      O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverAODOptionsInCommandLineCapability", "AOD options found in arguments. Populating from them.");
+      return true;
+    }
+  }
+  return false;
+};
+
 struct DiscoverMetadataInAODCapability : o2::framework::CapabilityPlugin {
   Capability* create() override
   {
@@ -68,6 +82,16 @@ struct DiscoverMetadataInCommandLineCapability : o2::framework::CapabilityPlugin
   }
 };
 
+struct DiscoverAODOptionsInCommandLineCapability : o2::framework::CapabilityPlugin {
+  Capability* create() override
+  {
+    return new Capability{
+      .name = "DiscoverAODOptionsInCommandLineCapability",
+      .checkIfNeeded = lookForCommandLineAODOptions,
+      .requiredPlugin = "O2Framework:DiscoverAODOptionsInCommandLine"};
+  }
+};
+
 struct DiscoverMetadataInCommandLine : o2::framework::ConfigDiscoveryPlugin {
   ConfigDiscovery* create() override
   {
@@ -99,9 +123,46 @@ struct DiscoverMetadataInCommandLine : o2::framework::ConfigDiscoveryPlugin {
       }};
   }
 };
+
+struct DiscoverAODOptionsInCommandLine : o2::framework::ConfigDiscoveryPlugin {
+  ConfigDiscovery* create() override
+  {
+    return new ConfigDiscovery{
+      .init = []() {},
+      .discover = [](ConfigParamRegistry& registry, int argc, char** argv) -> std::vector<ConfigParamSpec> {
+        O2_SIGNPOST_ID_GENERATE(sid, capabilities);
+        O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverAODOptionsInCommandLine",
+                               "Discovering AOD handling related options in commandline arguments.");
+        std::vector<ConfigParamSpec> results;
+        bool injectOption = true;
+        for (size_t i = 0; i < argc; i++) {
+          std::string_view arg = argv[i];
+          if (!arg.starts_with("--aod-writer-")) {
+            continue;
+          }
+          std::string key = arg.data() + 2;
+          std::string value = argv[i + 1];
+          O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverAODOptionsInCommandLine",
+                                 "Found %{public}s with value %{public}s.", key.c_str(), value.c_str());
+          if (key == "aod-writer-compression") {
+            int numericValue = std::stoi(value);
+            results.push_back(ConfigParamSpec{"aod-writer-compression", VariantType::Int, numericValue, {"AOD Compression options"}});
+            injectOption = false;
+          }
+        }
+        if (injectOption) {
+          results.push_back(ConfigParamSpec{"aod-writer-compression", VariantType::Int, 505, {"AOD Compression options"}});
+        }
+        return results;
+      }};
+  }
+};
+
 DEFINE_DPL_PLUGINS_BEGIN
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAODCapability, Capability);
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLineCapability, Capability);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverAODOptionsInCommandLineCapability, Capability);
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLine, ConfigDiscovery);
+DEFINE_DPL_PLUGIN_INSTANCE(DiscoverAODOptionsInCommandLine, ConfigDiscovery);
 DEFINE_DPL_PLUGINS_END
 } // namespace o2::framework
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 6349bd5889eba..3fe8fae19a3b5 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -606,7 +606,11 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
     // add TFNumber and TFFilename as input to the writer
     outputsInputsAOD.emplace_back(InputSpec{"tfn", "TFN", "TFNumber"});
     outputsInputsAOD.emplace_back(InputSpec{"tff", "TFF", "TFFilename"});
-    auto fileSink = AnalysisSupportHelpers::getGlobalAODSink(dod, outputsInputsAOD);
+    int compressionLevel = 505;
+    if (ctx.options().hasOption("aod-writer-compression")) {
+      compressionLevel = ctx.options().get<int>("aod-writer-compression");
+    }
+    auto fileSink = AnalysisSupportHelpers::getGlobalAODSink(dod, outputsInputsAOD, compressionLevel);
     extraSpecs.push_back(fileSink);
 
     auto it = std::find_if(outputsInputs.begin(), outputsInputs.end(), [](InputSpec& spec) -> bool {
diff --git a/Framework/TestWorkflows/src/o2TestHistograms.cxx b/Framework/TestWorkflows/src/o2TestHistograms.cxx
index e32149a8bfdc1..9986f52a1d940 100644
--- a/Framework/TestWorkflows/src/o2TestHistograms.cxx
+++ b/Framework/TestWorkflows/src/o2TestHistograms.cxx
@@ -22,26 +22,43 @@ using namespace o2;
 using namespace o2::framework;
 using namespace o2::framework::expressions;
 
+namespace o2::aod
+{
+namespace skimmedExampleTrack
+{
+DECLARE_SOA_COLUMN(Pt, pt, float);   //!
+DECLARE_SOA_COLUMN(Eta, eta, float); //!
+} // namespace skimmedExampleTrack
+
+DECLARE_SOA_TABLE(SkimmedExampleTrack, "AOD", "SKIMEXTRK", //!
+                  skimmedExampleTrack::Pt,
+                  skimmedExampleTrack::Eta);
+} // namespace o2::aod
+
 struct EtaAndClsHistogramsSimple {
   OutputObj<TH2F> etaClsH{TH2F("eta_vs_pt", "#eta vs pT", 102, -2.01, 2.01, 100, 0, 10)};
+  Produces<o2::aod::SkimmedExampleTrack> skimEx;
 
   void process(aod::Tracks const& tracks)
   {
     LOGP(info, "Invoking the simple one");
     for (auto& track : tracks) {
       etaClsH->Fill(track.eta(), track.pt(), 0);
+      skimEx(track.pt(), track.eta());
     }
   }
 };
 
 struct EtaAndClsHistogramsIUSimple {
   OutputObj<TH2F> etaClsH{TH2F("eta_vs_pt", "#eta vs pT", 102, -2.01, 2.01, 100, 0, 10)};
+  Produces<o2::aod::SkimmedExampleTrack> skimEx;
 
   void process(aod::TracksIU const& tracks)
   {
     LOGP(info, "Invoking the simple one");
     for (auto& track : tracks) {
       etaClsH->Fill(track.eta(), track.pt(), 0);
+      skimEx(track.pt(), track.eta());
     }
   }
 };

From f422114b5c49f1cf3c11e0fe6d1b9377e691fea2 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 7 Nov 2024 09:36:22 +0100
Subject: [PATCH 685/726] DPL: do not leak metadata file descriptor

---
 Framework/AnalysisSupport/src/Plugin.cxx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index baa1b36dd41a5..32a86d37aebb9 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -142,6 +142,7 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
         if (tables.empty() == false) {
           results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
         }
+        currentFile->Close();
         return results;
       }};
   }

From b91adae080d99d240c2f363cb7cf3b9c469f88b3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 7 Nov 2024 13:01:27 +0100
Subject: [PATCH 686/726] DPL: read metadata from parent files

If the metadata is not found in the main file and if there is a
list of parent files, try those as well.
---
 Framework/AnalysisSupport/src/Plugin.cxx | 103 +++++++++++++++++------
 1 file changed, 77 insertions(+), 26 deletions(-)

diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index 32a86d37aebb9..4f52c95a2a06d 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -22,6 +22,7 @@
 #include <TObjString.h>
 #include <TString.h>
 #include <fmt/format.h>
+#include <memory>
 
 O2_DECLARE_DYNAMIC_LOG(analysis_support);
 
@@ -65,7 +66,7 @@ struct RunSummary : o2::framework::ServicePlugin {
   }
 };
 
-std::vector<std::string> getListOfTables(TFile* f)
+std::vector<std::string> getListOfTables(std::unique_ptr<TFile>& f)
 {
   std::vector<std::string> r;
   TList* keyList = f->GetListOfKeys();
@@ -83,6 +84,32 @@ std::vector<std::string> getListOfTables(TFile* f)
   }
   return r;
 }
+auto readMetadata(std::unique_ptr<TFile>& currentFile) -> std::vector<ConfigParamSpec>
+{
+  // Get the metadata, if any
+  auto m = (TMap*)currentFile->Get("metaData");
+  if (!m) {
+    return {};
+  }
+  std::vector<ConfigParamSpec> results;
+  auto it = m->MakeIterator();
+
+  // Serialise metadata into a ; separated string with : separating key and value
+  bool first = true;
+  while (auto obj = it->Next()) {
+    if (first) {
+      LOGP(info, "Metadata for file \"{}\":", currentFile->GetName());
+      first = false;
+    }
+    auto objString = (TObjString*)m->GetValue(obj);
+    LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
+    std::string key = "aod-metadata-" + std::string(obj->GetName());
+    char const* value = strdup(objString->String());
+    results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
+  }
+
+  return results;
+}
 
 struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
   ConfigDiscovery* create() override
@@ -90,12 +117,11 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
     return new ConfigDiscovery{
       .init = []() {},
       .discover = [](ConfigParamRegistry& registry, int argc, char** argv) -> std::vector<ConfigParamSpec> {
+        std::string parentFileReplacement;
         auto filename = registry.get<std::string>("aod-file");
         if (filename.empty()) {
           return {};
         }
-        std::vector<ConfigParamSpec> results;
-        TFile* currentFile = nullptr;
         if (filename.at(0) == '@') {
           filename.erase(0, 1);
           // read the text file and set filename to the contents of the first line
@@ -110,39 +136,64 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
           TGrid::Connect("alien://");
         }
         LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
-        currentFile = TFile::Open(filename.c_str());
-        if (!currentFile) {
+        std::unique_ptr<TFile> currentFile{TFile::Open(filename.c_str())};
+        if (currentFile.get() == nullptr) {
           LOGP(fatal, "Couldn't open file \"{}\"!", filename);
         }
+        std::vector<ConfigParamSpec> results = readMetadata(currentFile);
+        // Found metadata already in the main file.
+        if (!results.empty()) {
+          auto tables = getListOfTables(currentFile);
+          if (tables.empty() == false) {
+            results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
+          }
+          results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
+          return results;
+        }
 
-        // Get the metadata, if any
-        auto m = (TMap*)currentFile->Get("metaData");
-        if (!m) {
+        // Lets try in parent files
+        auto parentFiles = (TMap*)currentFile->Get("parentFiles");
+        if (!parentFiles) {
           LOGP(info, "No metadata found in file \"{}\"", filename);
           results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
           return results;
         }
-        auto it = m->MakeIterator();
-
-        // Serialise metadata into a ; separated string with : separating key and value
-        bool first = true;
-        while (auto obj = it->Next()) {
-          if (first) {
-            LOGP(info, "Metadata for file \"{}\":", filename);
-            first = false;
+        for (auto* p : *parentFiles) {
+          std::string parentFilename = ((TPair*)p)->Value()->GetName();
+          // Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be
+          // a workflow option (because the metadata itself is potentially changing the topology).
+          if (registry.isSet("aod-parent-base-path-replacement")) {
+            parentFileReplacement = registry.get<std::string>("aod-parent-base-path-replacement");
+            auto pos = parentFileReplacement.find(';');
+            if (pos == std::string::npos) {
+              throw std::runtime_error(fmt::format("Invalid syntax in aod-parent-base-path-replacement: \"{}\"", parentFileReplacement.c_str()));
+            }
+            auto from = parentFileReplacement.substr(0, pos);
+            auto to = parentFileReplacement.substr(pos + 1);
+            pos = parentFilename.find(from);
+            if (pos != std::string::npos) {
+              parentFilename.replace(pos, from.length(), to);
+            }
           }
-          auto objString = (TObjString*)m->GetValue(obj);
-          LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
-          std::string key = "aod-metadata-" + std::string(obj->GetName());
-          char const* value = strdup(objString->String());
-          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
-        }
 
-        auto tables = getListOfTables(currentFile);
-        if (tables.empty() == false) {
-          results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
+          std::unique_ptr<TFile> parentFile{TFile::Open(parentFilename.c_str())};
+          if (parentFile.get() == nullptr) {
+            LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename);
+          }
+          results = readMetadata(parentFile);
+          // Found metadata already in the main file.
+          if (!results.empty()) {
+            auto tables = getListOfTables(parentFile);
+            if (tables.empty() == false) {
+              results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
+            }
+            results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
+            return results;
+          }
+          LOGP(info, "No metadata found in file \"{}\" nor in its parent file \"{}\"", filename, parentFilename);
+          break;
         }
-        currentFile->Close();
+        results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
         return results;
       }};
   }

From e2534853cd1b7d9dcaf0f5b33e6140f226761fe3 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 7 Nov 2024 13:21:57 +0100
Subject: [PATCH 687/726] DPL: read metadata from parent files

If the metadata is not found in the main file and if there is a
list of parent files, try those as well.
---
 Framework/AnalysisSupport/src/Plugin.cxx | 102 +++++++++++++++++------
 1 file changed, 76 insertions(+), 26 deletions(-)

diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index 32a86d37aebb9..ee1ed3c597ba0 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -22,6 +22,7 @@
 #include <TObjString.h>
 #include <TString.h>
 #include <fmt/format.h>
+#include <memory>
 
 O2_DECLARE_DYNAMIC_LOG(analysis_support);
 
@@ -65,7 +66,7 @@ struct RunSummary : o2::framework::ServicePlugin {
   }
 };
 
-std::vector<std::string> getListOfTables(TFile* f)
+std::vector<std::string> getListOfTables(std::unique_ptr<TFile>& f)
 {
   std::vector<std::string> r;
   TList* keyList = f->GetListOfKeys();
@@ -83,6 +84,32 @@ std::vector<std::string> getListOfTables(TFile* f)
   }
   return r;
 }
+auto readMetadata(std::unique_ptr<TFile>& currentFile) -> std::vector<ConfigParamSpec>
+{
+  // Get the metadata, if any
+  auto m = (TMap*)currentFile->Get("metaData");
+  if (!m) {
+    return {};
+  }
+  std::vector<ConfigParamSpec> results;
+  auto it = m->MakeIterator();
+
+  // Serialise metadata into a ; separated string with : separating key and value
+  bool first = true;
+  while (auto obj = it->Next()) {
+    if (first) {
+      LOGP(info, "Metadata for file \"{}\":", currentFile->GetName());
+      first = false;
+    }
+    auto objString = (TObjString*)m->GetValue(obj);
+    LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
+    std::string key = "aod-metadata-" + std::string(obj->GetName());
+    char const* value = strdup(objString->String());
+    results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
+  }
+
+  return results;
+}
 
 struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
   ConfigDiscovery* create() override
@@ -94,8 +121,6 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
         if (filename.empty()) {
           return {};
         }
-        std::vector<ConfigParamSpec> results;
-        TFile* currentFile = nullptr;
         if (filename.at(0) == '@') {
           filename.erase(0, 1);
           // read the text file and set filename to the contents of the first line
@@ -110,39 +135,64 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
           TGrid::Connect("alien://");
         }
         LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
-        currentFile = TFile::Open(filename.c_str());
-        if (!currentFile) {
+        std::unique_ptr<TFile> currentFile{TFile::Open(filename.c_str())};
+        if (currentFile.get() == nullptr) {
           LOGP(fatal, "Couldn't open file \"{}\"!", filename);
         }
+        std::vector<ConfigParamSpec> results = readMetadata(currentFile);
+        // Found metadata already in the main file.
+        if (!results.empty()) {
+          auto tables = getListOfTables(currentFile);
+          if (tables.empty() == false) {
+            results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
+          }
+          results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
+          return results;
+        }
 
-        // Get the metadata, if any
-        auto m = (TMap*)currentFile->Get("metaData");
-        if (!m) {
+        // Lets try in parent files
+        auto parentFiles = (TMap*)currentFile->Get("parentFiles");
+        if (!parentFiles) {
           LOGP(info, "No metadata found in file \"{}\"", filename);
           results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
           return results;
         }
-        auto it = m->MakeIterator();
-
-        // Serialise metadata into a ; separated string with : separating key and value
-        bool first = true;
-        while (auto obj = it->Next()) {
-          if (first) {
-            LOGP(info, "Metadata for file \"{}\":", filename);
-            first = false;
+        for (auto* p : *parentFiles) {
+          std::string parentFilename = ((TPair*)p)->Value()->GetName();
+          // Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be
+          // a workflow option (because the metadata itself is potentially changing the topology).
+          if (registry.isSet("aod-parent-base-path-replacement")) {
+            auto parentFileReplacement = registry.get<std::string>("aod-parent-base-path-replacement");
+            auto pos = parentFileReplacement.find(';');
+            if (pos == std::string::npos) {
+              throw std::runtime_error(fmt::format("Invalid syntax in aod-parent-base-path-replacement: \"{}\"", parentFileReplacement.c_str()));
+            }
+            auto from = parentFileReplacement.substr(0, pos);
+            auto to = parentFileReplacement.substr(pos + 1);
+            pos = parentFilename.find(from);
+            if (pos != std::string::npos) {
+              parentFilename.replace(pos, from.length(), to);
+            }
           }
-          auto objString = (TObjString*)m->GetValue(obj);
-          LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
-          std::string key = "aod-metadata-" + std::string(obj->GetName());
-          char const* value = strdup(objString->String());
-          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
-        }
 
-        auto tables = getListOfTables(currentFile);
-        if (tables.empty() == false) {
-          results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
+          std::unique_ptr<TFile> parentFile{TFile::Open(parentFilename.c_str())};
+          if (parentFile.get() == nullptr) {
+            LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename);
+          }
+          results = readMetadata(parentFile);
+          // Found metadata already in the main file.
+          if (!results.empty()) {
+            auto tables = getListOfTables(parentFile);
+            if (tables.empty() == false) {
+              results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
+            }
+            results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
+            return results;
+          }
+          LOGP(info, "No metadata found in file \"{}\" nor in its parent file \"{}\"", filename, parentFilename);
+          break;
         }
-        currentFile->Close();
+        results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
         return results;
       }};
   }

From a55f48275dd78f0453cf8c0b168928228375ae95 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 7 Nov 2024 15:40:41 +0100
Subject: [PATCH 688/726] DPL: pass ConfigContext to the PluginManager when
 creating AlgorithmSpec

This way a plugin can create more complex AlgorithmSpecs which depend on
the workflow options. This will be needed to properly read metadata from
parent files, and it opens the way to more service devices to be moved
in a plugin.
---
 Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx | 3 ++-
 Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h   | 2 +-
 Framework/AnalysisSupport/src/Plugin.cxx                 | 4 ++--
 Framework/CCDBSupport/src/Plugin.cxx                     | 2 +-
 Framework/Core/include/Framework/AlgorithmSpec.h         | 2 +-
 Framework/Core/include/Framework/PluginManager.h         | 5 +++--
 Framework/Core/src/PluginManager.cxx                     | 6 +++---
 Framework/Core/src/WorkflowHelpers.cxx                   | 4 ++--
 8 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index a8b708668dae1..016ed4f1df1ef 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -22,6 +22,7 @@
 #include "Framework/DeviceSpec.h"
 #include "Framework/RawDeviceService.h"
 #include "Framework/DataSpecUtils.h"
+#include "Framework/ConfigContext.h"
 #include "DataInputDirector.h"
 #include "Framework/SourceInfoHeader.h"
 #include "Framework/ChannelInfo.h"
@@ -117,7 +118,7 @@ static inline auto extractOriginalsTuple(framework::pack<Os...>, ProcessingConte
   return std::make_tuple(extractTypedOriginal<Os>(pc)...);
 }
 
-AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback()
+AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const& config)
 {
   auto callback = AlgorithmSpec{adaptStateful([](ConfigParamRegistry const& options,
                                                  DeviceSpec const& spec,
diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
index 4b9fd710aca14..e8d663d8fe0bb 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.h
@@ -24,7 +24,7 @@ namespace o2::framework::readers
 {
 
 struct AODJAlienReaderHelpers {
-  static AlgorithmSpec rootFileReaderCallback();
+  static AlgorithmSpec rootFileReaderCallback(ConfigContext const&context);
   static void dumpFileMetrics(o2::monitoring::Monitoring& monitoring, TFile* currentFile, uint64_t startedAt, uint64_t ioTime, int tfPerFile, int tfRead);
 };
 
diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index 32a86d37aebb9..9ab4dfa0a2a9f 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -26,9 +26,9 @@
 O2_DECLARE_DYNAMIC_LOG(analysis_support);
 
 struct ROOTFileReader : o2::framework::AlgorithmPlugin {
-  o2::framework::AlgorithmSpec create() override
+  o2::framework::AlgorithmSpec create(o2::framework::ConfigContext const& config) override
   {
-    return o2::framework::readers::AODJAlienReaderHelpers::rootFileReaderCallback();
+    return o2::framework::readers::AODJAlienReaderHelpers::rootFileReaderCallback(config);
   }
 };
 
diff --git a/Framework/CCDBSupport/src/Plugin.cxx b/Framework/CCDBSupport/src/Plugin.cxx
index 8769511f4849a..18aabc07ae4a4 100644
--- a/Framework/CCDBSupport/src/Plugin.cxx
+++ b/Framework/CCDBSupport/src/Plugin.cxx
@@ -13,7 +13,7 @@
 #include "CCDBHelpers.h"
 
 struct CCDBFetcherPlugin : o2::framework::AlgorithmPlugin {
-  o2::framework::AlgorithmSpec create() final
+  o2::framework::AlgorithmSpec create(o2::framework::ConfigContext const&) final
   {
     return o2::framework::CCDBHelpers::fetchFromCCDB();
   }
diff --git a/Framework/Core/include/Framework/AlgorithmSpec.h b/Framework/Core/include/Framework/AlgorithmSpec.h
index e08d829e489bd..7d56ba9f6ce68 100644
--- a/Framework/Core/include/Framework/AlgorithmSpec.h
+++ b/Framework/Core/include/Framework/AlgorithmSpec.h
@@ -81,7 +81,7 @@ struct AlgorithmSpec {
 
 /// Helper class for an algorithm which is loaded as a plugin.
 struct AlgorithmPlugin {
-  virtual AlgorithmSpec create() = 0;
+  virtual AlgorithmSpec create(ConfigContext const&) = 0;
 };
 // Allow fetching inputs from the context using a string literal.
 template <StringLiteral lit, typename T>
diff --git a/Framework/Core/include/Framework/PluginManager.h b/Framework/Core/include/Framework/PluginManager.h
index 4c6e965502500..d6b16f01ad713 100644
--- a/Framework/Core/include/Framework/PluginManager.h
+++ b/Framework/Core/include/Framework/PluginManager.h
@@ -51,8 +51,9 @@ struct PluginManager {
   /// the DPLPluginHandle provided by the library.
   static void load(std::vector<PluginInfo>& infos, const char* dso, std::function<void(DPLPluginHandle*)>& onSuccess);
   /// Load an called @plugin from a library called @a library and
-  /// return the associtated AlgorithmSpec.
-  static auto loadAlgorithmFromPlugin(std::string library, std::string plugin) -> AlgorithmSpec;
+  /// @return the associated AlgorithmSpec.
+  /// The config @a context can be used to determine the workflow options which affect such plugin.
+  static auto loadAlgorithmFromPlugin(std::string library, std::string plugin, ConfigContext const& context) -> AlgorithmSpec;
   /// Wrap an algorithm with some lambda @wrapper which will be called
   /// with the original callback and the ProcessingContext.
   static auto wrapAlgorithm(AlgorithmSpec const& spec, WrapperProcessCallback&& wrapper) -> AlgorithmSpec;
diff --git a/Framework/Core/src/PluginManager.cxx b/Framework/Core/src/PluginManager.cxx
index 96666722fc169..9faea85ad65e7 100644
--- a/Framework/Core/src/PluginManager.cxx
+++ b/Framework/Core/src/PluginManager.cxx
@@ -101,10 +101,10 @@ void PluginManager::load(std::vector<PluginInfo>& libs, const char* dso, std::fu
   onSuccess(pluginInstance);
 }
 
-auto PluginManager::loadAlgorithmFromPlugin(std::string library, std::string plugin) -> AlgorithmSpec
+auto PluginManager::loadAlgorithmFromPlugin(std::string library, std::string plugin, ConfigContext const& context) -> AlgorithmSpec
 {
   std::shared_ptr<AlgorithmSpec> algorithm{nullptr};
-  return AlgorithmSpec{[algorithm, library, plugin](InitContext& ic) mutable -> AlgorithmSpec::ProcessCallback {
+  return AlgorithmSpec{[algorithm, library, plugin, &context](InitContext& ic) mutable -> AlgorithmSpec::ProcessCallback {
     if (algorithm.get()) {
       return algorithm->onInit(ic);
     }
@@ -134,7 +134,7 @@ auto PluginManager::loadAlgorithmFromPlugin(std::string library, std::string plu
     if (!creator) {
       LOGP(fatal, "Could not find the {} plugin in {}.", plugin, libName);
     }
-    algorithm = std::make_shared<AlgorithmSpec>(creator->create());
+    algorithm = std::make_shared<AlgorithmSpec>(creator->create(context));
     return algorithm->onInit(ic);
   }};
 };
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 3fe8fae19a3b5..0366e39cf8976 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -432,7 +432,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
     auto mctracks2aod = std::find_if(workflow.begin(), workflow.end(), [](auto const& x) { return x.name == "mctracks-to-aod"; });
     if (mctracks2aod == workflow.end()) {
       // add normal reader
-      auto&& algo = PluginManager::loadAlgorithmFromPlugin("O2FrameworkAnalysisSupport", "ROOTFileReader");
+      auto&& algo = PluginManager::loadAlgorithmFromPlugin("O2FrameworkAnalysisSupport", "ROOTFileReader", ctx);
       if (internalRateLimiting) {
         aodReader.algorithm = CommonDataProcessors::wrapWithRateLimiting(algo);
       } else {
@@ -520,7 +520,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
     }
 
     // Load the CCDB backend from the plugin
-    ccdbBackend.algorithm = PluginManager::loadAlgorithmFromPlugin("O2FrameworkCCDBSupport", "CCDBFetcherPlugin");
+    ccdbBackend.algorithm = PluginManager::loadAlgorithmFromPlugin("O2FrameworkCCDBSupport", "CCDBFetcherPlugin", ctx);
     extraSpecs.push_back(ccdbBackend);
   } else {
     // If there is no CCDB requested, but we still ask for a FLP/DISTSUBTIMEFRAME/0xccdb

From 2d015adce39c98bbf385f98564a0144fcdf5415f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 7 Nov 2024 15:53:03 +0100
Subject: [PATCH 689/726] DPL: allow plugins to know about discoveries of other
 plugins

---
 Framework/Core/include/Framework/runDataProcessing.h | 1 -
 Framework/Core/src/ConfigParamDiscovery.cxx          | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/include/Framework/runDataProcessing.h b/Framework/Core/include/Framework/runDataProcessing.h
index fbf2843d4db01..eee4c4b6583d3 100644
--- a/Framework/Core/include/Framework/runDataProcessing.h
+++ b/Framework/Core/include/Framework/runDataProcessing.h
@@ -197,7 +197,6 @@ int mainNoCatch(int argc, char** argv)
   for (auto& extra : extraOptions) {
     workflowOptions.push_back(extra);
   }
-  workflowOptionsRegistry.loadExtra(extraOptions);
 
   ConfigContext configContext(workflowOptionsRegistry, argc, argv);
   o2::framework::WorkflowSpec specs = defineDataProcessing(configContext);
diff --git a/Framework/Core/src/ConfigParamDiscovery.cxx b/Framework/Core/src/ConfigParamDiscovery.cxx
index 9673f77ed0e42..63c38b7f0ac6c 100644
--- a/Framework/Core/src/ConfigParamDiscovery.cxx
+++ b/Framework/Core/src/ConfigParamDiscovery.cxx
@@ -75,6 +75,7 @@ std::vector<ConfigParamSpec> ConfigParamDiscovery::discover(ConfigParamRegistry&
     for (auto& extra : extras) {
       result.push_back(extra);
     }
+    registry.loadExtra(extras);
   }
   return result;
 }

From ed328b50751686876d2c0b320ded2115fa20b6e9 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 7 Nov 2024 15:53:21 +0100
Subject: [PATCH 690/726] DPL: read metadata from parent files

If the metadata is not found in the main file and if there is a
list of parent files, try those as well.
---
 .../src/AODJAlienReaderHelpers.cxx            |  20 ++--
 Framework/AnalysisSupport/src/Plugin.cxx      | 102 +++++++++++++-----
 Framework/Core/src/ConfigParamDiscovery.cxx   |   2 +-
 Framework/Core/src/Plugin.cxx                 |   9 +-
 Framework/Core/src/WorkflowHelpers.cxx        |   1 -
 5 files changed, 96 insertions(+), 38 deletions(-)

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 016ed4f1df1ef..90d88cb43626e 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -120,10 +120,17 @@ static inline auto extractOriginalsTuple(framework::pack<Os...>, ProcessingConte
 
 AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const& config)
 {
-  auto callback = AlgorithmSpec{adaptStateful([](ConfigParamRegistry const& options,
-                                                 DeviceSpec const& spec,
-                                                 Monitoring& monitoring,
-                                                 DataProcessingStats& stats) {
+  // aod-parent-base-path-replacement is now a workflow option, so it needs to be
+  // retrieved from the ConfigContext. This is because we do not allow workflow options
+  // to change over start-stop-start because they can affect the topology generation.
+  std::string parentFileReplacement;
+  if (config.options().isSet("aod-parent-base-path-replacement")) {
+    parentFileReplacement = config.options().get<std::string>("aod-parent-base-path-replacement");
+  }
+  auto callback = AlgorithmSpec{adaptStateful([parentFileReplacement](ConfigParamRegistry const& options,
+                                                                      DeviceSpec const& spec,
+                                                                      Monitoring& monitoring,
+                                                                      DataProcessingStats& stats) {
     // FIXME: not actually needed, since data processing stats can specify that we should
     // send the initial value.
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_BYTES_CREATED), DataProcessingStats::Op::Set, 0});
@@ -141,11 +148,6 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const
 
     auto maxRate = options.get<float>("aod-max-io-rate");
 
-    std::string parentFileReplacement;
-    if (options.isSet("aod-parent-base-path-replacement")) {
-      parentFileReplacement = options.get<std::string>("aod-parent-base-path-replacement");
-    }
-
     int parentAccessLevel = 0;
     if (options.isSet("aod-parent-access-level")) {
       parentAccessLevel = options.get<int>("aod-parent-access-level");
diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index 9ab4dfa0a2a9f..b899a52206422 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -22,6 +22,7 @@
 #include <TObjString.h>
 #include <TString.h>
 #include <fmt/format.h>
+#include <memory>
 
 O2_DECLARE_DYNAMIC_LOG(analysis_support);
 
@@ -65,7 +66,7 @@ struct RunSummary : o2::framework::ServicePlugin {
   }
 };
 
-std::vector<std::string> getListOfTables(TFile* f)
+std::vector<std::string> getListOfTables(std::unique_ptr<TFile>& f)
 {
   std::vector<std::string> r;
   TList* keyList = f->GetListOfKeys();
@@ -83,6 +84,32 @@ std::vector<std::string> getListOfTables(TFile* f)
   }
   return r;
 }
+auto readMetadata(std::unique_ptr<TFile>& currentFile) -> std::vector<ConfigParamSpec>
+{
+  // Get the metadata, if any
+  auto m = (TMap*)currentFile->Get("metaData");
+  if (!m) {
+    return {};
+  }
+  std::vector<ConfigParamSpec> results;
+  auto it = m->MakeIterator();
+
+  // Serialise metadata into a ; separated string with : separating key and value
+  bool first = true;
+  while (auto obj = it->Next()) {
+    if (first) {
+      LOGP(info, "Metadata for file \"{}\":", currentFile->GetName());
+      first = false;
+    }
+    auto objString = (TObjString*)m->GetValue(obj);
+    LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
+    std::string key = "aod-metadata-" + std::string(obj->GetName());
+    char const* value = strdup(objString->String());
+    results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
+  }
+
+  return results;
+}
 
 struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
   ConfigDiscovery* create() override
@@ -94,8 +121,6 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
         if (filename.empty()) {
           return {};
         }
-        std::vector<ConfigParamSpec> results;
-        TFile* currentFile = nullptr;
         if (filename.at(0) == '@') {
           filename.erase(0, 1);
           // read the text file and set filename to the contents of the first line
@@ -110,39 +135,64 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
           TGrid::Connect("alien://");
         }
         LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
-        currentFile = TFile::Open(filename.c_str());
-        if (!currentFile) {
+        std::unique_ptr<TFile> currentFile{TFile::Open(filename.c_str())};
+        if (currentFile.get() == nullptr) {
           LOGP(fatal, "Couldn't open file \"{}\"!", filename);
         }
+        std::vector<ConfigParamSpec> results = readMetadata(currentFile);
+        // Found metadata already in the main file.
+        if (!results.empty()) {
+          auto tables = getListOfTables(currentFile);
+          if (tables.empty() == false) {
+            results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
+          }
+          results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
+          return results;
+        }
 
-        // Get the metadata, if any
-        auto m = (TMap*)currentFile->Get("metaData");
-        if (!m) {
+        // Lets try in parent files
+        auto parentFiles = (TMap*)currentFile->Get("parentFiles");
+        if (!parentFiles) {
           LOGP(info, "No metadata found in file \"{}\"", filename);
           results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
           return results;
         }
-        auto it = m->MakeIterator();
-
-        // Serialise metadata into a ; separated string with : separating key and value
-        bool first = true;
-        while (auto obj = it->Next()) {
-          if (first) {
-            LOGP(info, "Metadata for file \"{}\":", filename);
-            first = false;
+        for (auto* p : *parentFiles) {
+          std::string parentFilename = ((TPair*)p)->Value()->GetName();
+          // Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be
+          // a workflow option (because the metadata itself is potentially changing the topology).
+          if (registry.isSet("aod-parent-base-path-replacement")) {
+            auto parentFileReplacement = registry.get<std::string>("aod-parent-base-path-replacement");
+            auto pos = parentFileReplacement.find(';');
+            if (pos == std::string::npos) {
+              throw std::runtime_error(fmt::format("Invalid syntax in aod-parent-base-path-replacement: \"{}\"", parentFileReplacement.c_str()));
+            }
+            auto from = parentFileReplacement.substr(0, pos);
+            auto to = parentFileReplacement.substr(pos + 1);
+            pos = parentFilename.find(from);
+            if (pos != std::string::npos) {
+              parentFilename.replace(pos, from.length(), to);
+            }
           }
-          auto objString = (TObjString*)m->GetValue(obj);
-          LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
-          std::string key = "aod-metadata-" + std::string(obj->GetName());
-          char const* value = strdup(objString->String());
-          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
-        }
 
-        auto tables = getListOfTables(currentFile);
-        if (tables.empty() == false) {
-          results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
+          std::unique_ptr<TFile> parentFile{TFile::Open(parentFilename.c_str())};
+          if (parentFile.get() == nullptr) {
+            LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename);
+          }
+          results = readMetadata(parentFile);
+          // Found metadata already in the main file.
+          if (!results.empty()) {
+            auto tables = getListOfTables(parentFile);
+            if (tables.empty() == false) {
+              results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
+            }
+            results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
+            return results;
+          }
+          LOGP(info, "No metadata found in file \"{}\" nor in its parent file \"{}\"", filename, parentFilename);
+          break;
         }
-        currentFile->Close();
+        results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
         return results;
       }};
   }
diff --git a/Framework/Core/src/ConfigParamDiscovery.cxx b/Framework/Core/src/ConfigParamDiscovery.cxx
index 63c38b7f0ac6c..fc8d6f2600bb4 100644
--- a/Framework/Core/src/ConfigParamDiscovery.cxx
+++ b/Framework/Core/src/ConfigParamDiscovery.cxx
@@ -25,9 +25,9 @@ namespace o2::framework
 std::vector<ConfigParamSpec> ConfigParamDiscovery::discover(ConfigParamRegistry& registry, int argc, char** argv)
 {
   std::vector<char const*> capabilitiesSpecs = {
+    "O2Framework:DiscoverAODOptionsInCommandLineCapability",
     "O2Framework:DiscoverMetadataInAODCapability",
     "O2Framework:DiscoverMetadataInCommandLineCapability",
-    "O2Framework:DiscoverAODOptionsInCommandLineCapability",
   };
 
   // Load all the requested plugins and discover what we can do.
diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx
index a98771a913d01..91c74bafff5ad 100644
--- a/Framework/Core/src/Plugin.cxx
+++ b/Framework/Core/src/Plugin.cxx
@@ -57,6 +57,10 @@ auto lookForCommandLineAODOptions = [](ConfigParamRegistry& registry, int argc,
       O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverAODOptionsInCommandLineCapability", "AOD options found in arguments. Populating from them.");
       return true;
     }
+    if (arg.starts_with("--aod-parent-base-path-replacement")) {
+      O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverAODOptionsInCommandLineCapability", "AOD options found in arguments. Populating from them.");
+      return true;
+    }
   }
   return false;
 };
@@ -137,7 +141,7 @@ struct DiscoverAODOptionsInCommandLine : o2::framework::ConfigDiscoveryPlugin {
         bool injectOption = true;
         for (size_t i = 0; i < argc; i++) {
           std::string_view arg = argv[i];
-          if (!arg.starts_with("--aod-writer-")) {
+          if (!arg.starts_with("--aod-writer-") && arg != "--aod-parent-base-path-replacement") {
             continue;
           }
           std::string key = arg.data() + 2;
@@ -149,6 +153,9 @@ struct DiscoverAODOptionsInCommandLine : o2::framework::ConfigDiscoveryPlugin {
             results.push_back(ConfigParamSpec{"aod-writer-compression", VariantType::Int, numericValue, {"AOD Compression options"}});
             injectOption = false;
           }
+          if (key == "aod-parent-base-path-replacement") {
+            results.push_back(ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, value, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}});
+          }
         }
         if (injectOption) {
           results.push_back(ConfigParamSpec{"aod-writer-compression", VariantType::Int, 505, {"AOD Compression options"}});
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 0366e39cf8976..56e9930e3b655 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -217,7 +217,6 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
                 ConfigParamSpec{"aod-max-io-rate", VariantType::Float, 0.f, {"Maximum I/O rate in MB/s"}},
                 ConfigParamSpec{"aod-reader-json", VariantType::String, {"json configuration file"}},
                 ConfigParamSpec{"aod-parent-access-level", VariantType::String, {"Allow parent file access up to specified level. Default: no (0)"}},
-                ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}},
                 ConfigParamSpec{"time-limit", VariantType::Int64, 0ll, {"Maximum run time limit in seconds"}},
                 ConfigParamSpec{"orbit-offset-enumeration", VariantType::Int64, 0ll, {"initial value for the orbit"}},
                 ConfigParamSpec{"orbit-multiplier-enumeration", VariantType::Int64, 0ll, {"multiplier to get the orbit from the counter"}},

From 6178630621d438857a975284bddbe7860d510328 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 7 Nov 2024 21:42:24 +0100
Subject: [PATCH 691/726] DPL: drop need for has_root_setowner

Simple inline constrain is much better.
---
 Framework/Core/include/Framework/DataRefUtils.h |  2 +-
 Framework/Core/include/Framework/TypeTraits.h   | 17 -----------------
 2 files changed, 1 insertion(+), 18 deletions(-)

diff --git a/Framework/Core/include/Framework/DataRefUtils.h b/Framework/Core/include/Framework/DataRefUtils.h
index c63b06357b8ed..2839860965357 100644
--- a/Framework/Core/include/Framework/DataRefUtils.h
+++ b/Framework/Core/include/Framework/DataRefUtils.h
@@ -122,7 +122,7 @@ struct DataRefUtils {
         // object only depends on the state at serialization of the original object. However,
         // all objects created during deserialization are new and must be owned by the collection
         // to avoid memory leak. So we call SetOwner if it is available for the type.
-        if constexpr (has_root_setowner<T>::value) {
+        if constexpr (requires(T t) { t.SetOwner(true); }) {
           result->SetOwner(true);
         }
       });
diff --git a/Framework/Core/include/Framework/TypeTraits.h b/Framework/Core/include/Framework/TypeTraits.h
index 19ca548835cdd..faa9055de3280 100644
--- a/Framework/Core/include/Framework/TypeTraits.h
+++ b/Framework/Core/include/Framework/TypeTraits.h
@@ -147,22 +147,5 @@ class has_root_dictionary<T, typename std::enable_if<is_container<T>::value>::ty
 {
 };
 
-// Detect whether a class is a ROOT class implementing SetOwner
-// This member detector idiom is implemented using SFINAE idiom to look for
-// a 'SetOwner()' method.
-template <typename T, typename _ = void>
-struct has_root_setowner : std::false_type {
-};
-
-template <typename T>
-struct has_root_setowner<
-  T,
-  std::conditional_t<
-    false,
-    class_member_checker<
-      decltype(std::declval<T>().SetOwner(true))>,
-    void>> : public std::true_type {
-};
-
 } // namespace o2::framework
 #endif // FRAMEWORK_TYPETRAITS_H

From b8c6f59992616a4dbae310cc97e8a87a05043f08 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 7 Nov 2024 21:43:29 +0100
Subject: [PATCH 692/726] DPL: drop need for has_root_setowner

Simple inline constrain is much better.
---
 Framework/Core/include/Framework/DataRefUtils.h |  4 ++--
 Framework/Core/include/Framework/TypeTraits.h   | 17 -----------------
 2 files changed, 2 insertions(+), 19 deletions(-)

diff --git a/Framework/Core/include/Framework/DataRefUtils.h b/Framework/Core/include/Framework/DataRefUtils.h
index c63b06357b8ed..4c1bd0ed7ed10 100644
--- a/Framework/Core/include/Framework/DataRefUtils.h
+++ b/Framework/Core/include/Framework/DataRefUtils.h
@@ -122,7 +122,7 @@ struct DataRefUtils {
         // object only depends on the state at serialization of the original object. However,
         // all objects created during deserialization are new and must be owned by the collection
         // to avoid memory leak. So we call SetOwner if it is available for the type.
-        if constexpr (has_root_setowner<T>::value) {
+        if constexpr (requires(T t) { t.SetOwner(true); }) {
           result->SetOwner(true);
         }
       });
@@ -159,7 +159,7 @@ struct DataRefUtils {
           throw runtime_error_f("Unable to extract class %s", cl == nullptr ? "<name not available>" : cl->GetName());
         }
         // workaround for ROOT feature, see above
-        if constexpr (has_root_setowner<T>::value) {
+        if constexpr (requires(T t) { t.SetOwner(true); }) {
           result->SetOwner(true);
         }
       });
diff --git a/Framework/Core/include/Framework/TypeTraits.h b/Framework/Core/include/Framework/TypeTraits.h
index 19ca548835cdd..faa9055de3280 100644
--- a/Framework/Core/include/Framework/TypeTraits.h
+++ b/Framework/Core/include/Framework/TypeTraits.h
@@ -147,22 +147,5 @@ class has_root_dictionary<T, typename std::enable_if<is_container<T>::value>::ty
 {
 };
 
-// Detect whether a class is a ROOT class implementing SetOwner
-// This member detector idiom is implemented using SFINAE idiom to look for
-// a 'SetOwner()' method.
-template <typename T, typename _ = void>
-struct has_root_setowner : std::false_type {
-};
-
-template <typename T>
-struct has_root_setowner<
-  T,
-  std::conditional_t<
-    false,
-    class_member_checker<
-      decltype(std::declval<T>().SetOwner(true))>,
-    void>> : public std::true_type {
-};
-
 } // namespace o2::framework
 #endif // FRAMEWORK_TYPETRAITS_H

From 843d2694cef65d7583ba0cde086cf0f9824ca41e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Nov 2024 09:57:25 +0100
Subject: [PATCH 693/726] DPL Analysis: fix missing connection to the grid

It's not a given that parent files and the original data are
on the same support, so we need to connect to the grid if needed.
---
 Framework/AnalysisSupport/src/Plugin.cxx | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index b899a52206422..e2a9a98ab90d3 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -175,6 +175,10 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
             }
           }
 
+          if (parentFilename.starts_with("alien://")) {
+            TGrid::Connect("alien://");
+          }
+
           std::unique_ptr<TFile> parentFile{TFile::Open(parentFilename.c_str())};
           if (parentFile.get() == nullptr) {
             LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename);

From 53c801ae1ab3f69cfd65afa324f684b7a74021e4 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Nov 2024 11:40:00 +0100
Subject: [PATCH 694/726] DPL: account for aod-parent-access-level == 0

In the case of self contained derived data, it is ok not to look in the parents
for missing metadata.
---
 .../src/AODJAlienReaderHelpers.cxx            | 23 +++++++++----------
 Framework/AnalysisSupport/src/Plugin.cxx      |  9 +++++++-
 Framework/Core/src/Plugin.cxx                 |  5 +++-
 Framework/Core/src/WorkflowHelpers.cxx        |  1 -
 4 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 90d88cb43626e..9c19de85739ce 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -118,19 +118,23 @@ static inline auto extractOriginalsTuple(framework::pack<Os...>, ProcessingConte
   return std::make_tuple(extractTypedOriginal<Os>(pc)...);
 }
 
-AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const& config)
+AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const& ctx)
 {
   // aod-parent-base-path-replacement is now a workflow option, so it needs to be
   // retrieved from the ConfigContext. This is because we do not allow workflow options
   // to change over start-stop-start because they can affect the topology generation.
   std::string parentFileReplacement;
-  if (config.options().isSet("aod-parent-base-path-replacement")) {
-    parentFileReplacement = config.options().get<std::string>("aod-parent-base-path-replacement");
+  if (ctx.options().isSet("aod-parent-base-path-replacement")) {
+    parentFileReplacement = ctx.options().get<std::string>("aod-parent-base-path-replacement");
   }
-  auto callback = AlgorithmSpec{adaptStateful([parentFileReplacement](ConfigParamRegistry const& options,
-                                                                      DeviceSpec const& spec,
-                                                                      Monitoring& monitoring,
-                                                                      DataProcessingStats& stats) {
+  int parentAccessLevel = 0;
+  if (ctx.options().isSet("aod-parent-access-level")) {
+    parentAccessLevel = ctx.options().get<int>("aod-parent-access-level");
+  }
+  auto callback = AlgorithmSpec{adaptStateful([parentFileReplacement, parentAccessLevel](ConfigParamRegistry const& options,
+                                                                                         DeviceSpec const& spec,
+                                                                                         Monitoring& monitoring,
+                                                                                         DataProcessingStats& stats) {
     // FIXME: not actually needed, since data processing stats can specify that we should
     // send the initial value.
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_BYTES_CREATED), DataProcessingStats::Op::Set, 0});
@@ -148,11 +152,6 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const
 
     auto maxRate = options.get<float>("aod-max-io-rate");
 
-    int parentAccessLevel = 0;
-    if (options.isSet("aod-parent-access-level")) {
-      parentAccessLevel = options.get<int>("aod-parent-access-level");
-    }
-
     // create a DataInputDirector
     auto didir = std::make_shared<DataInputDirector>(filename, &monitoring, parentAccessLevel, parentFileReplacement);
     if (options.isSet("aod-reader-json")) {
diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index b899a52206422..32babc5f83d33 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -150,13 +150,20 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
           return results;
         }
 
-        // Lets try in parent files
+        if (!registry.isSet("aod-parent-access-level") || registry.get<int>("aod-parent-access-level") == 0) {
+          LOGP(info, "No metadata found in file \"{}\" and parent level 0 prevents further lookup.", filename);
+          results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
+          return results;
+        }
+
+        // Lets try in parent file.
         auto parentFiles = (TMap*)currentFile->Get("parentFiles");
         if (!parentFiles) {
           LOGP(info, "No metadata found in file \"{}\"", filename);
           results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
           return results;
         }
+        LOGP(info, "No metadata found in file \"{}\", checking in its parents.", filename);
         for (auto* p : *parentFiles) {
           std::string parentFilename = ((TPair*)p)->Value()->GetName();
           // Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be
diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx
index 91c74bafff5ad..726b12ff68365 100644
--- a/Framework/Core/src/Plugin.cxx
+++ b/Framework/Core/src/Plugin.cxx
@@ -141,7 +141,7 @@ struct DiscoverAODOptionsInCommandLine : o2::framework::ConfigDiscoveryPlugin {
         bool injectOption = true;
         for (size_t i = 0; i < argc; i++) {
           std::string_view arg = argv[i];
-          if (!arg.starts_with("--aod-writer-") && arg != "--aod-parent-base-path-replacement") {
+          if (!arg.starts_with("--aod-writer-") && !arg.starts_with("--aod-parent-")) {
             continue;
           }
           std::string key = arg.data() + 2;
@@ -156,6 +156,9 @@ struct DiscoverAODOptionsInCommandLine : o2::framework::ConfigDiscoveryPlugin {
           if (key == "aod-parent-base-path-replacement") {
             results.push_back(ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, value, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}});
           }
+          if (key == "aod-parent-access-level") {
+            results.push_back(ConfigParamSpec{"aod-parent-access-level", VariantType::String, value, {"Allow parent file access up to specified level. Default: no (0)"}});
+          }
         }
         if (injectOption) {
           results.push_back(ConfigParamSpec{"aod-writer-compression", VariantType::Int, 505, {"AOD Compression options"}});
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 56e9930e3b655..da9a135dc5eb8 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -216,7 +216,6 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
     .options = {ConfigParamSpec{"aod-file-private", VariantType::String, ctx.options().get<std::string>("aod-file"), {"AOD file"}},
                 ConfigParamSpec{"aod-max-io-rate", VariantType::Float, 0.f, {"Maximum I/O rate in MB/s"}},
                 ConfigParamSpec{"aod-reader-json", VariantType::String, {"json configuration file"}},
-                ConfigParamSpec{"aod-parent-access-level", VariantType::String, {"Allow parent file access up to specified level. Default: no (0)"}},
                 ConfigParamSpec{"time-limit", VariantType::Int64, 0ll, {"Maximum run time limit in seconds"}},
                 ConfigParamSpec{"orbit-offset-enumeration", VariantType::Int64, 0ll, {"initial value for the orbit"}},
                 ConfigParamSpec{"orbit-multiplier-enumeration", VariantType::Int64, 0ll, {"multiplier to get the orbit from the counter"}},

From e2a70de4d06ce040b6580fa1fe6f4f766a8d498e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Nov 2024 11:51:34 +0100
Subject: [PATCH 695/726] DPL: account for aod-parent-access-level == 0

In the case of self contained derived data, it is ok not to look in the parents
for missing metadata.
---
 .../src/AODJAlienReaderHelpers.cxx            | 23 +++++++++----------
 Framework/AnalysisSupport/src/Plugin.cxx      |  9 +++++++-
 Framework/Core/src/Plugin.cxx                 |  5 +++-
 Framework/Core/src/WorkflowHelpers.cxx        |  1 -
 4 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 90d88cb43626e..9c19de85739ce 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -118,19 +118,23 @@ static inline auto extractOriginalsTuple(framework::pack<Os...>, ProcessingConte
   return std::make_tuple(extractTypedOriginal<Os>(pc)...);
 }
 
-AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const& config)
+AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const& ctx)
 {
   // aod-parent-base-path-replacement is now a workflow option, so it needs to be
   // retrieved from the ConfigContext. This is because we do not allow workflow options
   // to change over start-stop-start because they can affect the topology generation.
   std::string parentFileReplacement;
-  if (config.options().isSet("aod-parent-base-path-replacement")) {
-    parentFileReplacement = config.options().get<std::string>("aod-parent-base-path-replacement");
+  if (ctx.options().isSet("aod-parent-base-path-replacement")) {
+    parentFileReplacement = ctx.options().get<std::string>("aod-parent-base-path-replacement");
   }
-  auto callback = AlgorithmSpec{adaptStateful([parentFileReplacement](ConfigParamRegistry const& options,
-                                                                      DeviceSpec const& spec,
-                                                                      Monitoring& monitoring,
-                                                                      DataProcessingStats& stats) {
+  int parentAccessLevel = 0;
+  if (ctx.options().isSet("aod-parent-access-level")) {
+    parentAccessLevel = ctx.options().get<int>("aod-parent-access-level");
+  }
+  auto callback = AlgorithmSpec{adaptStateful([parentFileReplacement, parentAccessLevel](ConfigParamRegistry const& options,
+                                                                                         DeviceSpec const& spec,
+                                                                                         Monitoring& monitoring,
+                                                                                         DataProcessingStats& stats) {
     // FIXME: not actually needed, since data processing stats can specify that we should
     // send the initial value.
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_BYTES_CREATED), DataProcessingStats::Op::Set, 0});
@@ -148,11 +152,6 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const
 
     auto maxRate = options.get<float>("aod-max-io-rate");
 
-    int parentAccessLevel = 0;
-    if (options.isSet("aod-parent-access-level")) {
-      parentAccessLevel = options.get<int>("aod-parent-access-level");
-    }
-
     // create a DataInputDirector
     auto didir = std::make_shared<DataInputDirector>(filename, &monitoring, parentAccessLevel, parentFileReplacement);
     if (options.isSet("aod-reader-json")) {
diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index e2a9a98ab90d3..bba3499286e08 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -150,13 +150,20 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
           return results;
         }
 
-        // Lets try in parent files
+        if (!registry.isSet("aod-parent-access-level") || registry.get<int>("aod-parent-access-level") == 0) {
+          LOGP(info, "No metadata found in file \"{}\" and parent level 0 prevents further lookup.", filename);
+          results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
+          return results;
+        }
+
+        // Lets try in parent file.
         auto parentFiles = (TMap*)currentFile->Get("parentFiles");
         if (!parentFiles) {
           LOGP(info, "No metadata found in file \"{}\"", filename);
           results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
           return results;
         }
+        LOGP(info, "No metadata found in file \"{}\", checking in its parents.", filename);
         for (auto* p : *parentFiles) {
           std::string parentFilename = ((TPair*)p)->Value()->GetName();
           // Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be
diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx
index 91c74bafff5ad..726b12ff68365 100644
--- a/Framework/Core/src/Plugin.cxx
+++ b/Framework/Core/src/Plugin.cxx
@@ -141,7 +141,7 @@ struct DiscoverAODOptionsInCommandLine : o2::framework::ConfigDiscoveryPlugin {
         bool injectOption = true;
         for (size_t i = 0; i < argc; i++) {
           std::string_view arg = argv[i];
-          if (!arg.starts_with("--aod-writer-") && arg != "--aod-parent-base-path-replacement") {
+          if (!arg.starts_with("--aod-writer-") && !arg.starts_with("--aod-parent-")) {
             continue;
           }
           std::string key = arg.data() + 2;
@@ -156,6 +156,9 @@ struct DiscoverAODOptionsInCommandLine : o2::framework::ConfigDiscoveryPlugin {
           if (key == "aod-parent-base-path-replacement") {
             results.push_back(ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, value, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}});
           }
+          if (key == "aod-parent-access-level") {
+            results.push_back(ConfigParamSpec{"aod-parent-access-level", VariantType::String, value, {"Allow parent file access up to specified level. Default: no (0)"}});
+          }
         }
         if (injectOption) {
           results.push_back(ConfigParamSpec{"aod-writer-compression", VariantType::Int, 505, {"AOD Compression options"}});
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index 56e9930e3b655..da9a135dc5eb8 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -216,7 +216,6 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
     .options = {ConfigParamSpec{"aod-file-private", VariantType::String, ctx.options().get<std::string>("aod-file"), {"AOD file"}},
                 ConfigParamSpec{"aod-max-io-rate", VariantType::Float, 0.f, {"Maximum I/O rate in MB/s"}},
                 ConfigParamSpec{"aod-reader-json", VariantType::String, {"json configuration file"}},
-                ConfigParamSpec{"aod-parent-access-level", VariantType::String, {"Allow parent file access up to specified level. Default: no (0)"}},
                 ConfigParamSpec{"time-limit", VariantType::Int64, 0ll, {"Maximum run time limit in seconds"}},
                 ConfigParamSpec{"orbit-offset-enumeration", VariantType::Int64, 0ll, {"initial value for the orbit"}},
                 ConfigParamSpec{"orbit-multiplier-enumeration", VariantType::Int64, 0ll, {"multiplier to get the orbit from the counter"}},

From 86844587c9161b3d0b4b4b91ca6b2fae163225a1 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 8 Nov 2024 11:51:34 +0100
Subject: [PATCH 696/726] DPL Analysis: drop aod-file-private

This was a hack introduced to pass a workflow option to an AlgorithmSpec created
by a plugin. Not needed anymore now that we properly pass the ConfigContext to plugins
as well.
---
 .../src/AODJAlienReaderHelpers.cxx            | 21 +++++++++----------
 Framework/Core/src/WorkflowHelpers.cxx        |  3 +--
 2 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 9c19de85739ce..ebd355da0ac47 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -131,10 +131,16 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const
   if (ctx.options().isSet("aod-parent-access-level")) {
     parentAccessLevel = ctx.options().get<int>("aod-parent-access-level");
   }
-  auto callback = AlgorithmSpec{adaptStateful([parentFileReplacement, parentAccessLevel](ConfigParamRegistry const& options,
-                                                                                         DeviceSpec const& spec,
-                                                                                         Monitoring& monitoring,
-                                                                                         DataProcessingStats& stats) {
+  if (ctx.options().isSet("aod-file")) {
+    LOGP(fatal, "No input file defined!");
+    throw std::runtime_error("Processing is stopped!");
+  }
+  auto filename = ctx.options().get<std::string>("aod-file");
+
+  auto callback = AlgorithmSpec{adaptStateful([parentFileReplacement, parentAccessLevel, filename](ConfigParamRegistry const& options,
+                                                                                                   DeviceSpec const& spec,
+                                                                                                   Monitoring& monitoring,
+                                                                                                   DataProcessingStats& stats) {
     // FIXME: not actually needed, since data processing stats can specify that we should
     // send the initial value.
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_BYTES_CREATED), DataProcessingStats::Op::Set, 0});
@@ -143,13 +149,6 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_MESSAGES_DESTROYED), DataProcessingStats::Op::Set, 0});
     stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_BYTES_EXPIRED), DataProcessingStats::Op::Set, 0});
 
-    if (!options.isSet("aod-file-private")) {
-      LOGP(fatal, "No input file defined!");
-      throw std::runtime_error("Processing is stopped!");
-    }
-
-    auto filename = options.get<std::string>("aod-file-private");
-
     auto maxRate = options.get<float>("aod-max-io-rate");
 
     // create a DataInputDirector
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index da9a135dc5eb8..49cbcac948eb2 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -213,8 +213,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
                          static_cast<DataAllocator::SubSpecificationType>(compile_time_hash("internal-dpl-aod-reader")),
                          aodLifetime}},
     .algorithm = AlgorithmSpec::dummyAlgorithm(),
-    .options = {ConfigParamSpec{"aod-file-private", VariantType::String, ctx.options().get<std::string>("aod-file"), {"AOD file"}},
-                ConfigParamSpec{"aod-max-io-rate", VariantType::Float, 0.f, {"Maximum I/O rate in MB/s"}},
+    .options = {ConfigParamSpec{"aod-max-io-rate", VariantType::Float, 0.f, {"Maximum I/O rate in MB/s"}},
                 ConfigParamSpec{"aod-reader-json", VariantType::String, {"json configuration file"}},
                 ConfigParamSpec{"time-limit", VariantType::Int64, 0ll, {"Maximum run time limit in seconds"}},
                 ConfigParamSpec{"orbit-offset-enumeration", VariantType::Int64, 0ll, {"initial value for the orbit"}},

From 459428c9b34e469807c88f67c8235ce3701f1993 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 9 Nov 2024 10:35:11 +0100
Subject: [PATCH 697/726] DPL: introduce FragmentToBatch

---
 .../Core/include/Framework/DataAllocator.h    | 14 +++++
 .../Core/include/Framework/TableTreeHelpers.h | 17 ++++++
 Framework/Core/src/DataAllocator.cxx          | 60 +++++++++++++++++++
 Framework/Core/src/TableTreeHelpers.cxx       | 30 +++++++++-
 4 files changed, 120 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/include/Framework/DataAllocator.h b/Framework/Core/include/Framework/DataAllocator.h
index 9c563b06910b5..eb63b5469bb29 100644
--- a/Framework/Core/include/Framework/DataAllocator.h
+++ b/Framework/Core/include/Framework/DataAllocator.h
@@ -242,6 +242,15 @@ class DataAllocator
     return t2t;
   }
 
+  template <typename T, typename... Args>
+    requires(requires { static_cast<struct FragmentToBatch>(std::declval<std::decay_t<T>>()); })
+  decltype(auto) make(const Output& spec, Args... args)
+  {
+    auto f2b = std::move(LifetimeHolder<FragmentToBatch>(new std::decay_t<T>(args...)));
+    adopt(spec, f2b);
+    return f2b;
+  }
+
   template <typename T>
     requires is_messageable<T>::value && (!is_specialization_v<T, UninitializedVector>)
   decltype(auto) make(const Output& spec)
@@ -284,6 +293,11 @@ class DataAllocator
   void
     adopt(const Output& spec, LifetimeHolder<struct TreeToTable>&);
 
+  /// Adopt a Source2Batch in the framework and serialise / send
+  /// it as an Arrow Dataset to all consumers of @a spec once done
+  void
+    adopt(const Output& spec, LifetimeHolder<struct FragmentToBatch>&);
+
   /// Adopt an Arrow table and send it to all consumers of @a spec
   void
     adopt(const Output& spec, std::shared_ptr<class arrow::Table>);
diff --git a/Framework/Core/include/Framework/TableTreeHelpers.h b/Framework/Core/include/Framework/TableTreeHelpers.h
index ccc7035ba3435..9dc7038b83fe1 100644
--- a/Framework/Core/include/Framework/TableTreeHelpers.h
+++ b/Framework/Core/include/Framework/TableTreeHelpers.h
@@ -11,11 +11,14 @@
 #ifndef O2_FRAMEWORK_TABLETREEHELPERS_H_
 #define O2_FRAMEWORK_TABLETREEHELPERS_H_
 
+#include <arrow/record_batch.h>
 #include "TFile.h"
 #include "TTreeReader.h"
 #include "TTreeReaderValue.h"
 #include "TTreeReaderArray.h"
 #include "TableBuilder.h"
+#include <arrow/dataset/file_base.h>
+#include <memory>
 
 // =============================================================================
 namespace o2::framework
@@ -140,6 +143,20 @@ class TreeToTable
   void addReader(TBranch* branch, std::string const& name, bool VLA);
 };
 
+class FragmentToBatch
+{
+ public:
+  FragmentToBatch(arrow::MemoryPool* pool = arrow::default_memory_pool());
+  void setLabel(const char* label);
+  void fill(std::shared_ptr<arrow::dataset::FileFragment>, std::shared_ptr<arrow::Schema> dataSetSchema, std::shared_ptr<arrow::dataset::FileFormat>);
+  std::shared_ptr<arrow::RecordBatch> finalize();
+
+ private:
+  arrow::MemoryPool* mArrowMemoryPool = nullptr;
+  std::string mTableLabel;
+  std::shared_ptr<arrow::RecordBatch> mRecordBatch;
+};
+
 // -----------------------------------------------------------------------------
 } // namespace o2::framework
 
diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx
index bae40f2b47947..c310892c4c490 100644
--- a/Framework/Core/src/DataAllocator.cxx
+++ b/Framework/Core/src/DataAllocator.cxx
@@ -211,6 +211,34 @@ void doWriteTable(std::shared_ptr<FairMQResizableBuffer> b, arrow::Table* table)
   }
 }
 
+void doWriteBatch(std::shared_ptr<FairMQResizableBuffer> b, arrow::RecordBatch* batch)
+{
+  auto mock = std::make_shared<arrow::io::MockOutputStream>();
+  int64_t expectedSize = 0;
+  auto mockWriter = arrow::ipc::MakeStreamWriter(mock.get(), batch->schema());
+  arrow::Status outStatus = mockWriter.ValueOrDie()->WriteRecordBatch(*batch);
+
+  expectedSize = mock->Tell().ValueOrDie();
+  auto reserve = b->Reserve(expectedSize);
+  if (reserve.ok() == false) {
+    throw std::runtime_error("Unable to reserve memory for table");
+  }
+
+  auto stream = std::make_shared<FairMQOutputStream>(b);
+  // This is a copy maybe we can finally get rid of it by having using the
+  // dataset API?
+  auto outBatch = arrow::ipc::MakeStreamWriter(stream.get(), batch->schema());
+  if (outBatch.ok() == false) {
+    throw ::std::runtime_error("Unable to create batch writer");
+  }
+
+  outStatus = outBatch.ValueOrDie()->WriteRecordBatch(*batch);
+
+  if (outStatus.ok() == false) {
+    throw std::runtime_error("Unable to Write batch");
+  }
+}
+
 void DataAllocator::adopt(const Output& spec, LifetimeHolder<TableBuilder>& tb)
 {
   auto& timingInfo = mRegistry.get<TimingInfo>();
@@ -273,6 +301,38 @@ void DataAllocator::adopt(const Output& spec, LifetimeHolder<TreeToTable>& t2t)
   context.addBuffer(std::move(header), buffer, std::move(finalizer), routeIndex);
 }
 
+void DataAllocator::adopt(const Output& spec, LifetimeHolder<FragmentToBatch>& f2b)
+{
+  auto& timingInfo = mRegistry.get<TimingInfo>();
+  RouteIndex routeIndex = matchDataHeader(spec, timingInfo.timeslice);
+
+  auto header = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodArrow, 0);
+  auto& context = mRegistry.get<ArrowContext>();
+
+  auto creator = [transport = context.proxy().getOutputTransport(routeIndex)](size_t s) -> std::unique_ptr<fair::mq::Message> {
+    return transport->CreateMessage(s);
+  };
+  auto buffer = std::make_shared<FairMQResizableBuffer>(creator);
+
+  f2b.callback = [buffer = buffer, transport = context.proxy().getOutputTransport(routeIndex)](FragmentToBatch& source) {
+    // Serialization happens in here, so that we can
+    // get rid of the intermediate tree 2 table object, saving memory.
+    auto batch = source.finalize();
+    doWriteBatch(buffer, batch.get());
+    // deletion happens in the caller
+  };
+
+  /// To finalise this we write the table to the buffer.
+  /// FIXME: most likely not a great idea. We should probably write to the buffer
+  ///        directly in the TableBuilder, incrementally.
+  auto finalizer = [](std::shared_ptr<FairMQResizableBuffer> b) -> void {
+    // This is empty because we already serialised the object when
+    // the LifetimeHolder goes out of scope.
+  };
+
+  context.addBuffer(std::move(header), buffer, std::move(finalizer), routeIndex);
+}
+
 void DataAllocator::adopt(const Output& spec, std::shared_ptr<arrow::Table> ptr)
 {
   auto& timingInfo = mRegistry.get<TimingInfo>();
diff --git a/Framework/Core/src/TableTreeHelpers.cxx b/Framework/Core/src/TableTreeHelpers.cxx
index 23aa934c2ca8b..c20febaac517d 100644
--- a/Framework/Core/src/TableTreeHelpers.cxx
+++ b/Framework/Core/src/TableTreeHelpers.cxx
@@ -13,9 +13,13 @@
 #include "Framework/Endian.h"
 
 #include "arrow/type_traits.h"
+#include <arrow/dataset/file_base.h>
+#include <arrow/record_batch.h>
+#include <arrow/type.h>
 #include <arrow/util/key_value_metadata.h>
 #include <TBufferFile.h>
 
+#include <memory>
 #include <utility>
 namespace TableTreeHelpers
 {
@@ -407,7 +411,7 @@ std::shared_ptr<TTree> TableToTree::process()
 
   for (auto& reader : mColumnReaders) {
     int idealBasketSize = 1024 + reader->fieldSize() * reader->columnEntries(); // minimal additional size needed, otherwise we get 2 baskets
-    int basketSize = std::max(32000, idealBasketSize);        // keep a minimum value
+    int basketSize = std::max(32000, idealBasketSize);                          // keep a minimum value
     // std::cout << "Setting baskets size for " << reader->branchName() << " to " << basketSize << " =  1024 + "
     //           << reader->fieldSize() << " * " << reader->columnEntries() << ". mRows was " << mRows << std::endl;
     mTree->SetBasketSize(reader->branchName(), basketSize);
@@ -555,4 +559,28 @@ std::shared_ptr<arrow::Table> TreeToTable::finalize()
   return mTable;
 }
 
+FragmentToBatch::FragmentToBatch(arrow::MemoryPool* pool)
+  : mArrowMemoryPool{pool}
+{
+}
+
+void FragmentToBatch::setLabel(const char* label)
+{
+  mTableLabel = label;
+}
+
+void FragmentToBatch::fill(std::shared_ptr<arrow::dataset::FileFragment> fragment, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileFormat> format)
+{
+  auto options = std::make_shared<arrow::dataset::ScanOptions>();
+  options->dataset_schema = schema;
+  auto scanner = format->ScanBatchesAsync(options, fragment);
+  auto batch = (*scanner)();
+  mRecordBatch = *batch.result();
+}
+
+std::shared_ptr<arrow::RecordBatch> FragmentToBatch::finalize()
+{
+  return mRecordBatch;
+}
+
 } // namespace o2::framework

From 439a5310199b42bed784dbbc7e3cc8e3b9276da8 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sun, 10 Nov 2024 16:47:11 +0100
Subject: [PATCH 698/726] DPL: enable the plugin for any parent option

If we read metadata from parents, we might be affecting the workflow generation,
therefore we must consider all the parent affecting options as workflow ones.
---
 Framework/Core/src/Plugin.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx
index 726b12ff68365..0d225b81c0581 100644
--- a/Framework/Core/src/Plugin.cxx
+++ b/Framework/Core/src/Plugin.cxx
@@ -57,7 +57,7 @@ auto lookForCommandLineAODOptions = [](ConfigParamRegistry& registry, int argc,
       O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverAODOptionsInCommandLineCapability", "AOD options found in arguments. Populating from them.");
       return true;
     }
-    if (arg.starts_with("--aod-parent-base-path-replacement")) {
+    if (arg.starts_with("--aod-parent-")) {
       O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverAODOptionsInCommandLineCapability", "AOD options found in arguments. Populating from them.");
       return true;
     }

From f22bacf11e8517a7edd7db336488d00a3f8bf3cb Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 13 Nov 2024 14:06:56 +0100
Subject: [PATCH 699/726] DPL: move writers to plugin

This will allow us at some point to remove the dependency on TTree in the framework,
hopefully saving memory and allowing us to more easily customize the writing backend.
---
 Framework/AnalysisSupport/CMakeLists.txt      |   1 +
 .../AnalysisSupport/src/AODWriterHelpers.cxx  | 414 +++++++++++++++
 .../AnalysisSupport/src/AODWriterHelpers.h    |  28 +
 Framework/AnalysisSupport/src/Plugin.cxx      |  17 +
 .../Core/include/Framework/AnalysisContext.h  |  58 +++
 .../Framework}/AnalysisSupportHelpers.h       |  40 +-
 .../Core/include/Framework/ConfigContext.h    |  15 +-
 .../include/Framework/runDataProcessing.h     |   4 +-
 Framework/Core/src/AnalysisSupportHelpers.cxx | 484 ++++--------------
 Framework/Core/src/ArrowSupport.cxx           |  10 +-
 Framework/Core/src/ConfigContext.cxx          |   3 +
 Framework/Core/src/WorkflowHelpers.cxx        | 149 +-----
 Framework/Core/src/WorkflowHelpers.h          |   4 +-
 Framework/Core/test/Mocking.h                 |   3 +-
 .../Core/test/benchmark_WorkflowHelpers.cxx   |   3 +-
 Framework/Core/test/test_OverrideLabels.cxx   |   3 +-
 .../TestWorkflows/src/o2TestHistograms.cxx    |   5 +-
 17 files changed, 688 insertions(+), 553 deletions(-)
 create mode 100644 Framework/AnalysisSupport/src/AODWriterHelpers.cxx
 create mode 100644 Framework/AnalysisSupport/src/AODWriterHelpers.h
 create mode 100644 Framework/Core/include/Framework/AnalysisContext.h
 rename Framework/Core/{src => include/Framework}/AnalysisSupportHelpers.h (71%)

diff --git a/Framework/AnalysisSupport/CMakeLists.txt b/Framework/AnalysisSupport/CMakeLists.txt
index eb5706817704b..5fb1282469711 100644
--- a/Framework/AnalysisSupport/CMakeLists.txt
+++ b/Framework/AnalysisSupport/CMakeLists.txt
@@ -20,6 +20,7 @@ o2_add_library(FrameworkAnalysisSupport
                SOURCES src/Plugin.cxx
                        src/DataInputDirector.cxx
                        src/AODJAlienReaderHelpers.cxx
+                       src/AODWriterHelpers.cxx
                PRIVATE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/src
                PUBLIC_LINK_LIBRARIES O2::Framework ${EXTRA_TARGETS} ROOT::TreePlayer)
 
diff --git a/Framework/AnalysisSupport/src/AODWriterHelpers.cxx b/Framework/AnalysisSupport/src/AODWriterHelpers.cxx
new file mode 100644
index 0000000000000..fa10d4661f537
--- /dev/null
+++ b/Framework/AnalysisSupport/src/AODWriterHelpers.cxx
@@ -0,0 +1,414 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/AnalysisContext.h"
+#include "Framework/ConfigContext.h"
+#include "Framework/ControlService.h"
+#include "AODWriterHelpers.h"
+#include "Framework/OutputObjHeader.h"
+#include "Framework/EndOfStreamContext.h"
+#include "Framework/ProcessingContext.h"
+#include "Framework/InitContext.h"
+#include "Framework/CallbackService.h"
+#include "Framework/AnalysisSupportHelpers.h"
+#include "Framework/TableConsumer.h"
+#include "Framework/DataOutputDirector.h"
+#include "Framework/TableTreeHelpers.h"
+
+#include <TFile.h>
+#include <TFile.h>
+#include <TTree.h>
+#include <TMap.h>
+#include <TObjString.h>
+#include <arrow/table.h>
+
+namespace o2::framework::writers
+{
+
+struct InputObjectRoute {
+  std::string name;
+  uint32_t uniqueId;
+  std::string directory;
+  uint32_t taskHash;
+  OutputObjHandlingPolicy policy;
+  OutputObjSourceType sourceType;
+};
+
+struct InputObject {
+  TClass* kind = nullptr;
+  void* obj = nullptr;
+  std::string name;
+  int count = -1;
+};
+
+const static std::unordered_map<OutputObjHandlingPolicy, std::string> ROOTfileNames = {{OutputObjHandlingPolicy::AnalysisObject, "AnalysisResults.root"},
+                                                                                       {OutputObjHandlingPolicy::QAObject, "QAResults.root"}};
+
+AlgorithmSpec AODWriterHelpers::getOutputTTreeWriter(ConfigContext const& ctx)
+{
+  auto& ac = ctx.services().get<AnalysisContext>();
+  auto dod = AnalysisSupportHelpers::getDataOutputDirector(ctx);
+  int compressionLevel = 505;
+  if (ctx.options().hasOption("aod-writer-compression")) {
+    compressionLevel = ctx.options().get<int>("aod-writer-compression");
+  }
+  return AlgorithmSpec{[dod, outputInputs = ac.outputsInputsAOD, compressionLevel](InitContext& ic) -> std::function<void(ProcessingContext&)> {
+    LOGP(debug, "======== getGlobalAODSink::Init ==========");
+
+    // find out if any table needs to be saved
+    bool hasOutputsToWrite = false;
+    for (auto& outobj : outputInputs) {
+      auto ds = dod->getDataOutputDescriptors(outobj);
+      if (ds.size() > 0) {
+        hasOutputsToWrite = true;
+        break;
+      }
+    }
+
+    // if nothing needs to be saved then return a trivial functor
+    // this happens when nothing needs to be saved but there are dangling outputs
+    if (!hasOutputsToWrite) {
+      return [](ProcessingContext&) mutable -> void {
+        static bool once = false;
+        if (!once) {
+          LOG(info) << "No AODs to be saved.";
+          once = true;
+        }
+      };
+    }
+
+    // end of data functor is called at the end of the data stream
+    auto endofdatacb = [dod](EndOfStreamContext& context) {
+      dod->closeDataFiles();
+      context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
+    };
+
+    auto& callbacks = ic.services().get<CallbackService>();
+    callbacks.set<CallbackService::Id::EndOfStream>(endofdatacb);
+
+    // prepare map<uint64_t, uint64_t>(startTime, tfNumber)
+    std::map<uint64_t, uint64_t> tfNumbers;
+    std::map<uint64_t, std::string> tfFilenames;
+
+    std::vector<TString> aodMetaDataKeys;
+    std::vector<TString> aodMetaDataVals;
+
+    // this functor is called once per time frame
+    return [dod, tfNumbers, tfFilenames, aodMetaDataKeys, aodMetaDataVals, compressionLevel](ProcessingContext& pc) mutable -> void {
+      LOGP(debug, "======== getGlobalAODSink::processing ==========");
+      LOGP(debug, " processing data set with {} entries", pc.inputs().size());
+
+      // return immediately if pc.inputs() is empty. This should never happen!
+      if (pc.inputs().size() == 0) {
+        LOGP(info, "No inputs available!");
+        return;
+      }
+
+      // update tfNumbers
+      uint64_t startTime = 0;
+      uint64_t tfNumber = 0;
+      auto ref = pc.inputs().get("tfn");
+      if (ref.spec && ref.payload) {
+        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref)->startTime;
+        tfNumber = pc.inputs().get<uint64_t>("tfn");
+        tfNumbers.insert(std::pair<uint64_t, uint64_t>(startTime, tfNumber));
+      }
+      // update tfFilenames
+      std::string aodInputFile;
+      auto ref2 = pc.inputs().get("tff");
+      if (ref2.spec && ref2.payload) {
+        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref2)->startTime;
+        aodInputFile = pc.inputs().get<std::string>("tff");
+        tfFilenames.insert(std::pair<uint64_t, std::string>(startTime, aodInputFile));
+      }
+
+      // close all output files if one has reached size limit
+      dod->checkFileSizes();
+
+      // loop over the DataRefs which are contained in pc.inputs()
+      for (const auto& ref : pc.inputs()) {
+        if (!ref.spec) {
+          LOGP(debug, "Invalid input will be skipped!");
+          continue;
+        }
+
+        // get metadata
+        if (DataSpecUtils::partialMatch(*ref.spec, header::DataDescription("AODMetadataKeys"))) {
+          aodMetaDataKeys = pc.inputs().get<std::vector<TString>>(ref.spec->binding);
+        }
+        if (DataSpecUtils::partialMatch(*ref.spec, header::DataDescription("AODMetadataVals"))) {
+          aodMetaDataVals = pc.inputs().get<std::vector<TString>>(ref.spec->binding);
+        }
+
+        // skip non-AOD refs
+        if (!DataSpecUtils::partialMatch(*ref.spec, writableAODOrigins)) {
+          continue;
+        }
+        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref)->startTime;
+
+        // does this need to be saved?
+        auto dh = DataRefUtils::getHeader<header::DataHeader*>(ref);
+        auto tableName = dh->dataDescription.as<std::string>();
+        auto ds = dod->getDataOutputDescriptors(*dh);
+        if (ds.empty()) {
+          continue;
+        }
+
+        // get TF number from startTime
+        auto it = tfNumbers.find(startTime);
+        if (it != tfNumbers.end()) {
+          tfNumber = (it->second / dod->getNumberTimeFramesToMerge()) * dod->getNumberTimeFramesToMerge();
+        } else {
+          LOGP(fatal, "No time frame number found for output with start time {}", startTime);
+          throw std::runtime_error("Processing is stopped!");
+        }
+        // get aod input file from startTime
+        auto it2 = tfFilenames.find(startTime);
+        if (it2 != tfFilenames.end()) {
+          aodInputFile = it2->second;
+        }
+
+        // get the TableConsumer and corresponding arrow table
+        auto msg = pc.inputs().get(ref.spec->binding);
+        if (msg.header == nullptr) {
+          LOGP(error, "No header for message {}:{}", ref.spec->binding, DataSpecUtils::describe(*ref.spec));
+          continue;
+        }
+        auto s = pc.inputs().get<TableConsumer>(ref.spec->binding);
+        auto table = s->asArrowTable();
+        if (!table->Validate().ok()) {
+          LOGP(warning, "The table \"{}\" is not valid and will not be saved!", tableName);
+          continue;
+        }
+        if (table->schema()->fields().empty()) {
+          LOGP(debug, "The table \"{}\" is empty but will be saved anyway!", tableName);
+        }
+
+        // loop over all DataOutputDescriptors
+        // a table can be saved in multiple ways
+        // e.g. different selections of columns to different files
+        for (auto d : ds) {
+          auto fileAndFolder = dod->getFileFolder(d, tfNumber, aodInputFile, compressionLevel);
+          auto treename = fileAndFolder.folderName + "/" + d->treename;
+          TableToTree ta2tr(table,
+                            fileAndFolder.file,
+                            treename.c_str());
+
+          // update metadata
+          if (fileAndFolder.file->FindObjectAny("metaData")) {
+            LOGF(debug, "Metadata: target file %s already has metadata, preserving it", fileAndFolder.file->GetName());
+          } else if (!aodMetaDataKeys.empty() && !aodMetaDataVals.empty()) {
+            TMap aodMetaDataMap;
+            for (uint32_t imd = 0; imd < aodMetaDataKeys.size(); imd++) {
+              aodMetaDataMap.Add(new TObjString(aodMetaDataKeys[imd]), new TObjString(aodMetaDataVals[imd]));
+            }
+            fileAndFolder.file->WriteObject(&aodMetaDataMap, "metaData", "Overwrite");
+          }
+
+          if (!d->colnames.empty()) {
+            for (auto& cn : d->colnames) {
+              auto idx = table->schema()->GetFieldIndex(cn);
+              auto col = table->column(idx);
+              auto field = table->schema()->field(idx);
+              if (idx != -1) {
+                ta2tr.addBranch(col, field);
+              }
+            }
+          } else {
+            ta2tr.addAllBranches();
+          }
+          ta2tr.process();
+        }
+      }
+    };
+  }
+
+  };
+}
+
+AlgorithmSpec AODWriterHelpers::getOutputObjHistWriter(ConfigContext const& ctx)
+{
+  auto& ac = ctx.services().get<AnalysisContext>();
+  auto tskmap = ac.outTskMap;
+  auto objmap = ac.outObjHistMap;
+
+  return AlgorithmSpec{[objmap, tskmap](InitContext& ic) -> std::function<void(ProcessingContext&)> {
+    auto& callbacks = ic.services().get<CallbackService>();
+    auto inputObjects = std::make_shared<std::vector<std::pair<InputObjectRoute, InputObject>>>();
+
+    static TFile* f[OutputObjHandlingPolicy::numPolicies];
+    for (auto i = 0u; i < OutputObjHandlingPolicy::numPolicies; ++i) {
+      f[i] = nullptr;
+    }
+
+    static std::string currentDirectory = "";
+    static std::string currentFile = "";
+
+    auto endofdatacb = [inputObjects](EndOfStreamContext& context) {
+      LOG(debug) << "Writing merged objects and histograms to file";
+      if (inputObjects->empty()) {
+        LOG(error) << "Output object map is empty!";
+        context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
+        return;
+      }
+      for (auto i = 0u; i < OutputObjHandlingPolicy::numPolicies; ++i) {
+        if (f[i] != nullptr) {
+          f[i]->Close();
+        }
+      }
+      LOG(debug) << "All outputs merged in their respective target files";
+      context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
+    };
+
+    callbacks.set<CallbackService::Id::EndOfStream>(endofdatacb);
+    return [inputObjects, objmap, tskmap](ProcessingContext& pc) mutable -> void {
+      auto const& ref = pc.inputs().get("x");
+      if (!ref.header) {
+        LOG(error) << "Header not found";
+        return;
+      }
+      if (!ref.payload) {
+        LOG(error) << "Payload not found";
+        return;
+      }
+      auto datah = o2::header::get<o2::header::DataHeader*>(ref.header);
+      if (!datah) {
+        LOG(error) << "No data header in stack";
+        return;
+      }
+
+      auto objh = o2::header::get<o2::framework::OutputObjHeader*>(ref.header);
+      if (!objh) {
+        LOG(error) << "No output object header in stack";
+        return;
+      }
+
+      InputObject obj;
+      FairInputTBuffer tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
+      tm.InitMap();
+      obj.kind = tm.ReadClass();
+      tm.SetBufferOffset(0);
+      tm.ResetMap();
+      if (obj.kind == nullptr) {
+        LOG(error) << "Cannot read class info from buffer.";
+        return;
+      }
+
+      auto policy = objh->mPolicy;
+      auto sourceType = objh->mSourceType;
+      auto hash = objh->mTaskHash;
+
+      obj.obj = tm.ReadObjectAny(obj.kind);
+      auto* named = static_cast<TNamed*>(obj.obj);
+      obj.name = named->GetName();
+      auto hpos = std::find_if(tskmap.begin(), tskmap.end(), [&](auto&& x) { return x.id == hash; });
+      if (hpos == tskmap.end()) {
+        LOG(error) << "No task found for hash " << hash;
+        return;
+      }
+      auto taskname = hpos->name;
+      auto opos = std::find_if(objmap.begin(), objmap.end(), [&](auto&& x) { return x.id == hash; });
+      if (opos == objmap.end()) {
+        LOG(error) << "No object list found for task " << taskname << " (hash=" << hash << ")";
+        return;
+      }
+      auto objects = opos->bindings;
+      if (std::find(objects.begin(), objects.end(), obj.name) == objects.end()) {
+        LOG(error) << "No object " << obj.name << " in map for task " << taskname;
+        return;
+      }
+      auto nameHash = runtime_hash(obj.name.c_str());
+      InputObjectRoute key{obj.name, nameHash, taskname, hash, policy, sourceType};
+      auto existing = std::find_if(inputObjects->begin(), inputObjects->end(), [&](auto&& x) { return (x.first.uniqueId == nameHash) && (x.first.taskHash == hash); });
+      // If it's the first one, we just add it to the list.
+      if (existing == inputObjects->end()) {
+        obj.count = objh->mPipelineSize;
+        inputObjects->push_back(std::make_pair(key, obj));
+        existing = inputObjects->end() - 1;
+      } else {
+        obj.count = existing->second.count;
+        // Otherwise, we merge it with the existing one.
+        auto merger = existing->second.kind->GetMerge();
+        if (!merger) {
+          LOG(error) << "Already one unmergeable object found for " << obj.name;
+          return;
+        }
+        TList coll;
+        coll.Add(static_cast<TObject*>(obj.obj));
+        merger(existing->second.obj, &coll, nullptr);
+      }
+      // We expect as many objects as the pipeline size, for
+      // a given object name and task hash.
+      existing->second.count -= 1;
+
+      if (existing->second.count != 0) {
+        return;
+      }
+      // Write the object here.
+      auto route = existing->first;
+      auto entry = existing->second;
+      auto file = ROOTfileNames.find(route.policy);
+      if (file == ROOTfileNames.end()) {
+        return;
+      }
+      auto filename = file->second;
+      if (f[route.policy] == nullptr) {
+        f[route.policy] = TFile::Open(filename.c_str(), "RECREATE");
+      }
+      auto nextDirectory = route.directory;
+      if ((nextDirectory != currentDirectory) || (filename != currentFile)) {
+        if (!f[route.policy]->FindKey(nextDirectory.c_str())) {
+          f[route.policy]->mkdir(nextDirectory.c_str());
+        }
+        currentDirectory = nextDirectory;
+        currentFile = filename;
+      }
+
+      // translate the list-structure created by the registry into a directory structure within the file
+      std::function<void(TList*, TDirectory*)> writeListToFile;
+      writeListToFile = [&](TList* list, TDirectory* parentDir) {
+        TIter next(list);
+        TObject* object = nullptr;
+        while ((object = next())) {
+          if (object->InheritsFrom(TList::Class())) {
+            writeListToFile(static_cast<TList*>(object), parentDir->mkdir(object->GetName(), object->GetName(), true));
+          } else {
+            parentDir->WriteObjectAny(object, object->Class(), object->GetName());
+            auto* written = list->Remove(object);
+            delete written;
+          }
+        }
+      };
+
+      TDirectory* currentDir = f[route.policy]->GetDirectory(currentDirectory.c_str());
+      if (route.sourceType == OutputObjSourceType::HistogramRegistrySource) {
+        auto* outputList = static_cast<TList*>(entry.obj);
+        outputList->SetOwner(false);
+
+        // if registry should live in dedicated folder a TNamed object is appended to the list
+        if (outputList->Last() && outputList->Last()->IsA() == TNamed::Class()) {
+          delete outputList->Last();
+          outputList->RemoveLast();
+          currentDir = currentDir->mkdir(outputList->GetName(), outputList->GetName(), true);
+        }
+
+        writeListToFile(outputList, currentDir);
+        outputList->SetOwner();
+        delete outputList;
+        entry.obj = nullptr;
+      } else {
+        currentDir->WriteObjectAny(entry.obj, entry.kind, entry.name.c_str());
+        delete (TObject*)entry.obj;
+        entry.obj = nullptr;
+      }
+    };
+  }};
+}
+} // namespace o2::framework::writers
diff --git a/Framework/AnalysisSupport/src/AODWriterHelpers.h b/Framework/AnalysisSupport/src/AODWriterHelpers.h
new file mode 100644
index 0000000000000..7ae59a5cf3b01
--- /dev/null
+++ b/Framework/AnalysisSupport/src/AODWriterHelpers.h
@@ -0,0 +1,28 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_AODROOTWRITERHELPERS_H_
+#define O2_FRAMEWORK_AODROOTWRITERHELPERS_H_
+
+#include "Framework/AlgorithmSpec.h"
+#include <Monitoring/Monitoring.h>
+
+namespace o2::framework::writers
+{
+
+struct AODWriterHelpers {
+  static AlgorithmSpec getOutputObjHistWriter(ConfigContext const& context);
+  static AlgorithmSpec getOutputTTreeWriter(ConfigContext const& context);
+};
+
+} // namespace o2::framework::writers
+
+#endif // O2_FRAMEWORK_AODROOTWRITERHELPERS_H_
diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index bba3499286e08..52435375d7e9e 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -16,6 +16,7 @@
 #include "Framework/Capability.h"
 #include "Framework/Signpost.h"
 #include "AODJAlienReaderHelpers.h"
+#include "AODWriterHelpers.h"
 #include <TFile.h>
 #include <TMap.h>
 #include <TGrid.h>
@@ -33,6 +34,20 @@ struct ROOTFileReader : o2::framework::AlgorithmPlugin {
   }
 };
 
+struct ROOTObjWriter : o2::framework::AlgorithmPlugin {
+  o2::framework::AlgorithmSpec create(o2::framework::ConfigContext const& config) override
+  {
+    return o2::framework::writers::AODWriterHelpers::getOutputObjHistWriter(config);
+  }
+};
+
+struct ROOTTTreeWriter : o2::framework::AlgorithmPlugin {
+  o2::framework::AlgorithmSpec create(o2::framework::ConfigContext const& config) override
+  {
+    return o2::framework::writers::AODWriterHelpers::getOutputTTreeWriter(config);
+  }
+};
+
 using namespace o2::framework;
 struct RunSummary : o2::framework::ServicePlugin {
   o2::framework::ServiceSpec* create() final
@@ -211,6 +226,8 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
 
 DEFINE_DPL_PLUGINS_BEGIN
 DEFINE_DPL_PLUGIN_INSTANCE(ROOTFileReader, CustomAlgorithm);
+DEFINE_DPL_PLUGIN_INSTANCE(ROOTObjWriter, CustomAlgorithm);
+DEFINE_DPL_PLUGIN_INSTANCE(ROOTTTreeWriter, CustomAlgorithm);
 DEFINE_DPL_PLUGIN_INSTANCE(RunSummary, CustomService);
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAOD, ConfigDiscovery);
 DEFINE_DPL_PLUGINS_END
diff --git a/Framework/Core/include/Framework/AnalysisContext.h b/Framework/Core/include/Framework/AnalysisContext.h
new file mode 100644
index 0000000000000..0f62f952d0aaa
--- /dev/null
+++ b/Framework/Core/include/Framework/AnalysisContext.h
@@ -0,0 +1,58 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef O2_FRAMEWORK_ANALYSISCONTEXT_H_
+#define O2_FRAMEWORK_ANALYSISCONTEXT_H_
+
+#include <vector>
+#include "Framework/InputSpec.h"
+#include "Framework/OutputSpec.h"
+
+namespace o2::framework
+{
+class DataOutputDirector;
+
+struct OutputTaskInfo {
+  uint32_t id;
+  std::string name;
+};
+
+struct OutputObjectInfo {
+  uint32_t id;
+  std::vector<std::string> bindings;
+};
+
+//
+struct AnalysisContext {
+  std::vector<InputSpec> requestedAODs;
+  std::vector<OutputSpec> providedAODs;
+  std::vector<InputSpec> requestedDYNs;
+  std::vector<OutputSpec> providedDYNs;
+  std::vector<InputSpec> requestedIDXs;
+  std::vector<OutputSpec> providedOutputObjHist;
+  std::vector<InputSpec> spawnerInputs;
+
+  // Needed to created the hist writer
+  std::vector<OutputTaskInfo> outTskMap;
+  std::vector<OutputObjectInfo> outObjHistMap;
+
+  // Needed to create the output director
+  std::vector<InputSpec> outputsInputs;
+  std::vector<bool> isDangling;
+
+  // Needed to create the aod writer
+  std::vector<InputSpec> outputsInputsAOD;
+};
+} // namespace o2::framework
+
+extern template class std::vector<o2::framework::OutputObjectInfo>;
+extern template class std::vector<o2::framework::OutputTaskInfo>;
+
+#endif // O2_FRAMEWORK_ANALYSISCONTEXT_H_
diff --git a/Framework/Core/src/AnalysisSupportHelpers.h b/Framework/Core/include/Framework/AnalysisSupportHelpers.h
similarity index 71%
rename from Framework/Core/src/AnalysisSupportHelpers.h
rename to Framework/Core/include/Framework/AnalysisSupportHelpers.h
index ba5bcedb4bc67..4ae601dc9e4a2 100644
--- a/Framework/Core/src/AnalysisSupportHelpers.h
+++ b/Framework/Core/include/Framework/AnalysisSupportHelpers.h
@@ -14,6 +14,7 @@
 #include "Framework/OutputSpec.h"
 #include "Framework/InputSpec.h"
 #include "Framework/DataProcessorSpec.h"
+#include "Framework/AnalysisContext.h"
 #include "Headers/DataHeader.h"
 #include <array>
 
@@ -24,36 +25,7 @@ static constexpr std::array<header::DataOrigin, 5> extendedAODOrigins{header::Da
 static constexpr std::array<header::DataOrigin, 4> writableAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}};
 
 class DataOutputDirector;
-
-struct OutputTaskInfo {
-  uint32_t id;
-  std::string name;
-};
-
-struct OutputObjectInfo {
-  uint32_t id;
-  std::vector<std::string> bindings;
-};
-} // namespace o2::framework
-
-extern template class std::vector<o2::framework::OutputObjectInfo>;
-extern template class std::vector<o2::framework::OutputTaskInfo>;
-
-namespace o2::framework
-{
-//
-struct AnalysisContext {
-  std::vector<InputSpec> requestedAODs;
-  std::vector<OutputSpec> providedAODs;
-  std::vector<InputSpec> requestedDYNs;
-  std::vector<OutputSpec> providedDYNs;
-  std::vector<InputSpec> requestedIDXs;
-  std::vector<OutputSpec> providedOutputObjHist;
-  std::vector<InputSpec> spawnerInputs;
-
-  std::vector<OutputTaskInfo> outTskMap;
-  std::vector<OutputObjectInfo> outObjHistMap;
-};
+class ConfigContext;
 
 // Helper class to be moved in the AnalysisSupport plugin at some point
 struct AnalysisSupportHelpers {
@@ -74,11 +46,11 @@ struct AnalysisSupportHelpers {
 
   /// Match all inputs of kind ATSK and write them to a ROOT file,
   /// one root file per originating task.
-  static DataProcessorSpec getOutputObjHistSink(std::vector<OutputObjectInfo> const& objmap,
-                                                std::vector<OutputTaskInfo> const& tskmap);
+  static DataProcessorSpec getOutputObjHistSink(ConfigContext const&);
   /// writes inputs of kind AOD to file
-  static DataProcessorSpec getGlobalAODSink(std::shared_ptr<DataOutputDirector> dod,
-                                            std::vector<InputSpec> const& outputInputs, int compression);
+  static DataProcessorSpec getGlobalAODSink(ConfigContext const&);
+  /// Get the data director
+  static std::shared_ptr<DataOutputDirector> getDataOutputDirector(ConfigContext const& ctx);
 };
 
 }; // namespace o2::framework
diff --git a/Framework/Core/include/Framework/ConfigContext.h b/Framework/Core/include/Framework/ConfigContext.h
index 5790699fe68bb..87259f0519915 100644
--- a/Framework/Core/include/Framework/ConfigContext.h
+++ b/Framework/Core/include/Framework/ConfigContext.h
@@ -8,11 +8,11 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
-#ifndef FRAMEWORK_CONFIG_CONTEXT_H
-#define FRAMEWORK_CONFIG_CONTEXT_H
+#ifndef O2_FRAMEWORK_CONFIG_CONTEXT_H_
+#define O2_FRAMEWORK_CONFIG_CONTEXT_H_
 
 #include "Framework/ConfigParamRegistry.h"
-#include "Framework/ServiceRegistry.h"
+#include "Framework/ServiceRegistryRef.h"
 
 namespace o2::framework
 {
@@ -23,9 +23,10 @@ namespace o2::framework
 class ConfigContext
 {
  public:
-  ConfigContext(ConfigParamRegistry& options, int argc, char** argv) : mOptions{options}, mArgc{argc}, mArgv{argv} {}
+  ConfigContext(ConfigParamRegistry& options, ServiceRegistryRef services, int argc, char** argv);
 
   [[nodiscard]] ConfigParamRegistry& options() const { return mOptions; }
+  [[nodiscard]] ServiceRegistryRef services() const { return mServices; }
 
   [[nodiscard]] bool helpOnCommandLine() const;
 
@@ -34,11 +35,13 @@ class ConfigContext
 
  private:
   ConfigParamRegistry& mOptions;
+
+  ServiceRegistryRef mServices;
   // additionaly keep information about the original command line
   int mArgc = 0;
   char** mArgv = nullptr;
 };
 
-} // namespace o2
+} // namespace o2::framework
 
-#endif
+#endif // O2_FRAMEWORK_CONFIG_CONTEXT_H_
diff --git a/Framework/Core/include/Framework/runDataProcessing.h b/Framework/Core/include/Framework/runDataProcessing.h
index eee4c4b6583d3..8293bf0cf7039 100644
--- a/Framework/Core/include/Framework/runDataProcessing.h
+++ b/Framework/Core/include/Framework/runDataProcessing.h
@@ -30,6 +30,7 @@
 #include "Framework/CheckTypes.h"
 #include "Framework/StructToTuple.h"
 #include "Framework/ConfigParamDiscovery.h"
+#include "ServiceRegistryRef.h"
 #include <vector>
 
 namespace o2::framework
@@ -198,7 +199,8 @@ int mainNoCatch(int argc, char** argv)
     workflowOptions.push_back(extra);
   }
 
-  ConfigContext configContext(workflowOptionsRegistry, argc, argv);
+  ServiceRegistry configRegistry;
+  ConfigContext configContext(workflowOptionsRegistry, ServiceRegistryRef{configRegistry}, argc, argv);
   o2::framework::WorkflowSpec specs = defineDataProcessing(configContext);
   overrideCloning(configContext, specs);
   overridePipeline(configContext, specs);
diff --git a/Framework/Core/src/AnalysisSupportHelpers.cxx b/Framework/Core/src/AnalysisSupportHelpers.cxx
index e949f27a6eed6..eb17566fd6d31 100644
--- a/Framework/Core/src/AnalysisSupportHelpers.cxx
+++ b/Framework/Core/src/AnalysisSupportHelpers.cxx
@@ -9,18 +9,16 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
-#include "AnalysisSupportHelpers.h"
+#include "Framework/AnalysisSupportHelpers.h"
 #include "Framework/DataOutputDirector.h"
 #include "Framework/OutputObjHeader.h"
 #include "Framework/ControlService.h"
 #include "Framework/EndOfStreamContext.h"
 #include "Framework/DeviceSpec.h"
 #include "Framework/TableTreeHelpers.h"
-
-#include "TFile.h"
-#include "TTree.h"
-#include "TMap.h"
-#include "TObjString.h"
+#include "Framework/PluginManager.h"
+#include "Framework/ConfigContext.h"
+#include "WorkflowHelpers.h"
 
 template class std::vector<o2::framework::OutputObjectInfo>;
 template class std::vector<o2::framework::OutputTaskInfo>;
@@ -28,21 +26,105 @@ template class std::vector<o2::framework::OutputTaskInfo>;
 namespace o2::framework
 {
 
-struct InputObjectRoute {
-  std::string name;
-  uint32_t uniqueId;
-  std::string directory;
-  uint32_t taskHash;
-  OutputObjHandlingPolicy policy;
-  OutputObjSourceType sourceType;
-};
+std::shared_ptr<DataOutputDirector> AnalysisSupportHelpers::getDataOutputDirector(ConfigContext const& ctx)
+{
+  auto const& options = ctx.options();
+  auto const& OutputsInputs = ctx.services().get<AnalysisContext>().outputsInputs;
+  auto const& isDangling = ctx.services().get<AnalysisContext>().isDangling;
+
+  std::shared_ptr<DataOutputDirector> dod = std::make_shared<DataOutputDirector>();
+
+  // analyze options and take actions accordingly
+  // default values
+  std::string rdn, resdir("./");
+  std::string fnb, fnbase("AnalysisResults_trees");
+  float mfs, maxfilesize(-1.);
+  std::string fmo, filemode("RECREATE");
+  int ntfm, ntfmerge = 1;
+
+  // values from json
+  if (options.isSet("aod-writer-json")) {
+    auto fnjson = options.get<std::string>("aod-writer-json");
+    if (!fnjson.empty()) {
+      std::tie(rdn, fnb, fmo, mfs, ntfm) = dod->readJson(fnjson);
+      if (!rdn.empty()) {
+        resdir = rdn;
+      }
+      if (!fnb.empty()) {
+        fnbase = fnb;
+      }
+      if (!fmo.empty()) {
+        filemode = fmo;
+      }
+      if (mfs > 0.) {
+        maxfilesize = mfs;
+      }
+      if (ntfm > 0) {
+        ntfmerge = ntfm;
+      }
+    }
+  }
+
+  // values from command line options, information from json is overwritten
+  if (options.isSet("aod-writer-resdir")) {
+    rdn = options.get<std::string>("aod-writer-resdir");
+    if (!rdn.empty()) {
+      resdir = rdn;
+    }
+  }
+  if (options.isSet("aod-writer-resfile")) {
+    fnb = options.get<std::string>("aod-writer-resfile");
+    if (!fnb.empty()) {
+      fnbase = fnb;
+    }
+  }
+  if (options.isSet("aod-writer-resmode")) {
+    fmo = options.get<std::string>("aod-writer-resmode");
+    if (!fmo.empty()) {
+      filemode = fmo;
+    }
+  }
+  if (options.isSet("aod-writer-maxfilesize")) {
+    mfs = options.get<float>("aod-writer-maxfilesize");
+    if (mfs > 0) {
+      maxfilesize = mfs;
+    }
+  }
+  if (options.isSet("aod-writer-ntfmerge")) {
+    ntfm = options.get<int>("aod-writer-ntfmerge");
+    if (ntfm > 0) {
+      ntfmerge = ntfm;
+    }
+  }
+  // parse the keepString
+  if (options.isSet("aod-writer-keep")) {
+    auto keepString = options.get<std::string>("aod-writer-keep");
+    if (!keepString.empty()) {
+      dod->reset();
+      std::string d("dangling");
+      if (d.find(keepString) == 0) {
+        // use the dangling outputs
+        std::vector<InputSpec> danglingOutputs;
+        for (auto ii = 0u; ii < OutputsInputs.size(); ii++) {
+          if (DataSpecUtils::partialMatch(OutputsInputs[ii], writableAODOrigins) && isDangling[ii]) {
+            danglingOutputs.emplace_back(OutputsInputs[ii]);
+          }
+        }
+        dod->readSpecs(danglingOutputs);
+      } else {
+        // use the keep string
+        dod->readString(keepString);
+      }
+    }
+  }
+  dod->setResultDir(resdir);
+  dod->setFilenameBase(fnbase);
+  dod->setFileMode(filemode);
+  dod->setMaximumFileSize(maxfilesize);
+  dod->setNumberTimeFramesToMerge(ntfmerge);
 
-struct InputObject {
-  TClass* kind = nullptr;
-  void* obj = nullptr;
-  std::string name;
-  int count = -1;
-};
+  return dod;
+}
 
 void AnalysisSupportHelpers::addMissingOutputsToReader(std::vector<OutputSpec> const& providedOutputs,
                                                        std::vector<InputSpec> const& requestedInputs,
@@ -125,191 +207,16 @@ void AnalysisSupportHelpers::addMissingOutputsToBuilder(std::vector<InputSpec> c
   }
 }
 
-const static std::unordered_map<OutputObjHandlingPolicy, std::string> ROOTfileNames = {{OutputObjHandlingPolicy::AnalysisObject, "AnalysisResults.root"},
-                                                                                       {OutputObjHandlingPolicy::QAObject, "QAResults.root"}};
-
 // =============================================================================
-DataProcessorSpec AnalysisSupportHelpers::getOutputObjHistSink(std::vector<OutputObjectInfo> const& objmap, std::vector<OutputTaskInfo> const& tskmap)
+DataProcessorSpec AnalysisSupportHelpers::getOutputObjHistSink(ConfigContext const& ctx)
 {
-  auto writerFunction = [objmap, tskmap](InitContext& ic) -> std::function<void(ProcessingContext&)> {
-    auto& callbacks = ic.services().get<CallbackService>();
-    auto inputObjects = std::make_shared<std::vector<std::pair<InputObjectRoute, InputObject>>>();
-
-    static TFile* f[OutputObjHandlingPolicy::numPolicies];
-    for (auto i = 0u; i < OutputObjHandlingPolicy::numPolicies; ++i) {
-      f[i] = nullptr;
-    }
-
-    static std::string currentDirectory = "";
-    static std::string currentFile = "";
-
-    auto endofdatacb = [inputObjects](EndOfStreamContext& context) {
-      LOG(debug) << "Writing merged objects and histograms to file";
-      if (inputObjects->empty()) {
-        LOG(error) << "Output object map is empty!";
-        context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
-        return;
-      }
-      for (auto i = 0u; i < OutputObjHandlingPolicy::numPolicies; ++i) {
-        if (f[i] != nullptr) {
-          f[i]->Close();
-        }
-      }
-      LOG(debug) << "All outputs merged in their respective target files";
-      context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
-    };
-
-    callbacks.set<CallbackService::Id::EndOfStream>(endofdatacb);
-    return [inputObjects, objmap, tskmap](ProcessingContext& pc) mutable -> void {
-      auto const& ref = pc.inputs().get("x");
-      if (!ref.header) {
-        LOG(error) << "Header not found";
-        return;
-      }
-      if (!ref.payload) {
-        LOG(error) << "Payload not found";
-        return;
-      }
-      auto datah = o2::header::get<o2::header::DataHeader*>(ref.header);
-      if (!datah) {
-        LOG(error) << "No data header in stack";
-        return;
-      }
-
-      auto objh = o2::header::get<o2::framework::OutputObjHeader*>(ref.header);
-      if (!objh) {
-        LOG(error) << "No output object header in stack";
-        return;
-      }
-
-      InputObject obj;
-      FairInputTBuffer tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
-      tm.InitMap();
-      obj.kind = tm.ReadClass();
-      tm.SetBufferOffset(0);
-      tm.ResetMap();
-      if (obj.kind == nullptr) {
-        LOG(error) << "Cannot read class info from buffer.";
-        return;
-      }
-
-      auto policy = objh->mPolicy;
-      auto sourceType = objh->mSourceType;
-      auto hash = objh->mTaskHash;
-
-      obj.obj = tm.ReadObjectAny(obj.kind);
-      auto* named = static_cast<TNamed*>(obj.obj);
-      obj.name = named->GetName();
-      auto hpos = std::find_if(tskmap.begin(), tskmap.end(), [&](auto&& x) { return x.id == hash; });
-      if (hpos == tskmap.end()) {
-        LOG(error) << "No task found for hash " << hash;
-        return;
-      }
-      auto taskname = hpos->name;
-      auto opos = std::find_if(objmap.begin(), objmap.end(), [&](auto&& x) { return x.id == hash; });
-      if (opos == objmap.end()) {
-        LOG(error) << "No object list found for task " << taskname << " (hash=" << hash << ")";
-        return;
-      }
-      auto objects = opos->bindings;
-      if (std::find(objects.begin(), objects.end(), obj.name) == objects.end()) {
-        LOG(error) << "No object " << obj.name << " in map for task " << taskname;
-        return;
-      }
-      auto nameHash = runtime_hash(obj.name.c_str());
-      InputObjectRoute key{obj.name, nameHash, taskname, hash, policy, sourceType};
-      auto existing = std::find_if(inputObjects->begin(), inputObjects->end(), [&](auto&& x) { return (x.first.uniqueId == nameHash) && (x.first.taskHash == hash); });
-      // If it's the first one, we just add it to the list.
-      if (existing == inputObjects->end()) {
-        obj.count = objh->mPipelineSize;
-        inputObjects->push_back(std::make_pair(key, obj));
-        existing = inputObjects->end() - 1;
-      } else {
-        obj.count = existing->second.count;
-        // Otherwise, we merge it with the existing one.
-        auto merger = existing->second.kind->GetMerge();
-        if (!merger) {
-          LOG(error) << "Already one unmergeable object found for " << obj.name;
-          return;
-        }
-        TList coll;
-        coll.Add(static_cast<TObject*>(obj.obj));
-        merger(existing->second.obj, &coll, nullptr);
-      }
-      // We expect as many objects as the pipeline size, for
-      // a given object name and task hash.
-      existing->second.count -= 1;
-
-      if (existing->second.count != 0) {
-        return;
-      }
-      // Write the object here.
-      auto route = existing->first;
-      auto entry = existing->second;
-      auto file = ROOTfileNames.find(route.policy);
-      if (file == ROOTfileNames.end()) {
-        return;
-      }
-      auto filename = file->second;
-      if (f[route.policy] == nullptr) {
-        f[route.policy] = TFile::Open(filename.c_str(), "RECREATE");
-      }
-      auto nextDirectory = route.directory;
-      if ((nextDirectory != currentDirectory) || (filename != currentFile)) {
-        if (!f[route.policy]->FindKey(nextDirectory.c_str())) {
-          f[route.policy]->mkdir(nextDirectory.c_str());
-        }
-        currentDirectory = nextDirectory;
-        currentFile = filename;
-      }
-
-      // translate the list-structure created by the registry into a directory structure within the file
-      std::function<void(TList*, TDirectory*)> writeListToFile;
-      writeListToFile = [&](TList* list, TDirectory* parentDir) {
-        TIter next(list);
-        TObject* object = nullptr;
-        while ((object = next())) {
-          if (object->InheritsFrom(TList::Class())) {
-            writeListToFile(static_cast<TList*>(object), parentDir->mkdir(object->GetName(), object->GetName(), true));
-          } else {
-            parentDir->WriteObjectAny(object, object->Class(), object->GetName());
-            auto* written = list->Remove(object);
-            delete written;
-          }
-        }
-      };
-
-      TDirectory* currentDir = f[route.policy]->GetDirectory(currentDirectory.c_str());
-      if (route.sourceType == OutputObjSourceType::HistogramRegistrySource) {
-        auto* outputList = static_cast<TList*>(entry.obj);
-        outputList->SetOwner(false);
-
-        // if registry should live in dedicated folder a TNamed object is appended to the list
-        if (outputList->Last() && outputList->Last()->IsA() == TNamed::Class()) {
-          delete outputList->Last();
-          outputList->RemoveLast();
-          currentDir = currentDir->mkdir(outputList->GetName(), outputList->GetName(), true);
-        }
-
-        writeListToFile(outputList, currentDir);
-        outputList->SetOwner();
-        delete outputList;
-        entry.obj = nullptr;
-      } else {
-        currentDir->WriteObjectAny(entry.obj, entry.kind, entry.name.c_str());
-        delete (TObject*)entry.obj;
-        entry.obj = nullptr;
-      }
-    };
-  };
-
-  char const* name = "internal-dpl-aod-global-analysis-file-sink";
   // Lifetime is sporadic because we do not ask each analysis task to send its
   // results every timeframe.
   DataProcessorSpec spec{
-    .name = name,
+    .name = "internal-dpl-aod-global-analysis-file-sink",
     .inputs = {InputSpec("x", DataSpecUtils::dataDescriptorMatcherFrom(header::DataOrigin{"ATSK"}), Lifetime::Sporadic)},
-    .algorithm = {writerFunction},
+    .outputs = {},
+    .algorithm = PluginManager::loadAlgorithmFromPlugin("O2FrameworkAnalysisSupport", "ROOTObjWriter", ctx),
   };
 
   return spec;
@@ -317,188 +224,17 @@ DataProcessorSpec AnalysisSupportHelpers::getOutputObjHistSink(std::vector<Outpu
 
 // add sink for the AODs
 DataProcessorSpec
-  AnalysisSupportHelpers::getGlobalAODSink(std::shared_ptr<DataOutputDirector> dod,
-                                           std::vector<InputSpec> const& outputInputs, int compressionLevel)
+  AnalysisSupportHelpers::getGlobalAODSink(ConfigContext const& ctx)
 {
-
-  auto writerFunction = [dod, outputInputs, compressionLevel](InitContext& ic) -> std::function<void(ProcessingContext&)> {
-    LOGP(debug, "======== getGlobalAODSink::Init ==========");
-
-    // find out if any table needs to be saved
-    bool hasOutputsToWrite = false;
-    for (auto& outobj : outputInputs) {
-      auto ds = dod->getDataOutputDescriptors(outobj);
-      if (ds.size() > 0) {
-        hasOutputsToWrite = true;
-        break;
-      }
-    }
-
-    // if nothing needs to be saved then return a trivial functor
-    // this happens when nothing needs to be saved but there are dangling outputs
-    if (!hasOutputsToWrite) {
-      return [](ProcessingContext&) mutable -> void {
-        static bool once = false;
-        if (!once) {
-          LOG(info) << "No AODs to be saved.";
-          once = true;
-        }
-      };
-    }
-
-    // end of data functor is called at the end of the data stream
-    auto endofdatacb = [dod](EndOfStreamContext& context) {
-      dod->closeDataFiles();
-      context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
-    };
-
-    auto& callbacks = ic.services().get<CallbackService>();
-    callbacks.set<CallbackService::Id::EndOfStream>(endofdatacb);
-
-    // prepare map<uint64_t, uint64_t>(startTime, tfNumber)
-    std::map<uint64_t, uint64_t> tfNumbers;
-    std::map<uint64_t, std::string> tfFilenames;
-
-    std::vector<TString> aodMetaDataKeys;
-    std::vector<TString> aodMetaDataVals;
-
-    // this functor is called once per time frame
-    return [dod, tfNumbers, tfFilenames, aodMetaDataKeys, aodMetaDataVals, compressionLevel](ProcessingContext& pc) mutable -> void {
-      LOGP(debug, "======== getGlobalAODSink::processing ==========");
-      LOGP(debug, " processing data set with {} entries", pc.inputs().size());
-
-      // return immediately if pc.inputs() is empty. This should never happen!
-      if (pc.inputs().size() == 0) {
-        LOGP(info, "No inputs available!");
-        return;
-      }
-
-      // update tfNumbers
-      uint64_t startTime = 0;
-      uint64_t tfNumber = 0;
-      auto ref = pc.inputs().get("tfn");
-      if (ref.spec && ref.payload) {
-        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref)->startTime;
-        tfNumber = pc.inputs().get<uint64_t>("tfn");
-        tfNumbers.insert(std::pair<uint64_t, uint64_t>(startTime, tfNumber));
-      }
-      // update tfFilenames
-      std::string aodInputFile;
-      auto ref2 = pc.inputs().get("tff");
-      if (ref2.spec && ref2.payload) {
-        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref2)->startTime;
-        aodInputFile = pc.inputs().get<std::string>("tff");
-        tfFilenames.insert(std::pair<uint64_t, std::string>(startTime, aodInputFile));
-      }
-
-      // close all output files if one has reached size limit
-      dod->checkFileSizes();
-
-      // loop over the DataRefs which are contained in pc.inputs()
-      for (const auto& ref : pc.inputs()) {
-        if (!ref.spec) {
-          LOGP(debug, "Invalid input will be skipped!");
-          continue;
-        }
-
-        // get metadata
-        if (DataSpecUtils::partialMatch(*ref.spec, header::DataDescription("AODMetadataKeys"))) {
-          aodMetaDataKeys = pc.inputs().get<std::vector<TString>>(ref.spec->binding);
-        }
-        if (DataSpecUtils::partialMatch(*ref.spec, header::DataDescription("AODMetadataVals"))) {
-          aodMetaDataVals = pc.inputs().get<std::vector<TString>>(ref.spec->binding);
-        }
-
-        // skip non-AOD refs
-        if (!DataSpecUtils::partialMatch(*ref.spec, writableAODOrigins)) {
-          continue;
-        }
-        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref)->startTime;
-
-        // does this need to be saved?
-        auto dh = DataRefUtils::getHeader<header::DataHeader*>(ref);
-        auto tableName = dh->dataDescription.as<std::string>();
-        auto ds = dod->getDataOutputDescriptors(*dh);
-        if (ds.empty()) {
-          continue;
-        }
-
-        // get TF number from startTime
-        auto it = tfNumbers.find(startTime);
-        if (it != tfNumbers.end()) {
-          tfNumber = (it->second / dod->getNumberTimeFramesToMerge()) * dod->getNumberTimeFramesToMerge();
-        } else {
-          LOGP(fatal, "No time frame number found for output with start time {}", startTime);
-          throw std::runtime_error("Processing is stopped!");
-        }
-        // get aod input file from startTime
-        auto it2 = tfFilenames.find(startTime);
-        if (it2 != tfFilenames.end()) {
-          aodInputFile = it2->second;
-        }
-
-        // get the TableConsumer and corresponding arrow table
-        auto msg = pc.inputs().get(ref.spec->binding);
-        if (msg.header == nullptr) {
-          LOGP(error, "No header for message {}:{}", ref.spec->binding, DataSpecUtils::describe(*ref.spec));
-          continue;
-        }
-        auto s = pc.inputs().get<TableConsumer>(ref.spec->binding);
-        auto table = s->asArrowTable();
-        if (!table->Validate().ok()) {
-          LOGP(warning, "The table \"{}\" is not valid and will not be saved!", tableName);
-          continue;
-        }
-        if (table->schema()->fields().empty()) {
-          LOGP(debug, "The table \"{}\" is empty but will be saved anyway!", tableName);
-        }
-
-        // loop over all DataOutputDescriptors
-        // a table can be saved in multiple ways
-        // e.g. different selections of columns to different files
-        for (auto d : ds) {
-          auto fileAndFolder = dod->getFileFolder(d, tfNumber, aodInputFile, compressionLevel);
-          auto treename = fileAndFolder.folderName + "/" + d->treename;
-          TableToTree ta2tr(table,
-                            fileAndFolder.file,
-                            treename.c_str());
-
-          // update metadata
-          if (fileAndFolder.file->FindObjectAny("metaData")) {
-            LOGF(debug, "Metadata: target file %s already has metadata, preserving it", fileAndFolder.file->GetName());
-          } else if (!aodMetaDataKeys.empty() && !aodMetaDataVals.empty()) {
-            TMap aodMetaDataMap;
-            for (uint32_t imd = 0; imd < aodMetaDataKeys.size(); imd++) {
-              aodMetaDataMap.Add(new TObjString(aodMetaDataKeys[imd]), new TObjString(aodMetaDataVals[imd]));
-            }
-            fileAndFolder.file->WriteObject(&aodMetaDataMap, "metaData", "Overwrite");
-          }
-
-          if (!d->colnames.empty()) {
-            for (auto& cn : d->colnames) {
-              auto idx = table->schema()->GetFieldIndex(cn);
-              auto col = table->column(idx);
-              auto field = table->schema()->field(idx);
-              if (idx != -1) {
-                ta2tr.addBranch(col, field);
-              }
-            }
-          } else {
-            ta2tr.addAllBranches();
-          }
-          ta2tr.process();
-        }
-      }
-    };
-  }; // end of writerFunction
+  auto& ac = ctx.services().get<AnalysisContext>();
 
   // the command line options relevant for the writer are global
   // see runDataProcessing.h
   DataProcessorSpec spec{
     .name = "internal-dpl-aod-writer",
-    .inputs = outputInputs,
+    .inputs = ac.outputsInputsAOD,
     .outputs = {},
-    .algorithm = AlgorithmSpec{writerFunction},
+    .algorithm = PluginManager::loadAlgorithmFromPlugin("O2FrameworkAnalysisSupport", "ROOTTTreeWriter", ctx),
   };
 
   return spec;
diff --git a/Framework/Core/src/ArrowSupport.cxx b/Framework/Core/src/ArrowSupport.cxx
index 1a656e4d60080..230d708b47dc7 100644
--- a/Framework/Core/src/ArrowSupport.cxx
+++ b/Framework/Core/src/ArrowSupport.cxx
@@ -30,7 +30,7 @@
 #include "Framework/ServiceMetricsInfo.h"
 #include "WorkflowHelpers.h"
 #include "Framework/WorkflowSpecNode.h"
-#include "AnalysisSupportHelpers.h"
+#include "Framework/AnalysisSupportHelpers.h"
 
 #include "CommonMessageBackendsHelpers.h"
 #include <Monitoring/Monitoring.h>
@@ -516,7 +516,7 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
       auto [outputsInputs, isDangling] = WorkflowHelpers::analyzeOutputs(workflow);
 
       // create DataOutputDescriptor
-      std::shared_ptr<DataOutputDirector> dod = WorkflowHelpers::getDataOutputDirector(ctx.options(), outputsInputs, isDangling);
+      std::shared_ptr<DataOutputDirector> dod = AnalysisSupportHelpers::getDataOutputDirector(ctx);
 
       // select outputs of type AOD which need to be saved
       // ATTENTION: if there are dangling outputs the getGlobalAODSink
@@ -537,11 +537,7 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
         // add TFNumber and TFFilename as input to the writer
         outputsInputsAOD.emplace_back("tfn", "TFN", "TFNumber");
         outputsInputsAOD.emplace_back("tff", "TFF", "TFFilename");
-        int compression = 505; 
-        if (ctx.options().hasOption("aod-writer-compression")) {
-          compression = ctx.options().get<int>("aod-writer-compression");
-        }
-        workflow.push_back(AnalysisSupportHelpers::getGlobalAODSink(dod, outputsInputsAOD, compression));
+        workflow.push_back(AnalysisSupportHelpers::getGlobalAODSink(ctx));
       }
       // Move the dummy sink at the end, if needed
       for (size_t i = 0; i < workflow.size(); ++i) {
diff --git a/Framework/Core/src/ConfigContext.cxx b/Framework/Core/src/ConfigContext.cxx
index 726332e1d0ae3..9b121b1884998 100644
--- a/Framework/Core/src/ConfigContext.cxx
+++ b/Framework/Core/src/ConfigContext.cxx
@@ -14,6 +14,9 @@
 namespace o2::framework
 {
 
+ConfigContext::ConfigContext(ConfigParamRegistry& options, ServiceRegistryRef services, int argc, char** argv)
+  : mOptions{options}, mServices{services}, mArgc{argc}, mArgv{argv} {}
+
 bool ConfigContext::helpOnCommandLine() const
 {
   bool helpasked = false;
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index da9a135dc5eb8..3782c48e81c56 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -9,7 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 #include "WorkflowHelpers.h"
-#include "AnalysisSupportHelpers.h"
+#include "Framework/AnalysisSupportHelpers.h"
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/AODReaderHelpers.h"
 #include "Framework/ConfigParamSpec.h"
@@ -153,7 +153,7 @@ int defaultConditionQueryRateMultiplier()
   return getenv("DPL_CONDITION_QUERY_RATE_MULTIPLIER") ? std::stoi(getenv("DPL_CONDITION_QUERY_RATE_MULTIPLIER")) : 1;
 }
 
-void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext const& ctx)
+void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext& ctx)
 {
   auto fakeCallback = AlgorithmSpec{[](InitContext& ic) {
     LOG(info) << "This is not a real device, merely a placeholder for external inputs";
@@ -241,7 +241,9 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
     aodReader.options.emplace_back(ConfigParamSpec{"channel-config", VariantType::String, rateLimitingChannelConfigInput, {"how many timeframes can be in flight at the same time"}});
   }
 
-  AnalysisContext ac;
+  ctx.services().registerService(ServiceRegistryHelpers::handleForService<AnalysisContext>(new AnalysisContext));
+  auto& ac = ctx.services().get<AnalysisContext>();
+
   std::vector<InputSpec> requestedCCDBs;
   std::vector<OutputSpec> providedCCDBs;
 
@@ -573,7 +575,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
   // This is to inject a file sink so that any dangling ATSK object is written
   // to a ROOT file.
   if (ac.providedOutputObjHist.empty() == false) {
-    auto rootSink = AnalysisSupportHelpers::getOutputObjHistSink(ac.outObjHistMap, ac.outTskMap);
+    auto rootSink = AnalysisSupportHelpers::getOutputObjHistSink(ctx);
     extraSpecs.push_back(rootSink);
   }
 
@@ -581,41 +583,38 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
   extraSpecs.clear();
 
   /// Analyze all ouputs
-  auto [outputsInputs, isDangling] = analyzeOutputs(workflow);
+  auto [outputsInputsTmp, isDanglingTmp] = analyzeOutputs(workflow);
+  ac.isDangling = isDanglingTmp;
+  ac.outputsInputs = outputsInputsTmp;
 
   // create DataOutputDescriptor
-  std::shared_ptr<DataOutputDirector> dod = getDataOutputDirector(ctx.options(), outputsInputs, isDangling);
+  std::shared_ptr<DataOutputDirector> dod = AnalysisSupportHelpers::getDataOutputDirector(ctx);
 
   // select outputs of type AOD which need to be saved
   // ATTENTION: if there are dangling outputs the getGlobalAODSink
   // has to be created in any case!
-  std::vector<InputSpec> outputsInputsAOD;
-  for (auto ii = 0u; ii < outputsInputs.size(); ii++) {
-    if (DataSpecUtils::partialMatch(outputsInputs[ii], extendedAODOrigins)) {
-      auto ds = dod->getDataOutputDescriptors(outputsInputs[ii]);
-      if (ds.size() > 0 || isDangling[ii]) {
-        outputsInputsAOD.emplace_back(outputsInputs[ii]);
+  for (auto ii = 0u; ii < ac.outputsInputs.size(); ii++) {
+    if (DataSpecUtils::partialMatch(ac.outputsInputs[ii], extendedAODOrigins)) {
+      auto ds = dod->getDataOutputDescriptors(ac.outputsInputs[ii]);
+      if (ds.size() > 0 || ac.isDangling[ii]) {
+        ac.outputsInputsAOD.emplace_back(ac.outputsInputs[ii]);
       }
     }
   }
 
   // file sink for any AOD output
-  if (outputsInputsAOD.size() > 0) {
+  if (ac.outputsInputsAOD.size() > 0) {
     // add TFNumber and TFFilename as input to the writer
-    outputsInputsAOD.emplace_back(InputSpec{"tfn", "TFN", "TFNumber"});
-    outputsInputsAOD.emplace_back(InputSpec{"tff", "TFF", "TFFilename"});
-    int compressionLevel = 505;
-    if (ctx.options().hasOption("aod-writer-compression")) {
-      compressionLevel = ctx.options().get<int>("aod-writer-compression");
-    }
-    auto fileSink = AnalysisSupportHelpers::getGlobalAODSink(dod, outputsInputsAOD, compressionLevel);
+    ac.outputsInputsAOD.emplace_back(InputSpec{"tfn", "TFN", "TFNumber"});
+    ac.outputsInputsAOD.emplace_back(InputSpec{"tff", "TFF", "TFFilename"});
+    auto fileSink = AnalysisSupportHelpers::getGlobalAODSink(ctx);
     extraSpecs.push_back(fileSink);
 
-    auto it = std::find_if(outputsInputs.begin(), outputsInputs.end(), [](InputSpec& spec) -> bool {
+    auto it = std::find_if(ac.outputsInputs.begin(), ac.outputsInputs.end(), [](InputSpec& spec) -> bool {
       return DataSpecUtils::partialMatch(spec, o2::header::DataOrigin("TFN"));
     });
-    size_t ii = std::distance(outputsInputs.begin(), it);
-    isDangling[ii] = false;
+    size_t ii = std::distance(ac.outputsInputs.begin(), it);
+    ac.isDangling[ii] = false;
   }
 
   workflow.insert(workflow.end(), extraSpecs.begin(), extraSpecs.end());
@@ -623,20 +622,20 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
 
   // Select dangling outputs which are not of type AOD
   std::vector<InputSpec> redirectedOutputsInputs;
-  for (auto ii = 0u; ii < outputsInputs.size(); ii++) {
+  for (auto ii = 0u; ii < ac.outputsInputs.size(); ii++) {
     if (ctx.options().get<std::string>("forwarding-policy") == "none") {
       continue;
     }
     // We forward to the output proxy all the inputs only if they are dangling
     // or if the forwarding policy is "proxy".
-    if (!isDangling[ii] && (ctx.options().get<std::string>("forwarding-policy") != "all")) {
+    if (!ac.isDangling[ii] && (ctx.options().get<std::string>("forwarding-policy") != "all")) {
       continue;
     }
     // AODs are skipped in any case.
-    if (DataSpecUtils::partialMatch(outputsInputs[ii], extendedAODOrigins)) {
+    if (DataSpecUtils::partialMatch(ac.outputsInputs[ii], extendedAODOrigins)) {
       continue;
     }
-    redirectedOutputsInputs.emplace_back(outputsInputs[ii]);
+    redirectedOutputsInputs.emplace_back(ac.outputsInputs[ii]);
   }
 
   std::vector<InputSpec> unmatched;
@@ -985,102 +984,6 @@ struct DataMatcherId {
   size_t id;
 };
 
-std::shared_ptr<DataOutputDirector> WorkflowHelpers::getDataOutputDirector(ConfigParamRegistry const& options, std::vector<InputSpec> const& OutputsInputs, std::vector<bool> const& isDangling)
-{
-  std::shared_ptr<DataOutputDirector> dod = std::make_shared<DataOutputDirector>();
-
-  // analyze options and take actions accordingly
-  // default values
-  std::string rdn, resdir("./");
-  std::string fnb, fnbase("AnalysisResults_trees");
-  float mfs, maxfilesize(-1.);
-  std::string fmo, filemode("RECREATE");
-  int ntfm, ntfmerge = 1;
-
-  // values from json
-  if (options.isSet("aod-writer-json")) {
-    auto fnjson = options.get<std::string>("aod-writer-json");
-    if (!fnjson.empty()) {
-      std::tie(rdn, fnb, fmo, mfs, ntfm) = dod->readJson(fnjson);
-      if (!rdn.empty()) {
-        resdir = rdn;
-      }
-      if (!fnb.empty()) {
-        fnbase = fnb;
-      }
-      if (!fmo.empty()) {
-        filemode = fmo;
-      }
-      if (mfs > 0.) {
-        maxfilesize = mfs;
-      }
-      if (ntfm > 0) {
-        ntfmerge = ntfm;
-      }
-    }
-  }
-
-  // values from command line options, information from json is overwritten
-  if (options.isSet("aod-writer-resdir")) {
-    rdn = options.get<std::string>("aod-writer-resdir");
-    if (!rdn.empty()) {
-      resdir = rdn;
-    }
-  }
-  if (options.isSet("aod-writer-resfile")) {
-    fnb = options.get<std::string>("aod-writer-resfile");
-    if (!fnb.empty()) {
-      fnbase = fnb;
-    }
-  }
-  if (options.isSet("aod-writer-resmode")) {
-    fmo = options.get<std::string>("aod-writer-resmode");
-    if (!fmo.empty()) {
-      filemode = fmo;
-    }
-  }
-  if (options.isSet("aod-writer-maxfilesize")) {
-    mfs = options.get<float>("aod-writer-maxfilesize");
-    if (mfs > 0) {
-      maxfilesize = mfs;
-    }
-  }
-  if (options.isSet("aod-writer-ntfmerge")) {
-    ntfm = options.get<int>("aod-writer-ntfmerge");
-    if (ntfm > 0) {
-      ntfmerge = ntfm;
-    }
-  }
-  // parse the keepString
-  if (options.isSet("aod-writer-keep")) {
-    auto keepString = options.get<std::string>("aod-writer-keep");
-    if (!keepString.empty()) {
-      dod->reset();
-      std::string d("dangling");
-      if (d.find(keepString) == 0) {
-        // use the dangling outputs
-        std::vector<InputSpec> danglingOutputs;
-        for (auto ii = 0u; ii < OutputsInputs.size(); ii++) {
-          if (DataSpecUtils::partialMatch(OutputsInputs[ii], writableAODOrigins) && isDangling[ii]) {
-            danglingOutputs.emplace_back(OutputsInputs[ii]);
-          }
-        }
-        dod->readSpecs(danglingOutputs);
-      } else {
-        // use the keep string
-        dod->readString(keepString);
-      }
-    }
-  }
-  dod->setResultDir(resdir);
-  dod->setFilenameBase(fnbase);
-  dod->setFileMode(filemode);
-  dod->setMaximumFileSize(maxfilesize);
-  dod->setNumberTimeFramesToMerge(ntfmerge);
-
-  return dod;
-}
-
 std::tuple<std::vector<InputSpec>, std::vector<bool>> WorkflowHelpers::analyzeOutputs(WorkflowSpec const& workflow)
 {
   // compute total number of input/output
diff --git a/Framework/Core/src/WorkflowHelpers.h b/Framework/Core/src/WorkflowHelpers.h
index b20249b99edc8..b2a4d4cab55df 100644
--- a/Framework/Core/src/WorkflowHelpers.h
+++ b/Framework/Core/src/WorkflowHelpers.h
@@ -180,7 +180,7 @@ struct WorkflowHelpers {
   // dangling inputs are satisfied.
   // @a workflow the workflow to decorate
   // @a ctx the context for the configuration phase
-  static void injectServiceDevices(WorkflowSpec& workflow, ConfigContext const& ctx);
+  static void injectServiceDevices(WorkflowSpec& workflow, ConfigContext& ctx);
 
   // Final adjustments to @a workflow after service devices have been injected.
   static void adjustTopology(WorkflowSpec& workflow, ConfigContext const& ctx);
@@ -204,8 +204,6 @@ struct WorkflowHelpers {
     const std::vector<DeviceConnectionEdge>& edges,
     const std::vector<size_t>& index);
 
-  static std::shared_ptr<DataOutputDirector> getDataOutputDirector(ConfigParamRegistry const& options, std::vector<InputSpec> const& OutputsInputs, std::vector<bool> const& outputTypes);
-
   /// Given @a workflow it gathers all the OutputSpec and in addition provides
   /// the information whether and output is dangling and/or of type AOD
   /// An Output is dangling if it does not have a corresponding InputSpec.
diff --git a/Framework/Core/test/Mocking.h b/Framework/Core/test/Mocking.h
index b3e48ad3b2d0f..3b9a86b46e89c 100644
--- a/Framework/Core/test/Mocking.h
+++ b/Framework/Core/test/Mocking.h
@@ -34,7 +34,8 @@ std::unique_ptr<o2::framework::ConfigContext> makeEmptyConfigContext()
   store->preload();
   store->activate();
   static ConfigParamRegistry registry(std::move(store));
-  auto context = std::make_unique<ConfigContext>(registry, 0, nullptr);
+  static ServiceRegistry services;
+  auto context = std::make_unique<ConfigContext>(registry, ServiceRegistryRef{services}, 0, nullptr);
   return context;
 }
 
diff --git a/Framework/Core/test/benchmark_WorkflowHelpers.cxx b/Framework/Core/test/benchmark_WorkflowHelpers.cxx
index f1c070d8a0f4e..09a9ae0cca923 100644
--- a/Framework/Core/test/benchmark_WorkflowHelpers.cxx
+++ b/Framework/Core/test/benchmark_WorkflowHelpers.cxx
@@ -30,7 +30,8 @@ std::unique_ptr<ConfigContext> makeEmptyConfigContext()
   store->preload();
   store->activate();
   static ConfigParamRegistry registry(std::move(store));
-  auto context = std::make_unique<ConfigContext>(registry, 0, nullptr);
+  static ServiceRegistry services;
+  auto context = std::make_unique<ConfigContext>(registry, ServiceRegistryRef{services}, 0, nullptr);
   return context;
 }
 
diff --git a/Framework/Core/test/test_OverrideLabels.cxx b/Framework/Core/test/test_OverrideLabels.cxx
index 573bd13be797a..c5134c0c169c0 100644
--- a/Framework/Core/test/test_OverrideLabels.cxx
+++ b/Framework/Core/test/test_OverrideLabels.cxx
@@ -31,7 +31,8 @@ std::unique_ptr<o2::framework::ConfigContext> mockupLabels(std::string labelArg)
   store->preload();
   store->activate();
   registry = ConfigParamRegistry(std::move(store));
-  auto context = std::make_unique<ConfigContext>(registry, 0, nullptr);
+  static ServiceRegistry services;
+  auto context = std::make_unique<ConfigContext>(registry, ServiceRegistryRef{services}, 0, nullptr);
   return context;
 }
 
diff --git a/Framework/TestWorkflows/src/o2TestHistograms.cxx b/Framework/TestWorkflows/src/o2TestHistograms.cxx
index 9986f52a1d940..efac16f6da4f0 100644
--- a/Framework/TestWorkflows/src/o2TestHistograms.cxx
+++ b/Framework/TestWorkflows/src/o2TestHistograms.cxx
@@ -17,6 +17,7 @@
 #include "Framework/AnalysisTask.h"
 #include <TH2F.h>
 #include <cmath>
+#include <iostream>
 
 using namespace o2;
 using namespace o2::framework;
@@ -43,7 +44,7 @@ struct EtaAndClsHistogramsSimple {
   {
     LOGP(info, "Invoking the simple one");
     for (auto& track : tracks) {
-      etaClsH->Fill(track.eta(), track.pt(), 0);
+      etaClsH->Fill(track.eta(), track.pt());
       skimEx(track.pt(), track.eta());
     }
   }
@@ -57,7 +58,7 @@ struct EtaAndClsHistogramsIUSimple {
   {
     LOGP(info, "Invoking the simple one");
     for (auto& track : tracks) {
-      etaClsH->Fill(track.eta(), track.pt(), 0);
+      etaClsH->Fill(track.eta(), track.pt());
       skimEx(track.pt(), track.eta());
     }
   }

From 3db25287849f1f52355a83c428e3a650173430fb Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 13 Nov 2024 16:41:19 +0100
Subject: [PATCH 700/726] DPL: move writers to plugin

This will allow us at some point to remove the dependency on TTree in the framework,
hopefully saving memory and allowing us to more easily customize the writing backend.
---
 Framework/AnalysisSupport/CMakeLists.txt      |   1 +
 .../AnalysisSupport/src/AODWriterHelpers.cxx  | 414 +++++++++++++++
 .../AnalysisSupport/src/AODWriterHelpers.h    |  28 +
 Framework/AnalysisSupport/src/Plugin.cxx      |  17 +
 .../Core/include/Framework/AnalysisContext.h  |  58 +++
 .../Framework}/AnalysisSupportHelpers.h       |  40 +-
 .../Core/include/Framework/ConfigContext.h    |  15 +-
 .../include/Framework/runDataProcessing.h     |   4 +-
 Framework/Core/src/AnalysisSupportHelpers.cxx | 484 ++++--------------
 Framework/Core/src/ArrowSupport.cxx           |  10 +-
 Framework/Core/src/ConfigContext.cxx          |   3 +
 Framework/Core/src/WorkflowHelpers.cxx        | 149 +-----
 Framework/Core/src/WorkflowHelpers.h          |   4 +-
 Framework/Core/test/Mocking.h                 |   5 +-
 .../Core/test/benchmark_WorkflowHelpers.cxx   |   3 +-
 Framework/Core/test/test_OverrideLabels.cxx   |   3 +-
 .../TestWorkflows/src/o2TestHistograms.cxx    |   5 +-
 17 files changed, 690 insertions(+), 553 deletions(-)
 create mode 100644 Framework/AnalysisSupport/src/AODWriterHelpers.cxx
 create mode 100644 Framework/AnalysisSupport/src/AODWriterHelpers.h
 create mode 100644 Framework/Core/include/Framework/AnalysisContext.h
 rename Framework/Core/{src => include/Framework}/AnalysisSupportHelpers.h (71%)

diff --git a/Framework/AnalysisSupport/CMakeLists.txt b/Framework/AnalysisSupport/CMakeLists.txt
index eb5706817704b..5fb1282469711 100644
--- a/Framework/AnalysisSupport/CMakeLists.txt
+++ b/Framework/AnalysisSupport/CMakeLists.txt
@@ -20,6 +20,7 @@ o2_add_library(FrameworkAnalysisSupport
                SOURCES src/Plugin.cxx
                        src/DataInputDirector.cxx
                        src/AODJAlienReaderHelpers.cxx
+                       src/AODWriterHelpers.cxx
                PRIVATE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/src
                PUBLIC_LINK_LIBRARIES O2::Framework ${EXTRA_TARGETS} ROOT::TreePlayer)
 
diff --git a/Framework/AnalysisSupport/src/AODWriterHelpers.cxx b/Framework/AnalysisSupport/src/AODWriterHelpers.cxx
new file mode 100644
index 0000000000000..fa10d4661f537
--- /dev/null
+++ b/Framework/AnalysisSupport/src/AODWriterHelpers.cxx
@@ -0,0 +1,414 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/AnalysisContext.h"
+#include "Framework/ConfigContext.h"
+#include "Framework/ControlService.h"
+#include "AODWriterHelpers.h"
+#include "Framework/OutputObjHeader.h"
+#include "Framework/EndOfStreamContext.h"
+#include "Framework/ProcessingContext.h"
+#include "Framework/InitContext.h"
+#include "Framework/CallbackService.h"
+#include "Framework/AnalysisSupportHelpers.h"
+#include "Framework/TableConsumer.h"
+#include "Framework/DataOutputDirector.h"
+#include "Framework/TableTreeHelpers.h"
+
+#include <TFile.h>
+#include <TFile.h>
+#include <TTree.h>
+#include <TMap.h>
+#include <TObjString.h>
+#include <arrow/table.h>
+
+namespace o2::framework::writers
+{
+
+struct InputObjectRoute {
+  std::string name;
+  uint32_t uniqueId;
+  std::string directory;
+  uint32_t taskHash;
+  OutputObjHandlingPolicy policy;
+  OutputObjSourceType sourceType;
+};
+
+struct InputObject {
+  TClass* kind = nullptr;
+  void* obj = nullptr;
+  std::string name;
+  int count = -1;
+};
+
+const static std::unordered_map<OutputObjHandlingPolicy, std::string> ROOTfileNames = {{OutputObjHandlingPolicy::AnalysisObject, "AnalysisResults.root"},
+                                                                                       {OutputObjHandlingPolicy::QAObject, "QAResults.root"}};
+
+AlgorithmSpec AODWriterHelpers::getOutputTTreeWriter(ConfigContext const& ctx)
+{
+  auto& ac = ctx.services().get<AnalysisContext>();
+  auto dod = AnalysisSupportHelpers::getDataOutputDirector(ctx);
+  int compressionLevel = 505;
+  if (ctx.options().hasOption("aod-writer-compression")) {
+    compressionLevel = ctx.options().get<int>("aod-writer-compression");
+  }
+  return AlgorithmSpec{[dod, outputInputs = ac.outputsInputsAOD, compressionLevel](InitContext& ic) -> std::function<void(ProcessingContext&)> {
+    LOGP(debug, "======== getGlobalAODSink::Init ==========");
+
+    // find out if any table needs to be saved
+    bool hasOutputsToWrite = false;
+    for (auto& outobj : outputInputs) {
+      auto ds = dod->getDataOutputDescriptors(outobj);
+      if (ds.size() > 0) {
+        hasOutputsToWrite = true;
+        break;
+      }
+    }
+
+    // if nothing needs to be saved then return a trivial functor
+    // this happens when nothing needs to be saved but there are dangling outputs
+    if (!hasOutputsToWrite) {
+      return [](ProcessingContext&) mutable -> void {
+        static bool once = false;
+        if (!once) {
+          LOG(info) << "No AODs to be saved.";
+          once = true;
+        }
+      };
+    }
+
+    // end of data functor is called at the end of the data stream
+    auto endofdatacb = [dod](EndOfStreamContext& context) {
+      dod->closeDataFiles();
+      context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
+    };
+
+    auto& callbacks = ic.services().get<CallbackService>();
+    callbacks.set<CallbackService::Id::EndOfStream>(endofdatacb);
+
+    // prepare map<uint64_t, uint64_t>(startTime, tfNumber)
+    std::map<uint64_t, uint64_t> tfNumbers;
+    std::map<uint64_t, std::string> tfFilenames;
+
+    std::vector<TString> aodMetaDataKeys;
+    std::vector<TString> aodMetaDataVals;
+
+    // this functor is called once per time frame
+    return [dod, tfNumbers, tfFilenames, aodMetaDataKeys, aodMetaDataVals, compressionLevel](ProcessingContext& pc) mutable -> void {
+      LOGP(debug, "======== getGlobalAODSink::processing ==========");
+      LOGP(debug, " processing data set with {} entries", pc.inputs().size());
+
+      // return immediately if pc.inputs() is empty. This should never happen!
+      if (pc.inputs().size() == 0) {
+        LOGP(info, "No inputs available!");
+        return;
+      }
+
+      // update tfNumbers
+      uint64_t startTime = 0;
+      uint64_t tfNumber = 0;
+      auto ref = pc.inputs().get("tfn");
+      if (ref.spec && ref.payload) {
+        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref)->startTime;
+        tfNumber = pc.inputs().get<uint64_t>("tfn");
+        tfNumbers.insert(std::pair<uint64_t, uint64_t>(startTime, tfNumber));
+      }
+      // update tfFilenames
+      std::string aodInputFile;
+      auto ref2 = pc.inputs().get("tff");
+      if (ref2.spec && ref2.payload) {
+        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref2)->startTime;
+        aodInputFile = pc.inputs().get<std::string>("tff");
+        tfFilenames.insert(std::pair<uint64_t, std::string>(startTime, aodInputFile));
+      }
+
+      // close all output files if one has reached size limit
+      dod->checkFileSizes();
+
+      // loop over the DataRefs which are contained in pc.inputs()
+      for (const auto& ref : pc.inputs()) {
+        if (!ref.spec) {
+          LOGP(debug, "Invalid input will be skipped!");
+          continue;
+        }
+
+        // get metadata
+        if (DataSpecUtils::partialMatch(*ref.spec, header::DataDescription("AODMetadataKeys"))) {
+          aodMetaDataKeys = pc.inputs().get<std::vector<TString>>(ref.spec->binding);
+        }
+        if (DataSpecUtils::partialMatch(*ref.spec, header::DataDescription("AODMetadataVals"))) {
+          aodMetaDataVals = pc.inputs().get<std::vector<TString>>(ref.spec->binding);
+        }
+
+        // skip non-AOD refs
+        if (!DataSpecUtils::partialMatch(*ref.spec, writableAODOrigins)) {
+          continue;
+        }
+        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref)->startTime;
+
+        // does this need to be saved?
+        auto dh = DataRefUtils::getHeader<header::DataHeader*>(ref);
+        auto tableName = dh->dataDescription.as<std::string>();
+        auto ds = dod->getDataOutputDescriptors(*dh);
+        if (ds.empty()) {
+          continue;
+        }
+
+        // get TF number from startTime
+        auto it = tfNumbers.find(startTime);
+        if (it != tfNumbers.end()) {
+          tfNumber = (it->second / dod->getNumberTimeFramesToMerge()) * dod->getNumberTimeFramesToMerge();
+        } else {
+          LOGP(fatal, "No time frame number found for output with start time {}", startTime);
+          throw std::runtime_error("Processing is stopped!");
+        }
+        // get aod input file from startTime
+        auto it2 = tfFilenames.find(startTime);
+        if (it2 != tfFilenames.end()) {
+          aodInputFile = it2->second;
+        }
+
+        // get the TableConsumer and corresponding arrow table
+        auto msg = pc.inputs().get(ref.spec->binding);
+        if (msg.header == nullptr) {
+          LOGP(error, "No header for message {}:{}", ref.spec->binding, DataSpecUtils::describe(*ref.spec));
+          continue;
+        }
+        auto s = pc.inputs().get<TableConsumer>(ref.spec->binding);
+        auto table = s->asArrowTable();
+        if (!table->Validate().ok()) {
+          LOGP(warning, "The table \"{}\" is not valid and will not be saved!", tableName);
+          continue;
+        }
+        if (table->schema()->fields().empty()) {
+          LOGP(debug, "The table \"{}\" is empty but will be saved anyway!", tableName);
+        }
+
+        // loop over all DataOutputDescriptors
+        // a table can be saved in multiple ways
+        // e.g. different selections of columns to different files
+        for (auto d : ds) {
+          auto fileAndFolder = dod->getFileFolder(d, tfNumber, aodInputFile, compressionLevel);
+          auto treename = fileAndFolder.folderName + "/" + d->treename;
+          TableToTree ta2tr(table,
+                            fileAndFolder.file,
+                            treename.c_str());
+
+          // update metadata
+          if (fileAndFolder.file->FindObjectAny("metaData")) {
+            LOGF(debug, "Metadata: target file %s already has metadata, preserving it", fileAndFolder.file->GetName());
+          } else if (!aodMetaDataKeys.empty() && !aodMetaDataVals.empty()) {
+            TMap aodMetaDataMap;
+            for (uint32_t imd = 0; imd < aodMetaDataKeys.size(); imd++) {
+              aodMetaDataMap.Add(new TObjString(aodMetaDataKeys[imd]), new TObjString(aodMetaDataVals[imd]));
+            }
+            fileAndFolder.file->WriteObject(&aodMetaDataMap, "metaData", "Overwrite");
+          }
+
+          if (!d->colnames.empty()) {
+            for (auto& cn : d->colnames) {
+              auto idx = table->schema()->GetFieldIndex(cn);
+              auto col = table->column(idx);
+              auto field = table->schema()->field(idx);
+              if (idx != -1) {
+                ta2tr.addBranch(col, field);
+              }
+            }
+          } else {
+            ta2tr.addAllBranches();
+          }
+          ta2tr.process();
+        }
+      }
+    };
+  }
+
+  };
+}
+
+AlgorithmSpec AODWriterHelpers::getOutputObjHistWriter(ConfigContext const& ctx)
+{
+  auto& ac = ctx.services().get<AnalysisContext>();
+  auto tskmap = ac.outTskMap;
+  auto objmap = ac.outObjHistMap;
+
+  return AlgorithmSpec{[objmap, tskmap](InitContext& ic) -> std::function<void(ProcessingContext&)> {
+    auto& callbacks = ic.services().get<CallbackService>();
+    auto inputObjects = std::make_shared<std::vector<std::pair<InputObjectRoute, InputObject>>>();
+
+    static TFile* f[OutputObjHandlingPolicy::numPolicies];
+    for (auto i = 0u; i < OutputObjHandlingPolicy::numPolicies; ++i) {
+      f[i] = nullptr;
+    }
+
+    static std::string currentDirectory = "";
+    static std::string currentFile = "";
+
+    auto endofdatacb = [inputObjects](EndOfStreamContext& context) {
+      LOG(debug) << "Writing merged objects and histograms to file";
+      if (inputObjects->empty()) {
+        LOG(error) << "Output object map is empty!";
+        context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
+        return;
+      }
+      for (auto i = 0u; i < OutputObjHandlingPolicy::numPolicies; ++i) {
+        if (f[i] != nullptr) {
+          f[i]->Close();
+        }
+      }
+      LOG(debug) << "All outputs merged in their respective target files";
+      context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
+    };
+
+    callbacks.set<CallbackService::Id::EndOfStream>(endofdatacb);
+    return [inputObjects, objmap, tskmap](ProcessingContext& pc) mutable -> void {
+      auto const& ref = pc.inputs().get("x");
+      if (!ref.header) {
+        LOG(error) << "Header not found";
+        return;
+      }
+      if (!ref.payload) {
+        LOG(error) << "Payload not found";
+        return;
+      }
+      auto datah = o2::header::get<o2::header::DataHeader*>(ref.header);
+      if (!datah) {
+        LOG(error) << "No data header in stack";
+        return;
+      }
+
+      auto objh = o2::header::get<o2::framework::OutputObjHeader*>(ref.header);
+      if (!objh) {
+        LOG(error) << "No output object header in stack";
+        return;
+      }
+
+      InputObject obj;
+      FairInputTBuffer tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
+      tm.InitMap();
+      obj.kind = tm.ReadClass();
+      tm.SetBufferOffset(0);
+      tm.ResetMap();
+      if (obj.kind == nullptr) {
+        LOG(error) << "Cannot read class info from buffer.";
+        return;
+      }
+
+      auto policy = objh->mPolicy;
+      auto sourceType = objh->mSourceType;
+      auto hash = objh->mTaskHash;
+
+      obj.obj = tm.ReadObjectAny(obj.kind);
+      auto* named = static_cast<TNamed*>(obj.obj);
+      obj.name = named->GetName();
+      auto hpos = std::find_if(tskmap.begin(), tskmap.end(), [&](auto&& x) { return x.id == hash; });
+      if (hpos == tskmap.end()) {
+        LOG(error) << "No task found for hash " << hash;
+        return;
+      }
+      auto taskname = hpos->name;
+      auto opos = std::find_if(objmap.begin(), objmap.end(), [&](auto&& x) { return x.id == hash; });
+      if (opos == objmap.end()) {
+        LOG(error) << "No object list found for task " << taskname << " (hash=" << hash << ")";
+        return;
+      }
+      auto objects = opos->bindings;
+      if (std::find(objects.begin(), objects.end(), obj.name) == objects.end()) {
+        LOG(error) << "No object " << obj.name << " in map for task " << taskname;
+        return;
+      }
+      auto nameHash = runtime_hash(obj.name.c_str());
+      InputObjectRoute key{obj.name, nameHash, taskname, hash, policy, sourceType};
+      auto existing = std::find_if(inputObjects->begin(), inputObjects->end(), [&](auto&& x) { return (x.first.uniqueId == nameHash) && (x.first.taskHash == hash); });
+      // If it's the first one, we just add it to the list.
+      if (existing == inputObjects->end()) {
+        obj.count = objh->mPipelineSize;
+        inputObjects->push_back(std::make_pair(key, obj));
+        existing = inputObjects->end() - 1;
+      } else {
+        obj.count = existing->second.count;
+        // Otherwise, we merge it with the existing one.
+        auto merger = existing->second.kind->GetMerge();
+        if (!merger) {
+          LOG(error) << "Already one unmergeable object found for " << obj.name;
+          return;
+        }
+        TList coll;
+        coll.Add(static_cast<TObject*>(obj.obj));
+        merger(existing->second.obj, &coll, nullptr);
+      }
+      // We expect as many objects as the pipeline size, for
+      // a given object name and task hash.
+      existing->second.count -= 1;
+
+      if (existing->second.count != 0) {
+        return;
+      }
+      // Write the object here.
+      auto route = existing->first;
+      auto entry = existing->second;
+      auto file = ROOTfileNames.find(route.policy);
+      if (file == ROOTfileNames.end()) {
+        return;
+      }
+      auto filename = file->second;
+      if (f[route.policy] == nullptr) {
+        f[route.policy] = TFile::Open(filename.c_str(), "RECREATE");
+      }
+      auto nextDirectory = route.directory;
+      if ((nextDirectory != currentDirectory) || (filename != currentFile)) {
+        if (!f[route.policy]->FindKey(nextDirectory.c_str())) {
+          f[route.policy]->mkdir(nextDirectory.c_str());
+        }
+        currentDirectory = nextDirectory;
+        currentFile = filename;
+      }
+
+      // translate the list-structure created by the registry into a directory structure within the file
+      std::function<void(TList*, TDirectory*)> writeListToFile;
+      writeListToFile = [&](TList* list, TDirectory* parentDir) {
+        TIter next(list);
+        TObject* object = nullptr;
+        while ((object = next())) {
+          if (object->InheritsFrom(TList::Class())) {
+            writeListToFile(static_cast<TList*>(object), parentDir->mkdir(object->GetName(), object->GetName(), true));
+          } else {
+            parentDir->WriteObjectAny(object, object->Class(), object->GetName());
+            auto* written = list->Remove(object);
+            delete written;
+          }
+        }
+      };
+
+      TDirectory* currentDir = f[route.policy]->GetDirectory(currentDirectory.c_str());
+      if (route.sourceType == OutputObjSourceType::HistogramRegistrySource) {
+        auto* outputList = static_cast<TList*>(entry.obj);
+        outputList->SetOwner(false);
+
+        // if registry should live in dedicated folder a TNamed object is appended to the list
+        if (outputList->Last() && outputList->Last()->IsA() == TNamed::Class()) {
+          delete outputList->Last();
+          outputList->RemoveLast();
+          currentDir = currentDir->mkdir(outputList->GetName(), outputList->GetName(), true);
+        }
+
+        writeListToFile(outputList, currentDir);
+        outputList->SetOwner();
+        delete outputList;
+        entry.obj = nullptr;
+      } else {
+        currentDir->WriteObjectAny(entry.obj, entry.kind, entry.name.c_str());
+        delete (TObject*)entry.obj;
+        entry.obj = nullptr;
+      }
+    };
+  }};
+}
+} // namespace o2::framework::writers
diff --git a/Framework/AnalysisSupport/src/AODWriterHelpers.h b/Framework/AnalysisSupport/src/AODWriterHelpers.h
new file mode 100644
index 0000000000000..7ae59a5cf3b01
--- /dev/null
+++ b/Framework/AnalysisSupport/src/AODWriterHelpers.h
@@ -0,0 +1,28 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FRAMEWORK_AODROOTWRITERHELPERS_H_
+#define O2_FRAMEWORK_AODROOTWRITERHELPERS_H_
+
+#include "Framework/AlgorithmSpec.h"
+#include <Monitoring/Monitoring.h>
+
+namespace o2::framework::writers
+{
+
+struct AODWriterHelpers {
+  static AlgorithmSpec getOutputObjHistWriter(ConfigContext const& context);
+  static AlgorithmSpec getOutputTTreeWriter(ConfigContext const& context);
+};
+
+} // namespace o2::framework::writers
+
+#endif // O2_FRAMEWORK_AODROOTWRITERHELPERS_H_
diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx
index bba3499286e08..52435375d7e9e 100644
--- a/Framework/AnalysisSupport/src/Plugin.cxx
+++ b/Framework/AnalysisSupport/src/Plugin.cxx
@@ -16,6 +16,7 @@
 #include "Framework/Capability.h"
 #include "Framework/Signpost.h"
 #include "AODJAlienReaderHelpers.h"
+#include "AODWriterHelpers.h"
 #include <TFile.h>
 #include <TMap.h>
 #include <TGrid.h>
@@ -33,6 +34,20 @@ struct ROOTFileReader : o2::framework::AlgorithmPlugin {
   }
 };
 
+struct ROOTObjWriter : o2::framework::AlgorithmPlugin {
+  o2::framework::AlgorithmSpec create(o2::framework::ConfigContext const& config) override
+  {
+    return o2::framework::writers::AODWriterHelpers::getOutputObjHistWriter(config);
+  }
+};
+
+struct ROOTTTreeWriter : o2::framework::AlgorithmPlugin {
+  o2::framework::AlgorithmSpec create(o2::framework::ConfigContext const& config) override
+  {
+    return o2::framework::writers::AODWriterHelpers::getOutputTTreeWriter(config);
+  }
+};
+
 using namespace o2::framework;
 struct RunSummary : o2::framework::ServicePlugin {
   o2::framework::ServiceSpec* create() final
@@ -211,6 +226,8 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
 
 DEFINE_DPL_PLUGINS_BEGIN
 DEFINE_DPL_PLUGIN_INSTANCE(ROOTFileReader, CustomAlgorithm);
+DEFINE_DPL_PLUGIN_INSTANCE(ROOTObjWriter, CustomAlgorithm);
+DEFINE_DPL_PLUGIN_INSTANCE(ROOTTTreeWriter, CustomAlgorithm);
 DEFINE_DPL_PLUGIN_INSTANCE(RunSummary, CustomService);
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAOD, ConfigDiscovery);
 DEFINE_DPL_PLUGINS_END
diff --git a/Framework/Core/include/Framework/AnalysisContext.h b/Framework/Core/include/Framework/AnalysisContext.h
new file mode 100644
index 0000000000000..0f62f952d0aaa
--- /dev/null
+++ b/Framework/Core/include/Framework/AnalysisContext.h
@@ -0,0 +1,58 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef O2_FRAMEWORK_ANALYSISCONTEXT_H_
+#define O2_FRAMEWORK_ANALYSISCONTEXT_H_
+
+#include <vector>
+#include "Framework/InputSpec.h"
+#include "Framework/OutputSpec.h"
+
+namespace o2::framework
+{
+class DataOutputDirector;
+
+struct OutputTaskInfo {
+  uint32_t id;
+  std::string name;
+};
+
+struct OutputObjectInfo {
+  uint32_t id;
+  std::vector<std::string> bindings;
+};
+
+//
+struct AnalysisContext {
+  std::vector<InputSpec> requestedAODs;
+  std::vector<OutputSpec> providedAODs;
+  std::vector<InputSpec> requestedDYNs;
+  std::vector<OutputSpec> providedDYNs;
+  std::vector<InputSpec> requestedIDXs;
+  std::vector<OutputSpec> providedOutputObjHist;
+  std::vector<InputSpec> spawnerInputs;
+
+  // Needed to created the hist writer
+  std::vector<OutputTaskInfo> outTskMap;
+  std::vector<OutputObjectInfo> outObjHistMap;
+
+  // Needed to create the output director
+  std::vector<InputSpec> outputsInputs;
+  std::vector<bool> isDangling;
+
+  // Needed to create the aod writer
+  std::vector<InputSpec> outputsInputsAOD;
+};
+} // namespace o2::framework
+
+extern template class std::vector<o2::framework::OutputObjectInfo>;
+extern template class std::vector<o2::framework::OutputTaskInfo>;
+
+#endif // O2_FRAMEWORK_ANALYSISCONTEXT_H_
diff --git a/Framework/Core/src/AnalysisSupportHelpers.h b/Framework/Core/include/Framework/AnalysisSupportHelpers.h
similarity index 71%
rename from Framework/Core/src/AnalysisSupportHelpers.h
rename to Framework/Core/include/Framework/AnalysisSupportHelpers.h
index ba5bcedb4bc67..4ae601dc9e4a2 100644
--- a/Framework/Core/src/AnalysisSupportHelpers.h
+++ b/Framework/Core/include/Framework/AnalysisSupportHelpers.h
@@ -14,6 +14,7 @@
 #include "Framework/OutputSpec.h"
 #include "Framework/InputSpec.h"
 #include "Framework/DataProcessorSpec.h"
+#include "Framework/AnalysisContext.h"
 #include "Headers/DataHeader.h"
 #include <array>
 
@@ -24,36 +25,7 @@ static constexpr std::array<header::DataOrigin, 5> extendedAODOrigins{header::Da
 static constexpr std::array<header::DataOrigin, 4> writableAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}};
 
 class DataOutputDirector;
-
-struct OutputTaskInfo {
-  uint32_t id;
-  std::string name;
-};
-
-struct OutputObjectInfo {
-  uint32_t id;
-  std::vector<std::string> bindings;
-};
-} // namespace o2::framework
-
-extern template class std::vector<o2::framework::OutputObjectInfo>;
-extern template class std::vector<o2::framework::OutputTaskInfo>;
-
-namespace o2::framework
-{
-//
-struct AnalysisContext {
-  std::vector<InputSpec> requestedAODs;
-  std::vector<OutputSpec> providedAODs;
-  std::vector<InputSpec> requestedDYNs;
-  std::vector<OutputSpec> providedDYNs;
-  std::vector<InputSpec> requestedIDXs;
-  std::vector<OutputSpec> providedOutputObjHist;
-  std::vector<InputSpec> spawnerInputs;
-
-  std::vector<OutputTaskInfo> outTskMap;
-  std::vector<OutputObjectInfo> outObjHistMap;
-};
+class ConfigContext;
 
 // Helper class to be moved in the AnalysisSupport plugin at some point
 struct AnalysisSupportHelpers {
@@ -74,11 +46,11 @@ struct AnalysisSupportHelpers {
 
   /// Match all inputs of kind ATSK and write them to a ROOT file,
   /// one root file per originating task.
-  static DataProcessorSpec getOutputObjHistSink(std::vector<OutputObjectInfo> const& objmap,
-                                                std::vector<OutputTaskInfo> const& tskmap);
+  static DataProcessorSpec getOutputObjHistSink(ConfigContext const&);
   /// writes inputs of kind AOD to file
-  static DataProcessorSpec getGlobalAODSink(std::shared_ptr<DataOutputDirector> dod,
-                                            std::vector<InputSpec> const& outputInputs, int compression);
+  static DataProcessorSpec getGlobalAODSink(ConfigContext const&);
+  /// Get the data director
+  static std::shared_ptr<DataOutputDirector> getDataOutputDirector(ConfigContext const& ctx);
 };
 
 }; // namespace o2::framework
diff --git a/Framework/Core/include/Framework/ConfigContext.h b/Framework/Core/include/Framework/ConfigContext.h
index 5790699fe68bb..87259f0519915 100644
--- a/Framework/Core/include/Framework/ConfigContext.h
+++ b/Framework/Core/include/Framework/ConfigContext.h
@@ -8,11 +8,11 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
-#ifndef FRAMEWORK_CONFIG_CONTEXT_H
-#define FRAMEWORK_CONFIG_CONTEXT_H
+#ifndef O2_FRAMEWORK_CONFIG_CONTEXT_H_
+#define O2_FRAMEWORK_CONFIG_CONTEXT_H_
 
 #include "Framework/ConfigParamRegistry.h"
-#include "Framework/ServiceRegistry.h"
+#include "Framework/ServiceRegistryRef.h"
 
 namespace o2::framework
 {
@@ -23,9 +23,10 @@ namespace o2::framework
 class ConfigContext
 {
  public:
-  ConfigContext(ConfigParamRegistry& options, int argc, char** argv) : mOptions{options}, mArgc{argc}, mArgv{argv} {}
+  ConfigContext(ConfigParamRegistry& options, ServiceRegistryRef services, int argc, char** argv);
 
   [[nodiscard]] ConfigParamRegistry& options() const { return mOptions; }
+  [[nodiscard]] ServiceRegistryRef services() const { return mServices; }
 
   [[nodiscard]] bool helpOnCommandLine() const;
 
@@ -34,11 +35,13 @@ class ConfigContext
 
  private:
   ConfigParamRegistry& mOptions;
+
+  ServiceRegistryRef mServices;
   // additionaly keep information about the original command line
   int mArgc = 0;
   char** mArgv = nullptr;
 };
 
-} // namespace o2
+} // namespace o2::framework
 
-#endif
+#endif // O2_FRAMEWORK_CONFIG_CONTEXT_H_
diff --git a/Framework/Core/include/Framework/runDataProcessing.h b/Framework/Core/include/Framework/runDataProcessing.h
index eee4c4b6583d3..8293bf0cf7039 100644
--- a/Framework/Core/include/Framework/runDataProcessing.h
+++ b/Framework/Core/include/Framework/runDataProcessing.h
@@ -30,6 +30,7 @@
 #include "Framework/CheckTypes.h"
 #include "Framework/StructToTuple.h"
 #include "Framework/ConfigParamDiscovery.h"
+#include "ServiceRegistryRef.h"
 #include <vector>
 
 namespace o2::framework
@@ -198,7 +199,8 @@ int mainNoCatch(int argc, char** argv)
     workflowOptions.push_back(extra);
   }
 
-  ConfigContext configContext(workflowOptionsRegistry, argc, argv);
+  ServiceRegistry configRegistry;
+  ConfigContext configContext(workflowOptionsRegistry, ServiceRegistryRef{configRegistry}, argc, argv);
   o2::framework::WorkflowSpec specs = defineDataProcessing(configContext);
   overrideCloning(configContext, specs);
   overridePipeline(configContext, specs);
diff --git a/Framework/Core/src/AnalysisSupportHelpers.cxx b/Framework/Core/src/AnalysisSupportHelpers.cxx
index e949f27a6eed6..eb17566fd6d31 100644
--- a/Framework/Core/src/AnalysisSupportHelpers.cxx
+++ b/Framework/Core/src/AnalysisSupportHelpers.cxx
@@ -9,18 +9,16 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
-#include "AnalysisSupportHelpers.h"
+#include "Framework/AnalysisSupportHelpers.h"
 #include "Framework/DataOutputDirector.h"
 #include "Framework/OutputObjHeader.h"
 #include "Framework/ControlService.h"
 #include "Framework/EndOfStreamContext.h"
 #include "Framework/DeviceSpec.h"
 #include "Framework/TableTreeHelpers.h"
-
-#include "TFile.h"
-#include "TTree.h"
-#include "TMap.h"
-#include "TObjString.h"
+#include "Framework/PluginManager.h"
+#include "Framework/ConfigContext.h"
+#include "WorkflowHelpers.h"
 
 template class std::vector<o2::framework::OutputObjectInfo>;
 template class std::vector<o2::framework::OutputTaskInfo>;
@@ -28,21 +26,105 @@ template class std::vector<o2::framework::OutputTaskInfo>;
 namespace o2::framework
 {
 
-struct InputObjectRoute {
-  std::string name;
-  uint32_t uniqueId;
-  std::string directory;
-  uint32_t taskHash;
-  OutputObjHandlingPolicy policy;
-  OutputObjSourceType sourceType;
-};
+std::shared_ptr<DataOutputDirector> AnalysisSupportHelpers::getDataOutputDirector(ConfigContext const& ctx)
+{
+  auto const& options = ctx.options();
+  auto const& OutputsInputs = ctx.services().get<AnalysisContext>().outputsInputs;
+  auto const& isDangling = ctx.services().get<AnalysisContext>().isDangling;
+
+  std::shared_ptr<DataOutputDirector> dod = std::make_shared<DataOutputDirector>();
+
+  // analyze options and take actions accordingly
+  // default values
+  std::string rdn, resdir("./");
+  std::string fnb, fnbase("AnalysisResults_trees");
+  float mfs, maxfilesize(-1.);
+  std::string fmo, filemode("RECREATE");
+  int ntfm, ntfmerge = 1;
+
+  // values from json
+  if (options.isSet("aod-writer-json")) {
+    auto fnjson = options.get<std::string>("aod-writer-json");
+    if (!fnjson.empty()) {
+      std::tie(rdn, fnb, fmo, mfs, ntfm) = dod->readJson(fnjson);
+      if (!rdn.empty()) {
+        resdir = rdn;
+      }
+      if (!fnb.empty()) {
+        fnbase = fnb;
+      }
+      if (!fmo.empty()) {
+        filemode = fmo;
+      }
+      if (mfs > 0.) {
+        maxfilesize = mfs;
+      }
+      if (ntfm > 0) {
+        ntfmerge = ntfm;
+      }
+    }
+  }
+
+  // values from command line options, information from json is overwritten
+  if (options.isSet("aod-writer-resdir")) {
+    rdn = options.get<std::string>("aod-writer-resdir");
+    if (!rdn.empty()) {
+      resdir = rdn;
+    }
+  }
+  if (options.isSet("aod-writer-resfile")) {
+    fnb = options.get<std::string>("aod-writer-resfile");
+    if (!fnb.empty()) {
+      fnbase = fnb;
+    }
+  }
+  if (options.isSet("aod-writer-resmode")) {
+    fmo = options.get<std::string>("aod-writer-resmode");
+    if (!fmo.empty()) {
+      filemode = fmo;
+    }
+  }
+  if (options.isSet("aod-writer-maxfilesize")) {
+    mfs = options.get<float>("aod-writer-maxfilesize");
+    if (mfs > 0) {
+      maxfilesize = mfs;
+    }
+  }
+  if (options.isSet("aod-writer-ntfmerge")) {
+    ntfm = options.get<int>("aod-writer-ntfmerge");
+    if (ntfm > 0) {
+      ntfmerge = ntfm;
+    }
+  }
+  // parse the keepString
+  if (options.isSet("aod-writer-keep")) {
+    auto keepString = options.get<std::string>("aod-writer-keep");
+    if (!keepString.empty()) {
+      dod->reset();
+      std::string d("dangling");
+      if (d.find(keepString) == 0) {
+        // use the dangling outputs
+        std::vector<InputSpec> danglingOutputs;
+        for (auto ii = 0u; ii < OutputsInputs.size(); ii++) {
+          if (DataSpecUtils::partialMatch(OutputsInputs[ii], writableAODOrigins) && isDangling[ii]) {
+            danglingOutputs.emplace_back(OutputsInputs[ii]);
+          }
+        }
+        dod->readSpecs(danglingOutputs);
+      } else {
+        // use the keep string
+        dod->readString(keepString);
+      }
+    }
+  }
+  dod->setResultDir(resdir);
+  dod->setFilenameBase(fnbase);
+  dod->setFileMode(filemode);
+  dod->setMaximumFileSize(maxfilesize);
+  dod->setNumberTimeFramesToMerge(ntfmerge);
 
-struct InputObject {
-  TClass* kind = nullptr;
-  void* obj = nullptr;
-  std::string name;
-  int count = -1;
-};
+  return dod;
+}
 
 void AnalysisSupportHelpers::addMissingOutputsToReader(std::vector<OutputSpec> const& providedOutputs,
                                                        std::vector<InputSpec> const& requestedInputs,
@@ -125,191 +207,16 @@ void AnalysisSupportHelpers::addMissingOutputsToBuilder(std::vector<InputSpec> c
   }
 }
 
-const static std::unordered_map<OutputObjHandlingPolicy, std::string> ROOTfileNames = {{OutputObjHandlingPolicy::AnalysisObject, "AnalysisResults.root"},
-                                                                                       {OutputObjHandlingPolicy::QAObject, "QAResults.root"}};
-
 // =============================================================================
-DataProcessorSpec AnalysisSupportHelpers::getOutputObjHistSink(std::vector<OutputObjectInfo> const& objmap, std::vector<OutputTaskInfo> const& tskmap)
+DataProcessorSpec AnalysisSupportHelpers::getOutputObjHistSink(ConfigContext const& ctx)
 {
-  auto writerFunction = [objmap, tskmap](InitContext& ic) -> std::function<void(ProcessingContext&)> {
-    auto& callbacks = ic.services().get<CallbackService>();
-    auto inputObjects = std::make_shared<std::vector<std::pair<InputObjectRoute, InputObject>>>();
-
-    static TFile* f[OutputObjHandlingPolicy::numPolicies];
-    for (auto i = 0u; i < OutputObjHandlingPolicy::numPolicies; ++i) {
-      f[i] = nullptr;
-    }
-
-    static std::string currentDirectory = "";
-    static std::string currentFile = "";
-
-    auto endofdatacb = [inputObjects](EndOfStreamContext& context) {
-      LOG(debug) << "Writing merged objects and histograms to file";
-      if (inputObjects->empty()) {
-        LOG(error) << "Output object map is empty!";
-        context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
-        return;
-      }
-      for (auto i = 0u; i < OutputObjHandlingPolicy::numPolicies; ++i) {
-        if (f[i] != nullptr) {
-          f[i]->Close();
-        }
-      }
-      LOG(debug) << "All outputs merged in their respective target files";
-      context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
-    };
-
-    callbacks.set<CallbackService::Id::EndOfStream>(endofdatacb);
-    return [inputObjects, objmap, tskmap](ProcessingContext& pc) mutable -> void {
-      auto const& ref = pc.inputs().get("x");
-      if (!ref.header) {
-        LOG(error) << "Header not found";
-        return;
-      }
-      if (!ref.payload) {
-        LOG(error) << "Payload not found";
-        return;
-      }
-      auto datah = o2::header::get<o2::header::DataHeader*>(ref.header);
-      if (!datah) {
-        LOG(error) << "No data header in stack";
-        return;
-      }
-
-      auto objh = o2::header::get<o2::framework::OutputObjHeader*>(ref.header);
-      if (!objh) {
-        LOG(error) << "No output object header in stack";
-        return;
-      }
-
-      InputObject obj;
-      FairInputTBuffer tm(const_cast<char*>(ref.payload), static_cast<int>(datah->payloadSize));
-      tm.InitMap();
-      obj.kind = tm.ReadClass();
-      tm.SetBufferOffset(0);
-      tm.ResetMap();
-      if (obj.kind == nullptr) {
-        LOG(error) << "Cannot read class info from buffer.";
-        return;
-      }
-
-      auto policy = objh->mPolicy;
-      auto sourceType = objh->mSourceType;
-      auto hash = objh->mTaskHash;
-
-      obj.obj = tm.ReadObjectAny(obj.kind);
-      auto* named = static_cast<TNamed*>(obj.obj);
-      obj.name = named->GetName();
-      auto hpos = std::find_if(tskmap.begin(), tskmap.end(), [&](auto&& x) { return x.id == hash; });
-      if (hpos == tskmap.end()) {
-        LOG(error) << "No task found for hash " << hash;
-        return;
-      }
-      auto taskname = hpos->name;
-      auto opos = std::find_if(objmap.begin(), objmap.end(), [&](auto&& x) { return x.id == hash; });
-      if (opos == objmap.end()) {
-        LOG(error) << "No object list found for task " << taskname << " (hash=" << hash << ")";
-        return;
-      }
-      auto objects = opos->bindings;
-      if (std::find(objects.begin(), objects.end(), obj.name) == objects.end()) {
-        LOG(error) << "No object " << obj.name << " in map for task " << taskname;
-        return;
-      }
-      auto nameHash = runtime_hash(obj.name.c_str());
-      InputObjectRoute key{obj.name, nameHash, taskname, hash, policy, sourceType};
-      auto existing = std::find_if(inputObjects->begin(), inputObjects->end(), [&](auto&& x) { return (x.first.uniqueId == nameHash) && (x.first.taskHash == hash); });
-      // If it's the first one, we just add it to the list.
-      if (existing == inputObjects->end()) {
-        obj.count = objh->mPipelineSize;
-        inputObjects->push_back(std::make_pair(key, obj));
-        existing = inputObjects->end() - 1;
-      } else {
-        obj.count = existing->second.count;
-        // Otherwise, we merge it with the existing one.
-        auto merger = existing->second.kind->GetMerge();
-        if (!merger) {
-          LOG(error) << "Already one unmergeable object found for " << obj.name;
-          return;
-        }
-        TList coll;
-        coll.Add(static_cast<TObject*>(obj.obj));
-        merger(existing->second.obj, &coll, nullptr);
-      }
-      // We expect as many objects as the pipeline size, for
-      // a given object name and task hash.
-      existing->second.count -= 1;
-
-      if (existing->second.count != 0) {
-        return;
-      }
-      // Write the object here.
-      auto route = existing->first;
-      auto entry = existing->second;
-      auto file = ROOTfileNames.find(route.policy);
-      if (file == ROOTfileNames.end()) {
-        return;
-      }
-      auto filename = file->second;
-      if (f[route.policy] == nullptr) {
-        f[route.policy] = TFile::Open(filename.c_str(), "RECREATE");
-      }
-      auto nextDirectory = route.directory;
-      if ((nextDirectory != currentDirectory) || (filename != currentFile)) {
-        if (!f[route.policy]->FindKey(nextDirectory.c_str())) {
-          f[route.policy]->mkdir(nextDirectory.c_str());
-        }
-        currentDirectory = nextDirectory;
-        currentFile = filename;
-      }
-
-      // translate the list-structure created by the registry into a directory structure within the file
-      std::function<void(TList*, TDirectory*)> writeListToFile;
-      writeListToFile = [&](TList* list, TDirectory* parentDir) {
-        TIter next(list);
-        TObject* object = nullptr;
-        while ((object = next())) {
-          if (object->InheritsFrom(TList::Class())) {
-            writeListToFile(static_cast<TList*>(object), parentDir->mkdir(object->GetName(), object->GetName(), true));
-          } else {
-            parentDir->WriteObjectAny(object, object->Class(), object->GetName());
-            auto* written = list->Remove(object);
-            delete written;
-          }
-        }
-      };
-
-      TDirectory* currentDir = f[route.policy]->GetDirectory(currentDirectory.c_str());
-      if (route.sourceType == OutputObjSourceType::HistogramRegistrySource) {
-        auto* outputList = static_cast<TList*>(entry.obj);
-        outputList->SetOwner(false);
-
-        // if registry should live in dedicated folder a TNamed object is appended to the list
-        if (outputList->Last() && outputList->Last()->IsA() == TNamed::Class()) {
-          delete outputList->Last();
-          outputList->RemoveLast();
-          currentDir = currentDir->mkdir(outputList->GetName(), outputList->GetName(), true);
-        }
-
-        writeListToFile(outputList, currentDir);
-        outputList->SetOwner();
-        delete outputList;
-        entry.obj = nullptr;
-      } else {
-        currentDir->WriteObjectAny(entry.obj, entry.kind, entry.name.c_str());
-        delete (TObject*)entry.obj;
-        entry.obj = nullptr;
-      }
-    };
-  };
-
-  char const* name = "internal-dpl-aod-global-analysis-file-sink";
   // Lifetime is sporadic because we do not ask each analysis task to send its
   // results every timeframe.
   DataProcessorSpec spec{
-    .name = name,
+    .name = "internal-dpl-aod-global-analysis-file-sink",
     .inputs = {InputSpec("x", DataSpecUtils::dataDescriptorMatcherFrom(header::DataOrigin{"ATSK"}), Lifetime::Sporadic)},
-    .algorithm = {writerFunction},
+    .outputs = {},
+    .algorithm = PluginManager::loadAlgorithmFromPlugin("O2FrameworkAnalysisSupport", "ROOTObjWriter", ctx),
   };
 
   return spec;
@@ -317,188 +224,17 @@ DataProcessorSpec AnalysisSupportHelpers::getOutputObjHistSink(std::vector<Outpu
 
 // add sink for the AODs
 DataProcessorSpec
-  AnalysisSupportHelpers::getGlobalAODSink(std::shared_ptr<DataOutputDirector> dod,
-                                           std::vector<InputSpec> const& outputInputs, int compressionLevel)
+  AnalysisSupportHelpers::getGlobalAODSink(ConfigContext const& ctx)
 {
-
-  auto writerFunction = [dod, outputInputs, compressionLevel](InitContext& ic) -> std::function<void(ProcessingContext&)> {
-    LOGP(debug, "======== getGlobalAODSink::Init ==========");
-
-    // find out if any table needs to be saved
-    bool hasOutputsToWrite = false;
-    for (auto& outobj : outputInputs) {
-      auto ds = dod->getDataOutputDescriptors(outobj);
-      if (ds.size() > 0) {
-        hasOutputsToWrite = true;
-        break;
-      }
-    }
-
-    // if nothing needs to be saved then return a trivial functor
-    // this happens when nothing needs to be saved but there are dangling outputs
-    if (!hasOutputsToWrite) {
-      return [](ProcessingContext&) mutable -> void {
-        static bool once = false;
-        if (!once) {
-          LOG(info) << "No AODs to be saved.";
-          once = true;
-        }
-      };
-    }
-
-    // end of data functor is called at the end of the data stream
-    auto endofdatacb = [dod](EndOfStreamContext& context) {
-      dod->closeDataFiles();
-      context.services().get<ControlService>().readyToQuit(QuitRequest::Me);
-    };
-
-    auto& callbacks = ic.services().get<CallbackService>();
-    callbacks.set<CallbackService::Id::EndOfStream>(endofdatacb);
-
-    // prepare map<uint64_t, uint64_t>(startTime, tfNumber)
-    std::map<uint64_t, uint64_t> tfNumbers;
-    std::map<uint64_t, std::string> tfFilenames;
-
-    std::vector<TString> aodMetaDataKeys;
-    std::vector<TString> aodMetaDataVals;
-
-    // this functor is called once per time frame
-    return [dod, tfNumbers, tfFilenames, aodMetaDataKeys, aodMetaDataVals, compressionLevel](ProcessingContext& pc) mutable -> void {
-      LOGP(debug, "======== getGlobalAODSink::processing ==========");
-      LOGP(debug, " processing data set with {} entries", pc.inputs().size());
-
-      // return immediately if pc.inputs() is empty. This should never happen!
-      if (pc.inputs().size() == 0) {
-        LOGP(info, "No inputs available!");
-        return;
-      }
-
-      // update tfNumbers
-      uint64_t startTime = 0;
-      uint64_t tfNumber = 0;
-      auto ref = pc.inputs().get("tfn");
-      if (ref.spec && ref.payload) {
-        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref)->startTime;
-        tfNumber = pc.inputs().get<uint64_t>("tfn");
-        tfNumbers.insert(std::pair<uint64_t, uint64_t>(startTime, tfNumber));
-      }
-      // update tfFilenames
-      std::string aodInputFile;
-      auto ref2 = pc.inputs().get("tff");
-      if (ref2.spec && ref2.payload) {
-        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref2)->startTime;
-        aodInputFile = pc.inputs().get<std::string>("tff");
-        tfFilenames.insert(std::pair<uint64_t, std::string>(startTime, aodInputFile));
-      }
-
-      // close all output files if one has reached size limit
-      dod->checkFileSizes();
-
-      // loop over the DataRefs which are contained in pc.inputs()
-      for (const auto& ref : pc.inputs()) {
-        if (!ref.spec) {
-          LOGP(debug, "Invalid input will be skipped!");
-          continue;
-        }
-
-        // get metadata
-        if (DataSpecUtils::partialMatch(*ref.spec, header::DataDescription("AODMetadataKeys"))) {
-          aodMetaDataKeys = pc.inputs().get<std::vector<TString>>(ref.spec->binding);
-        }
-        if (DataSpecUtils::partialMatch(*ref.spec, header::DataDescription("AODMetadataVals"))) {
-          aodMetaDataVals = pc.inputs().get<std::vector<TString>>(ref.spec->binding);
-        }
-
-        // skip non-AOD refs
-        if (!DataSpecUtils::partialMatch(*ref.spec, writableAODOrigins)) {
-          continue;
-        }
-        startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref)->startTime;
-
-        // does this need to be saved?
-        auto dh = DataRefUtils::getHeader<header::DataHeader*>(ref);
-        auto tableName = dh->dataDescription.as<std::string>();
-        auto ds = dod->getDataOutputDescriptors(*dh);
-        if (ds.empty()) {
-          continue;
-        }
-
-        // get TF number from startTime
-        auto it = tfNumbers.find(startTime);
-        if (it != tfNumbers.end()) {
-          tfNumber = (it->second / dod->getNumberTimeFramesToMerge()) * dod->getNumberTimeFramesToMerge();
-        } else {
-          LOGP(fatal, "No time frame number found for output with start time {}", startTime);
-          throw std::runtime_error("Processing is stopped!");
-        }
-        // get aod input file from startTime
-        auto it2 = tfFilenames.find(startTime);
-        if (it2 != tfFilenames.end()) {
-          aodInputFile = it2->second;
-        }
-
-        // get the TableConsumer and corresponding arrow table
-        auto msg = pc.inputs().get(ref.spec->binding);
-        if (msg.header == nullptr) {
-          LOGP(error, "No header for message {}:{}", ref.spec->binding, DataSpecUtils::describe(*ref.spec));
-          continue;
-        }
-        auto s = pc.inputs().get<TableConsumer>(ref.spec->binding);
-        auto table = s->asArrowTable();
-        if (!table->Validate().ok()) {
-          LOGP(warning, "The table \"{}\" is not valid and will not be saved!", tableName);
-          continue;
-        }
-        if (table->schema()->fields().empty()) {
-          LOGP(debug, "The table \"{}\" is empty but will be saved anyway!", tableName);
-        }
-
-        // loop over all DataOutputDescriptors
-        // a table can be saved in multiple ways
-        // e.g. different selections of columns to different files
-        for (auto d : ds) {
-          auto fileAndFolder = dod->getFileFolder(d, tfNumber, aodInputFile, compressionLevel);
-          auto treename = fileAndFolder.folderName + "/" + d->treename;
-          TableToTree ta2tr(table,
-                            fileAndFolder.file,
-                            treename.c_str());
-
-          // update metadata
-          if (fileAndFolder.file->FindObjectAny("metaData")) {
-            LOGF(debug, "Metadata: target file %s already has metadata, preserving it", fileAndFolder.file->GetName());
-          } else if (!aodMetaDataKeys.empty() && !aodMetaDataVals.empty()) {
-            TMap aodMetaDataMap;
-            for (uint32_t imd = 0; imd < aodMetaDataKeys.size(); imd++) {
-              aodMetaDataMap.Add(new TObjString(aodMetaDataKeys[imd]), new TObjString(aodMetaDataVals[imd]));
-            }
-            fileAndFolder.file->WriteObject(&aodMetaDataMap, "metaData", "Overwrite");
-          }
-
-          if (!d->colnames.empty()) {
-            for (auto& cn : d->colnames) {
-              auto idx = table->schema()->GetFieldIndex(cn);
-              auto col = table->column(idx);
-              auto field = table->schema()->field(idx);
-              if (idx != -1) {
-                ta2tr.addBranch(col, field);
-              }
-            }
-          } else {
-            ta2tr.addAllBranches();
-          }
-          ta2tr.process();
-        }
-      }
-    };
-  }; // end of writerFunction
+  auto& ac = ctx.services().get<AnalysisContext>();
 
   // the command line options relevant for the writer are global
   // see runDataProcessing.h
   DataProcessorSpec spec{
     .name = "internal-dpl-aod-writer",
-    .inputs = outputInputs,
+    .inputs = ac.outputsInputsAOD,
     .outputs = {},
-    .algorithm = AlgorithmSpec{writerFunction},
+    .algorithm = PluginManager::loadAlgorithmFromPlugin("O2FrameworkAnalysisSupport", "ROOTTTreeWriter", ctx),
   };
 
   return spec;
diff --git a/Framework/Core/src/ArrowSupport.cxx b/Framework/Core/src/ArrowSupport.cxx
index 1a656e4d60080..230d708b47dc7 100644
--- a/Framework/Core/src/ArrowSupport.cxx
+++ b/Framework/Core/src/ArrowSupport.cxx
@@ -30,7 +30,7 @@
 #include "Framework/ServiceMetricsInfo.h"
 #include "WorkflowHelpers.h"
 #include "Framework/WorkflowSpecNode.h"
-#include "AnalysisSupportHelpers.h"
+#include "Framework/AnalysisSupportHelpers.h"
 
 #include "CommonMessageBackendsHelpers.h"
 #include <Monitoring/Monitoring.h>
@@ -516,7 +516,7 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
       auto [outputsInputs, isDangling] = WorkflowHelpers::analyzeOutputs(workflow);
 
       // create DataOutputDescriptor
-      std::shared_ptr<DataOutputDirector> dod = WorkflowHelpers::getDataOutputDirector(ctx.options(), outputsInputs, isDangling);
+      std::shared_ptr<DataOutputDirector> dod = AnalysisSupportHelpers::getDataOutputDirector(ctx);
 
       // select outputs of type AOD which need to be saved
       // ATTENTION: if there are dangling outputs the getGlobalAODSink
@@ -537,11 +537,7 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
         // add TFNumber and TFFilename as input to the writer
         outputsInputsAOD.emplace_back("tfn", "TFN", "TFNumber");
         outputsInputsAOD.emplace_back("tff", "TFF", "TFFilename");
-        int compression = 505; 
-        if (ctx.options().hasOption("aod-writer-compression")) {
-          compression = ctx.options().get<int>("aod-writer-compression");
-        }
-        workflow.push_back(AnalysisSupportHelpers::getGlobalAODSink(dod, outputsInputsAOD, compression));
+        workflow.push_back(AnalysisSupportHelpers::getGlobalAODSink(ctx));
       }
       // Move the dummy sink at the end, if needed
       for (size_t i = 0; i < workflow.size(); ++i) {
diff --git a/Framework/Core/src/ConfigContext.cxx b/Framework/Core/src/ConfigContext.cxx
index 726332e1d0ae3..9b121b1884998 100644
--- a/Framework/Core/src/ConfigContext.cxx
+++ b/Framework/Core/src/ConfigContext.cxx
@@ -14,6 +14,9 @@
 namespace o2::framework
 {
 
+ConfigContext::ConfigContext(ConfigParamRegistry& options, ServiceRegistryRef services, int argc, char** argv)
+  : mOptions{options}, mServices{services}, mArgc{argc}, mArgv{argv} {}
+
 bool ConfigContext::helpOnCommandLine() const
 {
   bool helpasked = false;
diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx
index da9a135dc5eb8..3782c48e81c56 100644
--- a/Framework/Core/src/WorkflowHelpers.cxx
+++ b/Framework/Core/src/WorkflowHelpers.cxx
@@ -9,7 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 #include "WorkflowHelpers.h"
-#include "AnalysisSupportHelpers.h"
+#include "Framework/AnalysisSupportHelpers.h"
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/AODReaderHelpers.h"
 #include "Framework/ConfigParamSpec.h"
@@ -153,7 +153,7 @@ int defaultConditionQueryRateMultiplier()
   return getenv("DPL_CONDITION_QUERY_RATE_MULTIPLIER") ? std::stoi(getenv("DPL_CONDITION_QUERY_RATE_MULTIPLIER")) : 1;
 }
 
-void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext const& ctx)
+void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext& ctx)
 {
   auto fakeCallback = AlgorithmSpec{[](InitContext& ic) {
     LOG(info) << "This is not a real device, merely a placeholder for external inputs";
@@ -241,7 +241,9 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
     aodReader.options.emplace_back(ConfigParamSpec{"channel-config", VariantType::String, rateLimitingChannelConfigInput, {"how many timeframes can be in flight at the same time"}});
   }
 
-  AnalysisContext ac;
+  ctx.services().registerService(ServiceRegistryHelpers::handleForService<AnalysisContext>(new AnalysisContext));
+  auto& ac = ctx.services().get<AnalysisContext>();
+
   std::vector<InputSpec> requestedCCDBs;
   std::vector<OutputSpec> providedCCDBs;
 
@@ -573,7 +575,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
   // This is to inject a file sink so that any dangling ATSK object is written
   // to a ROOT file.
   if (ac.providedOutputObjHist.empty() == false) {
-    auto rootSink = AnalysisSupportHelpers::getOutputObjHistSink(ac.outObjHistMap, ac.outTskMap);
+    auto rootSink = AnalysisSupportHelpers::getOutputObjHistSink(ctx);
     extraSpecs.push_back(rootSink);
   }
 
@@ -581,41 +583,38 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
   extraSpecs.clear();
 
   /// Analyze all ouputs
-  auto [outputsInputs, isDangling] = analyzeOutputs(workflow);
+  auto [outputsInputsTmp, isDanglingTmp] = analyzeOutputs(workflow);
+  ac.isDangling = isDanglingTmp;
+  ac.outputsInputs = outputsInputsTmp;
 
   // create DataOutputDescriptor
-  std::shared_ptr<DataOutputDirector> dod = getDataOutputDirector(ctx.options(), outputsInputs, isDangling);
+  std::shared_ptr<DataOutputDirector> dod = AnalysisSupportHelpers::getDataOutputDirector(ctx);
 
   // select outputs of type AOD which need to be saved
   // ATTENTION: if there are dangling outputs the getGlobalAODSink
   // has to be created in any case!
-  std::vector<InputSpec> outputsInputsAOD;
-  for (auto ii = 0u; ii < outputsInputs.size(); ii++) {
-    if (DataSpecUtils::partialMatch(outputsInputs[ii], extendedAODOrigins)) {
-      auto ds = dod->getDataOutputDescriptors(outputsInputs[ii]);
-      if (ds.size() > 0 || isDangling[ii]) {
-        outputsInputsAOD.emplace_back(outputsInputs[ii]);
+  for (auto ii = 0u; ii < ac.outputsInputs.size(); ii++) {
+    if (DataSpecUtils::partialMatch(ac.outputsInputs[ii], extendedAODOrigins)) {
+      auto ds = dod->getDataOutputDescriptors(ac.outputsInputs[ii]);
+      if (ds.size() > 0 || ac.isDangling[ii]) {
+        ac.outputsInputsAOD.emplace_back(ac.outputsInputs[ii]);
       }
     }
   }
 
   // file sink for any AOD output
-  if (outputsInputsAOD.size() > 0) {
+  if (ac.outputsInputsAOD.size() > 0) {
     // add TFNumber and TFFilename as input to the writer
-    outputsInputsAOD.emplace_back(InputSpec{"tfn", "TFN", "TFNumber"});
-    outputsInputsAOD.emplace_back(InputSpec{"tff", "TFF", "TFFilename"});
-    int compressionLevel = 505;
-    if (ctx.options().hasOption("aod-writer-compression")) {
-      compressionLevel = ctx.options().get<int>("aod-writer-compression");
-    }
-    auto fileSink = AnalysisSupportHelpers::getGlobalAODSink(dod, outputsInputsAOD, compressionLevel);
+    ac.outputsInputsAOD.emplace_back(InputSpec{"tfn", "TFN", "TFNumber"});
+    ac.outputsInputsAOD.emplace_back(InputSpec{"tff", "TFF", "TFFilename"});
+    auto fileSink = AnalysisSupportHelpers::getGlobalAODSink(ctx);
     extraSpecs.push_back(fileSink);
 
-    auto it = std::find_if(outputsInputs.begin(), outputsInputs.end(), [](InputSpec& spec) -> bool {
+    auto it = std::find_if(ac.outputsInputs.begin(), ac.outputsInputs.end(), [](InputSpec& spec) -> bool {
       return DataSpecUtils::partialMatch(spec, o2::header::DataOrigin("TFN"));
     });
-    size_t ii = std::distance(outputsInputs.begin(), it);
-    isDangling[ii] = false;
+    size_t ii = std::distance(ac.outputsInputs.begin(), it);
+    ac.isDangling[ii] = false;
   }
 
   workflow.insert(workflow.end(), extraSpecs.begin(), extraSpecs.end());
@@ -623,20 +622,20 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
 
   // Select dangling outputs which are not of type AOD
   std::vector<InputSpec> redirectedOutputsInputs;
-  for (auto ii = 0u; ii < outputsInputs.size(); ii++) {
+  for (auto ii = 0u; ii < ac.outputsInputs.size(); ii++) {
     if (ctx.options().get<std::string>("forwarding-policy") == "none") {
       continue;
     }
     // We forward to the output proxy all the inputs only if they are dangling
     // or if the forwarding policy is "proxy".
-    if (!isDangling[ii] && (ctx.options().get<std::string>("forwarding-policy") != "all")) {
+    if (!ac.isDangling[ii] && (ctx.options().get<std::string>("forwarding-policy") != "all")) {
       continue;
     }
     // AODs are skipped in any case.
-    if (DataSpecUtils::partialMatch(outputsInputs[ii], extendedAODOrigins)) {
+    if (DataSpecUtils::partialMatch(ac.outputsInputs[ii], extendedAODOrigins)) {
       continue;
     }
-    redirectedOutputsInputs.emplace_back(outputsInputs[ii]);
+    redirectedOutputsInputs.emplace_back(ac.outputsInputs[ii]);
   }
 
   std::vector<InputSpec> unmatched;
@@ -985,102 +984,6 @@ struct DataMatcherId {
   size_t id;
 };
 
-std::shared_ptr<DataOutputDirector> WorkflowHelpers::getDataOutputDirector(ConfigParamRegistry const& options, std::vector<InputSpec> const& OutputsInputs, std::vector<bool> const& isDangling)
-{
-  std::shared_ptr<DataOutputDirector> dod = std::make_shared<DataOutputDirector>();
-
-  // analyze options and take actions accordingly
-  // default values
-  std::string rdn, resdir("./");
-  std::string fnb, fnbase("AnalysisResults_trees");
-  float mfs, maxfilesize(-1.);
-  std::string fmo, filemode("RECREATE");
-  int ntfm, ntfmerge = 1;
-
-  // values from json
-  if (options.isSet("aod-writer-json")) {
-    auto fnjson = options.get<std::string>("aod-writer-json");
-    if (!fnjson.empty()) {
-      std::tie(rdn, fnb, fmo, mfs, ntfm) = dod->readJson(fnjson);
-      if (!rdn.empty()) {
-        resdir = rdn;
-      }
-      if (!fnb.empty()) {
-        fnbase = fnb;
-      }
-      if (!fmo.empty()) {
-        filemode = fmo;
-      }
-      if (mfs > 0.) {
-        maxfilesize = mfs;
-      }
-      if (ntfm > 0) {
-        ntfmerge = ntfm;
-      }
-    }
-  }
-
-  // values from command line options, information from json is overwritten
-  if (options.isSet("aod-writer-resdir")) {
-    rdn = options.get<std::string>("aod-writer-resdir");
-    if (!rdn.empty()) {
-      resdir = rdn;
-    }
-  }
-  if (options.isSet("aod-writer-resfile")) {
-    fnb = options.get<std::string>("aod-writer-resfile");
-    if (!fnb.empty()) {
-      fnbase = fnb;
-    }
-  }
-  if (options.isSet("aod-writer-resmode")) {
-    fmo = options.get<std::string>("aod-writer-resmode");
-    if (!fmo.empty()) {
-      filemode = fmo;
-    }
-  }
-  if (options.isSet("aod-writer-maxfilesize")) {
-    mfs = options.get<float>("aod-writer-maxfilesize");
-    if (mfs > 0) {
-      maxfilesize = mfs;
-    }
-  }
-  if (options.isSet("aod-writer-ntfmerge")) {
-    ntfm = options.get<int>("aod-writer-ntfmerge");
-    if (ntfm > 0) {
-      ntfmerge = ntfm;
-    }
-  }
-  // parse the keepString
-  if (options.isSet("aod-writer-keep")) {
-    auto keepString = options.get<std::string>("aod-writer-keep");
-    if (!keepString.empty()) {
-      dod->reset();
-      std::string d("dangling");
-      if (d.find(keepString) == 0) {
-        // use the dangling outputs
-        std::vector<InputSpec> danglingOutputs;
-        for (auto ii = 0u; ii < OutputsInputs.size(); ii++) {
-          if (DataSpecUtils::partialMatch(OutputsInputs[ii], writableAODOrigins) && isDangling[ii]) {
-            danglingOutputs.emplace_back(OutputsInputs[ii]);
-          }
-        }
-        dod->readSpecs(danglingOutputs);
-      } else {
-        // use the keep string
-        dod->readString(keepString);
-      }
-    }
-  }
-  dod->setResultDir(resdir);
-  dod->setFilenameBase(fnbase);
-  dod->setFileMode(filemode);
-  dod->setMaximumFileSize(maxfilesize);
-  dod->setNumberTimeFramesToMerge(ntfmerge);
-
-  return dod;
-}
-
 std::tuple<std::vector<InputSpec>, std::vector<bool>> WorkflowHelpers::analyzeOutputs(WorkflowSpec const& workflow)
 {
   // compute total number of input/output
diff --git a/Framework/Core/src/WorkflowHelpers.h b/Framework/Core/src/WorkflowHelpers.h
index b20249b99edc8..b2a4d4cab55df 100644
--- a/Framework/Core/src/WorkflowHelpers.h
+++ b/Framework/Core/src/WorkflowHelpers.h
@@ -180,7 +180,7 @@ struct WorkflowHelpers {
   // dangling inputs are satisfied.
   // @a workflow the workflow to decorate
   // @a ctx the context for the configuration phase
-  static void injectServiceDevices(WorkflowSpec& workflow, ConfigContext const& ctx);
+  static void injectServiceDevices(WorkflowSpec& workflow, ConfigContext& ctx);
 
   // Final adjustments to @a workflow after service devices have been injected.
   static void adjustTopology(WorkflowSpec& workflow, ConfigContext const& ctx);
@@ -204,8 +204,6 @@ struct WorkflowHelpers {
     const std::vector<DeviceConnectionEdge>& edges,
     const std::vector<size_t>& index);
 
-  static std::shared_ptr<DataOutputDirector> getDataOutputDirector(ConfigParamRegistry const& options, std::vector<InputSpec> const& OutputsInputs, std::vector<bool> const& outputTypes);
-
   /// Given @a workflow it gathers all the OutputSpec and in addition provides
   /// the information whether and output is dangling and/or of type AOD
   /// An Output is dangling if it does not have a corresponding InputSpec.
diff --git a/Framework/Core/test/Mocking.h b/Framework/Core/test/Mocking.h
index b3e48ad3b2d0f..a42a1b30a662f 100644
--- a/Framework/Core/test/Mocking.h
+++ b/Framework/Core/test/Mocking.h
@@ -34,7 +34,10 @@ std::unique_ptr<o2::framework::ConfigContext> makeEmptyConfigContext()
   store->preload();
   store->activate();
   static ConfigParamRegistry registry(std::move(store));
-  auto context = std::make_unique<ConfigContext>(registry, 0, nullptr);
+  static std::unique_ptr<ServiceRegistry> services;
+  // We need to reset it because we will inject services into it.
+  services = std::make_unique<ServiceRegistry>();
+  auto context = std::make_unique<ConfigContext>(registry, ServiceRegistryRef{*services}, 0, nullptr);
   return context;
 }
 
diff --git a/Framework/Core/test/benchmark_WorkflowHelpers.cxx b/Framework/Core/test/benchmark_WorkflowHelpers.cxx
index f1c070d8a0f4e..09a9ae0cca923 100644
--- a/Framework/Core/test/benchmark_WorkflowHelpers.cxx
+++ b/Framework/Core/test/benchmark_WorkflowHelpers.cxx
@@ -30,7 +30,8 @@ std::unique_ptr<ConfigContext> makeEmptyConfigContext()
   store->preload();
   store->activate();
   static ConfigParamRegistry registry(std::move(store));
-  auto context = std::make_unique<ConfigContext>(registry, 0, nullptr);
+  static ServiceRegistry services;
+  auto context = std::make_unique<ConfigContext>(registry, ServiceRegistryRef{services}, 0, nullptr);
   return context;
 }
 
diff --git a/Framework/Core/test/test_OverrideLabels.cxx b/Framework/Core/test/test_OverrideLabels.cxx
index 573bd13be797a..c5134c0c169c0 100644
--- a/Framework/Core/test/test_OverrideLabels.cxx
+++ b/Framework/Core/test/test_OverrideLabels.cxx
@@ -31,7 +31,8 @@ std::unique_ptr<o2::framework::ConfigContext> mockupLabels(std::string labelArg)
   store->preload();
   store->activate();
   registry = ConfigParamRegistry(std::move(store));
-  auto context = std::make_unique<ConfigContext>(registry, 0, nullptr);
+  static ServiceRegistry services;
+  auto context = std::make_unique<ConfigContext>(registry, ServiceRegistryRef{services}, 0, nullptr);
   return context;
 }
 
diff --git a/Framework/TestWorkflows/src/o2TestHistograms.cxx b/Framework/TestWorkflows/src/o2TestHistograms.cxx
index 9986f52a1d940..efac16f6da4f0 100644
--- a/Framework/TestWorkflows/src/o2TestHistograms.cxx
+++ b/Framework/TestWorkflows/src/o2TestHistograms.cxx
@@ -17,6 +17,7 @@
 #include "Framework/AnalysisTask.h"
 #include <TH2F.h>
 #include <cmath>
+#include <iostream>
 
 using namespace o2;
 using namespace o2::framework;
@@ -43,7 +44,7 @@ struct EtaAndClsHistogramsSimple {
   {
     LOGP(info, "Invoking the simple one");
     for (auto& track : tracks) {
-      etaClsH->Fill(track.eta(), track.pt(), 0);
+      etaClsH->Fill(track.eta(), track.pt());
       skimEx(track.pt(), track.eta());
     }
   }
@@ -57,7 +58,7 @@ struct EtaAndClsHistogramsIUSimple {
   {
     LOGP(info, "Invoking the simple one");
     for (auto& track : tracks) {
-      etaClsH->Fill(track.eta(), track.pt(), 0);
+      etaClsH->Fill(track.eta(), track.pt());
       skimEx(track.pt(), track.eta());
     }
   }

From 67955bf535152dd929c491cb246c5289697abc4b Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 14 Nov 2024 08:57:13 +0100
Subject: [PATCH 701/726] DPL: Cleanup unneeded headers

---
 Framework/Core/src/CommonDataProcessors.cxx | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/Framework/Core/src/CommonDataProcessors.cxx b/Framework/Core/src/CommonDataProcessors.cxx
index d893e16513f40..737e1b7e635c8 100644
--- a/Framework/Core/src/CommonDataProcessors.cxx
+++ b/Framework/Core/src/CommonDataProcessors.cxx
@@ -17,39 +17,26 @@
 #include "Framework/DataProcessingHeader.h"
 #include "Framework/DataDescriptorQueryBuilder.h"
 #include "Framework/DataDescriptorMatcher.h"
-#include "Framework/DataOutputDirector.h"
 #include "Framework/DataProcessorSpec.h"
 #include "Framework/DataProcessingStats.h"
 #include "Framework/DataSpecUtils.h"
-#include "Framework/TableBuilder.h"
-#include "Framework/EndOfStreamContext.h"
 #include "Framework/InitContext.h"
 #include "Framework/InputSpec.h"
-#include "Framework/Logger.h"
-#include "Framework/OutputSpec.h"
 #include "Framework/RawDeviceService.h"
 #include "Framework/TimesliceIndex.h"
 #include "Framework/Variant.h"
-#include "../../../Algorithm/include/Algorithm/HeaderStack.h"
-#include "Framework/OutputObjHeader.h"
-#include "Framework/StringHelpers.h"
 #include "Framework/ChannelSpec.h"
-#include "Framework/ChannelSpecHelpers.h"
 #include "Framework/ExternalFairMQDeviceProxy.h"
 #include "Framework/RuntimeError.h"
 #include "Framework/RateLimiter.h"
 #include "Framework/PluginManager.h"
-#include "Framework/DeviceSpec.h"
-#include "WorkflowHelpers.h"
 #include <Monitoring/Monitoring.h>
 
 #include <fairmq/Device.h>
-#include <chrono>
 #include <fstream>
 #include <functional>
 #include <memory>
 #include <string>
-#include <thread>
 
 using namespace o2::framework::data_matcher;
 

From 4f16f72e7516e6fe6e30d8ff0fed21c3501f3555 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 15 Nov 2024 10:15:31 +0100
Subject: [PATCH 702/726] DPL: Use AnalysisContext also in the case of amended
 topologies

---
 Framework/Core/src/ArrowSupport.cxx | 34 +++++++++++++++--------------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/Framework/Core/src/ArrowSupport.cxx b/Framework/Core/src/ArrowSupport.cxx
index 230d708b47dc7..b816265151a21 100644
--- a/Framework/Core/src/ArrowSupport.cxx
+++ b/Framework/Core/src/ArrowSupport.cxx
@@ -420,16 +420,18 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
       auto builder = std::find_if(workflow.begin(), workflow.end(), [](DataProcessorSpec const& spec) { return spec.name == "internal-dpl-aod-index-builder"; });
       auto reader = std::find_if(workflow.begin(), workflow.end(), [](DataProcessorSpec const& spec) { return spec.name == "internal-dpl-aod-reader"; });
       auto writer = std::find_if(workflow.begin(), workflow.end(), [](DataProcessorSpec const& spec) { return spec.name == "internal-dpl-aod-writer"; });
-      std::vector<InputSpec> requestedAODs;
-      std::vector<InputSpec> requestedDYNs;
-      std::vector<OutputSpec> providedDYNs;
+      auto &ac = ctx.services().get<AnalysisContext>();
+      ac.requestedAODs.clear();
+      ac.requestedDYNs.clear();
+      ac.providedDYNs.clear();
+
 
       auto inputSpecLessThan = [](InputSpec const& lhs, InputSpec const& rhs) { return DataSpecUtils::describe(lhs) < DataSpecUtils::describe(rhs); };
       auto outputSpecLessThan = [](OutputSpec const& lhs, OutputSpec const& rhs) { return DataSpecUtils::describe(lhs) < DataSpecUtils::describe(rhs); };
 
       if (builder != workflow.end()) {
         // collect currently requested IDXs
-        std::vector<InputSpec> requestedIDXs;
+        ac.requestedIDXs.clear();
         for (auto& d : workflow) {
           if (d.name == builder->name) {
             continue;
@@ -437,7 +439,7 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
           for (auto& i : d.inputs) {
             if (DataSpecUtils::partialMatch(i, header::DataOrigin{"IDX"})) {
               auto copy = i;
-              DataSpecUtils::updateInputList(requestedIDXs, std::move(copy));
+              DataSpecUtils::updateInputList(ac.requestedIDXs, std::move(copy));
             }
           }
         }
@@ -446,8 +448,8 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
         builder->outputs.clear();
         // replace AlgorithmSpec
         //  FIXME: it should be made more generic, so it does not need replacement...
-        builder->algorithm = readers::AODReaderHelpers::indexBuilderCallback(requestedIDXs);
-        AnalysisSupportHelpers::addMissingOutputsToBuilder(requestedIDXs, requestedAODs, requestedDYNs, *builder);
+        builder->algorithm = readers::AODReaderHelpers::indexBuilderCallback(ac.requestedIDXs);
+        AnalysisSupportHelpers::addMissingOutputsToBuilder(ac.requestedIDXs, ac.requestedAODs, ac.requestedDYNs, *builder);
       }
 
       if (spawner != workflow.end()) {
@@ -459,20 +461,20 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
           for (auto const& i : d.inputs) {
             if (DataSpecUtils::partialMatch(i, header::DataOrigin{"DYN"})) {
               auto copy = i;
-              DataSpecUtils::updateInputList(requestedDYNs, std::move(copy));
+              DataSpecUtils::updateInputList(ac.requestedDYNs, std::move(copy));
             }
           }
           for (auto const& o : d.outputs) {
             if (DataSpecUtils::partialMatch(o, header::DataOrigin{"DYN"})) {
-              providedDYNs.emplace_back(o);
+              ac.providedDYNs.emplace_back(o);
             }
           }
         }
-        std::sort(requestedDYNs.begin(), requestedDYNs.end(), inputSpecLessThan);
-        std::sort(providedDYNs.begin(), providedDYNs.end(), outputSpecLessThan);
+        std::sort(ac.requestedDYNs.begin(), ac.requestedDYNs.end(), inputSpecLessThan);
+        std::sort(ac.providedDYNs.begin(), ac.providedDYNs.end(), outputSpecLessThan);
         std::vector<InputSpec> spawnerInputs;
-        for (auto& input : requestedDYNs) {
-          if (std::none_of(providedDYNs.begin(), providedDYNs.end(), [&input](auto const& x) { return DataSpecUtils::match(input, x); })) {
+        for (auto& input : ac.requestedDYNs) {
+          if (std::none_of(ac.providedDYNs.begin(), ac.providedDYNs.end(), [&input](auto const& x) { return DataSpecUtils::match(input, x); })) {
             spawnerInputs.emplace_back(input);
           }
         }
@@ -482,7 +484,7 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
         // replace AlgorithmSpec
         // FIXME: it should be made more generic, so it does not need replacement...
         spawner->algorithm = readers::AODReaderHelpers::aodSpawnerCallback(spawnerInputs);
-        AnalysisSupportHelpers::addMissingOutputsToSpawner({}, spawnerInputs, requestedAODs, *spawner);
+        AnalysisSupportHelpers::addMissingOutputsToSpawner({}, spawnerInputs, ac.requestedAODs, *spawner);
       }
 
       if (writer != workflow.end()) {
@@ -496,14 +498,14 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
           for (auto const& i : d.inputs) {
             if (DataSpecUtils::partialMatch(i, AODOrigins)) {
               auto copy = i;
-              DataSpecUtils::updateInputList(requestedAODs, std::move(copy));
+              DataSpecUtils::updateInputList(ac.requestedAODs, std::move(copy));
             }
           }
         }
 
         // remove unmatched outputs
         auto o_end = std::remove_if(reader->outputs.begin(), reader->outputs.end(), [&](OutputSpec const& o) {
-          return !DataSpecUtils::partialMatch(o, o2::header::DataDescription{"TFNumber"}) && !DataSpecUtils::partialMatch(o, o2::header::DataDescription{"TFFilename"}) && std::none_of(requestedAODs.begin(), requestedAODs.end(), [&](InputSpec const& i) { return DataSpecUtils::match(i, o); });
+          return !DataSpecUtils::partialMatch(o, o2::header::DataDescription{"TFNumber"}) && !DataSpecUtils::partialMatch(o, o2::header::DataDescription{"TFFilename"}) && std::none_of(ac.requestedAODs.begin(), ac.requestedAODs.end(), [&](InputSpec const& i) { return DataSpecUtils::match(i, o); });
         });
         reader->outputs.erase(o_end, reader->outputs.end());
         if (reader->outputs.empty()) {

From 198a15e0073bf30acdd2431b187e6762943197aa Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 15 Nov 2024 10:17:55 +0100
Subject: [PATCH 703/726] DPL: Use AnalysisContext also in the case of amended
 topologies

---
 Framework/Core/src/ArrowSupport.cxx | 44 +++++++++++++++--------------
 1 file changed, 23 insertions(+), 21 deletions(-)

diff --git a/Framework/Core/src/ArrowSupport.cxx b/Framework/Core/src/ArrowSupport.cxx
index 230d708b47dc7..e6f8fb90c7af9 100644
--- a/Framework/Core/src/ArrowSupport.cxx
+++ b/Framework/Core/src/ArrowSupport.cxx
@@ -420,16 +420,18 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
       auto builder = std::find_if(workflow.begin(), workflow.end(), [](DataProcessorSpec const& spec) { return spec.name == "internal-dpl-aod-index-builder"; });
       auto reader = std::find_if(workflow.begin(), workflow.end(), [](DataProcessorSpec const& spec) { return spec.name == "internal-dpl-aod-reader"; });
       auto writer = std::find_if(workflow.begin(), workflow.end(), [](DataProcessorSpec const& spec) { return spec.name == "internal-dpl-aod-writer"; });
-      std::vector<InputSpec> requestedAODs;
-      std::vector<InputSpec> requestedDYNs;
-      std::vector<OutputSpec> providedDYNs;
+      auto &ac = ctx.services().get<AnalysisContext>();
+      ac.requestedAODs.clear();
+      ac.requestedDYNs.clear();
+      ac.providedDYNs.clear();
+
 
       auto inputSpecLessThan = [](InputSpec const& lhs, InputSpec const& rhs) { return DataSpecUtils::describe(lhs) < DataSpecUtils::describe(rhs); };
       auto outputSpecLessThan = [](OutputSpec const& lhs, OutputSpec const& rhs) { return DataSpecUtils::describe(lhs) < DataSpecUtils::describe(rhs); };
 
       if (builder != workflow.end()) {
         // collect currently requested IDXs
-        std::vector<InputSpec> requestedIDXs;
+        ac.requestedIDXs.clear();
         for (auto& d : workflow) {
           if (d.name == builder->name) {
             continue;
@@ -437,7 +439,7 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
           for (auto& i : d.inputs) {
             if (DataSpecUtils::partialMatch(i, header::DataOrigin{"IDX"})) {
               auto copy = i;
-              DataSpecUtils::updateInputList(requestedIDXs, std::move(copy));
+              DataSpecUtils::updateInputList(ac.requestedIDXs, std::move(copy));
             }
           }
         }
@@ -446,8 +448,8 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
         builder->outputs.clear();
         // replace AlgorithmSpec
         //  FIXME: it should be made more generic, so it does not need replacement...
-        builder->algorithm = readers::AODReaderHelpers::indexBuilderCallback(requestedIDXs);
-        AnalysisSupportHelpers::addMissingOutputsToBuilder(requestedIDXs, requestedAODs, requestedDYNs, *builder);
+        builder->algorithm = readers::AODReaderHelpers::indexBuilderCallback(ac.requestedIDXs);
+        AnalysisSupportHelpers::addMissingOutputsToBuilder(ac.requestedIDXs, ac.requestedAODs, ac.requestedDYNs, *builder);
       }
 
       if (spawner != workflow.end()) {
@@ -459,20 +461,20 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
           for (auto const& i : d.inputs) {
             if (DataSpecUtils::partialMatch(i, header::DataOrigin{"DYN"})) {
               auto copy = i;
-              DataSpecUtils::updateInputList(requestedDYNs, std::move(copy));
+              DataSpecUtils::updateInputList(ac.requestedDYNs, std::move(copy));
             }
           }
           for (auto const& o : d.outputs) {
             if (DataSpecUtils::partialMatch(o, header::DataOrigin{"DYN"})) {
-              providedDYNs.emplace_back(o);
+              ac.providedDYNs.emplace_back(o);
             }
           }
         }
-        std::sort(requestedDYNs.begin(), requestedDYNs.end(), inputSpecLessThan);
-        std::sort(providedDYNs.begin(), providedDYNs.end(), outputSpecLessThan);
+        std::sort(ac.requestedDYNs.begin(), ac.requestedDYNs.end(), inputSpecLessThan);
+        std::sort(ac.providedDYNs.begin(), ac.providedDYNs.end(), outputSpecLessThan);
         std::vector<InputSpec> spawnerInputs;
-        for (auto& input : requestedDYNs) {
-          if (std::none_of(providedDYNs.begin(), providedDYNs.end(), [&input](auto const& x) { return DataSpecUtils::match(input, x); })) {
+        for (auto& input : ac.requestedDYNs) {
+          if (std::none_of(ac.providedDYNs.begin(), ac.providedDYNs.end(), [&input](auto const& x) { return DataSpecUtils::match(input, x); })) {
             spawnerInputs.emplace_back(input);
           }
         }
@@ -482,7 +484,7 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
         // replace AlgorithmSpec
         // FIXME: it should be made more generic, so it does not need replacement...
         spawner->algorithm = readers::AODReaderHelpers::aodSpawnerCallback(spawnerInputs);
-        AnalysisSupportHelpers::addMissingOutputsToSpawner({}, spawnerInputs, requestedAODs, *spawner);
+        AnalysisSupportHelpers::addMissingOutputsToSpawner({}, spawnerInputs, ac.requestedAODs, *spawner);
       }
 
       if (writer != workflow.end()) {
@@ -496,14 +498,14 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
           for (auto const& i : d.inputs) {
             if (DataSpecUtils::partialMatch(i, AODOrigins)) {
               auto copy = i;
-              DataSpecUtils::updateInputList(requestedAODs, std::move(copy));
+              DataSpecUtils::updateInputList(ac.requestedAODs, std::move(copy));
             }
           }
         }
 
         // remove unmatched outputs
         auto o_end = std::remove_if(reader->outputs.begin(), reader->outputs.end(), [&](OutputSpec const& o) {
-          return !DataSpecUtils::partialMatch(o, o2::header::DataDescription{"TFNumber"}) && !DataSpecUtils::partialMatch(o, o2::header::DataDescription{"TFFilename"}) && std::none_of(requestedAODs.begin(), requestedAODs.end(), [&](InputSpec const& i) { return DataSpecUtils::match(i, o); });
+          return !DataSpecUtils::partialMatch(o, o2::header::DataDescription{"TFNumber"}) && !DataSpecUtils::partialMatch(o, o2::header::DataDescription{"TFFilename"}) && std::none_of(ac.requestedAODs.begin(), ac.requestedAODs.end(), [&](InputSpec const& i) { return DataSpecUtils::match(i, o); });
         });
         reader->outputs.erase(o_end, reader->outputs.end());
         if (reader->outputs.empty()) {
@@ -521,22 +523,22 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
       // select outputs of type AOD which need to be saved
       // ATTENTION: if there are dangling outputs the getGlobalAODSink
       // has to be created in any case!
-      std::vector<InputSpec> outputsInputsAOD;
+      ac.outputsInputsAOD.clear();
 
       for (auto ii = 0u; ii < outputsInputs.size(); ii++) {
         if (DataSpecUtils::partialMatch(outputsInputs[ii], extendedAODOrigins)) {
           auto ds = dod->getDataOutputDescriptors(outputsInputs[ii]);
           if (!ds.empty() || isDangling[ii]) {
-            outputsInputsAOD.emplace_back(outputsInputs[ii]);
+            ac.outputsInputsAOD.emplace_back(outputsInputs[ii]);
           }
         }
       }
 
       // file sink for any AOD output
-      if (!outputsInputsAOD.empty()) {
+      if (!ac.outputsInputsAOD.empty()) {
         // add TFNumber and TFFilename as input to the writer
-        outputsInputsAOD.emplace_back("tfn", "TFN", "TFNumber");
-        outputsInputsAOD.emplace_back("tff", "TFF", "TFFilename");
+        ac.outputsInputsAOD.emplace_back("tfn", "TFN", "TFNumber");
+        ac.outputsInputsAOD.emplace_back("tff", "TFF", "TFFilename");
         workflow.push_back(AnalysisSupportHelpers::getGlobalAODSink(ctx));
       }
       // Move the dummy sink at the end, if needed

From dfd9e31bcdf750fece4c1ae3021ae88f4bb8b734 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 18 Nov 2024 10:45:25 +0100
Subject: [PATCH 704/726] DPL: initialise the initial chunk length

---
 Framework/Core/include/Framework/TableTreeHelpers.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/include/Framework/TableTreeHelpers.h b/Framework/Core/include/Framework/TableTreeHelpers.h
index 9dc7038b83fe1..c6a769e579fb7 100644
--- a/Framework/Core/include/Framework/TableTreeHelpers.h
+++ b/Framework/Core/include/Framework/TableTreeHelpers.h
@@ -105,7 +105,7 @@ class ColumnToBranch
   arrow::Type::type mFieldType;
   std::vector<uint8_t> cache;
   std::shared_ptr<arrow::Array> mCurrentArray = nullptr;
-  int64_t mChunkLength;
+  int64_t mChunkLength = 0;
   int mFieldSize = 0;
 };
 

From 42a6068bfe1913e9932e12bba54169f6ce507d8e Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 19 Nov 2024 14:25:39 +0100
Subject: [PATCH 705/726] DPL: fix adjustment of spawnerInputs

---
 Framework/Core/src/ArrowSupport.cxx | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Framework/Core/src/ArrowSupport.cxx b/Framework/Core/src/ArrowSupport.cxx
index e6f8fb90c7af9..78a2445d68920 100644
--- a/Framework/Core/src/ArrowSupport.cxx
+++ b/Framework/Core/src/ArrowSupport.cxx
@@ -472,10 +472,9 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
         }
         std::sort(ac.requestedDYNs.begin(), ac.requestedDYNs.end(), inputSpecLessThan);
         std::sort(ac.providedDYNs.begin(), ac.providedDYNs.end(), outputSpecLessThan);
-        std::vector<InputSpec> spawnerInputs;
         for (auto& input : ac.requestedDYNs) {
           if (std::none_of(ac.providedDYNs.begin(), ac.providedDYNs.end(), [&input](auto const& x) { return DataSpecUtils::match(input, x); })) {
-            spawnerInputs.emplace_back(input);
+            ac.spawnerInputs.emplace_back(input);
           }
         }
         // recreate inputs and outputs
@@ -483,8 +482,8 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec()
         spawner->inputs.clear();
         // replace AlgorithmSpec
         // FIXME: it should be made more generic, so it does not need replacement...
-        spawner->algorithm = readers::AODReaderHelpers::aodSpawnerCallback(spawnerInputs);
-        AnalysisSupportHelpers::addMissingOutputsToSpawner({}, spawnerInputs, ac.requestedAODs, *spawner);
+        spawner->algorithm = readers::AODReaderHelpers::aodSpawnerCallback(ac.spawnerInputs);
+        AnalysisSupportHelpers::addMissingOutputsToSpawner({}, ac.spawnerInputs, ac.requestedAODs, *spawner);
       }
 
       if (writer != workflow.end()) {

From 6c8020b4155da1558b322f5c51d2e449af61b7a5 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 20 Nov 2024 13:48:46 +0100
Subject: [PATCH 706/726] Fix a few --compression options

---
 Framework/AODMerger/src/aodMerger.cxx   | 2 +-
 Framework/AODMerger/src/aodStrainer.cxx | 2 +-
 Framework/AODMerger/src/aodThinner.cxx  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Framework/AODMerger/src/aodMerger.cxx b/Framework/AODMerger/src/aodMerger.cxx
index aa27d4f617b3b..f474cb0d37e43 100644
--- a/Framework/AODMerger/src/aodMerger.cxx
+++ b/Framework/AODMerger/src/aodMerger.cxx
@@ -48,7 +48,7 @@ int main(int argc, char* argv[])
     {"max-size", required_argument, nullptr, 2},
     {"skip-non-existing-files", no_argument, nullptr, 3},
     {"skip-parent-files-list", no_argument, nullptr, 4},
-    {"compression", no_argument, nullptr, 5},
+    {"compression", required_argument, nullptr, 5},
     {"verbosity", required_argument, nullptr, 'v'},
     {"help", no_argument, nullptr, 'h'},
     {nullptr, 0, nullptr, 0}};
diff --git a/Framework/AODMerger/src/aodStrainer.cxx b/Framework/AODMerger/src/aodStrainer.cxx
index 0ecb8d0d81659..fc54aa9c533cf 100644
--- a/Framework/AODMerger/src/aodStrainer.cxx
+++ b/Framework/AODMerger/src/aodStrainer.cxx
@@ -71,7 +71,7 @@ int main(int argc, char* argv[])
     } else if (c == 4) {
       downsampling = atof(optarg);
     } else if (c == 5) {
-      compression = atof(optarg);
+      compression = atoi(optarg);
     } else if (c == 'h') {
       printf("AO2D strainer tool. Options: \n");
       printf("  --input <%s>      Contains path to files to be merged. Default: %s\n", inputAO2D.c_str(), inputAO2D.c_str());
diff --git a/Framework/AODMerger/src/aodThinner.cxx b/Framework/AODMerger/src/aodThinner.cxx
index f9fb31f9ad08d..5da17032c80d4 100644
--- a/Framework/AODMerger/src/aodThinner.cxx
+++ b/Framework/AODMerger/src/aodThinner.cxx
@@ -51,7 +51,7 @@ int main(int argc, char* argv[])
     {"input", required_argument, nullptr, 'i'},
     {"output", required_argument, nullptr, 'o'},
     {"overwrite", no_argument, nullptr, 'O'},
-    {"compression", no_argument, nullptr, 'c'},
+    {"compression", required_argument, nullptr, 'c'},
     {"help", no_argument, nullptr, 'h'},
     {nullptr, 0, nullptr, 0}};
 

From 897cc11c39347c3a5dfd868bffb8e9f2bccd7939 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 20 Nov 2024 14:32:35 +0100
Subject: [PATCH 707/726] DPL: write support for TTree using arrow::Dataset API

---
 .../include/Framework/RootArrowFilesystem.h   |  32 ++
 Framework/Core/src/RootArrowFilesystem.cxx    | 302 +++++++++++++++++-
 Framework/Core/src/TableTreeHelpers.cxx       |   2 +-
 Framework/Core/test/test_Root2ArrowTable.cxx  | 108 +++++++
 4 files changed, 441 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/include/Framework/RootArrowFilesystem.h b/Framework/Core/include/Framework/RootArrowFilesystem.h
index df00ce4fa8a76..7c8385ccd2b9d 100644
--- a/Framework/Core/include/Framework/RootArrowFilesystem.h
+++ b/Framework/Core/include/Framework/RootArrowFilesystem.h
@@ -87,6 +87,11 @@ class TTreeFileSystem : public VirtualRootFileSystemBase
   {
     return std::dynamic_pointer_cast<VirtualRootFileSystemBase>(shared_from_this());
   };
+
+  arrow::Result<std::shared_ptr<arrow::io::OutputStream>> OpenOutputStream(
+    const std::string& path,
+    const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override;
+
   virtual TTree* GetTree(arrow::dataset::FileSource source) = 0;
 };
 
@@ -128,6 +133,10 @@ class TFileFileSystem : public VirtualRootFileSystemBase
 
   std::shared_ptr<VirtualRootFileSystemBase> GetSubFilesystem(arrow::dataset::FileSource source) override;
 
+  arrow::Result<std::shared_ptr<arrow::io::OutputStream>> OpenOutputStream(
+    const std::string& path,
+    const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override;
+
   // We can go back to the TFile in case this is needed.
   TDirectoryFile* GetFile()
   {
@@ -218,6 +227,29 @@ class TTreeFileFormat : public arrow::dataset::FileFormat
     const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const override;
 };
 
+// An arrow outputstream which allows to write to a ttree
+class TTreeOutputStream : public arrow::io::OutputStream
+{
+ public:
+  TTreeOutputStream(TTree* t);
+
+  arrow::Status Close() override;
+
+  arrow::Result<int64_t> Tell() const override;
+
+  arrow::Status Write(const void* data, int64_t nbytes) override;
+
+  bool closed() const override;
+
+  TTree* GetTree()
+  {
+    return mTree;
+  }
+
+ private:
+  TTree* mTree;
+};
+
 } // namespace o2::framework
 
 #endif // O2_FRAMEWORK_ROOT_ARROW_FILESYSTEM_H_
diff --git a/Framework/Core/src/RootArrowFilesystem.cxx b/Framework/Core/src/RootArrowFilesystem.cxx
index 46489141c3173..7581ee57e5b9f 100644
--- a/Framework/Core/src/RootArrowFilesystem.cxx
+++ b/Framework/Core/src/RootArrowFilesystem.cxx
@@ -11,6 +11,7 @@
 #include "Framework/RootArrowFilesystem.h"
 #include "Framework/Endian.h"
 #include "Framework/RuntimeError.h"
+#include "Framework/Signpost.h"
 #include <Rtypes.h>
 #include <arrow/array/array_primitive.h>
 #include <arrow/array/builder_nested.h>
@@ -24,6 +25,13 @@
 #include <TDirectoryFile.h>
 #include <arrow/type.h>
 #include <arrow/type_fwd.h>
+#include <arrow/dataset/file_base.h>
+#include <arrow/result.h>
+#include <arrow/status.h>
+#include <fmt/format.h>
+
+
+O2_DECLARE_DYNAMIC_LOG(root_arrow_fs);
 
 namespace
 {
@@ -76,6 +84,7 @@ auto arrowTypeFromROOT(EDataType type, int size)
 }
 namespace o2::framework
 {
+using arrow::Status;
 
 TFileFileSystem::TFileFileSystem(TDirectoryFile* f, size_t readahead)
   : VirtualRootFileSystemBase(),
@@ -116,6 +125,15 @@ arrow::Result<arrow::fs::FileInfo> TFileFileSystem::GetFileInfo(const std::strin
   return result;
 }
 
+arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TFileFileSystem::OpenOutputStream(
+  const std::string& path,
+  const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
+{
+  auto* t = new TTree(path.c_str(), "should put a name here");
+  auto stream = std::make_shared<TTreeOutputStream>(t);
+  return stream;
+}
+
 arrow::Result<arrow::fs::FileInfo> VirtualRootFileSystemBase::GetFileInfo(std::string const&)
 {
   arrow::fs::FileInfo result;
@@ -267,9 +285,279 @@ arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> TTreeFileFormat::Ma
   return std::dynamic_pointer_cast<arrow::dataset::FileFragment>(fragment);
 }
 
+// An arrow outputstream which allows to write to a ttree
+TTreeOutputStream::TTreeOutputStream(TTree* t)
+  : mTree(t)
+{
+}
+
+arrow::Status TTreeOutputStream::Close()
+{
+  mTree->GetCurrentFile()->Close();
+  return arrow::Status::OK();
+}
+
+arrow::Result<int64_t> TTreeOutputStream::Tell() const
+{
+  return arrow::Result<int64_t>(arrow::Status::NotImplemented("Cannot move"));
+}
+
+arrow::Status TTreeOutputStream::Write(const void* data, int64_t nbytes)
+{
+  return arrow::Status::NotImplemented("Cannot write raw bytes to a TTree");
+}
+
+bool TTreeOutputStream::closed() const
+{
+  return mTree->GetCurrentFile()->IsOpen() == false;
+}
+
+char const* rootSuffixFromArrow(arrow::Type::type id)
+{
+  switch (id) {
+    case arrow::Type::BOOL:
+      return "/O";
+    case arrow::Type::UINT8:
+      return "/b";
+    case arrow::Type::UINT16:
+      return "/s";
+    case arrow::Type::UINT32:
+      return "/i";
+    case arrow::Type::UINT64:
+      return "/l";
+    case arrow::Type::INT8:
+      return "/B";
+    case arrow::Type::INT16:
+      return "/S";
+    case arrow::Type::INT32:
+      return "/I";
+    case arrow::Type::INT64:
+      return "/L";
+    case arrow::Type::FLOAT:
+      return "/F";
+    case arrow::Type::DOUBLE:
+      return "/D";
+    default:
+      throw runtime_error("Unsupported arrow column type");
+  }
+}
+
+class TTreeFileWriter : public arrow::dataset::FileWriter
+{
+  std::vector<TBranch*> branches;
+  std::vector<TBranch*> sizesBranches;
+  std::vector<std::shared_ptr<arrow::Array>> valueArrays;
+  std::vector<std::shared_ptr<arrow::Array>> sizeArrays;
+  std::vector<std::shared_ptr<arrow::DataType>> valueTypes;
+
+  std::vector<int64_t> valuesIdealBasketSize;
+  std::vector<int64_t> sizeIdealBasketSize;
+
+  std::vector<int64_t> typeSizes;
+  std::vector<int64_t> listSizes;
+  bool firstBasket = true;
+
+  // This is to create a batsket size according to the first batch.
+  void finaliseBasketSize(std::shared_ptr<arrow::RecordBatch> firstBatch)
+  {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, root_arrow_fs, this);
+    O2_SIGNPOST_START(root_arrow_fs, sid, "finaliseBasketSize", "First batch with %lli rows received and %zu columns",
+                      firstBatch->num_rows(), firstBatch->columns().size());
+    for (size_t i = 0; i < branches.size(); i++) {
+      auto* branch = branches[i];
+      auto* sizeBranch = sizesBranches[i];
+
+      int valueSize = valueTypes[i]->byte_width();
+      if (listSizes[i] == 1) {
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s exists and uses %d bytes per entry for %lli entries.",
+                               branch->GetName(), valueSize, firstBatch->num_rows());
+        assert(sizeBranch == nullptr);
+        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize);
+      } else if (listSizes[i] == -1) {
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s exists and uses %d bytes per entry.",
+                               branch->GetName(), valueSize);
+        // This should probably lookup the
+        auto column = firstBatch->GetColumnByName(branch->GetName());
+        auto list = std::static_pointer_cast<arrow::ListArray>(column);
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s needed. Associated size branch %s and there are %lli entries of size %d in that list.",
+                               branch->GetName(), sizeBranch->GetName(), list->length(), valueSize);
+        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize * list->length());
+        sizeBranch->SetBasketSize(1024 + firstBatch->num_rows() * 4);
+      } else {
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s needed. There are %lli entries per array of size %d in that list.",
+                               branch->GetName(), listSizes[i], valueSize);
+        assert(sizeBranch == nullptr);
+        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize * listSizes[i]);
+      }
+
+      auto field = firstBatch->schema()->field(i);
+      if (field->name().starts_with("fIndexArray")) {
+        // One int per array to keep track of the size
+        int idealBasketSize = 4 * firstBatch->num_rows() + 1024 + field->type()->byte_width() * firstBatch->num_rows(); // minimal additional size needed, otherwise we get 2 baskets
+        int basketSize = std::max(32000, idealBasketSize);                                                              // keep a minimum value
+        sizeBranch->SetBasketSize(basketSize);
+        branch->SetBasketSize(basketSize);
+      }
+    }
+    O2_SIGNPOST_END(root_arrow_fs, sid, "finaliseBasketSize", "Done");
+  }
+
+ public:
+  // Create the TTree based on the physical_schema, not the one in the batch.
+  // The write method will have to reconcile the two schemas.
+  TTreeFileWriter(std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options,
+                  std::shared_ptr<arrow::io::OutputStream> destination,
+                  arrow::fs::FileLocator destination_locator)
+    : FileWriter(schema, options, destination, destination_locator)
+  {
+    // Batches have the same number of entries for each column.
+    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
+    TTree* tree = treeStream->GetTree();
+
+    for (auto i = 0u; i < schema->fields().size(); ++i) {
+      auto& field = schema->field(i);
+      listSizes.push_back(1);
+
+      int valuesIdealBasketSize = 0;
+      // Construct all the needed branches.
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          listSizes.back() = std::static_pointer_cast<arrow::FixedSizeListType>(field->type())->list_size();
+          valuesIdealBasketSize = 1024 + valueTypes.back()->byte_width() * listSizes.back();
+          valueTypes.push_back(field->type()->field(0)->type());
+          sizesBranches.push_back(nullptr);
+          std::string leafList = fmt::format("{}[{}]{}", field->name(), listSizes.back(), rootSuffixFromArrow(valueTypes.back()->id()));
+          branches.push_back(tree->Branch(field->name().c_str(), (char*)nullptr, leafList.c_str()));
+        } break;
+        case arrow::Type::LIST: {
+          valueTypes.push_back(field->type()->field(0)->type());
+          listSizes.back() = 0; // VLA, we need to calculate it on the fly;
+          std::string leafList = fmt::format("{}[{}_size]{}", field->name(), field->name(), rootSuffixFromArrow(valueTypes.back()->id()));
+          std::string sizeLeafList = field->name() + "_size/I";
+          sizesBranches.push_back(tree->Branch((field->name() + "_size").c_str(), (char*)nullptr, sizeLeafList.c_str()));
+          branches.push_back(tree->Branch(field->name().c_str(), (char*)nullptr, leafList.c_str()));
+          // Notice that this could be replaced by a better guess of the
+          // average size of the list elements, but this is not trivial.
+        } break;
+        default: {
+          valueTypes.push_back(field->type());
+          std::string leafList = field->name() + rootSuffixFromArrow(valueTypes.back()->id());
+          sizesBranches.push_back(nullptr);
+          branches.push_back(tree->Branch(field->name().c_str(), (char*)nullptr, leafList.c_str()));
+        } break;
+      }
+    }
+    // We create the branches from the schema
+  }
+
+  arrow::Status Write(const std::shared_ptr<arrow::RecordBatch>& batch) override
+  {
+    if (firstBasket) {
+      firstBasket = false;
+      finaliseBasketSize(batch);
+    }
+
+    // Support writing empty tables
+    if (batch->columns().empty() || batch->num_rows() == 0) {
+      return arrow::Status::OK();
+    }
+
+    // Batches have the same number of entries for each column.
+    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
+    TTree* tree = treeStream->GetTree();
+
+    // Caches for the vectors of bools.
+    std::vector<std::shared_ptr<arrow::UInt8Array>> caches;
+
+    for (auto i = 0u; i < batch->columns().size(); ++i) {
+      auto column = batch->column(i);
+      auto& field = batch->schema()->field(i);
+
+      valueArrays.push_back(nullptr);
+
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          auto list = std::static_pointer_cast<arrow::FixedSizeListArray>(column);
+          valueArrays.back() = list->values();
+        } break;
+        case arrow::Type::LIST: {
+          auto list = std::static_pointer_cast<arrow::ListArray>(column);
+          valueArrays.back() = list;
+        } break;
+        default:
+          valueArrays.back() = column;
+      }
+    }
+
+    int64_t pos = 0;
+    while (pos < batch->num_rows()) {
+      for (size_t bi = 0; bi < branches.size(); ++bi) {
+        auto* branch = branches[bi];
+        auto* sizeBranch = sizesBranches[bi];
+        auto array = batch->column(bi);
+        auto& field = batch->schema()->field(bi);
+        auto& listSize = listSizes[bi];
+        auto valueType = valueTypes[bi];
+        auto valueArray = valueArrays[bi];
+
+        if (field->type()->id() == arrow::Type::BOOL) {
+          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(array);
+          int64_t length = boolArray->length();
+          arrow::UInt8Builder builder;
+          auto ok = builder.Reserve(length);
+
+          for (int64_t i = 0; i < length; ++i) {
+            if (boolArray->IsValid(i)) {
+              // Expand each boolean value (true/false) to uint8 (1/0)
+              uint8_t value = boolArray->Value(i) ? 1 : 0;
+              auto ok = builder.Append(value);
+            } else {
+              // Append null for invalid entries
+              auto ok = builder.AppendNull();
+            }
+          }
+
+          ok = builder.Finish(&caches[bi]);
+          branch->SetAddress((void*)(caches[bi]->values()->data()));
+          continue;
+        }
+        switch (field->type()->id()) {
+          case arrow::Type::LIST: {
+            auto list = std::static_pointer_cast<arrow::ListArray>(array);
+            listSize = list->value_length(pos);
+            uint8_t const* buffer = std::static_pointer_cast<arrow::PrimitiveArray>(valueArray)->values()->data() + array->offset() + list->value_offset(pos) * valueType->byte_width();
+            branch->SetAddress((void*)buffer);
+            sizeBranch->SetAddress(&listSize);
+          };
+            break;
+          case arrow::Type::FIXED_SIZE_LIST:
+          default: {
+            uint8_t const* buffer = std::static_pointer_cast<arrow::PrimitiveArray>(valueArray)->values()->data() + array->offset() + pos * listSize * valueType->byte_width();
+            branch->SetAddress((void*)buffer);
+          };
+        }
+      }
+      tree->Fill();
+      ++pos;
+    }
+    return arrow::Status::OK();
+  }
+
+  arrow::Future<> FinishInternal() override
+  {
+    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
+    TTree* tree = treeStream->GetTree();
+    tree->Write("", TObject::kOverwrite);
+    tree->SetDirectory(nullptr);
+
+    return {};
+  };
+};
+
 arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> TTreeFileFormat::MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options, arrow::fs::FileLocator destination_locator) const
 {
-  throw std::runtime_error("Unsupported operation");
+  auto writer = std::make_shared<TTreeFileWriter>(schema, options, destination, destination_locator);
+  return std::dynamic_pointer_cast<arrow::dataset::FileWriter>(writer);
 }
 
 std::shared_ptr<arrow::dataset::FileWriteOptions> TTreeFileFormat::DefaultWriteOptions()
@@ -401,8 +689,10 @@ arrow::Result<arrow::RecordBatchGenerator> TTreeFileFormat::ScanBatchesAsync(
         int64_t listSize = 1;
         if (auto fixedSizeList = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type())) {
           listSize = fixedSizeList->list_size();
+          typeSize = fixedSizeList->field(0)->type()->byte_width();
         } else if (auto vlaListType = std::dynamic_pointer_cast<arrow::ListType>(physicalField->type())) {
           listSize = -1;
+          typeSize = fixedSizeList->field(0)->type()->byte_width();
         }
         if (listSize == -1) {
           mSizeBranch = branch->GetTree()->GetBranch((std::string{branch->GetName()} + "_size").c_str());
@@ -474,6 +764,15 @@ arrow::Result<arrow::RecordBatchGenerator> TTreeFileFormat::ScanBatchesAsync(
   return generator;
 }
 
+
+arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenOutputStream(
+  const std::string& path,
+  const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
+{
+  auto stream = std::make_shared<TTreeOutputStream>(GetTree({path, shared_from_this()}));
+  return stream;
+}
+
 TBufferFileFS::TBufferFileFS(TBufferFile* f)
   : VirtualRootFileSystemBase(),
     mBuffer(f),
@@ -512,5 +811,4 @@ std::shared_ptr<VirtualRootFileSystemBase> TBufferFileFS::GetSubFilesystem(arrow
   }
   return mFilesystem;
 }
-
 } // namespace o2::framework
diff --git a/Framework/Core/src/TableTreeHelpers.cxx b/Framework/Core/src/TableTreeHelpers.cxx
index c20febaac517d..d0fdd0ced5779 100644
--- a/Framework/Core/src/TableTreeHelpers.cxx
+++ b/Framework/Core/src/TableTreeHelpers.cxx
@@ -512,7 +512,7 @@ void TreeToTable::addAllColumns(TTree* tree, std::vector<std::string>&& names)
     if (strncmp(reader->branch()->GetName(), "fIndexArray", strlen("fIndexArray")) == 0) {
       std::string sizeBranchName = reader->branch()->GetName();
       sizeBranchName += "_size";
-      TBranch* sizeBranch = (TBranch*)tree->GetBranch(sizeBranchName.c_str());
+      auto* sizeBranch = (TBranch*)tree->GetBranch(sizeBranchName.c_str());
       if (sizeBranch) {
         tree->AddBranchToCache(sizeBranch);
       }
diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx
index 599f1062c63a0..d7c1cc63728ec 100644
--- a/Framework/Core/test/test_Root2ArrowTable.cxx
+++ b/Framework/Core/test/test_Root2ArrowTable.cxx
@@ -358,4 +358,112 @@ TEST_CASE("RootTree2Dataset")
   REQUIRE(result.ok());
   REQUIRE((*result)->columns().size() == 7);
   REQUIRE((*result)->num_rows() == 100);
+
+  {
+    auto int_array = std::static_pointer_cast<arrow::Int32Array>((*result)->GetColumnByName("ev"));
+    for (int64_t j = 0; j < int_array->length(); j++) {
+      REQUIRE(int_array->Value(j) == j + 1);
+    }
+  }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>((*result)->GetColumnByName("xyz"));
+
+    // Iterate over the FixedSizeListArray
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      auto float_array = std::static_pointer_cast<arrow::FloatArray>(value_slice);
+
+      REQUIRE(float_array->Value(0) == 1);
+      REQUIRE(float_array->Value(1) == 2);
+      REQUIRE(float_array->Value(2) == i + 1);
+    }
+  }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>((*result)->GetColumnByName("ij"));
+
+    // Iterate over the FixedSizeListArray
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      auto int_array = std::static_pointer_cast<arrow::Int32Array>(value_slice);
+      REQUIRE(int_array->Value(0) == i);
+      REQUIRE(int_array->Value(1) == i + 1);
+    }
+  }
+
+  auto* output = new TMemFile("foo", "RECREATE");
+  auto outFs = std::make_shared<TFileFileSystem>(output, 0);
+  auto locator = arrow::fs::FileLocator(outFs, "/DF_3");
+
+  auto destination = outFs->OpenOutputStream(locator.path, {});
+  REQUIRE(destination.ok());
+
+  auto writer = format->MakeWriter(*destination, schema, {}, locator);
+  auto success = writer->get()->Write(*result);
+  auto rootDestination = std::dynamic_pointer_cast<TTreeOutputStream>(*destination);
+
+  REQUIRE(success.ok());
+  // Let's read it back...
+  arrow::dataset::FileSource source2("/DF_3", outFs);
+  auto newTreeFS = outFs->GetSubFilesystem(source2);
+
+  REQUIRE(format->IsSupported(source) == true);
+
+  auto schemaOptWritten = format->Inspect(source);
+  REQUIRE(schemaOptWritten.ok());
+  auto schemaWritten = *schemaOptWritten;
+  REQUIRE(schemaWritten->num_fields() == 7);
+  REQUIRE(schemaWritten->field(0)->type()->id() == arrow::float32()->id());
+  REQUIRE(schemaWritten->field(1)->type()->id() == arrow::float32()->id());
+  REQUIRE(schemaWritten->field(2)->type()->id() == arrow::float32()->id());
+  REQUIRE(schemaWritten->field(3)->type()->id() == arrow::float64()->id());
+  REQUIRE(schemaWritten->field(4)->type()->id() == arrow::int32()->id());
+  REQUIRE(schemaWritten->field(5)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id());
+  REQUIRE(schemaWritten->field(6)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id());
+
+  auto fragmentWritten = format->MakeFragment(source, {}, schema);
+  REQUIRE(fragmentWritten.ok());
+  auto optionsWritten = std::make_shared<arrow::dataset::ScanOptions>();
+  options->dataset_schema = schemaWritten;
+  auto scannerWritten = format->ScanBatchesAsync(optionsWritten, *fragment);
+  REQUIRE(scannerWritten.ok());
+  auto batchesWritten = (*scanner)();
+  auto resultWritten = batches.result();
+  REQUIRE(resultWritten.ok());
+  REQUIRE((*resultWritten)->columns().size() == 7);
+  REQUIRE((*resultWritten)->num_rows() == 100);
+
+  {
+    auto int_array = std::static_pointer_cast<arrow::Int32Array>((*resultWritten)->GetColumnByName("ev"));
+    for (int64_t j = 0; j < int_array->length(); j++) {
+      REQUIRE(int_array->Value(j) == j + 1);
+    }
+  }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>((*result)->GetColumnByName("xyz"));
+
+    // Iterate over the FixedSizeListArray
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      auto float_array = std::static_pointer_cast<arrow::FloatArray>(value_slice);
+
+      REQUIRE(float_array->Value(0) == 1);
+      REQUIRE(float_array->Value(1) == 2);
+      REQUIRE(float_array->Value(2) == i + 1);
+    }
+  }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>((*result)->GetColumnByName("ij"));
+
+    // Iterate over the FixedSizeListArray
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      auto int_array = std::static_pointer_cast<arrow::Int32Array>(value_slice);
+      REQUIRE(int_array->Value(0) == i);
+      REQUIRE(int_array->Value(1) == i + 1);
+    }
+  }
 }

From 7af865aa8570c59440dcf84de04aa11f53d9aa67 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 20 Nov 2024 17:01:30 +0100
Subject: [PATCH 708/726] DPL: write support for TTree using arrow::Dataset API

---
 .../include/Framework/RootArrowFilesystem.h   |  32 ++
 Framework/Core/src/RootArrowFilesystem.cxx    | 302 +++++++++++++++++-
 Framework/Core/src/TableTreeHelpers.cxx       |   2 +-
 Framework/Core/test/test_Root2ArrowTable.cxx  | 108 +++++++
 4 files changed, 441 insertions(+), 3 deletions(-)

diff --git a/Framework/Core/include/Framework/RootArrowFilesystem.h b/Framework/Core/include/Framework/RootArrowFilesystem.h
index df00ce4fa8a76..7c8385ccd2b9d 100644
--- a/Framework/Core/include/Framework/RootArrowFilesystem.h
+++ b/Framework/Core/include/Framework/RootArrowFilesystem.h
@@ -87,6 +87,11 @@ class TTreeFileSystem : public VirtualRootFileSystemBase
   {
     return std::dynamic_pointer_cast<VirtualRootFileSystemBase>(shared_from_this());
   };
+
+  arrow::Result<std::shared_ptr<arrow::io::OutputStream>> OpenOutputStream(
+    const std::string& path,
+    const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override;
+
   virtual TTree* GetTree(arrow::dataset::FileSource source) = 0;
 };
 
@@ -128,6 +133,10 @@ class TFileFileSystem : public VirtualRootFileSystemBase
 
   std::shared_ptr<VirtualRootFileSystemBase> GetSubFilesystem(arrow::dataset::FileSource source) override;
 
+  arrow::Result<std::shared_ptr<arrow::io::OutputStream>> OpenOutputStream(
+    const std::string& path,
+    const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override;
+
   // We can go back to the TFile in case this is needed.
   TDirectoryFile* GetFile()
   {
@@ -218,6 +227,29 @@ class TTreeFileFormat : public arrow::dataset::FileFormat
     const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const override;
 };
 
+// An arrow outputstream which allows to write to a ttree
+class TTreeOutputStream : public arrow::io::OutputStream
+{
+ public:
+  TTreeOutputStream(TTree* t);
+
+  arrow::Status Close() override;
+
+  arrow::Result<int64_t> Tell() const override;
+
+  arrow::Status Write(const void* data, int64_t nbytes) override;
+
+  bool closed() const override;
+
+  TTree* GetTree()
+  {
+    return mTree;
+  }
+
+ private:
+  TTree* mTree;
+};
+
 } // namespace o2::framework
 
 #endif // O2_FRAMEWORK_ROOT_ARROW_FILESYSTEM_H_
diff --git a/Framework/Core/src/RootArrowFilesystem.cxx b/Framework/Core/src/RootArrowFilesystem.cxx
index 46489141c3173..7581ee57e5b9f 100644
--- a/Framework/Core/src/RootArrowFilesystem.cxx
+++ b/Framework/Core/src/RootArrowFilesystem.cxx
@@ -11,6 +11,7 @@
 #include "Framework/RootArrowFilesystem.h"
 #include "Framework/Endian.h"
 #include "Framework/RuntimeError.h"
+#include "Framework/Signpost.h"
 #include <Rtypes.h>
 #include <arrow/array/array_primitive.h>
 #include <arrow/array/builder_nested.h>
@@ -24,6 +25,13 @@
 #include <TDirectoryFile.h>
 #include <arrow/type.h>
 #include <arrow/type_fwd.h>
+#include <arrow/dataset/file_base.h>
+#include <arrow/result.h>
+#include <arrow/status.h>
+#include <fmt/format.h>
+
+
+O2_DECLARE_DYNAMIC_LOG(root_arrow_fs);
 
 namespace
 {
@@ -76,6 +84,7 @@ auto arrowTypeFromROOT(EDataType type, int size)
 }
 namespace o2::framework
 {
+using arrow::Status;
 
 TFileFileSystem::TFileFileSystem(TDirectoryFile* f, size_t readahead)
   : VirtualRootFileSystemBase(),
@@ -116,6 +125,15 @@ arrow::Result<arrow::fs::FileInfo> TFileFileSystem::GetFileInfo(const std::strin
   return result;
 }
 
+arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TFileFileSystem::OpenOutputStream(
+  const std::string& path,
+  const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
+{
+  auto* t = new TTree(path.c_str(), "should put a name here");
+  auto stream = std::make_shared<TTreeOutputStream>(t);
+  return stream;
+}
+
 arrow::Result<arrow::fs::FileInfo> VirtualRootFileSystemBase::GetFileInfo(std::string const&)
 {
   arrow::fs::FileInfo result;
@@ -267,9 +285,279 @@ arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> TTreeFileFormat::Ma
   return std::dynamic_pointer_cast<arrow::dataset::FileFragment>(fragment);
 }
 
+// An arrow outputstream which allows to write to a ttree
+TTreeOutputStream::TTreeOutputStream(TTree* t)
+  : mTree(t)
+{
+}
+
+arrow::Status TTreeOutputStream::Close()
+{
+  mTree->GetCurrentFile()->Close();
+  return arrow::Status::OK();
+}
+
+arrow::Result<int64_t> TTreeOutputStream::Tell() const
+{
+  return arrow::Result<int64_t>(arrow::Status::NotImplemented("Cannot move"));
+}
+
+arrow::Status TTreeOutputStream::Write(const void* data, int64_t nbytes)
+{
+  return arrow::Status::NotImplemented("Cannot write raw bytes to a TTree");
+}
+
+bool TTreeOutputStream::closed() const
+{
+  return mTree->GetCurrentFile()->IsOpen() == false;
+}
+
+char const* rootSuffixFromArrow(arrow::Type::type id)
+{
+  switch (id) {
+    case arrow::Type::BOOL:
+      return "/O";
+    case arrow::Type::UINT8:
+      return "/b";
+    case arrow::Type::UINT16:
+      return "/s";
+    case arrow::Type::UINT32:
+      return "/i";
+    case arrow::Type::UINT64:
+      return "/l";
+    case arrow::Type::INT8:
+      return "/B";
+    case arrow::Type::INT16:
+      return "/S";
+    case arrow::Type::INT32:
+      return "/I";
+    case arrow::Type::INT64:
+      return "/L";
+    case arrow::Type::FLOAT:
+      return "/F";
+    case arrow::Type::DOUBLE:
+      return "/D";
+    default:
+      throw runtime_error("Unsupported arrow column type");
+  }
+}
+
+class TTreeFileWriter : public arrow::dataset::FileWriter
+{
+  std::vector<TBranch*> branches;
+  std::vector<TBranch*> sizesBranches;
+  std::vector<std::shared_ptr<arrow::Array>> valueArrays;
+  std::vector<std::shared_ptr<arrow::Array>> sizeArrays;
+  std::vector<std::shared_ptr<arrow::DataType>> valueTypes;
+
+  std::vector<int64_t> valuesIdealBasketSize;
+  std::vector<int64_t> sizeIdealBasketSize;
+
+  std::vector<int64_t> typeSizes;
+  std::vector<int64_t> listSizes;
+  bool firstBasket = true;
+
+  // This is to create a batsket size according to the first batch.
+  void finaliseBasketSize(std::shared_ptr<arrow::RecordBatch> firstBatch)
+  {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, root_arrow_fs, this);
+    O2_SIGNPOST_START(root_arrow_fs, sid, "finaliseBasketSize", "First batch with %lli rows received and %zu columns",
+                      firstBatch->num_rows(), firstBatch->columns().size());
+    for (size_t i = 0; i < branches.size(); i++) {
+      auto* branch = branches[i];
+      auto* sizeBranch = sizesBranches[i];
+
+      int valueSize = valueTypes[i]->byte_width();
+      if (listSizes[i] == 1) {
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s exists and uses %d bytes per entry for %lli entries.",
+                               branch->GetName(), valueSize, firstBatch->num_rows());
+        assert(sizeBranch == nullptr);
+        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize);
+      } else if (listSizes[i] == -1) {
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s exists and uses %d bytes per entry.",
+                               branch->GetName(), valueSize);
+        // This should probably lookup the
+        auto column = firstBatch->GetColumnByName(branch->GetName());
+        auto list = std::static_pointer_cast<arrow::ListArray>(column);
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s needed. Associated size branch %s and there are %lli entries of size %d in that list.",
+                               branch->GetName(), sizeBranch->GetName(), list->length(), valueSize);
+        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize * list->length());
+        sizeBranch->SetBasketSize(1024 + firstBatch->num_rows() * 4);
+      } else {
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s needed. There are %lli entries per array of size %d in that list.",
+                               branch->GetName(), listSizes[i], valueSize);
+        assert(sizeBranch == nullptr);
+        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize * listSizes[i]);
+      }
+
+      auto field = firstBatch->schema()->field(i);
+      if (field->name().starts_with("fIndexArray")) {
+        // One int per array to keep track of the size
+        int idealBasketSize = 4 * firstBatch->num_rows() + 1024 + field->type()->byte_width() * firstBatch->num_rows(); // minimal additional size needed, otherwise we get 2 baskets
+        int basketSize = std::max(32000, idealBasketSize);                                                              // keep a minimum value
+        sizeBranch->SetBasketSize(basketSize);
+        branch->SetBasketSize(basketSize);
+      }
+    }
+    O2_SIGNPOST_END(root_arrow_fs, sid, "finaliseBasketSize", "Done");
+  }
+
+ public:
+  // Create the TTree based on the physical_schema, not the one in the batch.
+  // The write method will have to reconcile the two schemas.
+  TTreeFileWriter(std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options,
+                  std::shared_ptr<arrow::io::OutputStream> destination,
+                  arrow::fs::FileLocator destination_locator)
+    : FileWriter(schema, options, destination, destination_locator)
+  {
+    // Batches have the same number of entries for each column.
+    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
+    TTree* tree = treeStream->GetTree();
+
+    for (auto i = 0u; i < schema->fields().size(); ++i) {
+      auto& field = schema->field(i);
+      listSizes.push_back(1);
+
+      int valuesIdealBasketSize = 0;
+      // Construct all the needed branches.
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          listSizes.back() = std::static_pointer_cast<arrow::FixedSizeListType>(field->type())->list_size();
+          valuesIdealBasketSize = 1024 + valueTypes.back()->byte_width() * listSizes.back();
+          valueTypes.push_back(field->type()->field(0)->type());
+          sizesBranches.push_back(nullptr);
+          std::string leafList = fmt::format("{}[{}]{}", field->name(), listSizes.back(), rootSuffixFromArrow(valueTypes.back()->id()));
+          branches.push_back(tree->Branch(field->name().c_str(), (char*)nullptr, leafList.c_str()));
+        } break;
+        case arrow::Type::LIST: {
+          valueTypes.push_back(field->type()->field(0)->type());
+          listSizes.back() = 0; // VLA, we need to calculate it on the fly;
+          std::string leafList = fmt::format("{}[{}_size]{}", field->name(), field->name(), rootSuffixFromArrow(valueTypes.back()->id()));
+          std::string sizeLeafList = field->name() + "_size/I";
+          sizesBranches.push_back(tree->Branch((field->name() + "_size").c_str(), (char*)nullptr, sizeLeafList.c_str()));
+          branches.push_back(tree->Branch(field->name().c_str(), (char*)nullptr, leafList.c_str()));
+          // Notice that this could be replaced by a better guess of the
+          // average size of the list elements, but this is not trivial.
+        } break;
+        default: {
+          valueTypes.push_back(field->type());
+          std::string leafList = field->name() + rootSuffixFromArrow(valueTypes.back()->id());
+          sizesBranches.push_back(nullptr);
+          branches.push_back(tree->Branch(field->name().c_str(), (char*)nullptr, leafList.c_str()));
+        } break;
+      }
+    }
+    // We create the branches from the schema
+  }
+
+  arrow::Status Write(const std::shared_ptr<arrow::RecordBatch>& batch) override
+  {
+    if (firstBasket) {
+      firstBasket = false;
+      finaliseBasketSize(batch);
+    }
+
+    // Support writing empty tables
+    if (batch->columns().empty() || batch->num_rows() == 0) {
+      return arrow::Status::OK();
+    }
+
+    // Batches have the same number of entries for each column.
+    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
+    TTree* tree = treeStream->GetTree();
+
+    // Caches for the vectors of bools.
+    std::vector<std::shared_ptr<arrow::UInt8Array>> caches;
+
+    for (auto i = 0u; i < batch->columns().size(); ++i) {
+      auto column = batch->column(i);
+      auto& field = batch->schema()->field(i);
+
+      valueArrays.push_back(nullptr);
+
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          auto list = std::static_pointer_cast<arrow::FixedSizeListArray>(column);
+          valueArrays.back() = list->values();
+        } break;
+        case arrow::Type::LIST: {
+          auto list = std::static_pointer_cast<arrow::ListArray>(column);
+          valueArrays.back() = list;
+        } break;
+        default:
+          valueArrays.back() = column;
+      }
+    }
+
+    int64_t pos = 0;
+    while (pos < batch->num_rows()) {
+      for (size_t bi = 0; bi < branches.size(); ++bi) {
+        auto* branch = branches[bi];
+        auto* sizeBranch = sizesBranches[bi];
+        auto array = batch->column(bi);
+        auto& field = batch->schema()->field(bi);
+        auto& listSize = listSizes[bi];
+        auto valueType = valueTypes[bi];
+        auto valueArray = valueArrays[bi];
+
+        if (field->type()->id() == arrow::Type::BOOL) {
+          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(array);
+          int64_t length = boolArray->length();
+          arrow::UInt8Builder builder;
+          auto ok = builder.Reserve(length);
+
+          for (int64_t i = 0; i < length; ++i) {
+            if (boolArray->IsValid(i)) {
+              // Expand each boolean value (true/false) to uint8 (1/0)
+              uint8_t value = boolArray->Value(i) ? 1 : 0;
+              auto ok = builder.Append(value);
+            } else {
+              // Append null for invalid entries
+              auto ok = builder.AppendNull();
+            }
+          }
+
+          ok = builder.Finish(&caches[bi]);
+          branch->SetAddress((void*)(caches[bi]->values()->data()));
+          continue;
+        }
+        switch (field->type()->id()) {
+          case arrow::Type::LIST: {
+            auto list = std::static_pointer_cast<arrow::ListArray>(array);
+            listSize = list->value_length(pos);
+            uint8_t const* buffer = std::static_pointer_cast<arrow::PrimitiveArray>(valueArray)->values()->data() + array->offset() + list->value_offset(pos) * valueType->byte_width();
+            branch->SetAddress((void*)buffer);
+            sizeBranch->SetAddress(&listSize);
+          };
+            break;
+          case arrow::Type::FIXED_SIZE_LIST:
+          default: {
+            uint8_t const* buffer = std::static_pointer_cast<arrow::PrimitiveArray>(valueArray)->values()->data() + array->offset() + pos * listSize * valueType->byte_width();
+            branch->SetAddress((void*)buffer);
+          };
+        }
+      }
+      tree->Fill();
+      ++pos;
+    }
+    return arrow::Status::OK();
+  }
+
+  arrow::Future<> FinishInternal() override
+  {
+    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
+    TTree* tree = treeStream->GetTree();
+    tree->Write("", TObject::kOverwrite);
+    tree->SetDirectory(nullptr);
+
+    return {};
+  };
+};
+
 arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> TTreeFileFormat::MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options, arrow::fs::FileLocator destination_locator) const
 {
-  throw std::runtime_error("Unsupported operation");
+  auto writer = std::make_shared<TTreeFileWriter>(schema, options, destination, destination_locator);
+  return std::dynamic_pointer_cast<arrow::dataset::FileWriter>(writer);
 }
 
 std::shared_ptr<arrow::dataset::FileWriteOptions> TTreeFileFormat::DefaultWriteOptions()
@@ -401,8 +689,10 @@ arrow::Result<arrow::RecordBatchGenerator> TTreeFileFormat::ScanBatchesAsync(
         int64_t listSize = 1;
         if (auto fixedSizeList = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type())) {
           listSize = fixedSizeList->list_size();
+          typeSize = fixedSizeList->field(0)->type()->byte_width();
         } else if (auto vlaListType = std::dynamic_pointer_cast<arrow::ListType>(physicalField->type())) {
           listSize = -1;
+          typeSize = fixedSizeList->field(0)->type()->byte_width();
         }
         if (listSize == -1) {
           mSizeBranch = branch->GetTree()->GetBranch((std::string{branch->GetName()} + "_size").c_str());
@@ -474,6 +764,15 @@ arrow::Result<arrow::RecordBatchGenerator> TTreeFileFormat::ScanBatchesAsync(
   return generator;
 }
 
+
+arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenOutputStream(
+  const std::string& path,
+  const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
+{
+  auto stream = std::make_shared<TTreeOutputStream>(GetTree({path, shared_from_this()}));
+  return stream;
+}
+
 TBufferFileFS::TBufferFileFS(TBufferFile* f)
   : VirtualRootFileSystemBase(),
     mBuffer(f),
@@ -512,5 +811,4 @@ std::shared_ptr<VirtualRootFileSystemBase> TBufferFileFS::GetSubFilesystem(arrow
   }
   return mFilesystem;
 }
-
 } // namespace o2::framework
diff --git a/Framework/Core/src/TableTreeHelpers.cxx b/Framework/Core/src/TableTreeHelpers.cxx
index c20febaac517d..d0fdd0ced5779 100644
--- a/Framework/Core/src/TableTreeHelpers.cxx
+++ b/Framework/Core/src/TableTreeHelpers.cxx
@@ -512,7 +512,7 @@ void TreeToTable::addAllColumns(TTree* tree, std::vector<std::string>&& names)
     if (strncmp(reader->branch()->GetName(), "fIndexArray", strlen("fIndexArray")) == 0) {
       std::string sizeBranchName = reader->branch()->GetName();
       sizeBranchName += "_size";
-      TBranch* sizeBranch = (TBranch*)tree->GetBranch(sizeBranchName.c_str());
+      auto* sizeBranch = (TBranch*)tree->GetBranch(sizeBranchName.c_str());
       if (sizeBranch) {
         tree->AddBranchToCache(sizeBranch);
       }
diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx
index 599f1062c63a0..03f0977a4c0c4 100644
--- a/Framework/Core/test/test_Root2ArrowTable.cxx
+++ b/Framework/Core/test/test_Root2ArrowTable.cxx
@@ -358,4 +358,112 @@ TEST_CASE("RootTree2Dataset")
   REQUIRE(result.ok());
   REQUIRE((*result)->columns().size() == 7);
   REQUIRE((*result)->num_rows() == 100);
+
+  {
+    auto int_array = std::static_pointer_cast<arrow::Int32Array>((*result)->GetColumnByName("ev"));
+    for (int64_t j = 0; j < int_array->length(); j++) {
+      REQUIRE(int_array->Value(j) == j + 1);
+    }
+  }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>((*result)->GetColumnByName("xyz"));
+
+    // Iterate over the FixedSizeListArray
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      auto float_array = std::static_pointer_cast<arrow::FloatArray>(value_slice);
+
+      REQUIRE(float_array->Value(0) == 1);
+      REQUIRE(float_array->Value(1) == 2);
+      REQUIRE(float_array->Value(2) == i + 1);
+    }
+  }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>((*result)->GetColumnByName("ij"));
+
+    // Iterate over the FixedSizeListArray
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      auto int_array = std::static_pointer_cast<arrow::Int32Array>(value_slice);
+      REQUIRE(int_array->Value(0) == i);
+      REQUIRE(int_array->Value(1) == i + 1);
+    }
+  }
+
+  auto* output = new TMemFile("foo", "RECREATE");
+  auto outFs = std::make_shared<TFileFileSystem>(output, 0);
+  arrow::fs::FileLocator locator{outFs, "/DF_3"};
+
+  auto destination = outFs->OpenOutputStream(locator.path, {});
+  REQUIRE(destination.ok());
+
+  auto writer = format->MakeWriter(*destination, schema, {}, locator);
+  auto success = writer->get()->Write(*result);
+  auto rootDestination = std::dynamic_pointer_cast<TTreeOutputStream>(*destination);
+
+  REQUIRE(success.ok());
+  // Let's read it back...
+  arrow::dataset::FileSource source2("/DF_3", outFs);
+  auto newTreeFS = outFs->GetSubFilesystem(source2);
+
+  REQUIRE(format->IsSupported(source) == true);
+
+  auto schemaOptWritten = format->Inspect(source);
+  REQUIRE(schemaOptWritten.ok());
+  auto schemaWritten = *schemaOptWritten;
+  REQUIRE(schemaWritten->num_fields() == 7);
+  REQUIRE(schemaWritten->field(0)->type()->id() == arrow::float32()->id());
+  REQUIRE(schemaWritten->field(1)->type()->id() == arrow::float32()->id());
+  REQUIRE(schemaWritten->field(2)->type()->id() == arrow::float32()->id());
+  REQUIRE(schemaWritten->field(3)->type()->id() == arrow::float64()->id());
+  REQUIRE(schemaWritten->field(4)->type()->id() == arrow::int32()->id());
+  REQUIRE(schemaWritten->field(5)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id());
+  REQUIRE(schemaWritten->field(6)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id());
+
+  auto fragmentWritten = format->MakeFragment(source, {}, schema);
+  REQUIRE(fragmentWritten.ok());
+  auto optionsWritten = std::make_shared<arrow::dataset::ScanOptions>();
+  options->dataset_schema = schemaWritten;
+  auto scannerWritten = format->ScanBatchesAsync(optionsWritten, *fragment);
+  REQUIRE(scannerWritten.ok());
+  auto batchesWritten = (*scanner)();
+  auto resultWritten = batches.result();
+  REQUIRE(resultWritten.ok());
+  REQUIRE((*resultWritten)->columns().size() == 7);
+  REQUIRE((*resultWritten)->num_rows() == 100);
+
+  {
+    auto int_array = std::static_pointer_cast<arrow::Int32Array>((*resultWritten)->GetColumnByName("ev"));
+    for (int64_t j = 0; j < int_array->length(); j++) {
+      REQUIRE(int_array->Value(j) == j + 1);
+    }
+  }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>((*result)->GetColumnByName("xyz"));
+
+    // Iterate over the FixedSizeListArray
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      auto float_array = std::static_pointer_cast<arrow::FloatArray>(value_slice);
+
+      REQUIRE(float_array->Value(0) == 1);
+      REQUIRE(float_array->Value(1) == 2);
+      REQUIRE(float_array->Value(2) == i + 1);
+    }
+  }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>((*result)->GetColumnByName("ij"));
+
+    // Iterate over the FixedSizeListArray
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      auto int_array = std::static_pointer_cast<arrow::Int32Array>(value_slice);
+      REQUIRE(int_array->Value(0) == i);
+      REQUIRE(int_array->Value(1) == i + 1);
+    }
+  }
 }

From 475d25f53cd5f7bcc33bac9eadfa56118c403ebe Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sat, 23 Nov 2024 10:49:13 +0100
Subject: [PATCH 709/726] DPL: improve arrow::Dataset integration

- Modularise filesystem to allow easier navigation and support for
  multiple formats.
- Add initial support to multiplex multiple tables on top of the same tree.
- Improve support for writing boolean fields.
---
 .../include/Framework/RootArrowFilesystem.h   |  36 ++++-
 Framework/Core/src/RootArrowFilesystem.cxx    | 152 +++++++++++++-----
 Framework/Core/test/test_Root2ArrowTable.cxx  | 145 ++++++++++++-----
 3 files changed, 252 insertions(+), 81 deletions(-)

diff --git a/Framework/Core/include/Framework/RootArrowFilesystem.h b/Framework/Core/include/Framework/RootArrowFilesystem.h
index 7c8385ccd2b9d..48d817bc9ddf2 100644
--- a/Framework/Core/include/Framework/RootArrowFilesystem.h
+++ b/Framework/Core/include/Framework/RootArrowFilesystem.h
@@ -17,6 +17,8 @@
 #include <arrow/type_fwd.h>
 #include <memory>
 
+class TFile;
+class TBranch;
 class TTree;
 class TBufferFile;
 class TDirectoryFile;
@@ -227,11 +229,38 @@ class TTreeFileFormat : public arrow::dataset::FileFormat
     const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const override;
 };
 
-// An arrow outputstream which allows to write to a ttree
+// An arrow outputstream which allows to write to a TDirectoryFile.
+// This will point to the location of the file itself. You can
+// specify the location of the actual object inside it by passing the
+// associated path to the Write() API.
+class TDirectoryFileOutputStream : public arrow::io::OutputStream
+{
+ public:
+  TDirectoryFileOutputStream(TDirectoryFile*);
+
+  arrow::Status Close() override;
+
+  arrow::Result<int64_t> Tell() const override;
+
+  arrow::Status Write(const void* data, int64_t nbytes) override;
+
+  bool closed() const override;
+
+  TDirectoryFile* GetDirectory()
+  {
+    return mDirectory;
+  }
+
+ private:
+  TDirectoryFile* mDirectory;
+};
+
+// An arrow outputstream which allows to write to a TTree. Eventually
+// with a prefix for the branches.
 class TTreeOutputStream : public arrow::io::OutputStream
 {
  public:
-  TTreeOutputStream(TTree* t);
+  TTreeOutputStream(TTree*, std::string branchPrefix);
 
   arrow::Status Close() override;
 
@@ -241,6 +270,8 @@ class TTreeOutputStream : public arrow::io::OutputStream
 
   bool closed() const override;
 
+  TBranch* CreateBranch(char const* branchName, char const* sizeBranch);
+
   TTree* GetTree()
   {
     return mTree;
@@ -248,6 +279,7 @@ class TTreeOutputStream : public arrow::io::OutputStream
 
  private:
   TTree* mTree;
+  std::string mBranchPrefix;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/RootArrowFilesystem.cxx b/Framework/Core/src/RootArrowFilesystem.cxx
index 7581ee57e5b9f..e5543471abd30 100644
--- a/Framework/Core/src/RootArrowFilesystem.cxx
+++ b/Framework/Core/src/RootArrowFilesystem.cxx
@@ -17,7 +17,6 @@
 #include <arrow/array/builder_nested.h>
 #include <arrow/array/builder_primitive.h>
 #include <memory>
-#include <stdexcept>
 #include <TFile.h>
 #include <TLeaf.h>
 #include <TBufferFile.h>
@@ -28,8 +27,11 @@
 #include <arrow/dataset/file_base.h>
 #include <arrow/result.h>
 #include <arrow/status.h>
+#include <arrow/util/key_value_metadata.h>
 #include <fmt/format.h>
 
+#include <stdexcept>
+#include <utility>
 
 O2_DECLARE_DYNAMIC_LOG(root_arrow_fs);
 
@@ -129,8 +131,15 @@ arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TFileFileSystem::OpenOut
   const std::string& path,
   const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
 {
-  auto* t = new TTree(path.c_str(), "should put a name here");
-  auto stream = std::make_shared<TTreeOutputStream>(t);
+  if (path == "/") {
+    return std::make_shared<TDirectoryFileOutputStream>(this->GetFile());
+  }
+
+  auto* dir = dynamic_cast<TDirectoryFile*>(this->GetFile()->Get(path.c_str()));
+  if (!dir) {
+    throw runtime_error_f("Unable to open directory %s in file %s", path.c_str(), GetFile()->GetName());
+  }
+  auto stream = std::make_shared<TDirectoryFileOutputStream>(dir);
   return stream;
 }
 
@@ -285,14 +294,48 @@ arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> TTreeFileFormat::Ma
   return std::dynamic_pointer_cast<arrow::dataset::FileFragment>(fragment);
 }
 
+
+// An arrow outputstream which allows to write to a ttree
+TDirectoryFileOutputStream::TDirectoryFileOutputStream(TDirectoryFile* f)
+  : mDirectory(f)
+{
+}
+
+arrow::Status TDirectoryFileOutputStream::Close()
+{
+  mDirectory->GetFile()->Close();
+  return arrow::Status::OK();
+}
+
+arrow::Result<int64_t> TDirectoryFileOutputStream::Tell() const
+{
+  return arrow::Result<int64_t>(arrow::Status::NotImplemented("Cannot move"));
+}
+
+arrow::Status TDirectoryFileOutputStream::Write(const void* data, int64_t nbytes)
+{
+  return arrow::Status::NotImplemented("Cannot write raw bytes to a TTree");
+}
+
+bool TDirectoryFileOutputStream::closed() const
+{
+  return mDirectory->GetFile()->IsOpen() == false;
+}
+
 // An arrow outputstream which allows to write to a ttree
-TTreeOutputStream::TTreeOutputStream(TTree* t)
-  : mTree(t)
+// @a branch prefix is to be used to identify a set of branches which all belong to
+// the same table.
+TTreeOutputStream::TTreeOutputStream(TTree* f, std::string branchPrefix)
+  : mTree(f),
+    mBranchPrefix(std::move(branchPrefix))
 {
 }
 
 arrow::Status TTreeOutputStream::Close()
 {
+  if (mTree->GetCurrentFile() == nullptr) {
+    return arrow::Status::Invalid("Cannot close a tree not attached to a file");
+  }
   mTree->GetCurrentFile()->Close();
   return arrow::Status::OK();
 }
@@ -309,9 +352,18 @@ arrow::Status TTreeOutputStream::Write(const void* data, int64_t nbytes)
 
 bool TTreeOutputStream::closed() const
 {
+  // A standalone tree is never closed.
+  if (mTree->GetCurrentFile() == nullptr) {
+    return false;
+  }
   return mTree->GetCurrentFile()->IsOpen() == false;
 }
 
+TBranch* TTreeOutputStream::CreateBranch(char const* branchName, char const* sizeBranch)
+{
+  return mTree->Branch((mBranchPrefix + "/" + branchName).c_str(), (char*)nullptr, (mBranchPrefix + sizeBranch).c_str());
+}
+
 char const* rootSuffixFromArrow(arrow::Type::type id)
 {
   switch (id) {
@@ -411,8 +463,24 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
     : FileWriter(schema, options, destination, destination_locator)
   {
     // Batches have the same number of entries for each column.
+    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
     auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
-    TTree* tree = treeStream->GetTree();
+
+    if (directoryStream.get()) {
+      TDirectoryFile* dir = directoryStream->GetDirectory();
+      dir->cd();
+      auto* tree = new TTree(destination_locator_.path.c_str(), "");
+      treeStream = std::make_shared<TTreeOutputStream>(tree, "");
+    } else if (treeStream.get()) {
+      // We already have a tree stream, let's derive a new one
+      // with the destination_locator_.path as prefix for the branches
+      // This way we can multiplex multiple tables in the same tree.
+      auto tree = treeStream->GetTree();
+      treeStream = std::make_shared<TTreeOutputStream>(tree, destination_locator_.path);
+    } else {
+      // I could simply set a prefix here to merge to an already existing tree.
+      throw std::runtime_error("Unsupported backend.");
+    }
 
     for (auto i = 0u; i < schema->fields().size(); ++i) {
       auto& field = schema->field(i);
@@ -427,15 +495,15 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
           valueTypes.push_back(field->type()->field(0)->type());
           sizesBranches.push_back(nullptr);
           std::string leafList = fmt::format("{}[{}]{}", field->name(), listSizes.back(), rootSuffixFromArrow(valueTypes.back()->id()));
-          branches.push_back(tree->Branch(field->name().c_str(), (char*)nullptr, leafList.c_str()));
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
         } break;
         case arrow::Type::LIST: {
           valueTypes.push_back(field->type()->field(0)->type());
           listSizes.back() = 0; // VLA, we need to calculate it on the fly;
           std::string leafList = fmt::format("{}[{}_size]{}", field->name(), field->name(), rootSuffixFromArrow(valueTypes.back()->id()));
           std::string sizeLeafList = field->name() + "_size/I";
-          sizesBranches.push_back(tree->Branch((field->name() + "_size").c_str(), (char*)nullptr, sizeLeafList.c_str()));
-          branches.push_back(tree->Branch(field->name().c_str(), (char*)nullptr, leafList.c_str()));
+          sizesBranches.push_back(treeStream->CreateBranch((field->name() + "_size").c_str(), sizeLeafList.c_str()));
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
           // Notice that this could be replaced by a better guess of the
           // average size of the list elements, but this is not trivial.
         } break;
@@ -443,7 +511,7 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
           valueTypes.push_back(field->type());
           std::string leafList = field->name() + rootSuffixFromArrow(valueTypes.back()->id());
           sizesBranches.push_back(nullptr);
-          branches.push_back(tree->Branch(field->name().c_str(), (char*)nullptr, leafList.c_str()));
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
         } break;
       }
     }
@@ -463,11 +531,18 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
     }
 
     // Batches have the same number of entries for each column.
+    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
+    TTree* tree = nullptr;
+    if (directoryStream.get()) {
+      TDirectoryFile* dir = directoryStream->GetDirectory();
+      tree = (TTree*)dir->Get(destination_locator_.path.c_str());
+    }
     auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
-    TTree* tree = treeStream->GetTree();
 
-    // Caches for the vectors of bools.
-    std::vector<std::shared_ptr<arrow::UInt8Array>> caches;
+    if (!tree) {
+      // I could simply set a prefix here to merge to an already existing tree.
+      throw std::runtime_error("Unsupported backend.");
+    }
 
     for (auto i = 0u; i < batch->columns().size(); ++i) {
       auto column = batch->column(i);
@@ -484,24 +559,12 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
           auto list = std::static_pointer_cast<arrow::ListArray>(column);
           valueArrays.back() = list;
         } break;
-        default:
-          valueArrays.back() = column;
-      }
-    }
-
-    int64_t pos = 0;
-    while (pos < batch->num_rows()) {
-      for (size_t bi = 0; bi < branches.size(); ++bi) {
-        auto* branch = branches[bi];
-        auto* sizeBranch = sizesBranches[bi];
-        auto array = batch->column(bi);
-        auto& field = batch->schema()->field(bi);
-        auto& listSize = listSizes[bi];
-        auto valueType = valueTypes[bi];
-        auto valueArray = valueArrays[bi];
+        case arrow::Type::BOOL: {
+          // In case of arrays of booleans, we need to go back to their
+          // char based representation for ROOT to save them.
+          valueArrays.back() = std::make_shared<arrow::UInt8Array>();
+          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(column);
 
-        if (field->type()->id() == arrow::Type::BOOL) {
-          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(array);
           int64_t length = boolArray->length();
           arrow::UInt8Builder builder;
           auto ok = builder.Reserve(length);
@@ -516,11 +579,24 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
               auto ok = builder.AppendNull();
             }
           }
-
-          ok = builder.Finish(&caches[bi]);
-          branch->SetAddress((void*)(caches[bi]->values()->data()));
-          continue;
+          ok = builder.Finish(&valueArrays.back());
         }
+        default:
+          valueArrays.back() = column;
+      }
+    }
+
+    int64_t pos = 0;
+    while (pos < batch->num_rows()) {
+      for (size_t bi = 0; bi < branches.size(); ++bi) {
+        auto* branch = branches[bi];
+        auto* sizeBranch = sizesBranches[bi];
+        auto array = batch->column(bi);
+        auto& field = batch->schema()->field(bi);
+        auto& listSize = listSizes[bi];
+        auto valueType = valueTypes[bi];
+        auto valueArray = valueArrays[bi];
+
         switch (field->type()->id()) {
           case arrow::Type::LIST: {
             auto list = std::static_pointer_cast<arrow::ListArray>(array);
@@ -769,8 +845,12 @@ arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenOut
   const std::string& path,
   const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
 {
-  auto stream = std::make_shared<TTreeOutputStream>(GetTree({path, shared_from_this()}));
-  return stream;
+  arrow::dataset::FileSource source{path, shared_from_this()};
+  auto prefix = metadata->Get("branch_prefix");
+  if (prefix.ok()) {
+    return std::make_shared<TTreeOutputStream>(GetTree(source), *prefix);
+  }
+  return std::make_shared<TTreeOutputStream>(GetTree(source), "");
 }
 
 TBufferFileFS::TBufferFileFS(TBufferFile* f)
diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx
index 03f0977a4c0c4..a0666deb4d8c3 100644
--- a/Framework/Core/test/test_Root2ArrowTable.cxx
+++ b/Framework/Core/test/test_Root2ArrowTable.cxx
@@ -20,13 +20,18 @@
 #include <ROOT/RArrowDS.hxx>
 #include <TBufferFile.h>
 #include <TClass.h>
+#include <TDirectoryFile.h>
 #include <TMemFile.h>
 #include <TDirectory.h>
 #include <TTree.h>
 #include <TRandom.h>
 #include <TFile.h>
+#include <memory>
 
+#include <arrow/array/array_primitive.h>
+#include <arrow/array/builder_primitive.h>
 #include <arrow/dataset/scanner.h>
+#include <arrow/record_batch.h>
 #include <arrow/table.h>
 #include <arrow/ipc/writer.h>
 #include <arrow/io/memory.h>
@@ -259,6 +264,82 @@ TEST_CASE("RootTree2Fragment")
   REQUIRE((*result)->num_rows() == 1000);
 }
 
+bool validateContents(std::shared_ptr<arrow::RecordBatch> batch)
+{
+  {
+    auto int_array = std::static_pointer_cast<arrow::Int32Array>(batch->GetColumnByName("ev"));
+    REQUIRE(int_array->length() == 100);
+    for (int64_t j = 0; j < int_array->length(); j++) {
+      REQUIRE(int_array->Value(j) == j + 1);
+    }
+  }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>(batch->GetColumnByName("xyz"));
+
+    REQUIRE(list_array->length() == 100);
+    // Iterate over the FixedSizeListArray
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      auto float_array = std::static_pointer_cast<arrow::FloatArray>(value_slice);
+
+      REQUIRE(float_array->Value(0) == 1);
+      REQUIRE(float_array->Value(1) == 2);
+      REQUIRE(float_array->Value(2) == i + 1);
+    }
+  }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>(batch->GetColumnByName("ij"));
+
+    REQUIRE(list_array->length() == 100);
+    // Iterate over the FixedSizeListArray
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      auto int_array = std::static_pointer_cast<arrow::Int32Array>(value_slice);
+      REQUIRE(int_array->Value(0) == i);
+      REQUIRE(int_array->Value(1) == i + 1);
+    }
+  }
+
+  {
+    auto bool_array = std::static_pointer_cast<arrow::BooleanArray>(batch->GetColumnByName("bools"));
+
+    REQUIRE(bool_array->length() == 100);
+    for (int64_t j = 0; j < bool_array->length(); j++) {
+      REQUIRE(bool_array->Value(j) == (j % 3 == 0));
+    }
+  }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>(batch->GetColumnByName("manyBools"));
+
+    REQUIRE(list_array->length() == 100);
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      auto bool_array = std::static_pointer_cast<arrow::BooleanArray>(value_slice);
+      REQUIRE(bool_array->Value(0) == (i % 4 == 0));
+      REQUIRE(bool_array->Value(1) == (i % 5 == 0));
+    }
+  }
+  return true;
+}
+
+bool validateSchema(std::shared_ptr<arrow::Schema> schema)
+{
+  REQUIRE(schema->num_fields() == 9);
+  REQUIRE(schema->field(0)->type()->id() == arrow::float32()->id());
+  REQUIRE(schema->field(1)->type()->id() == arrow::float32()->id());
+  REQUIRE(schema->field(2)->type()->id() == arrow::float32()->id());
+  REQUIRE(schema->field(3)->type()->id() == arrow::float64()->id());
+  REQUIRE(schema->field(4)->type()->id() == arrow::int32()->id());
+  REQUIRE(schema->field(5)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id());
+  REQUIRE(schema->field(6)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id());
+  REQUIRE(schema->field(7)->type()->id() == arrow::boolean()->id());
+  REQUIRE(schema->field(8)->type()->id() == arrow::fixed_size_list(arrow::boolean(), 2)->id());
+  return true;
+}
+
 TEST_CASE("RootTree2Dataset")
 {
   using namespace o2::framework;
@@ -307,6 +388,9 @@ TEST_CASE("RootTree2Dataset")
     Float_t px = 0, py = 1, pz = 2;
     Double_t random;
     Int_t ev;
+    bool oneBool;
+    bool manyBool[2];
+
     t->Branch("px", &px, "px/F");
     t->Branch("py", &py, "py/F");
     t->Branch("pz", &pz, "pz/F");
@@ -314,6 +398,8 @@ TEST_CASE("RootTree2Dataset")
     t->Branch("ev", &ev, "ev/I");
     t->Branch("xyz", xyz, "xyz[3]/F");
     t->Branch("ij", ij, "ij[2]/I");
+    t->Branch("bools", &oneBool, "bools/O");
+    t->Branch("manyBools", &manyBool, "manyBools[2]/O");
     // fill the tree
     for (Int_t i = 0; i < 100; i++) {
       xyz[0] = 1;
@@ -326,6 +412,9 @@ TEST_CASE("RootTree2Dataset")
       ij[1] = i + 1;
       random = gRandom->Rndm();
       ev = i + 1;
+      oneBool = (i % 3 == 0);
+      manyBool[0] = (i % 4 == 0);
+      manyBool[1] = (i % 5 == 0);
       t->Fill();
     }
   }
@@ -339,7 +428,7 @@ TEST_CASE("RootTree2Dataset")
   auto schemaOpt = format->Inspect(source);
   REQUIRE(schemaOpt.ok());
   auto schema = *schemaOpt;
-  REQUIRE(schema->num_fields() == 7);
+  REQUIRE(schema->num_fields() == 9);
   REQUIRE(schema->field(0)->type()->id() == arrow::float32()->id());
   REQUIRE(schema->field(1)->type()->id() == arrow::float32()->id());
   REQUIRE(schema->field(2)->type()->id() == arrow::float32()->id());
@@ -347,6 +436,9 @@ TEST_CASE("RootTree2Dataset")
   REQUIRE(schema->field(4)->type()->id() == arrow::int32()->id());
   REQUIRE(schema->field(5)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id());
   REQUIRE(schema->field(6)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id());
+  REQUIRE(schema->field(7)->type()->id() == arrow::boolean()->id());
+  REQUIRE(schema->field(8)->type()->id() == arrow::fixed_size_list(arrow::boolean(), 2)->id());
+
   auto fragment = format->MakeFragment(source, {}, schema);
   REQUIRE(fragment.ok());
   auto options = std::make_shared<arrow::dataset::ScanOptions>();
@@ -356,7 +448,7 @@ TEST_CASE("RootTree2Dataset")
   auto batches = (*scanner)();
   auto result = batches.result();
   REQUIRE(result.ok());
-  REQUIRE((*result)->columns().size() == 7);
+  REQUIRE((*result)->columns().size() == 9);
   REQUIRE((*result)->num_rows() == 100);
 
   {
@@ -394,14 +486,16 @@ TEST_CASE("RootTree2Dataset")
 
   auto* output = new TMemFile("foo", "RECREATE");
   auto outFs = std::make_shared<TFileFileSystem>(output, 0);
-  arrow::fs::FileLocator locator{outFs, "/DF_3"};
 
-  auto destination = outFs->OpenOutputStream(locator.path, {});
+  // Open a stream at toplevel
+  auto destination = outFs->OpenOutputStream("/", {});
   REQUIRE(destination.ok());
 
+  // Write to the /DF_3 tree at top level
+  arrow::fs::FileLocator locator{outFs, "/DF_3"};
   auto writer = format->MakeWriter(*destination, schema, {}, locator);
   auto success = writer->get()->Write(*result);
-  auto rootDestination = std::dynamic_pointer_cast<TTreeOutputStream>(*destination);
+  auto rootDestination = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(*destination);
 
   REQUIRE(success.ok());
   // Let's read it back...
@@ -413,14 +507,7 @@ TEST_CASE("RootTree2Dataset")
   auto schemaOptWritten = format->Inspect(source);
   REQUIRE(schemaOptWritten.ok());
   auto schemaWritten = *schemaOptWritten;
-  REQUIRE(schemaWritten->num_fields() == 7);
-  REQUIRE(schemaWritten->field(0)->type()->id() == arrow::float32()->id());
-  REQUIRE(schemaWritten->field(1)->type()->id() == arrow::float32()->id());
-  REQUIRE(schemaWritten->field(2)->type()->id() == arrow::float32()->id());
-  REQUIRE(schemaWritten->field(3)->type()->id() == arrow::float64()->id());
-  REQUIRE(schemaWritten->field(4)->type()->id() == arrow::int32()->id());
-  REQUIRE(schemaWritten->field(5)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id());
-  REQUIRE(schemaWritten->field(6)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id());
+  REQUIRE(validateSchema(schemaWritten));
 
   auto fragmentWritten = format->MakeFragment(source, {}, schema);
   REQUIRE(fragmentWritten.ok());
@@ -431,39 +518,11 @@ TEST_CASE("RootTree2Dataset")
   auto batchesWritten = (*scanner)();
   auto resultWritten = batches.result();
   REQUIRE(resultWritten.ok());
-  REQUIRE((*resultWritten)->columns().size() == 7);
+  REQUIRE((*resultWritten)->columns().size() == 9);
   REQUIRE((*resultWritten)->num_rows() == 100);
+  validateContents(*resultWritten);
 
-  {
-    auto int_array = std::static_pointer_cast<arrow::Int32Array>((*resultWritten)->GetColumnByName("ev"));
-    for (int64_t j = 0; j < int_array->length(); j++) {
-      REQUIRE(int_array->Value(j) == j + 1);
-    }
-  }
-
-  {
-    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>((*result)->GetColumnByName("xyz"));
-
-    // Iterate over the FixedSizeListArray
-    for (int64_t i = 0; i < list_array->length(); i++) {
-      auto value_slice = list_array->value_slice(i);
-      auto float_array = std::static_pointer_cast<arrow::FloatArray>(value_slice);
-
-      REQUIRE(float_array->Value(0) == 1);
-      REQUIRE(float_array->Value(1) == 2);
-      REQUIRE(float_array->Value(2) == i + 1);
-    }
-  }
 
   {
-    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>((*result)->GetColumnByName("ij"));
-
-    // Iterate over the FixedSizeListArray
-    for (int64_t i = 0; i < list_array->length(); i++) {
-      auto value_slice = list_array->value_slice(i);
-      auto int_array = std::static_pointer_cast<arrow::Int32Array>(value_slice);
-      REQUIRE(int_array->Value(0) == i);
-      REQUIRE(int_array->Value(1) == i + 1);
-    }
   }
 }

From f37b9d7630ccc9fd357a5939bfb6700084b04514 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sun, 24 Nov 2024 09:28:18 +0100
Subject: [PATCH 710/726] DPL: improve arrow::Dataset integration

- Modularise filesystem to allow easier navigation and support for
  multiple formats.
- Add initial support to multiplex multiple tables on top of the same tree.
- Improve support for writing boolean fields.
---
 .../include/Framework/RootArrowFilesystem.h   |  36 +++-
 Framework/Core/src/RootArrowFilesystem.cxx    | 154 +++++++++++++-----
 Framework/Core/test/test_Root2ArrowTable.cxx  | 146 ++++++++++++-----
 3 files changed, 251 insertions(+), 85 deletions(-)

diff --git a/Framework/Core/include/Framework/RootArrowFilesystem.h b/Framework/Core/include/Framework/RootArrowFilesystem.h
index 7c8385ccd2b9d..48d817bc9ddf2 100644
--- a/Framework/Core/include/Framework/RootArrowFilesystem.h
+++ b/Framework/Core/include/Framework/RootArrowFilesystem.h
@@ -17,6 +17,8 @@
 #include <arrow/type_fwd.h>
 #include <memory>
 
+class TFile;
+class TBranch;
 class TTree;
 class TBufferFile;
 class TDirectoryFile;
@@ -227,11 +229,38 @@ class TTreeFileFormat : public arrow::dataset::FileFormat
     const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const override;
 };
 
-// An arrow outputstream which allows to write to a ttree
+// An arrow outputstream which allows to write to a TDirectoryFile.
+// This will point to the location of the file itself. You can
+// specify the location of the actual object inside it by passing the
+// associated path to the Write() API.
+class TDirectoryFileOutputStream : public arrow::io::OutputStream
+{
+ public:
+  TDirectoryFileOutputStream(TDirectoryFile*);
+
+  arrow::Status Close() override;
+
+  arrow::Result<int64_t> Tell() const override;
+
+  arrow::Status Write(const void* data, int64_t nbytes) override;
+
+  bool closed() const override;
+
+  TDirectoryFile* GetDirectory()
+  {
+    return mDirectory;
+  }
+
+ private:
+  TDirectoryFile* mDirectory;
+};
+
+// An arrow outputstream which allows to write to a TTree. Eventually
+// with a prefix for the branches.
 class TTreeOutputStream : public arrow::io::OutputStream
 {
  public:
-  TTreeOutputStream(TTree* t);
+  TTreeOutputStream(TTree*, std::string branchPrefix);
 
   arrow::Status Close() override;
 
@@ -241,6 +270,8 @@ class TTreeOutputStream : public arrow::io::OutputStream
 
   bool closed() const override;
 
+  TBranch* CreateBranch(char const* branchName, char const* sizeBranch);
+
   TTree* GetTree()
   {
     return mTree;
@@ -248,6 +279,7 @@ class TTreeOutputStream : public arrow::io::OutputStream
 
  private:
   TTree* mTree;
+  std::string mBranchPrefix;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/RootArrowFilesystem.cxx b/Framework/Core/src/RootArrowFilesystem.cxx
index 7581ee57e5b9f..39f0474574a8b 100644
--- a/Framework/Core/src/RootArrowFilesystem.cxx
+++ b/Framework/Core/src/RootArrowFilesystem.cxx
@@ -17,7 +17,6 @@
 #include <arrow/array/builder_nested.h>
 #include <arrow/array/builder_primitive.h>
 #include <memory>
-#include <stdexcept>
 #include <TFile.h>
 #include <TLeaf.h>
 #include <TBufferFile.h>
@@ -28,8 +27,11 @@
 #include <arrow/dataset/file_base.h>
 #include <arrow/result.h>
 #include <arrow/status.h>
+#include <arrow/util/key_value_metadata.h>
 #include <fmt/format.h>
 
+#include <stdexcept>
+#include <utility>
 
 O2_DECLARE_DYNAMIC_LOG(root_arrow_fs);
 
@@ -100,7 +102,6 @@ std::shared_ptr<VirtualRootFileSystemBase> TFileFileSystem::GetSubFilesystem(arr
     return std::shared_ptr<VirtualRootFileSystemBase>(new SingleTreeFileSystem(tree));
   }
 
-
   auto directory = (TDirectoryFile*)mFile->GetObjectChecked(source.path().c_str(), TClass::GetClass<TDirectory>());
   if (directory) {
     return std::shared_ptr<VirtualRootFileSystemBase>(new TFileFileSystem(directory, 50 * 1024 * 1024));
@@ -129,8 +130,15 @@ arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TFileFileSystem::OpenOut
   const std::string& path,
   const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
 {
-  auto* t = new TTree(path.c_str(), "should put a name here");
-  auto stream = std::make_shared<TTreeOutputStream>(t);
+  if (path == "/") {
+    return std::make_shared<TDirectoryFileOutputStream>(this->GetFile());
+  }
+
+  auto* dir = dynamic_cast<TDirectoryFile*>(this->GetFile()->Get(path.c_str()));
+  if (!dir) {
+    throw runtime_error_f("Unable to open directory %s in file %s", path.c_str(), GetFile()->GetName());
+  }
+  auto stream = std::make_shared<TDirectoryFileOutputStream>(dir);
   return stream;
 }
 
@@ -286,13 +294,46 @@ arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> TTreeFileFormat::Ma
 }
 
 // An arrow outputstream which allows to write to a ttree
-TTreeOutputStream::TTreeOutputStream(TTree* t)
-  : mTree(t)
+TDirectoryFileOutputStream::TDirectoryFileOutputStream(TDirectoryFile* f)
+  : mDirectory(f)
+{
+}
+
+arrow::Status TDirectoryFileOutputStream::Close()
+{
+  mDirectory->GetFile()->Close();
+  return arrow::Status::OK();
+}
+
+arrow::Result<int64_t> TDirectoryFileOutputStream::Tell() const
+{
+  return arrow::Result<int64_t>(arrow::Status::NotImplemented("Cannot move"));
+}
+
+arrow::Status TDirectoryFileOutputStream::Write(const void* data, int64_t nbytes)
+{
+  return arrow::Status::NotImplemented("Cannot write raw bytes to a TTree");
+}
+
+bool TDirectoryFileOutputStream::closed() const
+{
+  return mDirectory->GetFile()->IsOpen() == false;
+}
+
+// An arrow outputstream which allows to write to a ttree
+// @a branch prefix is to be used to identify a set of branches which all belong to
+// the same table.
+TTreeOutputStream::TTreeOutputStream(TTree* f, std::string branchPrefix)
+  : mTree(f),
+    mBranchPrefix(std::move(branchPrefix))
 {
 }
 
 arrow::Status TTreeOutputStream::Close()
 {
+  if (mTree->GetCurrentFile() == nullptr) {
+    return arrow::Status::Invalid("Cannot close a tree not attached to a file");
+  }
   mTree->GetCurrentFile()->Close();
   return arrow::Status::OK();
 }
@@ -309,9 +350,18 @@ arrow::Status TTreeOutputStream::Write(const void* data, int64_t nbytes)
 
 bool TTreeOutputStream::closed() const
 {
+  // A standalone tree is never closed.
+  if (mTree->GetCurrentFile() == nullptr) {
+    return false;
+  }
   return mTree->GetCurrentFile()->IsOpen() == false;
 }
 
+TBranch* TTreeOutputStream::CreateBranch(char const* branchName, char const* sizeBranch)
+{
+  return mTree->Branch((mBranchPrefix + "/" + branchName).c_str(), (char*)nullptr, (mBranchPrefix + sizeBranch).c_str());
+}
+
 char const* rootSuffixFromArrow(arrow::Type::type id)
 {
   switch (id) {
@@ -411,8 +461,24 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
     : FileWriter(schema, options, destination, destination_locator)
   {
     // Batches have the same number of entries for each column.
+    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
     auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
-    TTree* tree = treeStream->GetTree();
+
+    if (directoryStream.get()) {
+      TDirectoryFile* dir = directoryStream->GetDirectory();
+      dir->cd();
+      auto* tree = new TTree(destination_locator_.path.c_str(), "");
+      treeStream = std::make_shared<TTreeOutputStream>(tree, "");
+    } else if (treeStream.get()) {
+      // We already have a tree stream, let's derive a new one
+      // with the destination_locator_.path as prefix for the branches
+      // This way we can multiplex multiple tables in the same tree.
+      auto tree = treeStream->GetTree();
+      treeStream = std::make_shared<TTreeOutputStream>(tree, destination_locator_.path);
+    } else {
+      // I could simply set a prefix here to merge to an already existing tree.
+      throw std::runtime_error("Unsupported backend.");
+    }
 
     for (auto i = 0u; i < schema->fields().size(); ++i) {
       auto& field = schema->field(i);
@@ -427,15 +493,15 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
           valueTypes.push_back(field->type()->field(0)->type());
           sizesBranches.push_back(nullptr);
           std::string leafList = fmt::format("{}[{}]{}", field->name(), listSizes.back(), rootSuffixFromArrow(valueTypes.back()->id()));
-          branches.push_back(tree->Branch(field->name().c_str(), (char*)nullptr, leafList.c_str()));
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
         } break;
         case arrow::Type::LIST: {
           valueTypes.push_back(field->type()->field(0)->type());
           listSizes.back() = 0; // VLA, we need to calculate it on the fly;
           std::string leafList = fmt::format("{}[{}_size]{}", field->name(), field->name(), rootSuffixFromArrow(valueTypes.back()->id()));
           std::string sizeLeafList = field->name() + "_size/I";
-          sizesBranches.push_back(tree->Branch((field->name() + "_size").c_str(), (char*)nullptr, sizeLeafList.c_str()));
-          branches.push_back(tree->Branch(field->name().c_str(), (char*)nullptr, leafList.c_str()));
+          sizesBranches.push_back(treeStream->CreateBranch((field->name() + "_size").c_str(), sizeLeafList.c_str()));
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
           // Notice that this could be replaced by a better guess of the
           // average size of the list elements, but this is not trivial.
         } break;
@@ -443,7 +509,7 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
           valueTypes.push_back(field->type());
           std::string leafList = field->name() + rootSuffixFromArrow(valueTypes.back()->id());
           sizesBranches.push_back(nullptr);
-          branches.push_back(tree->Branch(field->name().c_str(), (char*)nullptr, leafList.c_str()));
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
         } break;
       }
     }
@@ -463,11 +529,18 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
     }
 
     // Batches have the same number of entries for each column.
+    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
+    TTree* tree = nullptr;
+    if (directoryStream.get()) {
+      TDirectoryFile* dir = directoryStream->GetDirectory();
+      tree = (TTree*)dir->Get(destination_locator_.path.c_str());
+    }
     auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
-    TTree* tree = treeStream->GetTree();
 
-    // Caches for the vectors of bools.
-    std::vector<std::shared_ptr<arrow::UInt8Array>> caches;
+    if (!tree) {
+      // I could simply set a prefix here to merge to an already existing tree.
+      throw std::runtime_error("Unsupported backend.");
+    }
 
     for (auto i = 0u; i < batch->columns().size(); ++i) {
       auto column = batch->column(i);
@@ -484,24 +557,12 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
           auto list = std::static_pointer_cast<arrow::ListArray>(column);
           valueArrays.back() = list;
         } break;
-        default:
-          valueArrays.back() = column;
-      }
-    }
-
-    int64_t pos = 0;
-    while (pos < batch->num_rows()) {
-      for (size_t bi = 0; bi < branches.size(); ++bi) {
-        auto* branch = branches[bi];
-        auto* sizeBranch = sizesBranches[bi];
-        auto array = batch->column(bi);
-        auto& field = batch->schema()->field(bi);
-        auto& listSize = listSizes[bi];
-        auto valueType = valueTypes[bi];
-        auto valueArray = valueArrays[bi];
+        case arrow::Type::BOOL: {
+          // In case of arrays of booleans, we need to go back to their
+          // char based representation for ROOT to save them.
+          valueArrays.back() = std::make_shared<arrow::UInt8Array>();
+          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(column);
 
-        if (field->type()->id() == arrow::Type::BOOL) {
-          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(array);
           int64_t length = boolArray->length();
           arrow::UInt8Builder builder;
           auto ok = builder.Reserve(length);
@@ -516,11 +577,24 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
               auto ok = builder.AppendNull();
             }
           }
+          valueArrays.back() = *builder.Finish();
+        } break;
+        default:
+          valueArrays.back() = column;
+      }
+    }
+
+    int64_t pos = 0;
+    while (pos < batch->num_rows()) {
+      for (size_t bi = 0; bi < branches.size(); ++bi) {
+        auto* branch = branches[bi];
+        auto* sizeBranch = sizesBranches[bi];
+        auto array = batch->column(bi);
+        auto& field = batch->schema()->field(bi);
+        auto& listSize = listSizes[bi];
+        auto valueType = valueTypes[bi];
+        auto valueArray = valueArrays[bi];
 
-          ok = builder.Finish(&caches[bi]);
-          branch->SetAddress((void*)(caches[bi]->values()->data()));
-          continue;
-        }
         switch (field->type()->id()) {
           case arrow::Type::LIST: {
             auto list = std::static_pointer_cast<arrow::ListArray>(array);
@@ -764,13 +838,16 @@ arrow::Result<arrow::RecordBatchGenerator> TTreeFileFormat::ScanBatchesAsync(
   return generator;
 }
 
-
 arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenOutputStream(
   const std::string& path,
   const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
 {
-  auto stream = std::make_shared<TTreeOutputStream>(GetTree({path, shared_from_this()}));
-  return stream;
+  arrow::dataset::FileSource source{path, shared_from_this()};
+  auto prefix = metadata->Get("branch_prefix");
+  if (prefix.ok()) {
+    return std::make_shared<TTreeOutputStream>(GetTree(source), *prefix);
+  }
+  return std::make_shared<TTreeOutputStream>(GetTree(source), "");
 }
 
 TBufferFileFS::TBufferFileFS(TBufferFile* f)
@@ -782,7 +859,6 @@ TBufferFileFS::TBufferFileFS(TBufferFile* f)
 
 TTreeFileSystem::~TTreeFileSystem() = default;
 
-
 arrow::Result<arrow::fs::FileInfo> TBufferFileFS::GetFileInfo(const std::string& path)
 {
   arrow::fs::FileInfo result;
diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx
index 03f0977a4c0c4..a659d488ae24a 100644
--- a/Framework/Core/test/test_Root2ArrowTable.cxx
+++ b/Framework/Core/test/test_Root2ArrowTable.cxx
@@ -20,13 +20,18 @@
 #include <ROOT/RArrowDS.hxx>
 #include <TBufferFile.h>
 #include <TClass.h>
+#include <TDirectoryFile.h>
 #include <TMemFile.h>
 #include <TDirectory.h>
 #include <TTree.h>
 #include <TRandom.h>
 #include <TFile.h>
+#include <memory>
 
+#include <arrow/array/array_primitive.h>
+#include <arrow/array/builder_primitive.h>
 #include <arrow/dataset/scanner.h>
+#include <arrow/record_batch.h>
 #include <arrow/table.h>
 #include <arrow/ipc/writer.h>
 #include <arrow/io/memory.h>
@@ -259,6 +264,82 @@ TEST_CASE("RootTree2Fragment")
   REQUIRE((*result)->num_rows() == 1000);
 }
 
+bool validateContents(std::shared_ptr<arrow::RecordBatch> batch)
+{
+  {
+    auto int_array = std::static_pointer_cast<arrow::Int32Array>(batch->GetColumnByName("ev"));
+    REQUIRE(int_array->length() == 100);
+    for (int64_t j = 0; j < int_array->length(); j++) {
+      REQUIRE(int_array->Value(j) == j + 1);
+    }
+  }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>(batch->GetColumnByName("xyz"));
+
+    REQUIRE(list_array->length() == 100);
+    // Iterate over the FixedSizeListArray
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      auto float_array = std::static_pointer_cast<arrow::FloatArray>(value_slice);
+
+      REQUIRE(float_array->Value(0) == 1);
+      REQUIRE(float_array->Value(1) == 2);
+      REQUIRE(float_array->Value(2) == i + 1);
+    }
+  }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>(batch->GetColumnByName("ij"));
+
+    REQUIRE(list_array->length() == 100);
+    // Iterate over the FixedSizeListArray
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      auto int_array = std::static_pointer_cast<arrow::Int32Array>(value_slice);
+      REQUIRE(int_array->Value(0) == i);
+      REQUIRE(int_array->Value(1) == i + 1);
+    }
+  }
+
+  {
+    auto bool_array = std::static_pointer_cast<arrow::BooleanArray>(batch->GetColumnByName("bools"));
+
+    REQUIRE(bool_array->length() == 100);
+    for (int64_t j = 0; j < bool_array->length(); j++) {
+      REQUIRE(bool_array->Value(j) == (j % 3 == 0));
+    }
+  }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>(batch->GetColumnByName("manyBools"));
+
+    REQUIRE(list_array->length() == 100);
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      auto bool_array = std::static_pointer_cast<arrow::BooleanArray>(value_slice);
+      REQUIRE(bool_array->Value(0) == (i % 4 == 0));
+      REQUIRE(bool_array->Value(1) == (i % 5 == 0));
+    }
+  }
+  return true;
+}
+
+bool validateSchema(std::shared_ptr<arrow::Schema> schema)
+{
+  REQUIRE(schema->num_fields() == 9);
+  REQUIRE(schema->field(0)->type()->id() == arrow::float32()->id());
+  REQUIRE(schema->field(1)->type()->id() == arrow::float32()->id());
+  REQUIRE(schema->field(2)->type()->id() == arrow::float32()->id());
+  REQUIRE(schema->field(3)->type()->id() == arrow::float64()->id());
+  REQUIRE(schema->field(4)->type()->id() == arrow::int32()->id());
+  REQUIRE(schema->field(5)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id());
+  REQUIRE(schema->field(6)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id());
+  REQUIRE(schema->field(7)->type()->id() == arrow::boolean()->id());
+  REQUIRE(schema->field(8)->type()->id() == arrow::fixed_size_list(arrow::boolean(), 2)->id());
+  return true;
+}
+
 TEST_CASE("RootTree2Dataset")
 {
   using namespace o2::framework;
@@ -307,6 +388,9 @@ TEST_CASE("RootTree2Dataset")
     Float_t px = 0, py = 1, pz = 2;
     Double_t random;
     Int_t ev;
+    bool oneBool;
+    bool manyBool[2];
+
     t->Branch("px", &px, "px/F");
     t->Branch("py", &py, "py/F");
     t->Branch("pz", &pz, "pz/F");
@@ -314,6 +398,8 @@ TEST_CASE("RootTree2Dataset")
     t->Branch("ev", &ev, "ev/I");
     t->Branch("xyz", xyz, "xyz[3]/F");
     t->Branch("ij", ij, "ij[2]/I");
+    t->Branch("bools", &oneBool, "bools/O");
+    t->Branch("manyBools", &manyBool, "manyBools[2]/O");
     // fill the tree
     for (Int_t i = 0; i < 100; i++) {
       xyz[0] = 1;
@@ -326,6 +412,9 @@ TEST_CASE("RootTree2Dataset")
       ij[1] = i + 1;
       random = gRandom->Rndm();
       ev = i + 1;
+      oneBool = (i % 3 == 0);
+      manyBool[0] = (i % 4 == 0);
+      manyBool[1] = (i % 5 == 0);
       t->Fill();
     }
   }
@@ -339,7 +428,7 @@ TEST_CASE("RootTree2Dataset")
   auto schemaOpt = format->Inspect(source);
   REQUIRE(schemaOpt.ok());
   auto schema = *schemaOpt;
-  REQUIRE(schema->num_fields() == 7);
+  REQUIRE(schema->num_fields() == 9);
   REQUIRE(schema->field(0)->type()->id() == arrow::float32()->id());
   REQUIRE(schema->field(1)->type()->id() == arrow::float32()->id());
   REQUIRE(schema->field(2)->type()->id() == arrow::float32()->id());
@@ -347,6 +436,9 @@ TEST_CASE("RootTree2Dataset")
   REQUIRE(schema->field(4)->type()->id() == arrow::int32()->id());
   REQUIRE(schema->field(5)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id());
   REQUIRE(schema->field(6)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id());
+  REQUIRE(schema->field(7)->type()->id() == arrow::boolean()->id());
+  REQUIRE(schema->field(8)->type()->id() == arrow::fixed_size_list(arrow::boolean(), 2)->id());
+
   auto fragment = format->MakeFragment(source, {}, schema);
   REQUIRE(fragment.ok());
   auto options = std::make_shared<arrow::dataset::ScanOptions>();
@@ -356,7 +448,7 @@ TEST_CASE("RootTree2Dataset")
   auto batches = (*scanner)();
   auto result = batches.result();
   REQUIRE(result.ok());
-  REQUIRE((*result)->columns().size() == 7);
+  REQUIRE((*result)->columns().size() == 9);
   REQUIRE((*result)->num_rows() == 100);
 
   {
@@ -394,14 +486,16 @@ TEST_CASE("RootTree2Dataset")
 
   auto* output = new TMemFile("foo", "RECREATE");
   auto outFs = std::make_shared<TFileFileSystem>(output, 0);
-  arrow::fs::FileLocator locator{outFs, "/DF_3"};
 
-  auto destination = outFs->OpenOutputStream(locator.path, {});
+  // Open a stream at toplevel
+  auto destination = outFs->OpenOutputStream("/", {});
   REQUIRE(destination.ok());
 
+  // Write to the /DF_3 tree at top level
+  arrow::fs::FileLocator locator{outFs, "/DF_3"};
   auto writer = format->MakeWriter(*destination, schema, {}, locator);
   auto success = writer->get()->Write(*result);
-  auto rootDestination = std::dynamic_pointer_cast<TTreeOutputStream>(*destination);
+  auto rootDestination = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(*destination);
 
   REQUIRE(success.ok());
   // Let's read it back...
@@ -413,14 +507,7 @@ TEST_CASE("RootTree2Dataset")
   auto schemaOptWritten = format->Inspect(source);
   REQUIRE(schemaOptWritten.ok());
   auto schemaWritten = *schemaOptWritten;
-  REQUIRE(schemaWritten->num_fields() == 7);
-  REQUIRE(schemaWritten->field(0)->type()->id() == arrow::float32()->id());
-  REQUIRE(schemaWritten->field(1)->type()->id() == arrow::float32()->id());
-  REQUIRE(schemaWritten->field(2)->type()->id() == arrow::float32()->id());
-  REQUIRE(schemaWritten->field(3)->type()->id() == arrow::float64()->id());
-  REQUIRE(schemaWritten->field(4)->type()->id() == arrow::int32()->id());
-  REQUIRE(schemaWritten->field(5)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id());
-  REQUIRE(schemaWritten->field(6)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id());
+  REQUIRE(validateSchema(schemaWritten));
 
   auto fragmentWritten = format->MakeFragment(source, {}, schema);
   REQUIRE(fragmentWritten.ok());
@@ -431,39 +518,10 @@ TEST_CASE("RootTree2Dataset")
   auto batchesWritten = (*scanner)();
   auto resultWritten = batches.result();
   REQUIRE(resultWritten.ok());
-  REQUIRE((*resultWritten)->columns().size() == 7);
+  REQUIRE((*resultWritten)->columns().size() == 9);
   REQUIRE((*resultWritten)->num_rows() == 100);
+  validateContents(*resultWritten);
 
   {
-    auto int_array = std::static_pointer_cast<arrow::Int32Array>((*resultWritten)->GetColumnByName("ev"));
-    for (int64_t j = 0; j < int_array->length(); j++) {
-      REQUIRE(int_array->Value(j) == j + 1);
-    }
-  }
-
-  {
-    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>((*result)->GetColumnByName("xyz"));
-
-    // Iterate over the FixedSizeListArray
-    for (int64_t i = 0; i < list_array->length(); i++) {
-      auto value_slice = list_array->value_slice(i);
-      auto float_array = std::static_pointer_cast<arrow::FloatArray>(value_slice);
-
-      REQUIRE(float_array->Value(0) == 1);
-      REQUIRE(float_array->Value(1) == 2);
-      REQUIRE(float_array->Value(2) == i + 1);
-    }
-  }
-
-  {
-    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>((*result)->GetColumnByName("ij"));
-
-    // Iterate over the FixedSizeListArray
-    for (int64_t i = 0; i < list_array->length(); i++) {
-      auto value_slice = list_array->value_slice(i);
-      auto int_array = std::static_pointer_cast<arrow::Int32Array>(value_slice);
-      REQUIRE(int_array->Value(0) == i);
-      REQUIRE(int_array->Value(1) == i + 1);
-    }
   }
 }

From cc0b2f870a6f500f4c27a9c5d7e93dbd610a9020 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Sun, 24 Nov 2024 11:54:28 +0100
Subject: [PATCH 711/726] DPL: improve arrow::Dataset integration

- Modularise filesystem to allow easier navigation and support for
  multiple formats.
- Add initial support to multiplex multiple tables on top of the same tree.
- Improve support for writing boolean fields.
---
 .../include/Framework/RootArrowFilesystem.h   |  36 ++++-
 Framework/Core/src/RootArrowFilesystem.cxx    | 153 +++++++++++++-----
 Framework/Core/test/test_Root2ArrowTable.cxx  | 146 ++++++++++++-----
 3 files changed, 250 insertions(+), 85 deletions(-)

diff --git a/Framework/Core/include/Framework/RootArrowFilesystem.h b/Framework/Core/include/Framework/RootArrowFilesystem.h
index 7c8385ccd2b9d..48d817bc9ddf2 100644
--- a/Framework/Core/include/Framework/RootArrowFilesystem.h
+++ b/Framework/Core/include/Framework/RootArrowFilesystem.h
@@ -17,6 +17,8 @@
 #include <arrow/type_fwd.h>
 #include <memory>
 
+class TFile;
+class TBranch;
 class TTree;
 class TBufferFile;
 class TDirectoryFile;
@@ -227,11 +229,38 @@ class TTreeFileFormat : public arrow::dataset::FileFormat
     const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const override;
 };
 
-// An arrow outputstream which allows to write to a ttree
+// An arrow outputstream which allows to write to a TDirectoryFile.
+// This will point to the location of the file itself. You can
+// specify the location of the actual object inside it by passing the
+// associated path to the Write() API.
+class TDirectoryFileOutputStream : public arrow::io::OutputStream
+{
+ public:
+  TDirectoryFileOutputStream(TDirectoryFile*);
+
+  arrow::Status Close() override;
+
+  arrow::Result<int64_t> Tell() const override;
+
+  arrow::Status Write(const void* data, int64_t nbytes) override;
+
+  bool closed() const override;
+
+  TDirectoryFile* GetDirectory()
+  {
+    return mDirectory;
+  }
+
+ private:
+  TDirectoryFile* mDirectory;
+};
+
+// An arrow outputstream which allows to write to a TTree. Eventually
+// with a prefix for the branches.
 class TTreeOutputStream : public arrow::io::OutputStream
 {
  public:
-  TTreeOutputStream(TTree* t);
+  TTreeOutputStream(TTree*, std::string branchPrefix);
 
   arrow::Status Close() override;
 
@@ -241,6 +270,8 @@ class TTreeOutputStream : public arrow::io::OutputStream
 
   bool closed() const override;
 
+  TBranch* CreateBranch(char const* branchName, char const* sizeBranch);
+
   TTree* GetTree()
   {
     return mTree;
@@ -248,6 +279,7 @@ class TTreeOutputStream : public arrow::io::OutputStream
 
  private:
   TTree* mTree;
+  std::string mBranchPrefix;
 };
 
 } // namespace o2::framework
diff --git a/Framework/Core/src/RootArrowFilesystem.cxx b/Framework/Core/src/RootArrowFilesystem.cxx
index 7581ee57e5b9f..7e331814272a6 100644
--- a/Framework/Core/src/RootArrowFilesystem.cxx
+++ b/Framework/Core/src/RootArrowFilesystem.cxx
@@ -17,7 +17,6 @@
 #include <arrow/array/builder_nested.h>
 #include <arrow/array/builder_primitive.h>
 #include <memory>
-#include <stdexcept>
 #include <TFile.h>
 #include <TLeaf.h>
 #include <TBufferFile.h>
@@ -28,8 +27,11 @@
 #include <arrow/dataset/file_base.h>
 #include <arrow/result.h>
 #include <arrow/status.h>
+#include <arrow/util/key_value_metadata.h>
 #include <fmt/format.h>
 
+#include <stdexcept>
+#include <utility>
 
 O2_DECLARE_DYNAMIC_LOG(root_arrow_fs);
 
@@ -100,7 +102,6 @@ std::shared_ptr<VirtualRootFileSystemBase> TFileFileSystem::GetSubFilesystem(arr
     return std::shared_ptr<VirtualRootFileSystemBase>(new SingleTreeFileSystem(tree));
   }
 
-
   auto directory = (TDirectoryFile*)mFile->GetObjectChecked(source.path().c_str(), TClass::GetClass<TDirectory>());
   if (directory) {
     return std::shared_ptr<VirtualRootFileSystemBase>(new TFileFileSystem(directory, 50 * 1024 * 1024));
@@ -129,8 +130,15 @@ arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TFileFileSystem::OpenOut
   const std::string& path,
   const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
 {
-  auto* t = new TTree(path.c_str(), "should put a name here");
-  auto stream = std::make_shared<TTreeOutputStream>(t);
+  if (path == "/") {
+    return std::make_shared<TDirectoryFileOutputStream>(this->GetFile());
+  }
+
+  auto* dir = dynamic_cast<TDirectoryFile*>(this->GetFile()->Get(path.c_str()));
+  if (!dir) {
+    throw runtime_error_f("Unable to open directory %s in file %s", path.c_str(), GetFile()->GetName());
+  }
+  auto stream = std::make_shared<TDirectoryFileOutputStream>(dir);
   return stream;
 }
 
@@ -286,13 +294,46 @@ arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> TTreeFileFormat::Ma
 }
 
 // An arrow outputstream which allows to write to a ttree
-TTreeOutputStream::TTreeOutputStream(TTree* t)
-  : mTree(t)
+TDirectoryFileOutputStream::TDirectoryFileOutputStream(TDirectoryFile* f)
+  : mDirectory(f)
+{
+}
+
+arrow::Status TDirectoryFileOutputStream::Close()
+{
+  mDirectory->GetFile()->Close();
+  return arrow::Status::OK();
+}
+
+arrow::Result<int64_t> TDirectoryFileOutputStream::Tell() const
+{
+  return arrow::Result<int64_t>(arrow::Status::NotImplemented("Cannot move"));
+}
+
+arrow::Status TDirectoryFileOutputStream::Write(const void* data, int64_t nbytes)
+{
+  return arrow::Status::NotImplemented("Cannot write raw bytes to a TTree");
+}
+
+bool TDirectoryFileOutputStream::closed() const
+{
+  return mDirectory->GetFile()->IsOpen() == false;
+}
+
+// An arrow outputstream which allows to write to a ttree
+// @a branch prefix is to be used to identify a set of branches which all belong to
+// the same table.
+TTreeOutputStream::TTreeOutputStream(TTree* f, std::string branchPrefix)
+  : mTree(f),
+    mBranchPrefix(std::move(branchPrefix))
 {
 }
 
 arrow::Status TTreeOutputStream::Close()
 {
+  if (mTree->GetCurrentFile() == nullptr) {
+    return arrow::Status::Invalid("Cannot close a tree not attached to a file");
+  }
   mTree->GetCurrentFile()->Close();
   return arrow::Status::OK();
 }
@@ -309,9 +350,18 @@ arrow::Status TTreeOutputStream::Write(const void* data, int64_t nbytes)
 
 bool TTreeOutputStream::closed() const
 {
+  // A standalone tree is never closed.
+  if (mTree->GetCurrentFile() == nullptr) {
+    return false;
+  }
   return mTree->GetCurrentFile()->IsOpen() == false;
 }
 
+TBranch* TTreeOutputStream::CreateBranch(char const* branchName, char const* sizeBranch)
+{
+  return mTree->Branch((mBranchPrefix + "/" + branchName).c_str(), (char*)nullptr, (mBranchPrefix + sizeBranch).c_str());
+}
+
 char const* rootSuffixFromArrow(arrow::Type::type id)
 {
   switch (id) {
@@ -411,8 +461,24 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
     : FileWriter(schema, options, destination, destination_locator)
   {
     // Batches have the same number of entries for each column.
+    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
     auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
-    TTree* tree = treeStream->GetTree();
+
+    if (directoryStream.get()) {
+      TDirectoryFile* dir = directoryStream->GetDirectory();
+      dir->cd();
+      auto* tree = new TTree(destination_locator_.path.c_str(), "");
+      treeStream = std::make_shared<TTreeOutputStream>(tree, "");
+    } else if (treeStream.get()) {
+      // We already have a tree stream, let's derive a new one
+      // with the destination_locator_.path as prefix for the branches
+      // This way we can multiplex multiple tables in the same tree.
+      auto tree = treeStream->GetTree();
+      treeStream = std::make_shared<TTreeOutputStream>(tree, destination_locator_.path);
+    } else {
+      // I could simply set a prefix here to merge to an already existing tree.
+      throw std::runtime_error("Unsupported backend.");
+    }
 
     for (auto i = 0u; i < schema->fields().size(); ++i) {
       auto& field = schema->field(i);
@@ -427,15 +493,15 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
           valueTypes.push_back(field->type()->field(0)->type());
           sizesBranches.push_back(nullptr);
           std::string leafList = fmt::format("{}[{}]{}", field->name(), listSizes.back(), rootSuffixFromArrow(valueTypes.back()->id()));
-          branches.push_back(tree->Branch(field->name().c_str(), (char*)nullptr, leafList.c_str()));
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
         } break;
         case arrow::Type::LIST: {
           valueTypes.push_back(field->type()->field(0)->type());
           listSizes.back() = 0; // VLA, we need to calculate it on the fly;
           std::string leafList = fmt::format("{}[{}_size]{}", field->name(), field->name(), rootSuffixFromArrow(valueTypes.back()->id()));
           std::string sizeLeafList = field->name() + "_size/I";
-          sizesBranches.push_back(tree->Branch((field->name() + "_size").c_str(), (char*)nullptr, sizeLeafList.c_str()));
-          branches.push_back(tree->Branch(field->name().c_str(), (char*)nullptr, leafList.c_str()));
+          sizesBranches.push_back(treeStream->CreateBranch((field->name() + "_size").c_str(), sizeLeafList.c_str()));
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
           // Notice that this could be replaced by a better guess of the
           // average size of the list elements, but this is not trivial.
         } break;
@@ -443,7 +509,7 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
           valueTypes.push_back(field->type());
           std::string leafList = field->name() + rootSuffixFromArrow(valueTypes.back()->id());
           sizesBranches.push_back(nullptr);
-          branches.push_back(tree->Branch(field->name().c_str(), (char*)nullptr, leafList.c_str()));
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
         } break;
       }
     }
@@ -463,11 +529,18 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
     }
 
     // Batches have the same number of entries for each column.
+    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
+    TTree* tree = nullptr;
+    if (directoryStream.get()) {
+      TDirectoryFile* dir = directoryStream->GetDirectory();
+      tree = (TTree*)dir->Get(destination_locator_.path.c_str());
+    }
     auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
-    TTree* tree = treeStream->GetTree();
 
-    // Caches for the vectors of bools.
-    std::vector<std::shared_ptr<arrow::UInt8Array>> caches;
+    if (!tree) {
+      // I could simply set a prefix here to merge to an already existing tree.
+      throw std::runtime_error("Unsupported backend.");
+    }
 
     for (auto i = 0u; i < batch->columns().size(); ++i) {
       auto column = batch->column(i);
@@ -484,24 +557,11 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
           auto list = std::static_pointer_cast<arrow::ListArray>(column);
           valueArrays.back() = list;
         } break;
-        default:
-          valueArrays.back() = column;
-      }
-    }
-
-    int64_t pos = 0;
-    while (pos < batch->num_rows()) {
-      for (size_t bi = 0; bi < branches.size(); ++bi) {
-        auto* branch = branches[bi];
-        auto* sizeBranch = sizesBranches[bi];
-        auto array = batch->column(bi);
-        auto& field = batch->schema()->field(bi);
-        auto& listSize = listSizes[bi];
-        auto valueType = valueTypes[bi];
-        auto valueArray = valueArrays[bi];
+        case arrow::Type::BOOL: {
+          // In case of arrays of booleans, we need to go back to their
+          // char based representation for ROOT to save them.
+          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(column);
 
-        if (field->type()->id() == arrow::Type::BOOL) {
-          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(array);
           int64_t length = boolArray->length();
           arrow::UInt8Builder builder;
           auto ok = builder.Reserve(length);
@@ -516,11 +576,24 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
               auto ok = builder.AppendNull();
             }
           }
+          valueArrays.back() = *builder.Finish();
+        } break;
+        default:
+          valueArrays.back() = column;
+      }
+    }
+
+    int64_t pos = 0;
+    while (pos < batch->num_rows()) {
+      for (size_t bi = 0; bi < branches.size(); ++bi) {
+        auto* branch = branches[bi];
+        auto* sizeBranch = sizesBranches[bi];
+        auto array = batch->column(bi);
+        auto& field = batch->schema()->field(bi);
+        auto& listSize = listSizes[bi];
+        auto valueType = valueTypes[bi];
+        auto valueArray = valueArrays[bi];
 
-          ok = builder.Finish(&caches[bi]);
-          branch->SetAddress((void*)(caches[bi]->values()->data()));
-          continue;
-        }
         switch (field->type()->id()) {
           case arrow::Type::LIST: {
             auto list = std::static_pointer_cast<arrow::ListArray>(array);
@@ -764,13 +837,16 @@ arrow::Result<arrow::RecordBatchGenerator> TTreeFileFormat::ScanBatchesAsync(
   return generator;
 }
 
-
 arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenOutputStream(
   const std::string& path,
   const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
 {
-  auto stream = std::make_shared<TTreeOutputStream>(GetTree({path, shared_from_this()}));
-  return stream;
+  arrow::dataset::FileSource source{path, shared_from_this()};
+  auto prefix = metadata->Get("branch_prefix");
+  if (prefix.ok()) {
+    return std::make_shared<TTreeOutputStream>(GetTree(source), *prefix);
+  }
+  return std::make_shared<TTreeOutputStream>(GetTree(source), "");
 }
 
 TBufferFileFS::TBufferFileFS(TBufferFile* f)
@@ -782,7 +858,6 @@ TBufferFileFS::TBufferFileFS(TBufferFile* f)
 
 TTreeFileSystem::~TTreeFileSystem() = default;
 
-
 arrow::Result<arrow::fs::FileInfo> TBufferFileFS::GetFileInfo(const std::string& path)
 {
   arrow::fs::FileInfo result;
diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx
index 03f0977a4c0c4..a659d488ae24a 100644
--- a/Framework/Core/test/test_Root2ArrowTable.cxx
+++ b/Framework/Core/test/test_Root2ArrowTable.cxx
@@ -20,13 +20,18 @@
 #include <ROOT/RArrowDS.hxx>
 #include <TBufferFile.h>
 #include <TClass.h>
+#include <TDirectoryFile.h>
 #include <TMemFile.h>
 #include <TDirectory.h>
 #include <TTree.h>
 #include <TRandom.h>
 #include <TFile.h>
+#include <memory>
 
+#include <arrow/array/array_primitive.h>
+#include <arrow/array/builder_primitive.h>
 #include <arrow/dataset/scanner.h>
+#include <arrow/record_batch.h>
 #include <arrow/table.h>
 #include <arrow/ipc/writer.h>
 #include <arrow/io/memory.h>
@@ -259,6 +264,82 @@ TEST_CASE("RootTree2Fragment")
   REQUIRE((*result)->num_rows() == 1000);
 }
 
+bool validateContents(std::shared_ptr<arrow::RecordBatch> batch)
+{
+  {
+    auto int_array = std::static_pointer_cast<arrow::Int32Array>(batch->GetColumnByName("ev"));
+    REQUIRE(int_array->length() == 100);
+    for (int64_t j = 0; j < int_array->length(); j++) {
+      REQUIRE(int_array->Value(j) == j + 1);
+    }
+  }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>(batch->GetColumnByName("xyz"));
+
+    REQUIRE(list_array->length() == 100);
+    // Iterate over the FixedSizeListArray
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      auto float_array = std::static_pointer_cast<arrow::FloatArray>(value_slice);
+
+      REQUIRE(float_array->Value(0) == 1);
+      REQUIRE(float_array->Value(1) == 2);
+      REQUIRE(float_array->Value(2) == i + 1);
+    }
+  }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>(batch->GetColumnByName("ij"));
+
+    REQUIRE(list_array->length() == 100);
+    // Iterate over the FixedSizeListArray
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      auto int_array = std::static_pointer_cast<arrow::Int32Array>(value_slice);
+      REQUIRE(int_array->Value(0) == i);
+      REQUIRE(int_array->Value(1) == i + 1);
+    }
+  }
+
+  {
+    auto bool_array = std::static_pointer_cast<arrow::BooleanArray>(batch->GetColumnByName("bools"));
+
+    REQUIRE(bool_array->length() == 100);
+    for (int64_t j = 0; j < bool_array->length(); j++) {
+      REQUIRE(bool_array->Value(j) == (j % 3 == 0));
+    }
+  }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>(batch->GetColumnByName("manyBools"));
+
+    REQUIRE(list_array->length() == 100);
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      auto bool_array = std::static_pointer_cast<arrow::BooleanArray>(value_slice);
+      REQUIRE(bool_array->Value(0) == (i % 4 == 0));
+      REQUIRE(bool_array->Value(1) == (i % 5 == 0));
+    }
+  }
+  return true;
+}
+
+bool validateSchema(std::shared_ptr<arrow::Schema> schema)
+{
+  REQUIRE(schema->num_fields() == 9);
+  REQUIRE(schema->field(0)->type()->id() == arrow::float32()->id());
+  REQUIRE(schema->field(1)->type()->id() == arrow::float32()->id());
+  REQUIRE(schema->field(2)->type()->id() == arrow::float32()->id());
+  REQUIRE(schema->field(3)->type()->id() == arrow::float64()->id());
+  REQUIRE(schema->field(4)->type()->id() == arrow::int32()->id());
+  REQUIRE(schema->field(5)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id());
+  REQUIRE(schema->field(6)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id());
+  REQUIRE(schema->field(7)->type()->id() == arrow::boolean()->id());
+  REQUIRE(schema->field(8)->type()->id() == arrow::fixed_size_list(arrow::boolean(), 2)->id());
+  return true;
+}
+
 TEST_CASE("RootTree2Dataset")
 {
   using namespace o2::framework;
@@ -307,6 +388,9 @@ TEST_CASE("RootTree2Dataset")
     Float_t px = 0, py = 1, pz = 2;
     Double_t random;
     Int_t ev;
+    bool oneBool;
+    bool manyBool[2];
+
     t->Branch("px", &px, "px/F");
     t->Branch("py", &py, "py/F");
     t->Branch("pz", &pz, "pz/F");
@@ -314,6 +398,8 @@ TEST_CASE("RootTree2Dataset")
     t->Branch("ev", &ev, "ev/I");
     t->Branch("xyz", xyz, "xyz[3]/F");
     t->Branch("ij", ij, "ij[2]/I");
+    t->Branch("bools", &oneBool, "bools/O");
+    t->Branch("manyBools", &manyBool, "manyBools[2]/O");
     // fill the tree
     for (Int_t i = 0; i < 100; i++) {
       xyz[0] = 1;
@@ -326,6 +412,9 @@ TEST_CASE("RootTree2Dataset")
       ij[1] = i + 1;
       random = gRandom->Rndm();
       ev = i + 1;
+      oneBool = (i % 3 == 0);
+      manyBool[0] = (i % 4 == 0);
+      manyBool[1] = (i % 5 == 0);
       t->Fill();
     }
   }
@@ -339,7 +428,7 @@ TEST_CASE("RootTree2Dataset")
   auto schemaOpt = format->Inspect(source);
   REQUIRE(schemaOpt.ok());
   auto schema = *schemaOpt;
-  REQUIRE(schema->num_fields() == 7);
+  REQUIRE(schema->num_fields() == 9);
   REQUIRE(schema->field(0)->type()->id() == arrow::float32()->id());
   REQUIRE(schema->field(1)->type()->id() == arrow::float32()->id());
   REQUIRE(schema->field(2)->type()->id() == arrow::float32()->id());
@@ -347,6 +436,9 @@ TEST_CASE("RootTree2Dataset")
   REQUIRE(schema->field(4)->type()->id() == arrow::int32()->id());
   REQUIRE(schema->field(5)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id());
   REQUIRE(schema->field(6)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id());
+  REQUIRE(schema->field(7)->type()->id() == arrow::boolean()->id());
+  REQUIRE(schema->field(8)->type()->id() == arrow::fixed_size_list(arrow::boolean(), 2)->id());
+
   auto fragment = format->MakeFragment(source, {}, schema);
   REQUIRE(fragment.ok());
   auto options = std::make_shared<arrow::dataset::ScanOptions>();
@@ -356,7 +448,7 @@ TEST_CASE("RootTree2Dataset")
   auto batches = (*scanner)();
   auto result = batches.result();
   REQUIRE(result.ok());
-  REQUIRE((*result)->columns().size() == 7);
+  REQUIRE((*result)->columns().size() == 9);
   REQUIRE((*result)->num_rows() == 100);
 
   {
@@ -394,14 +486,16 @@ TEST_CASE("RootTree2Dataset")
 
   auto* output = new TMemFile("foo", "RECREATE");
   auto outFs = std::make_shared<TFileFileSystem>(output, 0);
-  arrow::fs::FileLocator locator{outFs, "/DF_3"};
 
-  auto destination = outFs->OpenOutputStream(locator.path, {});
+  // Open a stream at toplevel
+  auto destination = outFs->OpenOutputStream("/", {});
   REQUIRE(destination.ok());
 
+  // Write to the /DF_3 tree at top level
+  arrow::fs::FileLocator locator{outFs, "/DF_3"};
   auto writer = format->MakeWriter(*destination, schema, {}, locator);
   auto success = writer->get()->Write(*result);
-  auto rootDestination = std::dynamic_pointer_cast<TTreeOutputStream>(*destination);
+  auto rootDestination = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(*destination);
 
   REQUIRE(success.ok());
   // Let's read it back...
@@ -413,14 +507,7 @@ TEST_CASE("RootTree2Dataset")
   auto schemaOptWritten = format->Inspect(source);
   REQUIRE(schemaOptWritten.ok());
   auto schemaWritten = *schemaOptWritten;
-  REQUIRE(schemaWritten->num_fields() == 7);
-  REQUIRE(schemaWritten->field(0)->type()->id() == arrow::float32()->id());
-  REQUIRE(schemaWritten->field(1)->type()->id() == arrow::float32()->id());
-  REQUIRE(schemaWritten->field(2)->type()->id() == arrow::float32()->id());
-  REQUIRE(schemaWritten->field(3)->type()->id() == arrow::float64()->id());
-  REQUIRE(schemaWritten->field(4)->type()->id() == arrow::int32()->id());
-  REQUIRE(schemaWritten->field(5)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id());
-  REQUIRE(schemaWritten->field(6)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id());
+  REQUIRE(validateSchema(schemaWritten));
 
   auto fragmentWritten = format->MakeFragment(source, {}, schema);
   REQUIRE(fragmentWritten.ok());
@@ -431,39 +518,10 @@ TEST_CASE("RootTree2Dataset")
   auto batchesWritten = (*scanner)();
   auto resultWritten = batches.result();
   REQUIRE(resultWritten.ok());
-  REQUIRE((*resultWritten)->columns().size() == 7);
+  REQUIRE((*resultWritten)->columns().size() == 9);
   REQUIRE((*resultWritten)->num_rows() == 100);
+  validateContents(*resultWritten);
 
   {
-    auto int_array = std::static_pointer_cast<arrow::Int32Array>((*resultWritten)->GetColumnByName("ev"));
-    for (int64_t j = 0; j < int_array->length(); j++) {
-      REQUIRE(int_array->Value(j) == j + 1);
-    }
-  }
-
-  {
-    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>((*result)->GetColumnByName("xyz"));
-
-    // Iterate over the FixedSizeListArray
-    for (int64_t i = 0; i < list_array->length(); i++) {
-      auto value_slice = list_array->value_slice(i);
-      auto float_array = std::static_pointer_cast<arrow::FloatArray>(value_slice);
-
-      REQUIRE(float_array->Value(0) == 1);
-      REQUIRE(float_array->Value(1) == 2);
-      REQUIRE(float_array->Value(2) == i + 1);
-    }
-  }
-
-  {
-    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>((*result)->GetColumnByName("ij"));
-
-    // Iterate over the FixedSizeListArray
-    for (int64_t i = 0; i < list_array->length(); i++) {
-      auto value_slice = list_array->value_slice(i);
-      auto int_array = std::static_pointer_cast<arrow::Int32Array>(value_slice);
-      REQUIRE(int_array->Value(0) == i);
-      REQUIRE(int_array->Value(1) == i + 1);
-    }
   }
 }

From acbee814a787da79be6d5ac6e9590b9b8699c2c8 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 25 Nov 2024 17:06:03 +0100
Subject: [PATCH 712/726] DPL: add test for exception throwing in Variant

---
 Framework/Core/include/Framework/Variant.h |  2 +-
 Framework/Core/test/test_Variants.cxx      | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/include/Framework/Variant.h b/Framework/Core/include/Framework/Variant.h
index 54a91e90c3638..e69ca05b91d98 100644
--- a/Framework/Core/include/Framework/Variant.h
+++ b/Framework/Core/include/Framework/Variant.h
@@ -355,7 +355,7 @@ class Variant
   T get() const
   {
     if (mType != variant_trait_v<T>) {
-      throw runtime_error("Mismatch between types");
+      throw runtime_error_f("Variant::get: Mismatch between types %d %d.", mType, variant_trait_v<T>);
     }
     return variant_helper<T>::get(&mStore);
   }
diff --git a/Framework/Core/test/test_Variants.cxx b/Framework/Core/test/test_Variants.cxx
index a0edf40cf9f0d..a364b228871da 100644
--- a/Framework/Core/test/test_Variants.cxx
+++ b/Framework/Core/test/test_Variants.cxx
@@ -338,3 +338,15 @@ TEST_CASE("VariantJSONConversionsTest")
     REQUIRE(vstrings[i] == vvstra.get<std::string*>()[i]);
   }
 }
+
+TEST_CASE("VariantThrowing")
+{
+  Variant a("true");
+  REQUIRE_THROWS_AS(a.get<int>(), o2::framework::RuntimeErrorRef);
+  try {
+    a.get<int>();
+  } catch (RuntimeErrorRef& ref) {
+    RuntimeError& error = error_from_ref(ref);
+    REQUIRE(error.what == std::string("Variant::get: Mismatch between types 4 0."));
+  }
+}

From e1ff2ab51590a43c1347140869aab64cf328c8ee Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 2 Dec 2024 13:06:39 +0100
Subject: [PATCH 713/726] DPL Analysis: improve arrow::Dataset support for
 TTree

---
 Framework/Core/src/RootArrowFilesystem.cxx   |   7 +-
 Framework/Core/test/test_Root2ArrowTable.cxx | 120 ++++++++-----------
 2 files changed, 54 insertions(+), 73 deletions(-)

diff --git a/Framework/Core/src/RootArrowFilesystem.cxx b/Framework/Core/src/RootArrowFilesystem.cxx
index 7e331814272a6..5f2d21d942d37 100644
--- a/Framework/Core/src/RootArrowFilesystem.cxx
+++ b/Framework/Core/src/RootArrowFilesystem.cxx
@@ -13,6 +13,7 @@
 #include "Framework/RuntimeError.h"
 #include "Framework/Signpost.h"
 #include <Rtypes.h>
+#include <arrow/array/array_nested.h>
 #include <arrow/array/array_primitive.h>
 #include <arrow/array/builder_nested.h>
 #include <arrow/array/builder_primitive.h>
@@ -427,7 +428,7 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
         O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s exists and uses %d bytes per entry.",
                                branch->GetName(), valueSize);
         // This should probably lookup the
-        auto column = firstBatch->GetColumnByName(branch->GetName());
+        auto column = firstBatch->GetColumnByName(schema_->field(i)->name());
         auto list = std::static_pointer_cast<arrow::ListArray>(column);
         O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s needed. Associated size branch %s and there are %lli entries of size %d in that list.",
                                branch->GetName(), sizeBranch->GetName(), list->length(), valueSize);
@@ -497,8 +498,8 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
         } break;
         case arrow::Type::LIST: {
           valueTypes.push_back(field->type()->field(0)->type());
-          listSizes.back() = 0; // VLA, we need to calculate it on the fly;
           std::string leafList = fmt::format("{}[{}_size]{}", field->name(), field->name(), rootSuffixFromArrow(valueTypes.back()->id()));
+          listSizes.back() = -1; // VLA, we need to calculate it on the fly;
           std::string sizeLeafList = field->name() + "_size/I";
           sizesBranches.push_back(treeStream->CreateBranch((field->name() + "_size").c_str(), sizeLeafList.c_str()));
           branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
@@ -765,7 +766,7 @@ arrow::Result<arrow::RecordBatchGenerator> TTreeFileFormat::ScanBatchesAsync(
           typeSize = fixedSizeList->field(0)->type()->byte_width();
         } else if (auto vlaListType = std::dynamic_pointer_cast<arrow::ListType>(physicalField->type())) {
           listSize = -1;
-          typeSize = fixedSizeList->field(0)->type()->byte_width();
+          typeSize = vlaListType->field(0)->type()->byte_width();
         }
         if (listSize == -1) {
           mSizeBranch = branch->GetTree()->GetBranch((std::string{branch->GetName()} + "_size").c_str());
diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx
index a659d488ae24a..2b0ab9154250c 100644
--- a/Framework/Core/test/test_Root2ArrowTable.cxx
+++ b/Framework/Core/test/test_Root2ArrowTable.cxx
@@ -322,12 +322,26 @@ bool validateContents(std::shared_ptr<arrow::RecordBatch> batch)
       REQUIRE(bool_array->Value(1) == (i % 5 == 0));
     }
   }
+
+  {
+    auto list_array = std::static_pointer_cast<arrow::ListArray>(batch->GetColumnByName("vla"));
+
+    REQUIRE(list_array->length() == 100);
+    for (int64_t i = 0; i < list_array->length(); i++) {
+      auto value_slice = list_array->value_slice(i);
+      REQUIRE(value_slice->length() == (i % 10));
+      auto int_array = std::static_pointer_cast<arrow::Int32Array>(value_slice);
+      for (size_t j = 0; j < value_slice->length(); j++) {
+        REQUIRE(int_array->Value(j) == j);
+      }
+    }
+  }
   return true;
 }
 
 bool validateSchema(std::shared_ptr<arrow::Schema> schema)
 {
-  REQUIRE(schema->num_fields() == 9);
+  REQUIRE(schema->num_fields() == 10);
   REQUIRE(schema->field(0)->type()->id() == arrow::float32()->id());
   REQUIRE(schema->field(1)->type()->id() == arrow::float32()->id());
   REQUIRE(schema->field(2)->type()->id() == arrow::float32()->id());
@@ -337,6 +351,7 @@ bool validateSchema(std::shared_ptr<arrow::Schema> schema)
   REQUIRE(schema->field(6)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id());
   REQUIRE(schema->field(7)->type()->id() == arrow::boolean()->id());
   REQUIRE(schema->field(8)->type()->id() == arrow::fixed_size_list(arrow::boolean(), 2)->id());
+  REQUIRE(schema->field(9)->type()->id() == arrow::list(arrow::int32())->id());
   return true;
 }
 
@@ -390,6 +405,8 @@ TEST_CASE("RootTree2Dataset")
     Int_t ev;
     bool oneBool;
     bool manyBool[2];
+    int vla[10] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    int vlaSize = 0;
 
     t->Branch("px", &px, "px/F");
     t->Branch("py", &py, "py/F");
@@ -400,6 +417,8 @@ TEST_CASE("RootTree2Dataset")
     t->Branch("ij", ij, "ij[2]/I");
     t->Branch("bools", &oneBool, "bools/O");
     t->Branch("manyBools", &manyBool, "manyBools[2]/O");
+    t->Branch("vla_size", &vlaSize, "vla_size/I");
+    t->Branch("vla", vla, "vla[vla_size]/I");
     // fill the tree
     for (Int_t i = 0; i < 100; i++) {
       xyz[0] = 1;
@@ -415,9 +434,11 @@ TEST_CASE("RootTree2Dataset")
       oneBool = (i % 3 == 0);
       manyBool[0] = (i % 4 == 0);
       manyBool[1] = (i % 5 == 0);
+      vlaSize = i % 10;
       t->Fill();
     }
   }
+  f->Write();
 
   size_t totalSizeCompressed = 0;
   size_t totalSizeUncompressed = 0;
@@ -428,16 +449,7 @@ TEST_CASE("RootTree2Dataset")
   auto schemaOpt = format->Inspect(source);
   REQUIRE(schemaOpt.ok());
   auto schema = *schemaOpt;
-  REQUIRE(schema->num_fields() == 9);
-  REQUIRE(schema->field(0)->type()->id() == arrow::float32()->id());
-  REQUIRE(schema->field(1)->type()->id() == arrow::float32()->id());
-  REQUIRE(schema->field(2)->type()->id() == arrow::float32()->id());
-  REQUIRE(schema->field(3)->type()->id() == arrow::float64()->id());
-  REQUIRE(schema->field(4)->type()->id() == arrow::int32()->id());
-  REQUIRE(schema->field(5)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id());
-  REQUIRE(schema->field(6)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id());
-  REQUIRE(schema->field(7)->type()->id() == arrow::boolean()->id());
-  REQUIRE(schema->field(8)->type()->id() == arrow::fixed_size_list(arrow::boolean(), 2)->id());
+  validateSchema(schema);
 
   auto fragment = format->MakeFragment(source, {}, schema);
   REQUIRE(fragment.ok());
@@ -448,41 +460,9 @@ TEST_CASE("RootTree2Dataset")
   auto batches = (*scanner)();
   auto result = batches.result();
   REQUIRE(result.ok());
-  REQUIRE((*result)->columns().size() == 9);
+  REQUIRE((*result)->columns().size() == 10);
   REQUIRE((*result)->num_rows() == 100);
-
-  {
-    auto int_array = std::static_pointer_cast<arrow::Int32Array>((*result)->GetColumnByName("ev"));
-    for (int64_t j = 0; j < int_array->length(); j++) {
-      REQUIRE(int_array->Value(j) == j + 1);
-    }
-  }
-
-  {
-    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>((*result)->GetColumnByName("xyz"));
-
-    // Iterate over the FixedSizeListArray
-    for (int64_t i = 0; i < list_array->length(); i++) {
-      auto value_slice = list_array->value_slice(i);
-      auto float_array = std::static_pointer_cast<arrow::FloatArray>(value_slice);
-
-      REQUIRE(float_array->Value(0) == 1);
-      REQUIRE(float_array->Value(1) == 2);
-      REQUIRE(float_array->Value(2) == i + 1);
-    }
-  }
-
-  {
-    auto list_array = std::static_pointer_cast<arrow::FixedSizeListArray>((*result)->GetColumnByName("ij"));
-
-    // Iterate over the FixedSizeListArray
-    for (int64_t i = 0; i < list_array->length(); i++) {
-      auto value_slice = list_array->value_slice(i);
-      auto int_array = std::static_pointer_cast<arrow::Int32Array>(value_slice);
-      REQUIRE(int_array->Value(0) == i);
-      REQUIRE(int_array->Value(1) == i + 1);
-    }
-  }
+  validateContents(*result);
 
   auto* output = new TMemFile("foo", "RECREATE");
   auto outFs = std::make_shared<TFileFileSystem>(output, 0);
@@ -497,31 +477,31 @@ TEST_CASE("RootTree2Dataset")
   auto success = writer->get()->Write(*result);
   auto rootDestination = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(*destination);
 
-  REQUIRE(success.ok());
-  // Let's read it back...
-  arrow::dataset::FileSource source2("/DF_3", outFs);
-  auto newTreeFS = outFs->GetSubFilesystem(source2);
-
-  REQUIRE(format->IsSupported(source) == true);
-
-  auto schemaOptWritten = format->Inspect(source);
-  REQUIRE(schemaOptWritten.ok());
-  auto schemaWritten = *schemaOptWritten;
-  REQUIRE(validateSchema(schemaWritten));
-
-  auto fragmentWritten = format->MakeFragment(source, {}, schema);
-  REQUIRE(fragmentWritten.ok());
-  auto optionsWritten = std::make_shared<arrow::dataset::ScanOptions>();
-  options->dataset_schema = schemaWritten;
-  auto scannerWritten = format->ScanBatchesAsync(optionsWritten, *fragment);
-  REQUIRE(scannerWritten.ok());
-  auto batchesWritten = (*scanner)();
-  auto resultWritten = batches.result();
-  REQUIRE(resultWritten.ok());
-  REQUIRE((*resultWritten)->columns().size() == 9);
-  REQUIRE((*resultWritten)->num_rows() == 100);
-  validateContents(*resultWritten);
-
+  SECTION("Read tree")
   {
+    REQUIRE(success.ok());
+    // Let's read it back...
+    arrow::dataset::FileSource source2("/DF_3", outFs);
+    auto newTreeFS = outFs->GetSubFilesystem(source2);
+
+    REQUIRE(format->IsSupported(source) == true);
+
+    auto schemaOptWritten = format->Inspect(source);
+    REQUIRE(schemaOptWritten.ok());
+    auto schemaWritten = *schemaOptWritten;
+    REQUIRE(validateSchema(schemaWritten));
+
+    auto fragmentWritten = format->MakeFragment(source, {}, schema);
+    REQUIRE(fragmentWritten.ok());
+    auto optionsWritten = std::make_shared<arrow::dataset::ScanOptions>();
+    options->dataset_schema = schemaWritten;
+    auto scannerWritten = format->ScanBatchesAsync(optionsWritten, *fragment);
+    REQUIRE(scannerWritten.ok());
+    auto batchesWritten = (*scanner)();
+    auto resultWritten = batches.result();
+    REQUIRE(resultWritten.ok());
+    REQUIRE((*resultWritten)->columns().size() == 10);
+    REQUIRE((*resultWritten)->num_rows() == 100);
+    validateContents(*resultWritten);
   }
 }

From edd0bad949b02a38728391971838c5bac2b3e52f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Mon, 2 Dec 2024 15:47:50 +0100
Subject: [PATCH 714/726] DPL Analysis: add RNTuple arrow::Dataset support

---
 Framework/Core/CMakeLists.txt                 |   3 +
 .../include/Framework/RootArrowFilesystem.h   | 115 +++
 Framework/Core/src/RootArrowFilesystem.cxx    | 653 +++++++++++++++++-
 Framework/Core/test/test_Root2ArrowTable.cxx  |  43 ++
 4 files changed, 813 insertions(+), 1 deletion(-)

diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index 02367afdcc556..d1dde7e78fdd1 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -159,6 +159,8 @@ o2_add_library(Framework
                                      FairMQ::FairMQ
                                      ROOT::Tree
                                      ROOT::Hist
+                                     ROOT::ROOTNTuple
+                                     ROOT::ROOTNTupleUtil
                                      O2::FrameworkFoundation
                                      O2::CommonConstants
                                      O2::Headers
@@ -298,6 +300,7 @@ add_executable(o2-test-framework-root
 target_link_libraries(o2-test-framework-root PRIVATE O2::Framework)
 target_link_libraries(o2-test-framework-root PRIVATE O2::Catch2)
 target_link_libraries(o2-test-framework-root PRIVATE ROOT::ROOTDataFrame)
+target_link_libraries(o2-test-framework-root PRIVATE ROOT::ROOTNTuple)
 set_property(TARGET o2-test-framework-root PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
 add_test(NAME framework:root COMMAND o2-test-framework-root --skip-benchmarks)
 add_test(NAME framework:crash COMMAND  sh -e -c "PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:$PATH ${CMAKE_CURRENT_LIST_DIR}/test/test_AllCrashTypes.sh")
diff --git a/Framework/Core/include/Framework/RootArrowFilesystem.h b/Framework/Core/include/Framework/RootArrowFilesystem.h
index 48d817bc9ddf2..234757ea596b4 100644
--- a/Framework/Core/include/Framework/RootArrowFilesystem.h
+++ b/Framework/Core/include/Framework/RootArrowFilesystem.h
@@ -23,6 +23,11 @@ class TTree;
 class TBufferFile;
 class TDirectoryFile;
 
+namespace ROOT::Experimental
+{
+class RNTuple;
+} // namespace ROOT::Experimental
+
 namespace o2::framework
 {
 
@@ -35,6 +40,15 @@ class TTreeFileWriteOptions : public arrow::dataset::FileWriteOptions
   }
 };
 
+class RNTupleFileWriteOptions : public arrow::dataset::FileWriteOptions
+{
+ public:
+  RNTupleFileWriteOptions(std::shared_ptr<arrow::dataset::FileFormat> format)
+    : FileWriteOptions(format)
+  {
+  }
+};
+
 // This is to avoid having to implement a bunch of unimplemented methods
 // for all the possible virtual filesystem we can invent on top of ROOT
 // data structures.
@@ -97,6 +111,19 @@ class TTreeFileSystem : public VirtualRootFileSystemBase
   virtual TTree* GetTree(arrow::dataset::FileSource source) = 0;
 };
 
+// A filesystem which allows me to get a RNTuple
+class RNTupleFileSystem : public VirtualRootFileSystemBase
+{
+ public:
+  ~RNTupleFileSystem() override;
+
+  std::shared_ptr<VirtualRootFileSystemBase> GetSubFilesystem(arrow::dataset::FileSource source) override
+  {
+    return std::dynamic_pointer_cast<VirtualRootFileSystemBase>(shared_from_this());
+  };
+  virtual ROOT::Experimental::RNTuple* GetRNTuple(arrow::dataset::FileSource source) = 0;
+};
+
 class SingleTreeFileSystem : public TTreeFileSystem
 {
  public:
@@ -121,6 +148,30 @@ class SingleTreeFileSystem : public TTreeFileSystem
   TTree* mTree;
 };
 
+class SingleRNTupleFileSystem : public RNTupleFileSystem
+{
+ public:
+  SingleRNTupleFileSystem(ROOT::Experimental::RNTuple* tuple)
+    : RNTupleFileSystem(),
+      mTuple(tuple)
+  {
+  }
+
+  std::string type_name() const override
+  {
+    return "rntuple";
+  }
+
+  ROOT::Experimental::RNTuple* GetRNTuple(arrow::dataset::FileSource) override
+  {
+    // Simply return the only TTree we have
+    return mTuple;
+  }
+
+ private:
+  ROOT::Experimental::RNTuple* mTuple;
+};
+
 class TFileFileSystem : public VirtualRootFileSystemBase
 {
  public:
@@ -179,6 +230,70 @@ class TTreeFileFragment : public arrow::dataset::FileFragment
   }
 };
 
+class RNTupleFileFragment : public arrow::dataset::FileFragment
+{
+ public:
+  RNTupleFileFragment(arrow::dataset::FileSource source,
+                      std::shared_ptr<arrow::dataset::FileFormat> format,
+                      arrow::compute::Expression partition_expression,
+                      std::shared_ptr<arrow::Schema> physical_schema)
+    : FileFragment(std::move(source), std::move(format), std::move(partition_expression), std::move(physical_schema))
+  {
+  }
+};
+
+class RNTupleFileFormat : public arrow::dataset::FileFormat
+{
+  size_t& mTotCompressedSize;
+  size_t& mTotUncompressedSize;
+
+ public:
+  RNTupleFileFormat(size_t& totalCompressedSize, size_t& totalUncompressedSize)
+    : FileFormat({}),
+      mTotCompressedSize(totalCompressedSize),
+      mTotUncompressedSize(totalUncompressedSize)
+  {
+  }
+
+  ~RNTupleFileFormat() override = default;
+
+  std::string type_name() const override
+  {
+    return "rntuple";
+  }
+
+  bool Equals(const FileFormat& other) const override
+  {
+    return other.type_name() == this->type_name();
+  }
+
+  arrow::Result<bool> IsSupported(const arrow::dataset::FileSource& source) const override
+  {
+    auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
+    auto subFs = fs->GetSubFilesystem(source);
+    if (std::dynamic_pointer_cast<RNTupleFileSystem>(subFs)) {
+      return true;
+    }
+    return false;
+  }
+
+  arrow::Result<std::shared_ptr<arrow::Schema>> Inspect(const arrow::dataset::FileSource& source) const override;
+
+  arrow::Result<arrow::RecordBatchGenerator> ScanBatchesAsync(
+    const std::shared_ptr<arrow::dataset::ScanOptions>& options,
+    const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const override;
+
+  std::shared_ptr<arrow::dataset::FileWriteOptions> DefaultWriteOptions() override;
+
+  arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination,
+                                                                        std::shared_ptr<arrow::Schema> schema,
+                                                                        std::shared_ptr<arrow::dataset::FileWriteOptions> options,
+                                                                        arrow::fs::FileLocator destination_locator) const override;
+  arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> MakeFragment(
+    arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
+    std::shared_ptr<arrow::Schema> physical_schema) override;
+};
+
 class TTreeFileFormat : public arrow::dataset::FileFormat
 {
   size_t& mTotCompressedSize;
diff --git a/Framework/Core/src/RootArrowFilesystem.cxx b/Framework/Core/src/RootArrowFilesystem.cxx
index 5f2d21d942d37..9efd895847a9b 100644
--- a/Framework/Core/src/RootArrowFilesystem.cxx
+++ b/Framework/Core/src/RootArrowFilesystem.cxx
@@ -8,6 +8,7 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
+#include <ROOT/RVec.hxx>
 #include "Framework/RootArrowFilesystem.h"
 #include "Framework/Endian.h"
 #include "Framework/RuntimeError.h"
@@ -17,6 +18,8 @@
 #include <arrow/array/array_primitive.h>
 #include <arrow/array/builder_nested.h>
 #include <arrow/array/builder_primitive.h>
+#include <ROOT/RNTupleModel.hxx>
+#include <ROOT/RNTupleWriter.hxx>
 #include <memory>
 #include <TFile.h>
 #include <TLeaf.h>
@@ -30,10 +33,20 @@
 #include <arrow/status.h>
 #include <arrow/util/key_value_metadata.h>
 #include <fmt/format.h>
-
+#include <ROOT/RField.hxx>
+#include <ROOT/RNTuple.hxx>
+#include <ROOT/RNTupleReader.hxx>
+#include <ROOT/RFieldVisitor.hxx>
+#include <ROOT/RNTupleInspector.hxx>
 #include <stdexcept>
 #include <utility>
 
+template class
+  std::unique_ptr<ROOT::Experimental::RNTupleReader>;
+
+template class
+  std::shared_ptr<arrow::Array>;
+
 O2_DECLARE_DYNAMIC_LOG(root_arrow_fs);
 
 namespace
@@ -85,6 +98,61 @@ auto arrowTypeFromROOT(EDataType type, int size)
       throw o2::framework::runtime_error_f("Unsupported branch type: %d", static_cast<int>(type));
   }
 }
+
+struct RootNTupleVisitor : public ROOT::Experimental::Detail::RFieldVisitor {
+  void VisitArrayField(const ROOT::Experimental::RArrayField& field) override
+  {
+    int size = field.GetLength();
+    RootNTupleVisitor valueVisitor{};
+    auto valueField = field.GetSubFields()[0];
+    valueField->AcceptVisitor(valueVisitor);
+    auto type = valueVisitor.datatype;
+    this->datatype = arrow::fixed_size_list(type, size);
+  }
+
+  void VisitRVecField(const ROOT::Experimental::RRVecField& field) override
+  {
+    RootNTupleVisitor valueVisitor{};
+    auto valueField = field.GetSubFields()[0];
+    valueField->AcceptVisitor(valueVisitor);
+    auto type = valueVisitor.datatype;
+    this->datatype = arrow::list(type);
+  }
+
+  void VisitField(const ROOT::Experimental::RFieldBase& field) override
+  {
+    throw o2::framework::runtime_error_f("Unknown field %s with type %s", field.GetFieldName().c_str(), field.GetTypeName().c_str());
+  }
+
+  void VisitIntField(const ROOT::Experimental::RField<int>& field) override
+  {
+    this->datatype = arrow::int32();
+  }
+
+  void VisitBoolField(const ROOT::Experimental::RField<bool>& field) override
+  {
+    this->datatype = arrow::boolean();
+  }
+
+  void VisitFloatField(const ROOT::Experimental::RField<float>& field) override
+  {
+    this->datatype = arrow::float32();
+  }
+
+  void VisitDoubleField(const ROOT::Experimental::RField<double>& field) override
+  {
+    this->datatype = arrow::float64();
+  }
+  std::shared_ptr<arrow::DataType> datatype;
+};
+
+auto arrowTypeFromRNTuple(ROOT::Experimental::RFieldBase const& field, int size)
+{
+  RootNTupleVisitor visitor;
+  field.AcceptVisitor(visitor);
+  return visitor.datatype;
+}
+
 namespace o2::framework
 {
 using arrow::Status;
@@ -103,6 +171,11 @@ std::shared_ptr<VirtualRootFileSystemBase> TFileFileSystem::GetSubFilesystem(arr
     return std::shared_ptr<VirtualRootFileSystemBase>(new SingleTreeFileSystem(tree));
   }
 
+  auto rntuple = (ROOT::Experimental::RNTuple*)mFile->Get<ROOT::Experimental::RNTuple>(source.path().c_str());
+  if (rntuple) {
+    return std::shared_ptr<VirtualRootFileSystemBase>(new SingleRNTupleFileSystem(rntuple));
+  }
+
   auto directory = (TDirectoryFile*)mFile->GetObjectChecked(source.path().c_str(), TClass::GetClass<TDirectory>());
   if (directory) {
     return std::shared_ptr<VirtualRootFileSystemBase>(new TFileFileSystem(directory, 50 * 1024 * 1024));
@@ -294,6 +367,18 @@ arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> TTreeFileFormat::Ma
   return std::dynamic_pointer_cast<arrow::dataset::FileFragment>(fragment);
 }
 
+arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> RNTupleFileFormat::MakeFragment(
+  arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
+  std::shared_ptr<arrow::Schema> physical_schema)
+{
+  std::shared_ptr<arrow::dataset::FileFormat> format = std::make_shared<RNTupleFileFormat>(mTotCompressedSize, mTotUncompressedSize);
+
+  auto fragment = std::make_shared<RNTupleFileFragment>(std::move(source), std::move(format),
+                                                        std::move(partition_expression),
+                                                        std::move(physical_schema));
+  return std::dynamic_pointer_cast<arrow::dataset::FileFragment>(fragment);
+}
+
 // An arrow outputstream which allows to write to a ttree
 TDirectoryFileOutputStream::TDirectoryFileOutputStream(TDirectoryFile* f)
   : mDirectory(f)
@@ -393,6 +478,37 @@ char const* rootSuffixFromArrow(arrow::Type::type id)
   }
 }
 
+std::unique_ptr<ROOT::Experimental::RFieldBase> rootFieldFromArrow(std::shared_ptr<arrow::Field> field, std::string name)
+{
+  using namespace ROOT::Experimental;
+  switch (field->type()->id()) {
+    case arrow::Type::BOOL:
+      return std::make_unique<RField<bool>>(name);
+    case arrow::Type::UINT8:
+      return std::make_unique<RField<uint8_t>>(name);
+    case arrow::Type::UINT16:
+      return std::make_unique<RField<uint16_t>>(name);
+    case arrow::Type::UINT32:
+      return std::make_unique<RField<uint32_t>>(name);
+    case arrow::Type::UINT64:
+      return std::make_unique<RField<uint64_t>>(name);
+    case arrow::Type::INT8:
+      return std::make_unique<RField<int8_t>>(name);
+    case arrow::Type::INT16:
+      return std::make_unique<RField<int16_t>>(name);
+    case arrow::Type::INT32:
+      return std::make_unique<RField<int32_t>>(name);
+    case arrow::Type::INT64:
+      return std::make_unique<RField<int64_t>>(name);
+    case arrow::Type::FLOAT:
+      return std::make_unique<RField<float>>(name);
+    case arrow::Type::DOUBLE:
+      return std::make_unique<RField<double>>(name);
+    default:
+      throw runtime_error("Unsupported arrow column type");
+  }
+}
+
 class TTreeFileWriter : public arrow::dataset::FileWriter
 {
   std::vector<TBranch*> branches;
@@ -628,6 +744,212 @@ class TTreeFileWriter : public arrow::dataset::FileWriter
   };
 };
 
+class RNTupleFileWriter : public arrow::dataset::FileWriter
+{
+  std::shared_ptr<ROOT::Experimental::RNTupleWriter> mWriter;
+  bool firstBatch = true;
+  std::vector<std::shared_ptr<arrow::Array>> valueArrays;
+  std::vector<std::shared_ptr<arrow::DataType>> valueTypes;
+  std::vector<size_t> valueCount;
+
+ public:
+  RNTupleFileWriter(std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options,
+                    std::shared_ptr<arrow::io::OutputStream> destination,
+                    arrow::fs::FileLocator destination_locator)
+    : FileWriter(schema, options, destination, destination_locator)
+  {
+    using namespace ROOT::Experimental;
+
+    auto model = RNTupleModel::Create();
+    // Let's create a model from the physical schema
+    for (auto i = 0u; i < schema->fields().size(); ++i) {
+      auto& field = schema->field(i);
+
+      // Construct all the needed branches.
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          auto list = std::static_pointer_cast<arrow::FixedSizeListType>(field->type());
+          auto valueField = field->type()->field(0);
+          model->AddField(std::make_unique<RArrayField>(field->name(), rootFieldFromArrow(valueField, "_0"), list->list_size()));
+        } break;
+        case arrow::Type::LIST: {
+          auto valueField = field->type()->field(0);
+          model->AddField(std::make_unique<RRVecField>(field->name(), rootFieldFromArrow(valueField, "_0")));
+        } break;
+        default: {
+          model->AddField(rootFieldFromArrow(field, field->name()));
+        } break;
+      }
+    }
+    auto fileStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
+    auto* file = dynamic_cast<TFile*>(fileStream->GetDirectory());
+    mWriter = RNTupleWriter::Append(std::move(model), destination_locator_.path, *file, {});
+  }
+
+  arrow::Status Write(const std::shared_ptr<arrow::RecordBatch>& batch) override
+  {
+    if (firstBatch) {
+      firstBatch = false;
+    }
+
+    // Support writing empty tables
+    if (batch->columns().empty() || batch->num_rows() == 0) {
+      return arrow::Status::OK();
+    }
+
+    for (auto i = 0u; i < batch->columns().size(); ++i) {
+      auto column = batch->column(i);
+      auto& field = batch->schema()->field(i);
+
+      valueArrays.push_back(nullptr);
+      valueTypes.push_back(nullptr);
+      valueCount.push_back(1);
+
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          auto list = std::static_pointer_cast<arrow::FixedSizeListArray>(column);
+          auto listType = std::static_pointer_cast<arrow::FixedSizeListType>(field->type());
+          if (field->type()->field(0)->type()->id() == arrow::Type::BOOL) {
+            auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(list->values());
+            int64_t length = boolArray->length();
+            arrow::UInt8Builder builder;
+            auto ok = builder.Reserve(length);
+
+            for (int64_t i = 0; i < length; ++i) {
+              if (boolArray->IsValid(i)) {
+                // Expand each boolean value (true/false) to uint8 (1/0)
+                uint8_t value = boolArray->Value(i) ? 1 : 0;
+                auto ok = builder.Append(value);
+              } else {
+                // Append null for invalid entries
+                auto ok = builder.AppendNull();
+              }
+            }
+            valueArrays.back() = *builder.Finish();
+            valueTypes.back() = valueArrays.back()->type();
+          } else {
+            valueArrays.back() = list->values();
+            valueTypes.back() = field->type()->field(0)->type();
+          }
+          valueCount.back() = listType->list_size();
+        } break;
+        case arrow::Type::LIST: {
+          auto list = std::static_pointer_cast<arrow::ListArray>(column);
+          valueArrays.back() = list;
+          valueTypes.back() = field->type()->field(0)->type();
+          valueCount.back() = -1;
+        } break;
+        case arrow::Type::BOOL: {
+          // We unpack the array
+          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(column);
+          int64_t length = boolArray->length();
+          arrow::UInt8Builder builder;
+          auto ok = builder.Reserve(length);
+
+          for (int64_t i = 0; i < length; ++i) {
+            if (boolArray->IsValid(i)) {
+              // Expand each boolean value (true/false) to uint8 (1/0)
+              uint8_t value = boolArray->Value(i) ? 1 : 0;
+              auto ok = builder.Append(value);
+            } else {
+              // Append null for invalid entries
+              auto ok = builder.AppendNull();
+            }
+          }
+          valueArrays.back() = *builder.Finish();
+          valueTypes.back() = valueArrays.back()->type();
+        } break;
+        default:
+          valueArrays.back() = column;
+          valueTypes.back() = field->type();
+          break;
+      }
+    }
+
+    int64_t pos = 0;
+
+    auto entry = mWriter->CreateEntry();
+    while (pos < batch->num_rows()) {
+      for (size_t ci = 0; ci < batch->columns().size(); ++ci) {
+        auto type = batch->column(ci)->type();
+        auto field = batch->schema()->field(ci);
+        auto token = entry->GetToken(field->name());
+
+        switch (type->id()) {
+          case arrow::Type::LIST: {
+            auto list = std::static_pointer_cast<arrow::ListArray>(valueArrays[ci]);
+            auto value_slice = list->value_slice(pos);
+
+            valueCount[ci] = value_slice->length();
+            auto bindValue = [&vc = valueCount, ci, token](auto array, std::unique_ptr<ROOT::Experimental::REntry>& entry) -> void {
+              using value_type = std::decay_t<decltype(*array.get())>::value_type;
+              auto v = std::make_shared<ROOT::VecOps::RVec<value_type>>((value_type*)array->raw_values(), vc[ci]);
+              entry->BindValue(token, v);
+            };
+            switch (valueTypes[ci]->id()) {
+              case arrow::Type::FLOAT: {
+                bindValue(std::static_pointer_cast<arrow::FloatArray>(value_slice), entry);
+              } break;
+              case arrow::Type::DOUBLE: {
+                bindValue(std::static_pointer_cast<arrow::DoubleArray>(value_slice), entry);
+              } break;
+              case arrow::Type::INT8: {
+                bindValue(std::static_pointer_cast<arrow::Int8Array>(value_slice), entry);
+              } break;
+              case arrow::Type::INT16: {
+                bindValue(std::static_pointer_cast<arrow::Int16Array>(value_slice), entry);
+              } break;
+              case arrow::Type::INT32: {
+                bindValue(std::static_pointer_cast<arrow::Int32Array>(value_slice), entry);
+              } break;
+              case arrow::Type::INT64: {
+                bindValue(std::static_pointer_cast<arrow::Int64Array>(value_slice), entry);
+              } break;
+              case arrow::Type::UINT8: {
+                bindValue(std::static_pointer_cast<arrow::UInt8Array>(value_slice), entry);
+              } break;
+              case arrow::Type::UINT16: {
+                bindValue(std::static_pointer_cast<arrow::UInt16Array>(value_slice), entry);
+              } break;
+              case arrow::Type::UINT32: {
+                bindValue(std::static_pointer_cast<arrow::UInt32Array>(value_slice), entry);
+              } break;
+              case arrow::Type::UINT64: {
+                bindValue(std::static_pointer_cast<arrow::UInt64Array>(value_slice), entry);
+              } break;
+              default: {
+                throw runtime_error("Unsupported kind of VLA");
+              } break;
+            }
+          } break;
+          case arrow::Type::FIXED_SIZE_LIST: {
+            entry->BindRawPtr<void>(token, (void*)(valueArrays[ci]->data()->buffers[1]->data() + pos * valueCount[ci] * valueTypes[ci]->byte_width()));
+          } break;
+          case arrow::Type::BOOL: {
+            // Not sure we actually need this
+            entry->BindRawPtr<bool>(token, (bool*)(valueArrays[ci]->data()->buffers[1]->data() + pos * 1));
+          } break;
+          default:
+            // By default we consider things scalars.
+            entry->BindRawPtr<void>(token, (void*)(valueArrays[ci]->data()->buffers[1]->data() + pos * valueTypes[ci]->byte_width()));
+            break;
+        }
+      }
+      mWriter->Fill(*entry);
+      ++pos;
+    }
+    mWriter->CommitCluster();
+
+    return arrow::Status::OK();
+  }
+
+  arrow::Future<>
+    FinishInternal() override
+  {
+    return {};
+  };
+};
+
 arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> TTreeFileFormat::MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options, arrow::fs::FileLocator destination_locator) const
 {
   auto writer = std::make_shared<TTreeFileWriter>(schema, options, destination, destination_locator);
@@ -838,6 +1160,314 @@ arrow::Result<arrow::RecordBatchGenerator> TTreeFileFormat::ScanBatchesAsync(
   return generator;
 }
 
+arrow::Result<std::shared_ptr<arrow::Schema>> RNTupleFileFormat::Inspect(const arrow::dataset::FileSource& source) const
+{
+
+  auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
+  // Actually get the TTree from the ROOT file.
+  auto ntupleFs = std::dynamic_pointer_cast<RNTupleFileSystem>(fs->GetSubFilesystem(source));
+  if (!ntupleFs.get()) {
+    throw runtime_error_f("Unknown filesystem %s\n", source.filesystem()->type_name().c_str());
+  }
+  ROOT::Experimental::RNTuple* rntuple = ntupleFs->GetRNTuple(source);
+
+  auto inspector = ROOT::Experimental::RNTupleInspector::Create(rntuple);
+
+  auto reader = ROOT::Experimental::RNTupleReader::Open(rntuple);
+
+  auto& tupleField0 = reader->GetModel().GetFieldZero();
+  std::vector<std::shared_ptr<arrow::Field>> fields;
+  for (auto& tupleField : tupleField0.GetSubFields()) {
+    auto field = std::make_shared<arrow::Field>(tupleField->GetFieldName(), arrowTypeFromRNTuple(*tupleField, tupleField->GetValueSize()));
+    fields.push_back(field);
+  }
+
+  return std::make_shared<arrow::Schema>(fields);
+}
+
+arrow::Result<arrow::RecordBatchGenerator> RNTupleFileFormat::ScanBatchesAsync(
+  const std::shared_ptr<arrow::dataset::ScanOptions>& options,
+  const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const
+{
+  auto dataset_schema = options->dataset_schema;
+  auto ntupleFragment = std::dynamic_pointer_cast<RNTupleFileFragment>(fragment);
+
+  auto generator = [pool = options->pool, ntupleFragment, dataset_schema, &totalCompressedSize = mTotCompressedSize,
+                    &totalUncompressedSize = mTotUncompressedSize]() -> arrow::Future<std::shared_ptr<arrow::RecordBatch>> {
+    using namespace ROOT::Experimental;
+    std::vector<std::shared_ptr<arrow::Array>> columns;
+    std::vector<std::shared_ptr<arrow::Field>> fields = dataset_schema->fields();
+
+    auto containerFS = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(ntupleFragment->source().filesystem());
+    auto fs = std::dynamic_pointer_cast<RNTupleFileSystem>(containerFS->GetSubFilesystem(ntupleFragment->source()));
+
+    int64_t rows = -1;
+    ROOT::Experimental::RNTuple* rntuple = fs->GetRNTuple(ntupleFragment->source());
+    auto reader = ROOT::Experimental::RNTupleReader::Open(rntuple);
+    auto& model = reader->GetModel();
+    for (auto& physicalField : fields) {
+      auto bulk = model.CreateBulk(physicalField->name());
+
+      auto listType = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type());
+
+      auto& descriptor = reader->GetDescriptor();
+      auto totalEntries = reader->GetNEntries();
+
+      if (rows == -1) {
+        rows = totalEntries;
+      }
+      if (rows != totalEntries) {
+        throw runtime_error_f("Unmatching number of rows for branch %s", physicalField->name().c_str());
+      }
+      arrow::Status status;
+      int readEntries = 0;
+      std::shared_ptr<arrow::Array> array;
+      if (physicalField->type() == arrow::boolean() ||
+          (listType && physicalField->type()->field(0)->type() == arrow::boolean())) {
+        if (listType) {
+          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
+          auto status = arrow::MakeBuilder(pool, physicalField->type()->field(0)->type(), &builder);
+          if (!status.ok()) {
+            throw runtime_error("Cannot create value builder");
+          }
+          auto listBuilder = std::make_unique<arrow::FixedSizeListBuilder>(pool, std::move(builder), listType->list_size());
+          auto valueBuilder = listBuilder.get()->value_builder();
+          // boolean array special case: we need to use builder to create the bitmap
+          status = valueBuilder->Reserve(totalEntries * listType->list_size());
+          status &= listBuilder->Reserve(totalEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to reserve memory for array builder");
+          }
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          // No adoption for now...
+          // bulk.AdoptBuffer(buffer, totalEntries)
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            void* ptr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries());
+            int readLast = index.GetNEntries();
+            readEntries += readLast;
+            status &= static_cast<arrow::BooleanBuilder*>(valueBuilder)->AppendValues(reinterpret_cast<uint8_t const*>(ptr), readLast * listType->list_size());
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+          status &= static_cast<arrow::FixedSizeListBuilder*>(listBuilder.get())->AppendValues(readEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to append values to array");
+          }
+          status &= listBuilder->Finish(&array);
+          if (!status.ok()) {
+            throw runtime_error("Failed to create array");
+          }
+        } else if (listType == nullptr) {
+          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
+          auto status = arrow::MakeBuilder(pool, physicalField->type(), &builder);
+          if (!status.ok()) {
+            throw runtime_error("Cannot create builder");
+          }
+          auto valueBuilder = static_cast<arrow::BooleanBuilder*>(builder.get());
+          // boolean array special case: we need to use builder to create the bitmap
+          status = valueBuilder->Reserve(totalEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to reserve memory for array builder");
+          }
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            void* ptr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries());
+            int readLast = index.GetNEntries();
+            readEntries += readLast;
+            status &= valueBuilder->AppendValues(reinterpret_cast<uint8_t const*>(ptr), readLast);
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+          if (!status.ok()) {
+            throw runtime_error("Failed to append values to array");
+          }
+          status &= valueBuilder->Finish(&array);
+          if (!status.ok()) {
+            throw runtime_error("Failed to create array");
+          }
+        }
+      } else {
+        // other types: use serialized read to build arrays directly.
+        auto typeSize = physicalField->type()->byte_width();
+        // FIXME: for now...
+        auto bytes = 0;
+        auto branchSize = bytes ? bytes : 1000000;
+        auto&& result = arrow::AllocateResizableBuffer(branchSize, pool);
+        if (!result.ok()) {
+          throw runtime_error("Cannot allocate values buffer");
+        }
+        std::shared_ptr<arrow::Buffer> arrowValuesBuffer = std::move(result).ValueUnsafe();
+        auto ptr = arrowValuesBuffer->mutable_data();
+        if (ptr == nullptr) {
+          throw runtime_error("Invalid buffer");
+        }
+
+        std::unique_ptr<TBufferFile> offsetBuffer = nullptr;
+
+        std::shared_ptr<arrow::Buffer> arrowOffsetBuffer;
+        std::span<int> offsets;
+        int size = 0;
+        uint32_t totalSize = 0;
+        int64_t listSize = 1;
+        if (auto fixedSizeList = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type())) {
+          listSize = fixedSizeList->list_size();
+          typeSize = fixedSizeList->field(0)->type()->byte_width();
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            void* inPtr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries());
+
+            int readLast = index.GetNEntries();
+            if (listSize == -1) {
+              size = offsets[readEntries + readLast] - offsets[readEntries];
+            } else {
+              size = readLast * listSize;
+            }
+            readEntries += readLast;
+            memcpy(ptr, inPtr, size * typeSize);
+            ptr += (ptrdiff_t)(size * typeSize);
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+        } else if (auto vlaListType = std::dynamic_pointer_cast<arrow::ListType>(physicalField->type())) {
+          listSize = -1;
+          typeSize = vlaListType->field(0)->type()->byte_width();
+          offsetBuffer = std::make_unique<TBufferFile>(TBuffer::EMode::kWrite, 4 * 1024 * 1024);
+          result = arrow::AllocateResizableBuffer((totalEntries + 1) * (int64_t)sizeof(int), pool);
+          if (!result.ok()) {
+            throw runtime_error("Cannot allocate offset buffer");
+          }
+          arrowOffsetBuffer = std::move(result).ValueUnsafe();
+
+          // Offset bulk
+          auto offsetBulk = model.CreateBulk(physicalField->name());
+          // Actual values are in a different place...
+          bulk = model.CreateBulk(physicalField->name());
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          auto* ptrOffset = reinterpret_cast<int*>(arrowOffsetBuffer->mutable_data());
+          auto* tPtrOffset = reinterpret_cast<int*>(ptrOffset);
+          offsets = std::span<int>{tPtrOffset, tPtrOffset + totalEntries + 1};
+
+          auto copyOffsets = [&arrowValuesBuffer, &pool, &ptrOffset, &ptr, &totalSize](auto inPtr, size_t total) {
+            using value_type = typename std::decay_t<decltype(*inPtr)>::value_type;
+            for (size_t i = 0; i < total; i++) {
+              *ptrOffset++ = totalSize;
+              totalSize += inPtr[i].size();
+            }
+            *ptrOffset = totalSize;
+            auto&& result = arrow::AllocateResizableBuffer(totalSize * sizeof(value_type), pool);
+            if (!result.ok()) {
+              throw runtime_error("Cannot allocate values buffer");
+            }
+            arrowValuesBuffer = std::move(result).ValueUnsafe();
+            ptr = (uint8_t*)(arrowValuesBuffer->mutable_data());
+            // Calculate the size of the buffer here.
+            for (size_t i = 0; i < total; i++) {
+              int vlaSizeInBytes = inPtr[i].size() * sizeof(value_type);
+              if (vlaSizeInBytes == 0) {
+                continue;
+              }
+              memcpy(ptr, inPtr[i].data(), vlaSizeInBytes);
+              ptr += vlaSizeInBytes;
+            }
+          };
+
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            int readLast = index.GetNEntries();
+            switch (vlaListType->field(0)->type()->id()) {
+              case arrow::Type::FLOAT: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<float>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::DOUBLE: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<double>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::INT8: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<int8_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::INT16: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<int16_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::INT32: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<int32_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::INT64: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<int64_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::UINT8: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<uint8_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::UINT16: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<uint16_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::UINT32: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<uint32_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::UINT64: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<uint64_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              default: {
+                throw runtime_error("Unsupported kind of VLA");
+              } break;
+            }
+
+            readEntries += readLast;
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+        } else {
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            void* inPtr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries());
+
+            int readLast = index.GetNEntries();
+            if (listSize == -1) {
+              size = offsets[readEntries + readLast] - offsets[readEntries];
+            } else {
+              size = readLast * listSize;
+            }
+            readEntries += readLast;
+            memcpy(ptr, inPtr, size * typeSize);
+            ptr += (ptrdiff_t)(size * typeSize);
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+        }
+        switch (listSize) {
+          case -1: {
+            auto varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
+            array = std::make_shared<arrow::ListArray>(physicalField->type(), readEntries, arrowOffsetBuffer, varray);
+          } break;
+          case 1: {
+            totalSize = readEntries * listSize;
+            array = std::make_shared<arrow::PrimitiveArray>(physicalField->type(), readEntries, arrowValuesBuffer);
+
+          } break;
+          default: {
+            totalSize = readEntries * listSize;
+            auto varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
+            array = std::make_shared<arrow::FixedSizeListArray>(physicalField->type(), readEntries, varray);
+          }
+        }
+      }
+      columns.push_back(array);
+    }
+
+    auto batch = arrow::RecordBatch::Make(dataset_schema, rows, columns);
+    return batch;
+  };
+
+  return generator;
+}
+
 arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenOutputStream(
   const std::string& path,
   const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
@@ -850,6 +1480,21 @@ arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenOut
   return std::make_shared<TTreeOutputStream>(GetTree(source), "");
 }
 
+std::shared_ptr<arrow::dataset::FileWriteOptions>
+  RNTupleFileFormat::DefaultWriteOptions()
+{
+  return std::make_shared<RNTupleFileWriteOptions>(shared_from_this());
+}
+
+arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> RNTupleFileFormat::MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination,
+                                                                                         std::shared_ptr<arrow::Schema> schema,
+                                                                                         std::shared_ptr<arrow::dataset::FileWriteOptions> options,
+                                                                                         arrow::fs::FileLocator destination_locator) const
+{
+  auto writer = std::make_shared<RNTupleFileWriter>(schema, options, destination, destination_locator);
+  return std::dynamic_pointer_cast<arrow::dataset::FileWriter>(writer);
+}
+
 TBufferFileFS::TBufferFileFS(TBufferFile* f)
   : VirtualRootFileSystemBase(),
     mBuffer(f),
@@ -859,6 +1504,8 @@ TBufferFileFS::TBufferFileFS(TBufferFile* f)
 
 TTreeFileSystem::~TTreeFileSystem() = default;
 
+RNTupleFileSystem::~RNTupleFileSystem() = default;
+
 arrow::Result<arrow::fs::FileInfo> TBufferFileFS::GetFileInfo(const std::string& path)
 {
   arrow::fs::FileInfo result;
@@ -876,6 +1523,10 @@ arrow::Result<arrow::fs::FileInfo> TBufferFileFS::GetFileInfo(const std::string&
     result.set_type(arrow::fs::FileType::File);
     return result;
   }
+  if (std::dynamic_pointer_cast<SingleRNTupleFileSystem>(mFilesystem)) {
+    result.set_type(arrow::fs::FileType::File);
+    return result;
+  }
   return result;
 }
 
diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx
index 2b0ab9154250c..cd9a4b4685e7a 100644
--- a/Framework/Core/test/test_Root2ArrowTable.cxx
+++ b/Framework/Core/test/test_Root2ArrowTable.cxx
@@ -26,6 +26,13 @@
 #include <TTree.h>
 #include <TRandom.h>
 #include <TFile.h>
+#include <ROOT/RField.hxx>
+#include <ROOT/RNTuple.hxx>
+#include <ROOT/RNTupleDescriptor.hxx>
+#include <ROOT/RNTupleModel.hxx>
+#include <ROOT/RNTupleReader.hxx>
+#include <ROOT/RNTupleUtil.hxx>
+#include <ROOT/RNTupleWriter.hxx>
 #include <memory>
 
 #include <arrow/array/array_primitive.h>
@@ -504,4 +511,40 @@ TEST_CASE("RootTree2Dataset")
     REQUIRE((*resultWritten)->num_rows() == 100);
     validateContents(*resultWritten);
   }
+  // Let's write back an RNTuple
+  auto rNtupleFormat = std::make_shared<RNTupleFileFormat>(totalSizeCompressed, totalSizeUncompressed);
+  arrow::fs::FileLocator rnTupleLocator{outFs, "/rntuple"};
+  // We write an RNTuple in the same TMemFile, using /rntuple as a location
+  auto rntupleDestination = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(*destination);
+
+  {
+    auto rNtupleWriter = rNtupleFormat->MakeWriter(*destination, schema, {}, rnTupleLocator);
+    auto rNtupleSuccess = rNtupleWriter->get()->Write(*result);
+    REQUIRE(rNtupleSuccess.ok());
+  }
+
+  // And now we can read back the RNTuple into a RecordBatch
+  arrow::dataset::FileSource writtenRntupleSource("/rntuple", outFs);
+  auto newRNTupleFS = outFs->GetSubFilesystem(writtenRntupleSource);
+
+  REQUIRE(rNtupleFormat->IsSupported(writtenRntupleSource) == true);
+
+  auto rntupleSchemaOpt = rNtupleFormat->Inspect(writtenRntupleSource);
+  REQUIRE(rntupleSchemaOpt.ok());
+  auto rntupleSchemaWritten = *rntupleSchemaOpt;
+  REQUIRE(validateSchema(rntupleSchemaWritten));
+
+  auto rntupleFragmentWritten = rNtupleFormat->MakeFragment(writtenRntupleSource, {}, rntupleSchemaWritten);
+  REQUIRE(rntupleFragmentWritten.ok());
+  auto rntupleOptionsWritten = std::make_shared<arrow::dataset::ScanOptions>();
+  rntupleOptionsWritten->dataset_schema = rntupleSchemaWritten;
+  auto rntupleScannerWritten = rNtupleFormat->ScanBatchesAsync(rntupleOptionsWritten, *rntupleFragmentWritten);
+  REQUIRE(rntupleScannerWritten.ok());
+  auto rntupleBatchesWritten = (*rntupleScannerWritten)();
+  auto rntupleResultWritten = rntupleBatchesWritten.result();
+  REQUIRE(rntupleResultWritten.ok());
+  REQUIRE((*rntupleResultWritten)->columns().size() == 10);
+  REQUIRE(validateSchema((*rntupleResultWritten)->schema()));
+  REQUIRE((*rntupleResultWritten)->num_rows() == 100);
+  REQUIRE(validateContents(*rntupleResultWritten));
 }

From 19424d5c1a476703e8eadde29f44cec08f519184 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Tue, 3 Dec 2024 15:43:42 +0100
Subject: [PATCH 715/726] DPL: Move DataInputDirector to arrow::Dataset API

---
 .../src/AODJAlienReaderHelpers.cxx            |  16 ++-
 .../AnalysisSupport/src/DataInputDirector.cxx | 103 ++++++++++--------
 .../AnalysisSupport/src/DataInputDirector.h   |  14 +--
 3 files changed, 75 insertions(+), 58 deletions(-)

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 9c19de85739ce..f8a9705e4eb62 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -10,10 +10,12 @@
 // or submit itself to any jurisdiction.
 
 #include "AODJAlienReaderHelpers.h"
+#include <memory>
 #include "Framework/TableTreeHelpers.h"
 #include "Framework/AnalysisHelpers.h"
 #include "Framework/DataProcessingStats.h"
 #include "Framework/RootTableBuilderHelpers.h"
+#include "Framework/RootArrowFilesystem.h"
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/ConfigParamRegistry.h"
 #include "Framework/ControlService.h"
@@ -41,6 +43,8 @@
 #include <arrow/io/interfaces.h>
 #include <arrow/table.h>
 #include <arrow/util/key_value_metadata.h>
+#include <arrow/dataset/dataset.h>
+#include <arrow/dataset/file_base.h>
 
 using namespace o2;
 using namespace o2::aod;
@@ -272,11 +276,13 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const
             // Origin file name for derived output map
             auto o2 = Output(TFFileNameHeader);
             auto fileAndFolder = didir->getFileFolder(dh, fcnt, ntf);
-            std::string currentFilename(fileAndFolder.file->GetName());
-            if (strcmp(fileAndFolder.file->GetEndpointUrl()->GetProtocol(), "file") == 0 && fileAndFolder.file->GetEndpointUrl()->GetFile()[0] != '/') {
+            auto rootFS = std::dynamic_pointer_cast<TFileFileSystem>(fileAndFolder.filesystem());
+            auto* f = dynamic_cast<TFile*>(rootFS->GetFile());
+            std::string currentFilename(f->GetFile()->GetName());
+            if (strcmp(f->GetEndpointUrl()->GetProtocol(), "file") == 0 && f->GetEndpointUrl()->GetFile()[0] != '/') {
               // This is not an absolute local path. Make it absolute.
               static std::string pwd = gSystem->pwd() + std::string("/");
-              currentFilename = pwd + std::string(fileAndFolder.file->GetName());
+              currentFilename = pwd + std::string(f->GetName());
             }
             outputs.make<std::string>(o2) = currentFilename;
           }
@@ -312,7 +318,9 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const
       auto concrete = DataSpecUtils::asConcreteDataMatcher(firstRoute.matcher);
       auto dh = header::DataHeader(concrete.description, concrete.origin, concrete.subSpec);
       auto fileAndFolder = didir->getFileFolder(dh, fcnt, ntf);
-      if (!fileAndFolder.file) {
+
+      // In case the filesource is empty, move to the next one.
+      if (fileAndFolder.path().empty()) {
         fcnt += 1;
         ntf = 0;
         if (didir->atEnd(fcnt)) {
diff --git a/Framework/AnalysisSupport/src/DataInputDirector.cxx b/Framework/AnalysisSupport/src/DataInputDirector.cxx
index 172ecd66c0e64..7e86f87ca8490 100644
--- a/Framework/AnalysisSupport/src/DataInputDirector.cxx
+++ b/Framework/AnalysisSupport/src/DataInputDirector.cxx
@@ -11,6 +11,7 @@
 #include "DataInputDirector.h"
 #include "Framework/DataDescriptorQueryBuilder.h"
 #include "Framework/Logger.h"
+#include "Framework/RootArrowFilesystem.h"
 #include "Framework/AnalysisDataModelHelpers.h"
 #include "Framework/Output.h"
 #include "Headers/DataHeader.h"
@@ -26,8 +27,12 @@
 #include "TGrid.h"
 #include "TObjString.h"
 #include "TMap.h"
+#include "TFile.h"
 
+#include <arrow/dataset/file_base.h>
+#include <arrow/dataset/dataset.h>
 #include <uv.h>
+#include <memory>
 
 #if __has_include(<TJAlienFile.h>)
 #include <TJAlienFile.h>
@@ -108,20 +113,22 @@ bool DataInputDescriptor::setFile(int counter)
 
   // open file
   auto filename = mfilenames[counter]->fileName;
-  if (mcurrentFile) {
-    if (mcurrentFile->GetName() == filename) {
+  auto rootFS = std::dynamic_pointer_cast<TFileFileSystem>(mCurrentFilesystem);
+  if (rootFS.get()) {
+    if (rootFS->GetFile()->GetName() == filename) {
       return true;
     }
     closeInputFile();
   }
-  mcurrentFile = TFile::Open(filename.c_str());
-  if (!mcurrentFile) {
+
+  mCurrentFilesystem = std::make_shared<TFileFileSystem>(TFile::Open(filename.c_str()), 50 * 1024 * 1024);
+  if (!mCurrentFilesystem.get()) {
     throw std::runtime_error(fmt::format("Couldn't open file \"{}\"!", filename));
   }
-  mcurrentFile->SetReadaheadSize(50 * 1024 * 1024);
+  rootFS = std::dynamic_pointer_cast<TFileFileSystem>(mCurrentFilesystem);
 
   // get the parent file map if exists
-  mParentFileMap = (TMap*)mcurrentFile->Get("parentFiles"); // folder name (DF_XXX) --> parent file (absolute path)
+  mParentFileMap = (TMap*)rootFS->GetFile()->Get("parentFiles"); // folder name (DF_XXX) --> parent file (absolute path)
   if (mParentFileMap && !mParentFileReplacement.empty()) {
     auto pos = mParentFileReplacement.find(';');
     if (pos == std::string::npos) {
@@ -141,7 +148,7 @@ bool DataInputDescriptor::setFile(int counter)
   // get the directory names
   if (mfilenames[counter]->numberOfTimeFrames <= 0) {
     std::regex TFRegex = std::regex("DF_[0-9]+");
-    TList* keyList = mcurrentFile->GetListOfKeys();
+    TList* keyList = rootFS->GetFile()->GetListOfKeys();
 
     // extract TF numbers and sort accordingly
     for (auto key : *keyList) {
@@ -193,26 +200,21 @@ uint64_t DataInputDescriptor::getTimeFrameNumber(int counter, int numTF)
   return (mfilenames[counter]->listOfTimeFrameNumbers)[numTF];
 }
 
-FileAndFolder DataInputDescriptor::getFileFolder(int counter, int numTF)
+arrow::dataset::FileSource DataInputDescriptor::getFileFolder(int counter, int numTF)
 {
-  FileAndFolder fileAndFolder;
-
   // open file
   if (!setFile(counter)) {
-    return fileAndFolder;
+    return {};
   }
 
   // no TF left
   if (mfilenames[counter]->numberOfTimeFrames > 0 && numTF >= mfilenames[counter]->numberOfTimeFrames) {
-    return fileAndFolder;
+    return {};
   }
 
-  fileAndFolder.file = mcurrentFile;
-  fileAndFolder.folderName = (mfilenames[counter]->listOfTimeFrameKeys)[numTF];
-
   mfilenames[counter]->alreadyRead[numTF] = true;
 
-  return fileAndFolder;
+  return {(mfilenames[counter]->listOfTimeFrameKeys)[numTF], mCurrentFilesystem};
 }
 
 DataInputDescriptor* DataInputDescriptor::getParentFile(int counter, int numTF, std::string treename)
@@ -223,15 +225,17 @@ DataInputDescriptor* DataInputDescriptor::getParentFile(int counter, int numTF,
   }
   auto folderName = (mfilenames[counter]->listOfTimeFrameKeys)[numTF];
   auto parentFileName = (TObjString*)mParentFileMap->GetValue(folderName.c_str());
+  // The current DF is not found in the parent map (this should not happen and is a fatal error)
+  auto rootFS = std::dynamic_pointer_cast<TFileFileSystem>(mCurrentFilesystem);
   if (!parentFileName) {
-    // The current DF is not found in the parent map (this should not happen and is a fatal error)
-    throw std::runtime_error(fmt::format(R"(parent file map exists but does not contain the current DF "{}" in file "{}")", folderName.c_str(), mcurrentFile->GetName()));
+    throw std::runtime_error(fmt::format(R"(parent file map exists but does not contain the current DF "{}" in file "{}")", folderName.c_str(), rootFS->GetFile()->GetName()));
     return nullptr;
   }
 
   if (mParentFile) {
     // Is this still the corresponding to the correct file?
-    if (parentFileName->GetString().CompareTo(mParentFile->mcurrentFile->GetName()) == 0) {
+    auto parentRootFS = std::dynamic_pointer_cast<TFileFileSystem>(mParentFile->mCurrentFilesystem);
+    if (parentFileName->GetString().CompareTo(parentRootFS->GetFile()->GetName()) == 0) {
       return mParentFile;
     } else {
       mParentFile->closeInputFile();
@@ -241,7 +245,8 @@ DataInputDescriptor* DataInputDescriptor::getParentFile(int counter, int numTF,
   }
 
   if (mLevel == mAllowedParentLevel) {
-    throw std::runtime_error(fmt::format(R"(while looking for tree "{}", the parent file was requested but we are already at level {} of maximal allowed level {} for DF "{}" in file "{}")", treename.c_str(), mLevel, mAllowedParentLevel, folderName.c_str(), mcurrentFile->GetName()));
+    throw std::runtime_error(fmt::format(R"(while looking for tree "{}", the parent file was requested but we are already at level {} of maximal allowed level {} for DF "{}" in file "{}")", treename.c_str(), mLevel, mAllowedParentLevel, folderName.c_str(),
+                                         rootFS->GetFile()->GetName()));
   }
 
   LOGP(info, "Opening parent file {} for DF {}", parentFileName->GetString().Data(), folderName.c_str());
@@ -270,11 +275,13 @@ void DataInputDescriptor::printFileStatistics()
   if (wait_time < 0) {
     wait_time = 0;
   }
-  std::string monitoringInfo(fmt::format("lfn={},size={},total_df={},read_df={},read_bytes={},read_calls={},io_time={:.1f},wait_time={:.1f},level={}", mcurrentFile->GetName(),
-                                         mcurrentFile->GetSize(), getTimeFramesInFile(mCurrentFileID), getReadTimeFramesInFile(mCurrentFileID), mcurrentFile->GetBytesRead(), mcurrentFile->GetReadCalls(),
+  auto rootFS = std::dynamic_pointer_cast<TFileFileSystem>(mCurrentFilesystem);
+  auto f = dynamic_cast<TFile*>(rootFS->GetFile());
+  std::string monitoringInfo(fmt::format("lfn={},size={},total_df={},read_df={},read_bytes={},read_calls={},io_time={:.1f},wait_time={:.1f},level={}", f->GetName(),
+                                         f->GetSize(), getTimeFramesInFile(mCurrentFileID), getReadTimeFramesInFile(mCurrentFileID), f->GetBytesRead(), f->GetReadCalls(),
                                          ((float)mIOTime / 1e9), ((float)wait_time / 1e9), mLevel));
 #if __has_include(<TJAlienFile.h>)
-  auto alienFile = dynamic_cast<TJAlienFile*>(mcurrentFile);
+  auto alienFile = dynamic_cast<TJAlienFile*>(f);
   if (alienFile) {
     monitoringInfo += fmt::format(",se={},open_time={:.1f}", alienFile->GetSE(), alienFile->GetElapsed());
   }
@@ -285,7 +292,7 @@ void DataInputDescriptor::printFileStatistics()
 
 void DataInputDescriptor::closeInputFile()
 {
-  if (mcurrentFile) {
+  if (mCurrentFilesystem.get()) {
     if (mParentFile) {
       mParentFile->closeInputFile();
       delete mParentFile;
@@ -296,9 +303,7 @@ void DataInputDescriptor::closeInputFile()
     mParentFileMap = nullptr;
 
     printFileStatistics();
-    mcurrentFile->Close();
-    delete mcurrentFile;
-    mcurrentFile = nullptr;
+    mCurrentFilesystem.reset();
   }
 }
 
@@ -358,40 +363,46 @@ bool DataInputDescriptor::readTree(DataAllocator& outputs, header::DataHeader dh
 {
   auto ioStart = uv_hrtime();
 
-  auto fileAndFolder = getFileFolder(counter, numTF);
-  if (!fileAndFolder.file) {
+  auto folder = getFileFolder(counter, numTF);
+  if (!folder.filesystem()) {
     return false;
   }
 
-  auto fullpath = fileAndFolder.folderName + "/" + treename;
-  auto tree = (TTree*)fileAndFolder.file->Get(fullpath.c_str());
+  auto format = std::make_shared<TTreeFileFormat>(totalSizeCompressed, totalSizeUncompressed);
+  auto fullpath = arrow::dataset::FileSource{folder.path() + "/" + treename, folder.filesystem()};
+  auto schemaOpt = format->Inspect(fullpath);
+  auto schema = *schemaOpt;
 
-  if (!tree) {
-    LOGP(debug, "Could not find tree {}. Trying in parent file.", fullpath.c_str());
+  auto fragment = format->MakeFragment(fullpath, {}, schema);
+
+  if (!fragment.ok()) {
+    LOGP(debug, "Could not find tree {}. Trying in parent file.", fullpath.path());
     auto parentFile = getParentFile(counter, numTF, treename);
     if (parentFile != nullptr) {
-      int parentNumTF = parentFile->findDFNumber(0, fileAndFolder.folderName);
+      int parentNumTF = parentFile->findDFNumber(0, folder.path());
       if (parentNumTF == -1) {
-        throw std::runtime_error(fmt::format(R"(DF {} listed in parent file map but not found in the corresponding file "{}")", fileAndFolder.folderName, parentFile->mcurrentFile->GetName()));
+        auto parentRootFS = std::dynamic_pointer_cast<TFileFileSystem>(parentFile->mCurrentFilesystem);
+        throw std::runtime_error(fmt::format(R"(DF {} listed in parent file map but not found in the corresponding file "{}")", folder.path(), parentRootFS->GetFile()->GetName()));
       }
       // first argument is 0 as the parent file object contains only 1 file
       return parentFile->readTree(outputs, dh, 0, parentNumTF, treename, totalSizeCompressed, totalSizeUncompressed);
     }
-    throw std::runtime_error(fmt::format(R"(Couldn't get TTree "{}" from "{}". Please check https://aliceo2group.github.io/analysis-framework/docs/troubleshooting/#tree-not-found for more information.)", fileAndFolder.folderName + "/" + treename, fileAndFolder.file->GetName()));
+    auto rootFS = std::dynamic_pointer_cast<TFileFileSystem>(mCurrentFilesystem);
+    throw std::runtime_error(fmt::format(R"(Couldn't get TTree "{}" from "{}". Please check https://aliceo2group.github.io/analysis-framework/docs/troubleshooting/#tree-not-found for more information.)", fullpath.path(), rootFS->GetFile()->GetName()));
   }
 
   // create table output
   auto o = Output(dh);
-  auto t2t = outputs.make<TreeToTable>(o);
 
-  // add branches to read
-  // fill the table
-  t2t->setLabel(tree->GetName());
-  totalSizeCompressed += tree->GetZipBytes();
-  totalSizeUncompressed += tree->GetTotBytes();
-  t2t->addAllColumns(tree);
-  t2t->fill(tree);
-  delete tree;
+  // FIXME: This should allow me to create a memory pool
+  // which I can then use to scan the dataset.
+  //
+  auto f2b = outputs.make<FragmentToBatch>(o);
+
+  //// add branches to read
+  //// fill the table
+  f2b->setLabel(treename.c_str());
+  f2b->fill(*fragment, schema, format);
 
   mIOTime += (uv_hrtime() - ioStart);
 
@@ -693,7 +704,7 @@ DataInputDescriptor* DataInputDirector::getDataInputDescriptor(header::DataHeade
   return result;
 }
 
-FileAndFolder DataInputDirector::getFileFolder(header::DataHeader dh, int counter, int numTF)
+arrow::dataset::FileSource DataInputDirector::getFileFolder(header::DataHeader dh, int counter, int numTF)
 {
   auto didesc = getDataInputDescriptor(dh);
   // if NOT match then use defaultDataInputDescriptor
diff --git a/Framework/AnalysisSupport/src/DataInputDirector.h b/Framework/AnalysisSupport/src/DataInputDirector.h
index eca0ef195d111..c03104be4ffba 100644
--- a/Framework/AnalysisSupport/src/DataInputDirector.h
+++ b/Framework/AnalysisSupport/src/DataInputDirector.h
@@ -16,6 +16,9 @@
 #include "Framework/DataDescriptorMatcher.h"
 #include "Framework/DataAllocator.h"
 
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/dataset/dataset.h>
+
 #include <regex>
 #include "rapidjson/fwd.h"
 
@@ -36,11 +39,6 @@ struct FileNameHolder {
 };
 FileNameHolder* makeFileNameHolder(std::string fileName);
 
-struct FileAndFolder {
-  TFile* file = nullptr;
-  std::string folderName = "";
-};
-
 class DataInputDescriptor
 {
   /// Holds information concerning the reading of an aod table.
@@ -78,7 +76,7 @@ class DataInputDescriptor
   int findDFNumber(int file, std::string dfName);
 
   uint64_t getTimeFrameNumber(int counter, int numTF);
-  FileAndFolder getFileFolder(int counter, int numTF);
+  arrow::dataset::FileSource getFileFolder(int counter, int numTF);
   DataInputDescriptor* getParentFile(int counter, int numTF, std::string treename);
   int getTimeFramesInFile(int counter);
   int getReadTimeFramesInFile(int counter);
@@ -98,7 +96,7 @@ class DataInputDescriptor
   std::string mParentFileReplacement;
   std::vector<FileNameHolder*> mfilenames;
   std::vector<FileNameHolder*>* mdefaultFilenamesPtr = nullptr;
-  TFile* mcurrentFile = nullptr;
+  std::shared_ptr<arrow::fs::FileSystem> mCurrentFilesystem;
   int mCurrentFileID = -1;
   bool mAlienSupport = false;
 
@@ -143,7 +141,7 @@ class DataInputDirector
 
   bool readTree(DataAllocator& outputs, header::DataHeader dh, int counter, int numTF, size_t& totalSizeCompressed, size_t& totalSizeUncompressed);
   uint64_t getTimeFrameNumber(header::DataHeader dh, int counter, int numTF);
-  FileAndFolder getFileFolder(header::DataHeader dh, int counter, int numTF);
+  arrow::dataset::FileSource getFileFolder(header::DataHeader dh, int counter, int numTF);
   int getTimeFramesInFile(header::DataHeader dh, int counter);
 
   uint64_t getTotalSizeCompressed();

From 18a9be8a2ce3c201af37a3a6740e8f68b74b605f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 4 Dec 2024 11:57:29 +0100
Subject: [PATCH 716/726] DPL: Simplify prune_voids_pack

Use the requirements directly on the template arguments.
---
 Framework/Foundation/include/Framework/Pack.h | 24 ++++++++++---------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Pack.h b/Framework/Foundation/include/Framework/Pack.h
index 2355ffed6b35b..78511876917b2 100644
--- a/Framework/Foundation/include/Framework/Pack.h
+++ b/Framework/Foundation/include/Framework/Pack.h
@@ -114,42 +114,44 @@ consteval auto prune_voids_pack(Result result, pack<>)
   return result;
 }
 
+template<typename T>
+concept void_pack_element = std::is_void_v<T>;
+
+template<typename T>
+concept nonvoid_pack_element = !void_pack_element<T>;
+
+
 // The first one is non void, but one of the others is void
-template <typename... Rs, typename T, typename... Ts>
-  requires(!std::is_void_v<T>)
+template <typename... Rs, nonvoid_pack_element T, typename... Ts>
 consteval auto prune_voids_pack(pack<Rs...> result, pack<T, Ts...>)
 {
   return prune_voids_pack(pack<Rs..., T>{}, pack<Ts...>{});
 }
 
 // The first one is void
-template <typename... Rs, typename V, typename... Ts>
-  requires(std::is_void_v<V>)
+template <typename... Rs, void_pack_element V, typename... Ts>
 consteval auto prune_voids_pack(pack<Rs...> result, pack<V, Ts...>)
 {
   return prune_voids_pack(pack<Rs...>{}, pack<Ts...>{});
 }
 
 // The first one is non void, but one of the others is void
-template <typename... Rs, typename T1, typename T2, typename... Ts>
-  requires(!std::is_void_v<T1> && !std::is_void_v<T2>)
+template <typename... Rs, nonvoid_pack_element T1, nonvoid_pack_element T2, typename... Ts>
 consteval auto prune_voids_pack(pack<Rs...> result, pack<T1, T2, Ts...>)
 {
   return prune_voids_pack(pack<Rs..., T1, T2>{}, pack<Ts...>{});
 }
 
 // Eats 4 types at the time
-template <typename... Rs, typename T1, typename T2, typename T3, typename T4, typename... Ts>
-  requires(!std::is_void_v<T1> && !std::is_void_v<T2> && !std::is_void_v<T3> && !std::is_void_v<T4>)
+template <typename... Rs, nonvoid_pack_element T1, nonvoid_pack_element T2, nonvoid_pack_element T3, nonvoid_pack_element T4, typename... Ts>
 consteval auto prune_voids_pack(pack<Rs...> result, pack<T1, T2, T3, T4, Ts...>)
 {
   return prune_voids_pack(pack<Rs..., T1, T2, T3, T4>{}, pack<Ts...>{});
 }
 
 // Eats 8 types at the time
-template <typename... Rs, typename T1, typename T2, typename T3, typename T4,
-          typename T5, typename T6, typename T7, typename T8, typename... Ts>
-  requires(!std::is_void_v<T1> && !std::is_void_v<T2> && !std::is_void_v<T3> && !std::is_void_v<T4> && !std::is_void_v<T5> && !std::is_void_v<T6> && !std::is_void_v<T7> && !std::is_void_v<T8>)
+template <typename... Rs, nonvoid_pack_element T1, nonvoid_pack_element T2, nonvoid_pack_element T3, nonvoid_pack_element T4,
+          nonvoid_pack_element T5, nonvoid_pack_element T6, nonvoid_pack_element T7, nonvoid_pack_element T8, typename... Ts>
 consteval auto prune_voids_pack(pack<Rs...> result, pack<T1, T2, T3, T4, T5, T6, T7, T8, Ts...>)
 {
   return prune_voids_pack(pack<Rs..., T1, T2, T3, T4, T5, T6, T7, T8>{}, pack<Ts...>{});

From e089b2ffab660169828c25fc529f00bb2fbf0a89 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 4 Dec 2024 12:11:30 +0100
Subject: [PATCH 717/726] DPL: use constraints directly on arguments in place
 of requires

---
 Framework/Foundation/include/Framework/Endian.h | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Endian.h b/Framework/Foundation/include/Framework/Endian.h
index a8fee915ee340..3df7912cb093a 100644
--- a/Framework/Foundation/include/Framework/Endian.h
+++ b/Framework/Foundation/include/Framework/Endian.h
@@ -35,22 +35,19 @@
 
 
 template <typename T>
-  requires std::same_as<T, uint16_t>
-inline uint16_t doSwap(uint16_t x)
+inline uint16_t doSwap(std::same_as<uint16_t> auto x)
 {
   return swap16_(x);
 }
 
 template <typename T>
-  requires std::same_as<T, uint32_t>
-inline uint32_t doSwap(uint32_t x)
+inline uint32_t doSwap(std::same_as<uint32_t> auto x)
 {
   return swap32_(x);
 }
 
 template <typename T>
-  requires std::same_as<T, uint64_t>
-inline uint64_t doSwap(uint64_t x)
+inline uint64_t doSwap(std::same_as<uint64_t> auto x)
 {
   return swap64_(x);
 }

From 2ae1d33489c22ce773d53b35fcd7842086a4daea Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 4 Dec 2024 12:40:25 +0100
Subject: [PATCH 718/726] DPL: Simplify prune_voids_pack

Use the requirements directly on the template arguments.
---
 Framework/Foundation/include/Framework/Pack.h | 23 ++++++++++---------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Pack.h b/Framework/Foundation/include/Framework/Pack.h
index 2355ffed6b35b..4fcf796a3ec62 100644
--- a/Framework/Foundation/include/Framework/Pack.h
+++ b/Framework/Foundation/include/Framework/Pack.h
@@ -114,42 +114,43 @@ consteval auto prune_voids_pack(Result result, pack<>)
   return result;
 }
 
+template <typename T>
+concept void_pack_element = std::is_void_v<T>;
+
+template <typename T>
+concept nonvoid_pack_element = !void_pack_element<T>;
+
 // The first one is non void, but one of the others is void
-template <typename... Rs, typename T, typename... Ts>
-  requires(!std::is_void_v<T>)
+template <typename... Rs, nonvoid_pack_element T, typename... Ts>
 consteval auto prune_voids_pack(pack<Rs...> result, pack<T, Ts...>)
 {
   return prune_voids_pack(pack<Rs..., T>{}, pack<Ts...>{});
 }
 
 // The first one is void
-template <typename... Rs, typename V, typename... Ts>
-  requires(std::is_void_v<V>)
+template <typename... Rs, void_pack_element V, typename... Ts>
 consteval auto prune_voids_pack(pack<Rs...> result, pack<V, Ts...>)
 {
   return prune_voids_pack(pack<Rs...>{}, pack<Ts...>{});
 }
 
 // The first one is non void, but one of the others is void
-template <typename... Rs, typename T1, typename T2, typename... Ts>
-  requires(!std::is_void_v<T1> && !std::is_void_v<T2>)
+template <typename... Rs, nonvoid_pack_element T1, nonvoid_pack_element T2, typename... Ts>
 consteval auto prune_voids_pack(pack<Rs...> result, pack<T1, T2, Ts...>)
 {
   return prune_voids_pack(pack<Rs..., T1, T2>{}, pack<Ts...>{});
 }
 
 // Eats 4 types at the time
-template <typename... Rs, typename T1, typename T2, typename T3, typename T4, typename... Ts>
-  requires(!std::is_void_v<T1> && !std::is_void_v<T2> && !std::is_void_v<T3> && !std::is_void_v<T4>)
+template <typename... Rs, nonvoid_pack_element T1, nonvoid_pack_element T2, nonvoid_pack_element T3, nonvoid_pack_element T4, typename... Ts>
 consteval auto prune_voids_pack(pack<Rs...> result, pack<T1, T2, T3, T4, Ts...>)
 {
   return prune_voids_pack(pack<Rs..., T1, T2, T3, T4>{}, pack<Ts...>{});
 }
 
 // Eats 8 types at the time
-template <typename... Rs, typename T1, typename T2, typename T3, typename T4,
-          typename T5, typename T6, typename T7, typename T8, typename... Ts>
-  requires(!std::is_void_v<T1> && !std::is_void_v<T2> && !std::is_void_v<T3> && !std::is_void_v<T4> && !std::is_void_v<T5> && !std::is_void_v<T6> && !std::is_void_v<T7> && !std::is_void_v<T8>)
+template <typename... Rs, nonvoid_pack_element T1, nonvoid_pack_element T2, nonvoid_pack_element T3, nonvoid_pack_element T4,
+          nonvoid_pack_element T5, nonvoid_pack_element T6, nonvoid_pack_element T7, nonvoid_pack_element T8, typename... Ts>
 consteval auto prune_voids_pack(pack<Rs...> result, pack<T1, T2, T3, T4, T5, T6, T7, T8, Ts...>)
 {
   return prune_voids_pack(pack<Rs..., T1, T2, T3, T4, T5, T6, T7, T8>{}, pack<Ts...>{});

From f4f29a3806e1371773c4ab7605fdd5845377dd45 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 4 Dec 2024 12:40:25 +0100
Subject: [PATCH 719/726] DPL: use constraints directly on arguments in place
 of requires

---
 Framework/Foundation/include/Framework/Endian.h | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/Framework/Foundation/include/Framework/Endian.h b/Framework/Foundation/include/Framework/Endian.h
index a8fee915ee340..5b69591030249 100644
--- a/Framework/Foundation/include/Framework/Endian.h
+++ b/Framework/Foundation/include/Framework/Endian.h
@@ -33,24 +33,20 @@
 #define O2_BIG_ENDIAN __BIG_ENDIAN
 #define O2_LITTLE_ENDIAN __LITTLE_ENDIAN
 
-
 template <typename T>
-  requires std::same_as<T, uint16_t>
-inline uint16_t doSwap(uint16_t x)
+inline uint16_t doSwap(std::same_as<uint16_t> auto x)
 {
   return swap16_(x);
 }
 
 template <typename T>
-  requires std::same_as<T, uint32_t>
-inline uint32_t doSwap(uint32_t x)
+inline uint32_t doSwap(std::same_as<uint32_t> auto x)
 {
   return swap32_(x);
 }
 
 template <typename T>
-  requires std::same_as<T, uint64_t>
-inline uint64_t doSwap(uint64_t x)
+inline uint64_t doSwap(std::same_as<uint64_t> auto x)
 {
   return swap64_(x);
 }

From 8c955dd74747abf9fb152a13dbad6895d0d450fe Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 4 Dec 2024 15:00:22 +0100
Subject: [PATCH 720/726] DPL: Move DataInputDirector to arrow::Dataset API

---
 .../src/AODJAlienReaderHelpers.cxx            |  16 ++-
 .../AnalysisSupport/src/DataInputDirector.cxx | 105 ++++++++++--------
 .../AnalysisSupport/src/DataInputDirector.h   |  14 +--
 3 files changed, 77 insertions(+), 58 deletions(-)

diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
index 9c19de85739ce..f8a9705e4eb62 100644
--- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
+++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
@@ -10,10 +10,12 @@
 // or submit itself to any jurisdiction.
 
 #include "AODJAlienReaderHelpers.h"
+#include <memory>
 #include "Framework/TableTreeHelpers.h"
 #include "Framework/AnalysisHelpers.h"
 #include "Framework/DataProcessingStats.h"
 #include "Framework/RootTableBuilderHelpers.h"
+#include "Framework/RootArrowFilesystem.h"
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/ConfigParamRegistry.h"
 #include "Framework/ControlService.h"
@@ -41,6 +43,8 @@
 #include <arrow/io/interfaces.h>
 #include <arrow/table.h>
 #include <arrow/util/key_value_metadata.h>
+#include <arrow/dataset/dataset.h>
+#include <arrow/dataset/file_base.h>
 
 using namespace o2;
 using namespace o2::aod;
@@ -272,11 +276,13 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const
             // Origin file name for derived output map
             auto o2 = Output(TFFileNameHeader);
             auto fileAndFolder = didir->getFileFolder(dh, fcnt, ntf);
-            std::string currentFilename(fileAndFolder.file->GetName());
-            if (strcmp(fileAndFolder.file->GetEndpointUrl()->GetProtocol(), "file") == 0 && fileAndFolder.file->GetEndpointUrl()->GetFile()[0] != '/') {
+            auto rootFS = std::dynamic_pointer_cast<TFileFileSystem>(fileAndFolder.filesystem());
+            auto* f = dynamic_cast<TFile*>(rootFS->GetFile());
+            std::string currentFilename(f->GetFile()->GetName());
+            if (strcmp(f->GetEndpointUrl()->GetProtocol(), "file") == 0 && f->GetEndpointUrl()->GetFile()[0] != '/') {
               // This is not an absolute local path. Make it absolute.
               static std::string pwd = gSystem->pwd() + std::string("/");
-              currentFilename = pwd + std::string(fileAndFolder.file->GetName());
+              currentFilename = pwd + std::string(f->GetName());
             }
             outputs.make<std::string>(o2) = currentFilename;
           }
@@ -312,7 +318,9 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const
       auto concrete = DataSpecUtils::asConcreteDataMatcher(firstRoute.matcher);
       auto dh = header::DataHeader(concrete.description, concrete.origin, concrete.subSpec);
       auto fileAndFolder = didir->getFileFolder(dh, fcnt, ntf);
-      if (!fileAndFolder.file) {
+
+      // In case the filesource is empty, move to the next one.
+      if (fileAndFolder.path().empty()) {
         fcnt += 1;
         ntf = 0;
         if (didir->atEnd(fcnt)) {
diff --git a/Framework/AnalysisSupport/src/DataInputDirector.cxx b/Framework/AnalysisSupport/src/DataInputDirector.cxx
index 172ecd66c0e64..85ae51a68f574 100644
--- a/Framework/AnalysisSupport/src/DataInputDirector.cxx
+++ b/Framework/AnalysisSupport/src/DataInputDirector.cxx
@@ -11,6 +11,7 @@
 #include "DataInputDirector.h"
 #include "Framework/DataDescriptorQueryBuilder.h"
 #include "Framework/Logger.h"
+#include "Framework/RootArrowFilesystem.h"
 #include "Framework/AnalysisDataModelHelpers.h"
 #include "Framework/Output.h"
 #include "Headers/DataHeader.h"
@@ -26,8 +27,12 @@
 #include "TGrid.h"
 #include "TObjString.h"
 #include "TMap.h"
+#include "TFile.h"
 
+#include <arrow/dataset/file_base.h>
+#include <arrow/dataset/dataset.h>
 #include <uv.h>
+#include <memory>
 
 #if __has_include(<TJAlienFile.h>)
 #include <TJAlienFile.h>
@@ -108,20 +113,22 @@ bool DataInputDescriptor::setFile(int counter)
 
   // open file
   auto filename = mfilenames[counter]->fileName;
-  if (mcurrentFile) {
-    if (mcurrentFile->GetName() == filename) {
+  auto rootFS = std::dynamic_pointer_cast<TFileFileSystem>(mCurrentFilesystem);
+  if (rootFS.get()) {
+    if (rootFS->GetFile()->GetName() == filename) {
       return true;
     }
     closeInputFile();
   }
-  mcurrentFile = TFile::Open(filename.c_str());
-  if (!mcurrentFile) {
+
+  mCurrentFilesystem = std::make_shared<TFileFileSystem>(TFile::Open(filename.c_str()), 50 * 1024 * 1024);
+  if (!mCurrentFilesystem.get()) {
     throw std::runtime_error(fmt::format("Couldn't open file \"{}\"!", filename));
   }
-  mcurrentFile->SetReadaheadSize(50 * 1024 * 1024);
+  rootFS = std::dynamic_pointer_cast<TFileFileSystem>(mCurrentFilesystem);
 
   // get the parent file map if exists
-  mParentFileMap = (TMap*)mcurrentFile->Get("parentFiles"); // folder name (DF_XXX) --> parent file (absolute path)
+  mParentFileMap = (TMap*)rootFS->GetFile()->Get("parentFiles"); // folder name (DF_XXX) --> parent file (absolute path)
   if (mParentFileMap && !mParentFileReplacement.empty()) {
     auto pos = mParentFileReplacement.find(';');
     if (pos == std::string::npos) {
@@ -141,7 +148,7 @@ bool DataInputDescriptor::setFile(int counter)
   // get the directory names
   if (mfilenames[counter]->numberOfTimeFrames <= 0) {
     std::regex TFRegex = std::regex("DF_[0-9]+");
-    TList* keyList = mcurrentFile->GetListOfKeys();
+    TList* keyList = rootFS->GetFile()->GetListOfKeys();
 
     // extract TF numbers and sort accordingly
     for (auto key : *keyList) {
@@ -193,26 +200,21 @@ uint64_t DataInputDescriptor::getTimeFrameNumber(int counter, int numTF)
   return (mfilenames[counter]->listOfTimeFrameNumbers)[numTF];
 }
 
-FileAndFolder DataInputDescriptor::getFileFolder(int counter, int numTF)
+arrow::dataset::FileSource DataInputDescriptor::getFileFolder(int counter, int numTF)
 {
-  FileAndFolder fileAndFolder;
-
   // open file
   if (!setFile(counter)) {
-    return fileAndFolder;
+    return {};
   }
 
   // no TF left
   if (mfilenames[counter]->numberOfTimeFrames > 0 && numTF >= mfilenames[counter]->numberOfTimeFrames) {
-    return fileAndFolder;
+    return {};
   }
 
-  fileAndFolder.file = mcurrentFile;
-  fileAndFolder.folderName = (mfilenames[counter]->listOfTimeFrameKeys)[numTF];
-
   mfilenames[counter]->alreadyRead[numTF] = true;
 
-  return fileAndFolder;
+  return {(mfilenames[counter]->listOfTimeFrameKeys)[numTF], mCurrentFilesystem};
 }
 
 DataInputDescriptor* DataInputDescriptor::getParentFile(int counter, int numTF, std::string treename)
@@ -223,15 +225,17 @@ DataInputDescriptor* DataInputDescriptor::getParentFile(int counter, int numTF,
   }
   auto folderName = (mfilenames[counter]->listOfTimeFrameKeys)[numTF];
   auto parentFileName = (TObjString*)mParentFileMap->GetValue(folderName.c_str());
+  // The current DF is not found in the parent map (this should not happen and is a fatal error)
+  auto rootFS = std::dynamic_pointer_cast<TFileFileSystem>(mCurrentFilesystem);
   if (!parentFileName) {
-    // The current DF is not found in the parent map (this should not happen and is a fatal error)
-    throw std::runtime_error(fmt::format(R"(parent file map exists but does not contain the current DF "{}" in file "{}")", folderName.c_str(), mcurrentFile->GetName()));
+    throw std::runtime_error(fmt::format(R"(parent file map exists but does not contain the current DF "{}" in file "{}")", folderName.c_str(), rootFS->GetFile()->GetName()));
     return nullptr;
   }
 
   if (mParentFile) {
     // Is this still the corresponding to the correct file?
-    if (parentFileName->GetString().CompareTo(mParentFile->mcurrentFile->GetName()) == 0) {
+    auto parentRootFS = std::dynamic_pointer_cast<TFileFileSystem>(mParentFile->mCurrentFilesystem);
+    if (parentFileName->GetString().CompareTo(parentRootFS->GetFile()->GetName()) == 0) {
       return mParentFile;
     } else {
       mParentFile->closeInputFile();
@@ -241,7 +245,8 @@ DataInputDescriptor* DataInputDescriptor::getParentFile(int counter, int numTF,
   }
 
   if (mLevel == mAllowedParentLevel) {
-    throw std::runtime_error(fmt::format(R"(while looking for tree "{}", the parent file was requested but we are already at level {} of maximal allowed level {} for DF "{}" in file "{}")", treename.c_str(), mLevel, mAllowedParentLevel, folderName.c_str(), mcurrentFile->GetName()));
+    throw std::runtime_error(fmt::format(R"(while looking for tree "{}", the parent file was requested but we are already at level {} of maximal allowed level {} for DF "{}" in file "{}")", treename.c_str(), mLevel, mAllowedParentLevel, folderName.c_str(),
+                                         rootFS->GetFile()->GetName()));
   }
 
   LOGP(info, "Opening parent file {} for DF {}", parentFileName->GetString().Data(), folderName.c_str());
@@ -270,11 +275,13 @@ void DataInputDescriptor::printFileStatistics()
   if (wait_time < 0) {
     wait_time = 0;
   }
-  std::string monitoringInfo(fmt::format("lfn={},size={},total_df={},read_df={},read_bytes={},read_calls={},io_time={:.1f},wait_time={:.1f},level={}", mcurrentFile->GetName(),
-                                         mcurrentFile->GetSize(), getTimeFramesInFile(mCurrentFileID), getReadTimeFramesInFile(mCurrentFileID), mcurrentFile->GetBytesRead(), mcurrentFile->GetReadCalls(),
+  auto rootFS = std::dynamic_pointer_cast<TFileFileSystem>(mCurrentFilesystem);
+  auto f = dynamic_cast<TFile*>(rootFS->GetFile());
+  std::string monitoringInfo(fmt::format("lfn={},size={},total_df={},read_df={},read_bytes={},read_calls={},io_time={:.1f},wait_time={:.1f},level={}", f->GetName(),
+                                         f->GetSize(), getTimeFramesInFile(mCurrentFileID), getReadTimeFramesInFile(mCurrentFileID), f->GetBytesRead(), f->GetReadCalls(),
                                          ((float)mIOTime / 1e9), ((float)wait_time / 1e9), mLevel));
 #if __has_include(<TJAlienFile.h>)
-  auto alienFile = dynamic_cast<TJAlienFile*>(mcurrentFile);
+  auto alienFile = dynamic_cast<TJAlienFile*>(f);
   if (alienFile) {
     monitoringInfo += fmt::format(",se={},open_time={:.1f}", alienFile->GetSE(), alienFile->GetElapsed());
   }
@@ -285,7 +292,7 @@ void DataInputDescriptor::printFileStatistics()
 
 void DataInputDescriptor::closeInputFile()
 {
-  if (mcurrentFile) {
+  if (mCurrentFilesystem.get()) {
     if (mParentFile) {
       mParentFile->closeInputFile();
       delete mParentFile;
@@ -296,9 +303,9 @@ void DataInputDescriptor::closeInputFile()
     mParentFileMap = nullptr;
 
     printFileStatistics();
-    mcurrentFile->Close();
-    delete mcurrentFile;
-    mcurrentFile = nullptr;
+    auto rootFS = std::dynamic_pointer_cast<TFileFileSystem>(mCurrentFilesystem);
+    rootFS->GetFile()->Close();
+    mCurrentFilesystem.reset();
   }
 }
 
@@ -358,40 +365,46 @@ bool DataInputDescriptor::readTree(DataAllocator& outputs, header::DataHeader dh
 {
   auto ioStart = uv_hrtime();
 
-  auto fileAndFolder = getFileFolder(counter, numTF);
-  if (!fileAndFolder.file) {
+  auto folder = getFileFolder(counter, numTF);
+  if (!folder.filesystem()) {
     return false;
   }
 
-  auto fullpath = fileAndFolder.folderName + "/" + treename;
-  auto tree = (TTree*)fileAndFolder.file->Get(fullpath.c_str());
+  auto format = std::make_shared<TTreeFileFormat>(totalSizeCompressed, totalSizeUncompressed);
+  auto fullpath = arrow::dataset::FileSource{folder.path() + "/" + treename, folder.filesystem()};
+  auto schemaOpt = format->Inspect(fullpath);
+  auto schema = *schemaOpt;
 
-  if (!tree) {
-    LOGP(debug, "Could not find tree {}. Trying in parent file.", fullpath.c_str());
+  auto fragment = format->MakeFragment(fullpath, {}, schema);
+
+  if (!fragment.ok()) {
+    LOGP(debug, "Could not find tree {}. Trying in parent file.", fullpath.path());
     auto parentFile = getParentFile(counter, numTF, treename);
     if (parentFile != nullptr) {
-      int parentNumTF = parentFile->findDFNumber(0, fileAndFolder.folderName);
+      int parentNumTF = parentFile->findDFNumber(0, folder.path());
       if (parentNumTF == -1) {
-        throw std::runtime_error(fmt::format(R"(DF {} listed in parent file map but not found in the corresponding file "{}")", fileAndFolder.folderName, parentFile->mcurrentFile->GetName()));
+        auto parentRootFS = std::dynamic_pointer_cast<TFileFileSystem>(parentFile->mCurrentFilesystem);
+        throw std::runtime_error(fmt::format(R"(DF {} listed in parent file map but not found in the corresponding file "{}")", folder.path(), parentRootFS->GetFile()->GetName()));
       }
       // first argument is 0 as the parent file object contains only 1 file
       return parentFile->readTree(outputs, dh, 0, parentNumTF, treename, totalSizeCompressed, totalSizeUncompressed);
     }
-    throw std::runtime_error(fmt::format(R"(Couldn't get TTree "{}" from "{}". Please check https://aliceo2group.github.io/analysis-framework/docs/troubleshooting/#tree-not-found for more information.)", fileAndFolder.folderName + "/" + treename, fileAndFolder.file->GetName()));
+    auto rootFS = std::dynamic_pointer_cast<TFileFileSystem>(mCurrentFilesystem);
+    throw std::runtime_error(fmt::format(R"(Couldn't get TTree "{}" from "{}". Please check https://aliceo2group.github.io/analysis-framework/docs/troubleshooting/#tree-not-found for more information.)", fullpath.path(), rootFS->GetFile()->GetName()));
   }
 
   // create table output
   auto o = Output(dh);
-  auto t2t = outputs.make<TreeToTable>(o);
 
-  // add branches to read
-  // fill the table
-  t2t->setLabel(tree->GetName());
-  totalSizeCompressed += tree->GetZipBytes();
-  totalSizeUncompressed += tree->GetTotBytes();
-  t2t->addAllColumns(tree);
-  t2t->fill(tree);
-  delete tree;
+  // FIXME: This should allow me to create a memory pool
+  // which I can then use to scan the dataset.
+  //
+  auto f2b = outputs.make<FragmentToBatch>(o);
+
+  //// add branches to read
+  //// fill the table
+  f2b->setLabel(treename.c_str());
+  f2b->fill(*fragment, schema, format);
 
   mIOTime += (uv_hrtime() - ioStart);
 
@@ -693,7 +706,7 @@ DataInputDescriptor* DataInputDirector::getDataInputDescriptor(header::DataHeade
   return result;
 }
 
-FileAndFolder DataInputDirector::getFileFolder(header::DataHeader dh, int counter, int numTF)
+arrow::dataset::FileSource DataInputDirector::getFileFolder(header::DataHeader dh, int counter, int numTF)
 {
   auto didesc = getDataInputDescriptor(dh);
   // if NOT match then use defaultDataInputDescriptor
diff --git a/Framework/AnalysisSupport/src/DataInputDirector.h b/Framework/AnalysisSupport/src/DataInputDirector.h
index eca0ef195d111..c03104be4ffba 100644
--- a/Framework/AnalysisSupport/src/DataInputDirector.h
+++ b/Framework/AnalysisSupport/src/DataInputDirector.h
@@ -16,6 +16,9 @@
 #include "Framework/DataDescriptorMatcher.h"
 #include "Framework/DataAllocator.h"
 
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/dataset/dataset.h>
+
 #include <regex>
 #include "rapidjson/fwd.h"
 
@@ -36,11 +39,6 @@ struct FileNameHolder {
 };
 FileNameHolder* makeFileNameHolder(std::string fileName);
 
-struct FileAndFolder {
-  TFile* file = nullptr;
-  std::string folderName = "";
-};
-
 class DataInputDescriptor
 {
   /// Holds information concerning the reading of an aod table.
@@ -78,7 +76,7 @@ class DataInputDescriptor
   int findDFNumber(int file, std::string dfName);
 
   uint64_t getTimeFrameNumber(int counter, int numTF);
-  FileAndFolder getFileFolder(int counter, int numTF);
+  arrow::dataset::FileSource getFileFolder(int counter, int numTF);
   DataInputDescriptor* getParentFile(int counter, int numTF, std::string treename);
   int getTimeFramesInFile(int counter);
   int getReadTimeFramesInFile(int counter);
@@ -98,7 +96,7 @@ class DataInputDescriptor
   std::string mParentFileReplacement;
   std::vector<FileNameHolder*> mfilenames;
   std::vector<FileNameHolder*>* mdefaultFilenamesPtr = nullptr;
-  TFile* mcurrentFile = nullptr;
+  std::shared_ptr<arrow::fs::FileSystem> mCurrentFilesystem;
   int mCurrentFileID = -1;
   bool mAlienSupport = false;
 
@@ -143,7 +141,7 @@ class DataInputDirector
 
   bool readTree(DataAllocator& outputs, header::DataHeader dh, int counter, int numTF, size_t& totalSizeCompressed, size_t& totalSizeUncompressed);
   uint64_t getTimeFrameNumber(header::DataHeader dh, int counter, int numTF);
-  FileAndFolder getFileFolder(header::DataHeader dh, int counter, int numTF);
+  arrow::dataset::FileSource getFileFolder(header::DataHeader dh, int counter, int numTF);
   int getTimeFramesInFile(header::DataHeader dh, int counter);
 
   uint64_t getTotalSizeCompressed();

From 1e7eb118faa37b2477f9e34ddbf304475fe1f23d Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Fri, 6 Dec 2024 11:12:13 +0100
Subject: [PATCH 721/726] Do not compile tests which need GLFW if it's not
 found

---
 Detectors/MUON/MCH/Geometry/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Detectors/MUON/MCH/Geometry/CMakeLists.txt b/Detectors/MUON/MCH/Geometry/CMakeLists.txt
index 5e367fa87bbf4..000364af028c6 100644
--- a/Detectors/MUON/MCH/Geometry/CMakeLists.txt
+++ b/Detectors/MUON/MCH/Geometry/CMakeLists.txt
@@ -12,7 +12,7 @@
 add_subdirectory(MisAligner)
 add_subdirectory(Transformer)
 add_subdirectory(Creator)
-if (BUILD_TESTING)
+if (BUILD_TESTING AND GLFW_FOUND)
 add_subdirectory(Test)
 endif()
 

From dd0cc3fe24e54e0be617103cc0283bfc5f2d9a77 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 11 Dec 2024 11:52:19 +0100
Subject: [PATCH 722/726] Make sure only the signposts matching the provided
 string are enabled

---
 Framework/Core/src/runDataProcessing.cxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx
index 0861ae1a65a0c..4bfbc3232822a 100644
--- a/Framework/Core/src/runDataProcessing.cxx
+++ b/Framework/Core/src/runDataProcessing.cxx
@@ -2786,7 +2786,7 @@ void enableSignposts(std::string const& signpostsToEnable)
     }
 
     auto fullName = prefix + std::string{selectedName, last ? last - selectedName : strlen(selectedName)};
-    if (strncmp(name, fullName.data(), fullName.size()) == 0) {
+    if (fullName == name) {
       LOGP(info, "Enabling signposts for stream \"{}\" with depth {}.", fullName, maxDepth);
       _o2_log_set_stacktrace(log, maxDepth);
       return false;

From 68c93ae26f6b2781cc221ed0cb52a30c82d68c24 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Wed, 11 Dec 2024 17:15:44 +0100
Subject: [PATCH 723/726] DPL: extra signposts for InitTask callback

---
 Framework/Core/src/DataProcessingDevice.cxx | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Framework/Core/src/DataProcessingDevice.cxx b/Framework/Core/src/DataProcessingDevice.cxx
index f4123de250e79..da92c73e1e16a 100644
--- a/Framework/Core/src/DataProcessingDevice.cxx
+++ b/Framework/Core/src/DataProcessingDevice.cxx
@@ -1008,6 +1008,9 @@ void DataProcessingDevice::InitTask()
   auto ref = ServiceRegistryRef{mServiceRegistry};
   auto& deviceContext = ref.get<DeviceContext>();
   auto& context = ref.get<DataProcessorContext>();
+
+  O2_SIGNPOST_ID_FROM_POINTER(cid, device, &context);
+  O2_SIGNPOST_START(device, cid, "InitTask", "Entering InitTask callback.");
   auto& spec = getRunningDevice(mRunningDevice, mServiceRegistry);
   auto distinct = DataRelayerHelpers::createDistinctRouteIndex(spec.inputs);
   auto& state = ref.get<DeviceState>();
@@ -1098,10 +1101,13 @@ void DataProcessingDevice::InitTask()
   // We will get there.
   this->fillContext(mServiceRegistry.get<DataProcessorContext>(ServiceRegistry::globalDeviceSalt()), deviceContext);
 
+  O2_SIGNPOST_END(device, cid, "InitTask", "Exiting InitTask callback waiting for the remaining region callbacks.");
+
   auto hasPendingEvents = [&mutex = mRegionInfoMutex, &pendingRegionInfos = mPendingRegionInfos](DeviceContext& deviceContext) {
     std::lock_guard<std::mutex> lock(mutex);
     return (pendingRegionInfos.empty() == false) || deviceContext.expectedRegionCallbacks > 0;
   };
+  O2_SIGNPOST_START(device, cid, "InitTask", "Waiting for registation events.");
   /// We now run an event loop also in InitTask. This is needed to:
   /// * Make sure region registration callbacks are invoked
   /// on the main thread.
@@ -1111,10 +1117,12 @@ void DataProcessingDevice::InitTask()
     uv_run(state.loop, UV_RUN_ONCE);
     // Handle callbacks if any
     {
+      O2_SIGNPOST_EVENT_EMIT(device, cid, "InitTask", "Memory registration event received.");
       std::lock_guard<std::mutex> lock(mRegionInfoMutex);
       handleRegionCallbacks(mServiceRegistry, mPendingRegionInfos);
     }
   }
+  O2_SIGNPOST_END(device, cid, "InitTask", "Done waiting for registration events.");
 }
 
 void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceContext& deviceContext)

From 319d2cf6baa17cd623ed57323e714c726f618aec Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 12 Dec 2024 12:18:23 +0100
Subject: [PATCH 724/726] DPL Analysis: add RNTuple arrow::Dataset support

As part of the changes, move the actual logic which serialises /
deserialised things to plugins so that we do not need to depend
or RNTuple in production code.
---
 Framework/AnalysisSupport/CMakeLists.txt      |  10 +
 .../AnalysisSupport/src/RNTuplePlugin.cxx     | 816 +++++++++++++++++
 Framework/AnalysisSupport/src/TTreePlugin.cxx | 862 ++++++++++++++++++
 Framework/Core/CMakeLists.txt                 |   3 +
 Framework/Core/include/Framework/Plugins.h    |   8 +
 .../include/Framework/RootArrowFilesystem.h   | 174 +---
 Framework/Core/src/Plugin.cxx                 |  69 ++
 Framework/Core/src/RootArrowFilesystem.cxx    | 721 +--------------
 Framework/Core/test/test_Root2ArrowTable.cxx  | 105 ++-
 9 files changed, 1941 insertions(+), 827 deletions(-)
 create mode 100644 Framework/AnalysisSupport/src/RNTuplePlugin.cxx
 create mode 100644 Framework/AnalysisSupport/src/TTreePlugin.cxx

diff --git a/Framework/AnalysisSupport/CMakeLists.txt b/Framework/AnalysisSupport/CMakeLists.txt
index 5fb1282469711..05cd8075a49d8 100644
--- a/Framework/AnalysisSupport/CMakeLists.txt
+++ b/Framework/AnalysisSupport/CMakeLists.txt
@@ -24,6 +24,16 @@ o2_add_library(FrameworkAnalysisSupport
                PRIVATE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/src
                PUBLIC_LINK_LIBRARIES O2::Framework ${EXTRA_TARGETS} ROOT::TreePlayer)
 
+o2_add_library(FrameworkAnalysisRNTupleSupport
+               SOURCES src/RNTuplePlugin.cxx
+               PRIVATE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/src
+               PUBLIC_LINK_LIBRARIES O2::Framework ${EXTRA_TARGETS} ROOT::ROOTNTuple)
+
+o2_add_library(FrameworkAnalysisTTreeSupport
+               SOURCES src/TTreePlugin.cxx
+               PRIVATE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/src
+               PUBLIC_LINK_LIBRARIES O2::Framework ${EXTRA_TARGETS} ROOT::TreePlayer)
+
 o2_add_test(DataInputDirector NAME test_Framework_test_DataInputDirector
                SOURCES test/test_DataInputDirector.cxx
                COMPONENT_NAME Framework
diff --git a/Framework/AnalysisSupport/src/RNTuplePlugin.cxx b/Framework/AnalysisSupport/src/RNTuplePlugin.cxx
new file mode 100644
index 0000000000000..5e73c1790e63a
--- /dev/null
+++ b/Framework/AnalysisSupport/src/RNTuplePlugin.cxx
@@ -0,0 +1,816 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "Framework/RuntimeError.h"
+#include "Framework/RootArrowFilesystem.h"
+#include "Framework/Plugins.h"
+#include <ROOT/RNTupleModel.hxx>
+#include <ROOT/RNTupleWriteOptions.hxx>
+#include <ROOT/RNTupleWriter.hxx>
+#include <ROOT/RField.hxx>
+#include <ROOT/RNTuple.hxx>
+#include <ROOT/RNTupleReader.hxx>
+#include <ROOT/RFieldVisitor.hxx>
+#include <ROOT/RNTupleInspector.hxx>
+#include <ROOT/RVec.hxx>
+#include <TBufferFile.h>
+
+#include <TDirectory.h>
+#include <arrow/array/array_nested.h>
+#include <arrow/array/array_primitive.h>
+#include <arrow/array/builder_nested.h>
+#include <arrow/array/builder_primitive.h>
+#include <arrow/dataset/file_base.h>
+
+template class
+  std::unique_ptr<ROOT::Experimental::RNTupleReader>;
+
+namespace o2::framework
+{
+
+class RNTupleFileWriteOptions : public arrow::dataset::FileWriteOptions
+{
+ public:
+  RNTupleFileWriteOptions(std::shared_ptr<arrow::dataset::FileFormat> format)
+    : FileWriteOptions(format)
+  {
+  }
+};
+
+// A filesystem which allows me to get a RNTuple
+class RNTupleFileSystem : public VirtualRootFileSystemBase
+{
+ public:
+  ~RNTupleFileSystem() override;
+
+  std::shared_ptr<VirtualRootFileSystemBase> GetSubFilesystem(arrow::dataset::FileSource source) override
+  {
+    return std::dynamic_pointer_cast<VirtualRootFileSystemBase>(shared_from_this());
+  };
+  virtual ROOT::Experimental::RNTuple* GetRNTuple(arrow::dataset::FileSource source) = 0;
+};
+
+class SingleRNTupleFileSystem : public RNTupleFileSystem
+{
+ public:
+  SingleRNTupleFileSystem(ROOT::Experimental::RNTuple* tuple)
+    : RNTupleFileSystem(),
+      mTuple(tuple)
+  {
+  }
+
+  arrow::Result<arrow::fs::FileInfo> GetFileInfo(std::string const& path) override;
+
+  std::string type_name() const override
+  {
+    return "rntuple";
+  }
+
+  ROOT::Experimental::RNTuple* GetRNTuple(arrow::dataset::FileSource) override
+  {
+    // Simply return the only TTree we have
+    return mTuple;
+  }
+
+ private:
+  ROOT::Experimental::RNTuple* mTuple;
+};
+
+arrow::Result<arrow::fs::FileInfo> SingleRNTupleFileSystem::GetFileInfo(std::string const& path)
+{
+  arrow::dataset::FileSource source(path, shared_from_this());
+  arrow::fs::FileInfo result;
+  result.set_path(path);
+  result.set_type(arrow::fs::FileType::File);
+  return result;
+}
+
+class RNTupleFileFragment : public arrow::dataset::FileFragment
+{
+ public:
+  RNTupleFileFragment(arrow::dataset::FileSource source,
+                      std::shared_ptr<arrow::dataset::FileFormat> format,
+                      arrow::compute::Expression partition_expression,
+                      std::shared_ptr<arrow::Schema> physical_schema)
+    : FileFragment(std::move(source), std::move(format), std::move(partition_expression), std::move(physical_schema))
+  {
+  }
+};
+
+class RNTupleFileFormat : public arrow::dataset::FileFormat
+{
+  size_t& mTotCompressedSize;
+  size_t& mTotUncompressedSize;
+
+ public:
+  RNTupleFileFormat(size_t& totalCompressedSize, size_t& totalUncompressedSize)
+    : FileFormat({}),
+      mTotCompressedSize(totalCompressedSize),
+      mTotUncompressedSize(totalUncompressedSize)
+  {
+  }
+
+  ~RNTupleFileFormat() override = default;
+
+  std::string type_name() const override
+  {
+    return "rntuple";
+  }
+
+  bool Equals(const FileFormat& other) const override
+  {
+    return other.type_name() == this->type_name();
+  }
+
+  arrow::Result<bool> IsSupported(const arrow::dataset::FileSource& source) const override
+  {
+    auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
+    auto subFs = fs->GetSubFilesystem(source);
+    if (std::dynamic_pointer_cast<RNTupleFileSystem>(subFs)) {
+      return true;
+    }
+    return false;
+  }
+
+  arrow::Result<std::shared_ptr<arrow::Schema>> Inspect(const arrow::dataset::FileSource& source) const override;
+
+  arrow::Result<arrow::RecordBatchGenerator> ScanBatchesAsync(
+    const std::shared_ptr<arrow::dataset::ScanOptions>& options,
+    const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const override;
+
+  std::shared_ptr<arrow::dataset::FileWriteOptions> DefaultWriteOptions() override;
+
+  arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination,
+                                                                        std::shared_ptr<arrow::Schema> schema,
+                                                                        std::shared_ptr<arrow::dataset::FileWriteOptions> options,
+                                                                        arrow::fs::FileLocator destination_locator) const override;
+  arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> MakeFragment(
+    arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
+    std::shared_ptr<arrow::Schema> physical_schema) override;
+};
+
+struct RootNTupleVisitor : public ROOT::Experimental::Detail::RFieldVisitor {
+  void VisitArrayField(const ROOT::Experimental::RArrayField& field) override
+  {
+    int size = field.GetLength();
+    RootNTupleVisitor valueVisitor{};
+    auto valueField = field.GetSubFields()[0];
+    valueField->AcceptVisitor(valueVisitor);
+    auto type = valueVisitor.datatype;
+    this->datatype = arrow::fixed_size_list(type, size);
+  }
+
+  void VisitRVecField(const ROOT::Experimental::RRVecField& field) override
+  {
+    RootNTupleVisitor valueVisitor{};
+    auto valueField = field.GetSubFields()[0];
+    valueField->AcceptVisitor(valueVisitor);
+    auto type = valueVisitor.datatype;
+    this->datatype = arrow::list(type);
+  }
+
+  void VisitField(const ROOT::Experimental::RFieldBase& field) override
+  {
+    throw o2::framework::runtime_error_f("Unknown field %s with type %s", field.GetFieldName().c_str(), field.GetTypeName().c_str());
+  }
+
+  void VisitIntField(const ROOT::Experimental::RField<int>& field) override
+  {
+    this->datatype = arrow::int32();
+  }
+
+  void VisitBoolField(const ROOT::Experimental::RField<bool>& field) override
+  {
+    this->datatype = arrow::boolean();
+  }
+
+  void VisitFloatField(const ROOT::Experimental::RField<float>& field) override
+  {
+    this->datatype = arrow::float32();
+  }
+
+  void VisitDoubleField(const ROOT::Experimental::RField<double>& field) override
+  {
+    this->datatype = arrow::float64();
+  }
+  std::shared_ptr<arrow::DataType> datatype;
+};
+} // namespace o2::framework
+
+auto arrowTypeFromRNTuple(ROOT::Experimental::RFieldBase const& field, int size)
+{
+  o2::framework::RootNTupleVisitor visitor;
+  field.AcceptVisitor(visitor);
+  return visitor.datatype;
+}
+
+namespace o2::framework
+{
+std::unique_ptr<ROOT::Experimental::RFieldBase> rootFieldFromArrow(std::shared_ptr<arrow::Field> field, std::string name)
+{
+  using namespace ROOT::Experimental;
+  switch (field->type()->id()) {
+    case arrow::Type::BOOL:
+      return std::make_unique<RField<bool>>(name);
+    case arrow::Type::UINT8:
+      return std::make_unique<RField<uint8_t>>(name);
+    case arrow::Type::UINT16:
+      return std::make_unique<RField<uint16_t>>(name);
+    case arrow::Type::UINT32:
+      return std::make_unique<RField<uint32_t>>(name);
+    case arrow::Type::UINT64:
+      return std::make_unique<RField<uint64_t>>(name);
+    case arrow::Type::INT8:
+      return std::make_unique<RField<int8_t>>(name);
+    case arrow::Type::INT16:
+      return std::make_unique<RField<int16_t>>(name);
+    case arrow::Type::INT32:
+      return std::make_unique<RField<int32_t>>(name);
+    case arrow::Type::INT64:
+      return std::make_unique<RField<int64_t>>(name);
+    case arrow::Type::FLOAT:
+      return std::make_unique<RField<float>>(name);
+    case arrow::Type::DOUBLE:
+      return std::make_unique<RField<double>>(name);
+    default:
+      throw runtime_error("Unsupported arrow column type");
+  }
+}
+
+class RNTupleFileWriter : public arrow::dataset::FileWriter
+{
+  std::shared_ptr<ROOT::Experimental::RNTupleWriter> mWriter;
+  bool firstBatch = true;
+  std::vector<std::shared_ptr<arrow::Array>> valueArrays;
+  std::vector<std::shared_ptr<arrow::DataType>> valueTypes;
+  std::vector<size_t> valueCount;
+
+ public:
+  RNTupleFileWriter(std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options,
+                    std::shared_ptr<arrow::io::OutputStream> destination,
+                    arrow::fs::FileLocator destination_locator)
+    : FileWriter(schema, options, destination, destination_locator)
+  {
+    using namespace ROOT::Experimental;
+
+    auto model = RNTupleModel::CreateBare();
+    // Let's create a model from the physical schema
+    for (auto i = 0u; i < schema->fields().size(); ++i) {
+      auto& field = schema->field(i);
+
+      // Construct all the needed branches.
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          auto list = std::static_pointer_cast<arrow::FixedSizeListType>(field->type());
+          auto valueField = field->type()->field(0);
+          model->AddField(std::make_unique<RArrayField>(field->name(), rootFieldFromArrow(valueField, "_0"), list->list_size()));
+        } break;
+        case arrow::Type::LIST: {
+          auto valueField = field->type()->field(0);
+          model->AddField(std::make_unique<RRVecField>(field->name(), rootFieldFromArrow(valueField, "_0")));
+        } break;
+        default: {
+          model->AddField(rootFieldFromArrow(field, field->name()));
+        } break;
+      }
+    }
+    auto fileStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
+    auto* file = dynamic_cast<TFile*>(fileStream->GetDirectory());
+    mWriter = RNTupleWriter::Append(std::move(model), destination_locator_.path, *file, {});
+  }
+
+  arrow::Status Write(const std::shared_ptr<arrow::RecordBatch>& batch) override
+  {
+    if (firstBatch) {
+      firstBatch = false;
+    }
+
+    // Support writing empty tables
+    if (batch->columns().empty() || batch->num_rows() == 0) {
+      return arrow::Status::OK();
+    }
+
+    for (auto i = 0u; i < batch->columns().size(); ++i) {
+      auto column = batch->column(i);
+      auto& field = batch->schema()->field(i);
+
+      valueArrays.push_back(nullptr);
+      valueTypes.push_back(nullptr);
+      valueCount.push_back(1);
+
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          auto list = std::static_pointer_cast<arrow::FixedSizeListArray>(column);
+          auto listType = std::static_pointer_cast<arrow::FixedSizeListType>(field->type());
+          if (field->type()->field(0)->type()->id() == arrow::Type::BOOL) {
+            auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(list->values());
+            int64_t length = boolArray->length();
+            arrow::UInt8Builder builder;
+            auto ok = builder.Reserve(length);
+
+            for (int64_t i = 0; i < length; ++i) {
+              if (boolArray->IsValid(i)) {
+                // Expand each boolean value (true/false) to uint8 (1/0)
+                uint8_t value = boolArray->Value(i) ? 1 : 0;
+                auto ok = builder.Append(value);
+              } else {
+                // Append null for invalid entries
+                auto ok = builder.AppendNull();
+              }
+            }
+            valueArrays.back() = *builder.Finish();
+            valueTypes.back() = valueArrays.back()->type();
+          } else {
+            valueArrays.back() = list->values();
+            valueTypes.back() = field->type()->field(0)->type();
+          }
+          valueCount.back() = listType->list_size();
+        } break;
+        case arrow::Type::LIST: {
+          auto list = std::static_pointer_cast<arrow::ListArray>(column);
+          valueArrays.back() = list;
+          valueTypes.back() = field->type()->field(0)->type();
+          valueCount.back() = -1;
+        } break;
+        case arrow::Type::BOOL: {
+          // We unpack the array
+          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(column);
+          int64_t length = boolArray->length();
+          arrow::UInt8Builder builder;
+          auto ok = builder.Reserve(length);
+
+          for (int64_t i = 0; i < length; ++i) {
+            if (boolArray->IsValid(i)) {
+              // Expand each boolean value (true/false) to uint8 (1/0)
+              uint8_t value = boolArray->Value(i) ? 1 : 0;
+              auto ok = builder.Append(value);
+            } else {
+              // Append null for invalid entries
+              auto ok = builder.AppendNull();
+            }
+          }
+          valueArrays.back() = *builder.Finish();
+          valueTypes.back() = valueArrays.back()->type();
+        } break;
+        default:
+          valueArrays.back() = column;
+          valueTypes.back() = field->type();
+          break;
+      }
+    }
+
+    int64_t pos = 0;
+
+    auto entry = mWriter->CreateEntry();
+
+    while (pos < batch->num_rows()) {
+      for (size_t ci = 0; ci < batch->columns().size(); ++ci) {
+        auto type = batch->column(ci)->type();
+        auto field = batch->schema()->field(ci);
+        auto token = entry->GetToken(field->name());
+
+        switch (type->id()) {
+          case arrow::Type::LIST: {
+            auto list = std::static_pointer_cast<arrow::ListArray>(valueArrays[ci]);
+            auto value_slice = list->value_slice(pos);
+
+            valueCount[ci] = value_slice->length();
+            auto bindValue = [&vc = valueCount, ci, token](auto array, std::unique_ptr<ROOT::Experimental::REntry>& entry) -> void {
+              using value_type = std::decay_t<decltype(*array.get())>::value_type;
+              auto v = std::make_shared<ROOT::VecOps::RVec<value_type>>((value_type*)array->raw_values(), vc[ci]);
+              entry->BindValue(token, v);
+            };
+            switch (valueTypes[ci]->id()) {
+              case arrow::Type::FLOAT: {
+                bindValue(std::static_pointer_cast<arrow::FloatArray>(value_slice), entry);
+              } break;
+              case arrow::Type::DOUBLE: {
+                bindValue(std::static_pointer_cast<arrow::DoubleArray>(value_slice), entry);
+              } break;
+              case arrow::Type::INT8: {
+                bindValue(std::static_pointer_cast<arrow::Int8Array>(value_slice), entry);
+              } break;
+              case arrow::Type::INT16: {
+                bindValue(std::static_pointer_cast<arrow::Int16Array>(value_slice), entry);
+              } break;
+              case arrow::Type::INT32: {
+                bindValue(std::static_pointer_cast<arrow::Int32Array>(value_slice), entry);
+              } break;
+              case arrow::Type::INT64: {
+                bindValue(std::static_pointer_cast<arrow::Int64Array>(value_slice), entry);
+              } break;
+              case arrow::Type::UINT8: {
+                bindValue(std::static_pointer_cast<arrow::UInt8Array>(value_slice), entry);
+              } break;
+              case arrow::Type::UINT16: {
+                bindValue(std::static_pointer_cast<arrow::UInt16Array>(value_slice), entry);
+              } break;
+              case arrow::Type::UINT32: {
+                bindValue(std::static_pointer_cast<arrow::UInt32Array>(value_slice), entry);
+              } break;
+              case arrow::Type::UINT64: {
+                bindValue(std::static_pointer_cast<arrow::UInt64Array>(value_slice), entry);
+              } break;
+              default: {
+                throw runtime_error("Unsupported kind of VLA");
+              } break;
+            }
+          } break;
+          case arrow::Type::FIXED_SIZE_LIST: {
+            entry->BindRawPtr<void>(token, (void*)(valueArrays[ci]->data()->buffers[1]->data() + pos * valueCount[ci] * valueTypes[ci]->byte_width()));
+          } break;
+          case arrow::Type::BOOL: {
+            // Not sure we actually need this
+            entry->BindRawPtr<bool>(token, (bool*)(valueArrays[ci]->data()->buffers[1]->data() + pos * 1));
+          } break;
+          default:
+            // By default we consider things scalars.
+            entry->BindRawPtr<void>(token, (void*)(valueArrays[ci]->data()->buffers[1]->data() + pos * valueTypes[ci]->byte_width()));
+            break;
+        }
+      }
+      mWriter->Fill(*entry);
+      ++pos;
+    }
+    // mWriter->CommitCluster();
+
+    return arrow::Status::OK();
+  }
+
+  arrow::Future<>
+    FinishInternal() override
+  {
+    return {};
+  };
+};
+
+arrow::Result<std::shared_ptr<arrow::Schema>> RNTupleFileFormat::Inspect(const arrow::dataset::FileSource& source) const
+{
+
+  auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
+  // Actually get the TTree from the ROOT file.
+  auto ntupleFs = std::dynamic_pointer_cast<RNTupleFileSystem>(fs->GetSubFilesystem(source));
+  if (!ntupleFs.get()) {
+    throw runtime_error_f("Unknown filesystem %s\n", source.filesystem()->type_name().c_str());
+  }
+  ROOT::Experimental::RNTuple* rntuple = ntupleFs->GetRNTuple(source);
+
+  auto inspector = ROOT::Experimental::RNTupleInspector::Create(rntuple);
+
+  auto reader = ROOT::Experimental::RNTupleReader::Open(rntuple);
+
+  auto& tupleField0 = reader->GetModel().GetFieldZero();
+  std::vector<std::shared_ptr<arrow::Field>> fields;
+  for (auto& tupleField : tupleField0.GetSubFields()) {
+    auto field = std::make_shared<arrow::Field>(tupleField->GetFieldName(), arrowTypeFromRNTuple(*tupleField, tupleField->GetValueSize()));
+    fields.push_back(field);
+  }
+
+  return std::make_shared<arrow::Schema>(fields);
+}
+
+arrow::Result<arrow::RecordBatchGenerator> RNTupleFileFormat::ScanBatchesAsync(
+  const std::shared_ptr<arrow::dataset::ScanOptions>& options,
+  const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const
+{
+  auto dataset_schema = options->dataset_schema;
+  auto ntupleFragment = std::dynamic_pointer_cast<RNTupleFileFragment>(fragment);
+
+  auto generator = [pool = options->pool, ntupleFragment, dataset_schema, &totalCompressedSize = mTotCompressedSize,
+                    &totalUncompressedSize = mTotUncompressedSize]() -> arrow::Future<std::shared_ptr<arrow::RecordBatch>> {
+    using namespace ROOT::Experimental;
+    std::vector<std::shared_ptr<arrow::Array>> columns;
+    std::vector<std::shared_ptr<arrow::Field>> fields = dataset_schema->fields();
+
+    auto containerFS = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(ntupleFragment->source().filesystem());
+    auto fs = std::dynamic_pointer_cast<RNTupleFileSystem>(containerFS->GetSubFilesystem(ntupleFragment->source()));
+
+    int64_t rows = -1;
+    ROOT::Experimental::RNTuple* rntuple = fs->GetRNTuple(ntupleFragment->source());
+    auto reader = ROOT::Experimental::RNTupleReader::Open(rntuple);
+    auto& model = reader->GetModel();
+    for (auto& physicalField : fields) {
+      auto bulk = model.CreateBulk(physicalField->name());
+
+      auto listType = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type());
+
+      auto& descriptor = reader->GetDescriptor();
+      auto totalEntries = reader->GetNEntries();
+
+      if (rows == -1) {
+        rows = totalEntries;
+      }
+      if (rows != totalEntries) {
+        throw runtime_error_f("Unmatching number of rows for branch %s", physicalField->name().c_str());
+      }
+      arrow::Status status;
+      int readEntries = 0;
+      std::shared_ptr<arrow::Array> array;
+      if (physicalField->type() == arrow::boolean() ||
+          (listType && physicalField->type()->field(0)->type() == arrow::boolean())) {
+        if (listType) {
+          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
+          auto status = arrow::MakeBuilder(pool, physicalField->type()->field(0)->type(), &builder);
+          if (!status.ok()) {
+            throw runtime_error("Cannot create value builder");
+          }
+          auto listBuilder = std::make_unique<arrow::FixedSizeListBuilder>(pool, std::move(builder), listType->list_size());
+          auto valueBuilder = listBuilder.get()->value_builder();
+          // boolean array special case: we need to use builder to create the bitmap
+          status = valueBuilder->Reserve(totalEntries * listType->list_size());
+          status &= listBuilder->Reserve(totalEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to reserve memory for array builder");
+          }
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          // No adoption for now...
+          // bulk.AdoptBuffer(buffer, totalEntries)
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            void* ptr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries());
+            int readLast = index.GetNEntries();
+            readEntries += readLast;
+            status &= static_cast<arrow::BooleanBuilder*>(valueBuilder)->AppendValues(reinterpret_cast<uint8_t const*>(ptr), readLast * listType->list_size());
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+          status &= static_cast<arrow::FixedSizeListBuilder*>(listBuilder.get())->AppendValues(readEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to append values to array");
+          }
+          status &= listBuilder->Finish(&array);
+          if (!status.ok()) {
+            throw runtime_error("Failed to create array");
+          }
+        } else if (listType == nullptr) {
+          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
+          auto status = arrow::MakeBuilder(pool, physicalField->type(), &builder);
+          if (!status.ok()) {
+            throw runtime_error("Cannot create builder");
+          }
+          auto valueBuilder = static_cast<arrow::BooleanBuilder*>(builder.get());
+          // boolean array special case: we need to use builder to create the bitmap
+          status = valueBuilder->Reserve(totalEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to reserve memory for array builder");
+          }
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            void* ptr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries());
+            int readLast = index.GetNEntries();
+            readEntries += readLast;
+            status &= valueBuilder->AppendValues(reinterpret_cast<uint8_t const*>(ptr), readLast);
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+          if (!status.ok()) {
+            throw runtime_error("Failed to append values to array");
+          }
+          status &= valueBuilder->Finish(&array);
+          if (!status.ok()) {
+            throw runtime_error("Failed to create array");
+          }
+        }
+      } else {
+        // other types: use serialized read to build arrays directly.
+        auto typeSize = physicalField->type()->byte_width();
+        // FIXME: for now...
+        auto bytes = 0;
+        auto branchSize = bytes ? bytes : 1000000;
+        auto&& result = arrow::AllocateResizableBuffer(branchSize, pool);
+        if (!result.ok()) {
+          throw runtime_error("Cannot allocate values buffer");
+        }
+        std::shared_ptr<arrow::Buffer> arrowValuesBuffer = std::move(result).ValueUnsafe();
+        auto ptr = arrowValuesBuffer->mutable_data();
+        if (ptr == nullptr) {
+          throw runtime_error("Invalid buffer");
+        }
+
+        std::unique_ptr<TBufferFile> offsetBuffer = nullptr;
+
+        std::shared_ptr<arrow::Buffer> arrowOffsetBuffer;
+        std::span<int> offsets;
+        int size = 0;
+        uint32_t totalSize = 0;
+        int64_t listSize = 1;
+        if (auto fixedSizeList = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type())) {
+          listSize = fixedSizeList->list_size();
+          typeSize = fixedSizeList->field(0)->type()->byte_width();
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            void* inPtr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries());
+
+            int readLast = index.GetNEntries();
+            if (listSize == -1) {
+              size = offsets[readEntries + readLast] - offsets[readEntries];
+            } else {
+              size = readLast * listSize;
+            }
+            readEntries += readLast;
+            memcpy(ptr, inPtr, size * typeSize);
+            ptr += (ptrdiff_t)(size * typeSize);
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+        } else if (auto vlaListType = std::dynamic_pointer_cast<arrow::ListType>(physicalField->type())) {
+          listSize = -1;
+          typeSize = vlaListType->field(0)->type()->byte_width();
+          offsetBuffer = std::make_unique<TBufferFile>(TBuffer::EMode::kWrite, 4 * 1024 * 1024);
+          result = arrow::AllocateResizableBuffer((totalEntries + 1) * (int64_t)sizeof(int), pool);
+          if (!result.ok()) {
+            throw runtime_error("Cannot allocate offset buffer");
+          }
+          arrowOffsetBuffer = std::move(result).ValueUnsafe();
+
+          // Offset bulk
+          auto offsetBulk = model.CreateBulk(physicalField->name());
+          // Actual values are in a different place...
+          bulk = model.CreateBulk(physicalField->name());
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          auto* ptrOffset = reinterpret_cast<int*>(arrowOffsetBuffer->mutable_data());
+          auto* tPtrOffset = reinterpret_cast<int*>(ptrOffset);
+          offsets = std::span<int>{tPtrOffset, tPtrOffset + totalEntries + 1};
+
+          auto copyOffsets = [&arrowValuesBuffer, &pool, &ptrOffset, &ptr, &totalSize](auto inPtr, size_t total) {
+            using value_type = typename std::decay_t<decltype(*inPtr)>::value_type;
+            for (size_t i = 0; i < total; i++) {
+              *ptrOffset++ = totalSize;
+              totalSize += inPtr[i].size();
+            }
+            *ptrOffset = totalSize;
+            auto&& result = arrow::AllocateResizableBuffer(totalSize * sizeof(value_type), pool);
+            if (!result.ok()) {
+              throw runtime_error("Cannot allocate values buffer");
+            }
+            arrowValuesBuffer = std::move(result).ValueUnsafe();
+            ptr = (uint8_t*)(arrowValuesBuffer->mutable_data());
+            // Calculate the size of the buffer here.
+            for (size_t i = 0; i < total; i++) {
+              int vlaSizeInBytes = inPtr[i].size() * sizeof(value_type);
+              if (vlaSizeInBytes == 0) {
+                continue;
+              }
+              memcpy(ptr, inPtr[i].data(), vlaSizeInBytes);
+              ptr += vlaSizeInBytes;
+            }
+          };
+
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            int readLast = index.GetNEntries();
+            switch (vlaListType->field(0)->type()->id()) {
+              case arrow::Type::FLOAT: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<float>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::DOUBLE: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<double>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::INT8: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<int8_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::INT16: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<int16_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::INT32: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<int32_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::INT64: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<int64_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::UINT8: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<uint8_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::UINT16: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<uint16_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::UINT32: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<uint32_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::UINT64: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<uint64_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              default: {
+                throw runtime_error("Unsupported kind of VLA");
+              } break;
+            }
+
+            readEntries += readLast;
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+        } else {
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            void* inPtr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries());
+
+            int readLast = index.GetNEntries();
+            if (listSize == -1) {
+              size = offsets[readEntries + readLast] - offsets[readEntries];
+            } else {
+              size = readLast * listSize;
+            }
+            readEntries += readLast;
+            memcpy(ptr, inPtr, size * typeSize);
+            ptr += (ptrdiff_t)(size * typeSize);
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+        }
+        switch (listSize) {
+          case -1: {
+            auto varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
+            array = std::make_shared<arrow::ListArray>(physicalField->type(), readEntries, arrowOffsetBuffer, varray);
+          } break;
+          case 1: {
+            totalSize = readEntries * listSize;
+            array = std::make_shared<arrow::PrimitiveArray>(physicalField->type(), readEntries, arrowValuesBuffer);
+
+          } break;
+          default: {
+            totalSize = readEntries * listSize;
+            auto varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
+            array = std::make_shared<arrow::FixedSizeListArray>(physicalField->type(), readEntries, varray);
+          }
+        }
+      }
+      columns.push_back(array);
+    }
+
+    auto batch = arrow::RecordBatch::Make(dataset_schema, rows, columns);
+    return batch;
+  };
+
+  return generator;
+}
+
+arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> RNTupleFileFormat::MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination,
+                                                                                         std::shared_ptr<arrow::Schema> schema,
+                                                                                         std::shared_ptr<arrow::dataset::FileWriteOptions> options,
+                                                                                         arrow::fs::FileLocator destination_locator) const
+{
+  auto writer = std::make_shared<RNTupleFileWriter>(schema, options, destination, destination_locator);
+  return std::dynamic_pointer_cast<arrow::dataset::FileWriter>(writer);
+}
+
+arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> RNTupleFileFormat::MakeFragment(
+  arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
+  std::shared_ptr<arrow::Schema> physical_schema)
+{
+  std::shared_ptr<arrow::dataset::FileFormat> format = std::make_shared<RNTupleFileFormat>(mTotCompressedSize, mTotUncompressedSize);
+
+  auto fragment = std::make_shared<RNTupleFileFragment>(std::move(source), std::move(format),
+                                                        std::move(partition_expression),
+                                                        std::move(physical_schema));
+  return std::dynamic_pointer_cast<arrow::dataset::FileFragment>(fragment);
+}
+
+RNTupleFileSystem::~RNTupleFileSystem() = default;
+
+std::shared_ptr<arrow::dataset::FileWriteOptions>
+  RNTupleFileFormat::DefaultWriteOptions()
+{
+  return std::make_shared<RNTupleFileWriteOptions>(shared_from_this());
+}
+
+struct RNTuplePluginContext {
+  size_t totalCompressedSize = 0;
+  size_t totalUncompressedSize = 0;
+  std::shared_ptr<o2::framework::RNTupleFileFormat> format = nullptr;
+};
+
+struct RNTupleObjectReadingImplementation : public RootArrowFactoryPlugin {
+  RootArrowFactory* create() override
+  {
+    auto context = new RNTuplePluginContext;
+    context->format = std::make_shared<o2::framework::RNTupleFileFormat>(context->totalCompressedSize, context->totalUncompressedSize);
+    return new RootArrowFactory{
+      .options = [context]() { return context->format->DefaultWriteOptions(); },
+      .format = [context]() { return context->format; },
+      .getSubFilesystem = [](void* handle) {
+        auto rntuple = (ROOT::Experimental::RNTuple*)handle;
+        return std::shared_ptr<VirtualRootFileSystemBase>(new SingleRNTupleFileSystem(rntuple)); },
+    };
+  }
+};
+
+DEFINE_DPL_PLUGINS_BEGIN
+DEFINE_DPL_PLUGIN_INSTANCE(RNTupleObjectReadingImplementation, RootObjectReadingImplementation);
+DEFINE_DPL_PLUGINS_END
+} // namespace o2::framework
diff --git a/Framework/AnalysisSupport/src/TTreePlugin.cxx b/Framework/AnalysisSupport/src/TTreePlugin.cxx
new file mode 100644
index 0000000000000..e376ed8b96268
--- /dev/null
+++ b/Framework/AnalysisSupport/src/TTreePlugin.cxx
@@ -0,0 +1,862 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "Framework/RootArrowFilesystem.h"
+#include "Framework/Plugins.h"
+#include "Framework/Signpost.h"
+#include "Framework/Endian.h"
+#include <arrow/dataset/file_base.h>
+#include <arrow/util/key_value_metadata.h>
+#include <arrow/array/array_nested.h>
+#include <arrow/array/array_primitive.h>
+#include <arrow/array/builder_nested.h>
+#include <arrow/array/builder_primitive.h>
+#include <TTree.h>
+#include <TBranch.h>
+#include <TFile.h>
+#include <TLeaf.h>
+
+O2_DECLARE_DYNAMIC_LOG(root_arrow_fs);
+
+namespace o2::framework
+{
+
+class TTreeFileWriteOptions : public arrow::dataset::FileWriteOptions
+{
+ public:
+  TTreeFileWriteOptions(std::shared_ptr<arrow::dataset::FileFormat> format)
+    : FileWriteOptions(format)
+  {
+  }
+};
+
+// A filesystem which allows me to get a TTree
+class TTreeFileSystem : public VirtualRootFileSystemBase
+{
+ public:
+  ~TTreeFileSystem() override;
+
+  std::shared_ptr<VirtualRootFileSystemBase> GetSubFilesystem(arrow::dataset::FileSource source) override
+  {
+    return std::dynamic_pointer_cast<VirtualRootFileSystemBase>(shared_from_this());
+  };
+
+  arrow::Result<std::shared_ptr<arrow::io::OutputStream>> OpenOutputStream(
+    const std::string& path,
+    const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override;
+
+  virtual TTree* GetTree(arrow::dataset::FileSource source) = 0;
+};
+
+class SingleTreeFileSystem : public TTreeFileSystem
+{
+ public:
+  SingleTreeFileSystem(TTree* tree)
+    : TTreeFileSystem(),
+      mTree(tree)
+  {
+  }
+
+  arrow::Result<arrow::fs::FileInfo> GetFileInfo(std::string const& path) override;
+
+  std::string type_name() const override
+  {
+    return "ttree";
+  }
+
+  TTree* GetTree(arrow::dataset::FileSource) override
+  {
+    // Simply return the only TTree we have
+    return mTree;
+  }
+
+ private:
+  TTree* mTree;
+};
+
+arrow::Result<arrow::fs::FileInfo> SingleTreeFileSystem::GetFileInfo(std::string const& path)
+{
+  arrow::dataset::FileSource source(path, shared_from_this());
+  arrow::fs::FileInfo result;
+  result.set_path(path);
+  result.set_type(arrow::fs::FileType::File);
+  return result;
+}
+
+class TTreeFileFragment : public arrow::dataset::FileFragment
+{
+ public:
+  TTreeFileFragment(arrow::dataset::FileSource source,
+                    std::shared_ptr<arrow::dataset::FileFormat> format,
+                    arrow::compute::Expression partition_expression,
+                    std::shared_ptr<arrow::Schema> physical_schema)
+    : FileFragment(std::move(source), std::move(format), std::move(partition_expression), std::move(physical_schema))
+  {
+  }
+};
+
+class TTreeFileFormat : public arrow::dataset::FileFormat
+{
+  size_t& mTotCompressedSize;
+  size_t& mTotUncompressedSize;
+
+ public:
+  TTreeFileFormat(size_t& totalCompressedSize, size_t& totalUncompressedSize)
+    : FileFormat({}),
+      mTotCompressedSize(totalCompressedSize),
+      mTotUncompressedSize(totalUncompressedSize)
+  {
+  }
+
+  ~TTreeFileFormat() override = default;
+
+  std::string type_name() const override
+  {
+    return "ttree";
+  }
+
+  bool Equals(const FileFormat& other) const override
+  {
+    return other.type_name() == this->type_name();
+  }
+
+  arrow::Result<bool> IsSupported(const arrow::dataset::FileSource& source) const override
+  {
+    auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
+    auto subFs = fs->GetSubFilesystem(source);
+    if (std::dynamic_pointer_cast<TTreeFileSystem>(subFs)) {
+      return true;
+    }
+    return false;
+  }
+
+  arrow::Result<std::shared_ptr<arrow::Schema>> Inspect(const arrow::dataset::FileSource& source) const override;
+  /// \brief Create a FileFragment for a FileSource.
+  arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> MakeFragment(
+    arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
+    std::shared_ptr<arrow::Schema> physical_schema) override;
+
+  arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options, arrow::fs::FileLocator destination_locator) const override;
+
+  std::shared_ptr<arrow::dataset::FileWriteOptions> DefaultWriteOptions() override;
+
+  arrow::Result<arrow::RecordBatchGenerator> ScanBatchesAsync(
+    const std::shared_ptr<arrow::dataset::ScanOptions>& options,
+    const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const override;
+};
+
+// An arrow outputstream which allows to write to a TTree. Eventually
+// with a prefix for the branches.
+class TTreeOutputStream : public arrow::io::OutputStream
+{
+ public:
+  TTreeOutputStream(TTree*, std::string branchPrefix);
+
+  arrow::Status Close() override;
+
+  arrow::Result<int64_t> Tell() const override;
+
+  arrow::Status Write(const void* data, int64_t nbytes) override;
+
+  bool closed() const override;
+
+  TBranch* CreateBranch(char const* branchName, char const* sizeBranch);
+
+  TTree* GetTree()
+  {
+    return mTree;
+  }
+
+ private:
+  TTree* mTree;
+  std::string mBranchPrefix;
+};
+
+// An arrow outputstream which allows to write to a ttree
+// @a branch prefix is to be used to identify a set of branches which all belong to
+// the same table.
+TTreeOutputStream::TTreeOutputStream(TTree* f, std::string branchPrefix)
+  : mTree(f),
+    mBranchPrefix(std::move(branchPrefix))
+{
+}
+
+arrow::Status TTreeOutputStream::Close()
+{
+  if (mTree->GetCurrentFile() == nullptr) {
+    return arrow::Status::Invalid("Cannot close a tree not attached to a file");
+  }
+  mTree->GetCurrentFile()->Close();
+  return arrow::Status::OK();
+}
+
+arrow::Result<int64_t> TTreeOutputStream::Tell() const
+{
+  return arrow::Result<int64_t>(arrow::Status::NotImplemented("Cannot move"));
+}
+
+arrow::Status TTreeOutputStream::Write(const void* data, int64_t nbytes)
+{
+  return arrow::Status::NotImplemented("Cannot write raw bytes to a TTree");
+}
+
+bool TTreeOutputStream::closed() const
+{
+  // A standalone tree is never closed.
+  if (mTree->GetCurrentFile() == nullptr) {
+    return false;
+  }
+  return mTree->GetCurrentFile()->IsOpen() == false;
+}
+
+TBranch* TTreeOutputStream::CreateBranch(char const* branchName, char const* sizeBranch)
+{
+  return mTree->Branch((mBranchPrefix + "/" + branchName).c_str(), (char*)nullptr, (mBranchPrefix + sizeBranch).c_str());
+}
+
+struct TTreePluginContext {
+  size_t totalCompressedSize = 0;
+  size_t totalUncompressedSize = 0;
+  std::shared_ptr<o2::framework::TTreeFileFormat> format = nullptr;
+};
+
+struct TTreeObjectReadingImplementation : public RootArrowFactoryPlugin {
+  RootArrowFactory* create() override
+  {
+    auto context = new TTreePluginContext;
+    context->format = std::make_shared<o2::framework::TTreeFileFormat>(context->totalCompressedSize, context->totalUncompressedSize);
+    return new RootArrowFactory{
+      .options = [context]() { return context->format->DefaultWriteOptions(); },
+      .format = [context]() { return context->format; },
+      .getSubFilesystem = [](void* handle) {
+        auto tree = (TTree*)handle;
+        return std::shared_ptr<VirtualRootFileSystemBase>(new SingleTreeFileSystem(tree)); },
+    };
+  }
+};
+
+arrow::Result<arrow::RecordBatchGenerator> TTreeFileFormat::ScanBatchesAsync(
+  const std::shared_ptr<arrow::dataset::ScanOptions>& options,
+  const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const
+{
+  // Get the fragment as a TTreeFragment. This might be PART of a TTree.
+  auto treeFragment = std::dynamic_pointer_cast<TTreeFileFragment>(fragment);
+  // This is the schema we want to read
+  auto dataset_schema = options->dataset_schema;
+
+  auto generator = [pool = options->pool, treeFragment, dataset_schema, &totalCompressedSize = mTotCompressedSize,
+                    &totalUncompressedSize = mTotUncompressedSize]() -> arrow::Future<std::shared_ptr<arrow::RecordBatch>> {
+    auto schema = treeFragment->format()->Inspect(treeFragment->source());
+
+    std::vector<std::shared_ptr<arrow::Array>> columns;
+    std::vector<std::shared_ptr<arrow::Field>> fields = dataset_schema->fields();
+    auto physical_schema = *treeFragment->ReadPhysicalSchema();
+
+    static TBufferFile buffer{TBuffer::EMode::kWrite, 4 * 1024 * 1024};
+    auto containerFS = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(treeFragment->source().filesystem());
+    auto fs = std::dynamic_pointer_cast<TTreeFileSystem>(containerFS->GetSubFilesystem(treeFragment->source()));
+
+    int64_t rows = -1;
+    TTree* tree = fs->GetTree(treeFragment->source());
+    for (auto& field : fields) {
+      // The field actually on disk
+      auto physicalField = physical_schema->GetFieldByName(field->name());
+      TBranch* branch = tree->GetBranch(physicalField->name().c_str());
+      assert(branch);
+      buffer.Reset();
+      auto totalEntries = branch->GetEntries();
+      if (rows == -1) {
+        rows = totalEntries;
+      }
+      if (rows != totalEntries) {
+        throw runtime_error_f("Unmatching number of rows for branch %s", branch->GetName());
+      }
+      arrow::Status status;
+      int readEntries = 0;
+      std::shared_ptr<arrow::Array> array;
+      auto listType = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type());
+      if (physicalField->type() == arrow::boolean() ||
+          (listType && physicalField->type()->field(0)->type() == arrow::boolean())) {
+        if (listType) {
+          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
+          auto status = arrow::MakeBuilder(pool, physicalField->type()->field(0)->type(), &builder);
+          if (!status.ok()) {
+            throw runtime_error("Cannot create value builder");
+          }
+          auto listBuilder = std::make_unique<arrow::FixedSizeListBuilder>(pool, std::move(builder), listType->list_size());
+          auto valueBuilder = listBuilder.get()->value_builder();
+          // boolean array special case: we need to use builder to create the bitmap
+          status = valueBuilder->Reserve(totalEntries * listType->list_size());
+          status &= listBuilder->Reserve(totalEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to reserve memory for array builder");
+          }
+          while (readEntries < totalEntries) {
+            auto readLast = branch->GetBulkRead().GetBulkEntries(readEntries, buffer);
+            readEntries += readLast;
+            status &= static_cast<arrow::BooleanBuilder*>(valueBuilder)->AppendValues(reinterpret_cast<uint8_t const*>(buffer.GetCurrent()), readLast * listType->list_size());
+          }
+          status &= static_cast<arrow::FixedSizeListBuilder*>(listBuilder.get())->AppendValues(readEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to append values to array");
+          }
+          status &= listBuilder->Finish(&array);
+          if (!status.ok()) {
+            throw runtime_error("Failed to create array");
+          }
+        } else if (listType == nullptr) {
+          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
+          auto status = arrow::MakeBuilder(pool, physicalField->type(), &builder);
+          if (!status.ok()) {
+            throw runtime_error("Cannot create builder");
+          }
+          auto valueBuilder = static_cast<arrow::BooleanBuilder*>(builder.get());
+          // boolean array special case: we need to use builder to create the bitmap
+          status = valueBuilder->Reserve(totalEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to reserve memory for array builder");
+          }
+          while (readEntries < totalEntries) {
+            auto readLast = branch->GetBulkRead().GetBulkEntries(readEntries, buffer);
+            readEntries += readLast;
+            status &= valueBuilder->AppendValues(reinterpret_cast<uint8_t const*>(buffer.GetCurrent()), readLast);
+          }
+          if (!status.ok()) {
+            throw runtime_error("Failed to append values to array");
+          }
+          status &= valueBuilder->Finish(&array);
+          if (!status.ok()) {
+            throw runtime_error("Failed to create array");
+          }
+        }
+      } else {
+        // other types: use serialized read to build arrays directly.
+        auto typeSize = physicalField->type()->byte_width();
+        // This is needed for branches which have not been persisted.
+        auto bytes = branch->GetTotBytes();
+        auto branchSize = bytes ? bytes : 1000000;
+        auto&& result = arrow::AllocateResizableBuffer(branchSize, pool);
+        if (!result.ok()) {
+          throw runtime_error("Cannot allocate values buffer");
+        }
+        std::shared_ptr<arrow::Buffer> arrowValuesBuffer = std::move(result).ValueUnsafe();
+        auto ptr = arrowValuesBuffer->mutable_data();
+        if (ptr == nullptr) {
+          throw runtime_error("Invalid buffer");
+        }
+
+        std::unique_ptr<TBufferFile> offsetBuffer = nullptr;
+
+        uint32_t offset = 0;
+        int count = 0;
+        std::shared_ptr<arrow::Buffer> arrowOffsetBuffer;
+        std::span<int> offsets;
+        int size = 0;
+        uint32_t totalSize = 0;
+        TBranch* mSizeBranch = nullptr;
+        int64_t listSize = 1;
+        if (auto fixedSizeList = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type())) {
+          listSize = fixedSizeList->list_size();
+          typeSize = fixedSizeList->field(0)->type()->byte_width();
+        } else if (auto vlaListType = std::dynamic_pointer_cast<arrow::ListType>(physicalField->type())) {
+          listSize = -1;
+          typeSize = vlaListType->field(0)->type()->byte_width();
+        }
+        if (listSize == -1) {
+          mSizeBranch = branch->GetTree()->GetBranch((std::string{branch->GetName()} + "_size").c_str());
+          offsetBuffer = std::make_unique<TBufferFile>(TBuffer::EMode::kWrite, 4 * 1024 * 1024);
+          result = arrow::AllocateResizableBuffer((totalEntries + 1) * (int64_t)sizeof(int), pool);
+          if (!result.ok()) {
+            throw runtime_error("Cannot allocate offset buffer");
+          }
+          arrowOffsetBuffer = std::move(result).ValueUnsafe();
+          unsigned char* ptrOffset = arrowOffsetBuffer->mutable_data();
+          auto* tPtrOffset = reinterpret_cast<int*>(ptrOffset);
+          offsets = std::span<int>{tPtrOffset, tPtrOffset + totalEntries + 1};
+
+          // read sizes first
+          while (readEntries < totalEntries) {
+            auto readLast = mSizeBranch->GetBulkRead().GetEntriesSerialized(readEntries, *offsetBuffer);
+            readEntries += readLast;
+            for (auto i = 0; i < readLast; ++i) {
+              offsets[count++] = (int)offset;
+              offset += swap32_(reinterpret_cast<uint32_t*>(offsetBuffer->GetCurrent())[i]);
+            }
+          }
+          offsets[count] = (int)offset;
+          totalSize = offset;
+          readEntries = 0;
+        }
+
+        while (readEntries < totalEntries) {
+          auto readLast = branch->GetBulkRead().GetEntriesSerialized(readEntries, buffer);
+          if (listSize == -1) {
+            size = offsets[readEntries + readLast] - offsets[readEntries];
+          } else {
+            size = readLast * listSize;
+          }
+          readEntries += readLast;
+          swapCopy(ptr, buffer.GetCurrent(), size, typeSize);
+          ptr += (ptrdiff_t)(size * typeSize);
+        }
+        if (listSize >= 1) {
+          totalSize = readEntries * listSize;
+        }
+        std::shared_ptr<arrow::PrimitiveArray> varray;
+        switch (listSize) {
+          case -1:
+            varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
+            array = std::make_shared<arrow::ListArray>(physicalField->type(), readEntries, arrowOffsetBuffer, varray);
+            break;
+          case 1:
+            array = std::make_shared<arrow::PrimitiveArray>(physicalField->type(), readEntries, arrowValuesBuffer);
+            break;
+          default:
+            varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
+            array = std::make_shared<arrow::FixedSizeListArray>(physicalField->type(), readEntries, varray);
+        }
+      }
+
+      branch->SetStatus(false);
+      branch->DropBaskets("all");
+      branch->Reset();
+      branch->GetTransientBuffer(0)->Expand(0);
+
+      columns.push_back(array);
+    }
+    auto batch = arrow::RecordBatch::Make(dataset_schema, rows, columns);
+    totalCompressedSize += tree->GetZipBytes();
+    totalUncompressedSize += tree->GetTotBytes();
+    return batch;
+  };
+  return generator;
+}
+
+char const* rootSuffixFromArrow(arrow::Type::type id)
+{
+  switch (id) {
+    case arrow::Type::BOOL:
+      return "/O";
+    case arrow::Type::UINT8:
+      return "/b";
+    case arrow::Type::UINT16:
+      return "/s";
+    case arrow::Type::UINT32:
+      return "/i";
+    case arrow::Type::UINT64:
+      return "/l";
+    case arrow::Type::INT8:
+      return "/B";
+    case arrow::Type::INT16:
+      return "/S";
+    case arrow::Type::INT32:
+      return "/I";
+    case arrow::Type::INT64:
+      return "/L";
+    case arrow::Type::FLOAT:
+      return "/F";
+    case arrow::Type::DOUBLE:
+      return "/D";
+    default:
+      throw runtime_error("Unsupported arrow column type");
+  }
+}
+
+arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenOutputStream(
+  const std::string& path,
+  const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
+{
+  arrow::dataset::FileSource source{path, shared_from_this()};
+  auto prefix = metadata->Get("branch_prefix");
+  if (prefix.ok()) {
+    return std::make_shared<TTreeOutputStream>(GetTree(source), *prefix);
+  }
+  return std::make_shared<TTreeOutputStream>(GetTree(source), "");
+}
+
+namespace
+{
+struct BranchInfo {
+  std::string name;
+  TBranch* ptr;
+  bool mVLA;
+};
+} // namespace
+
+auto arrowTypeFromROOT(EDataType type, int size)
+{
+  auto typeGenerator = [](std::shared_ptr<arrow::DataType> const& type, int size) -> std::shared_ptr<arrow::DataType> {
+    switch (size) {
+      case -1:
+        return arrow::list(type);
+      case 1:
+        return std::move(type);
+      default:
+        return arrow::fixed_size_list(type, size);
+    }
+  };
+
+  switch (type) {
+    case EDataType::kBool_t:
+      return typeGenerator(arrow::boolean(), size);
+    case EDataType::kUChar_t:
+      return typeGenerator(arrow::uint8(), size);
+    case EDataType::kUShort_t:
+      return typeGenerator(arrow::uint16(), size);
+    case EDataType::kUInt_t:
+      return typeGenerator(arrow::uint32(), size);
+    case EDataType::kULong64_t:
+      return typeGenerator(arrow::uint64(), size);
+    case EDataType::kChar_t:
+      return typeGenerator(arrow::int8(), size);
+    case EDataType::kShort_t:
+      return typeGenerator(arrow::int16(), size);
+    case EDataType::kInt_t:
+      return typeGenerator(arrow::int32(), size);
+    case EDataType::kLong64_t:
+      return typeGenerator(arrow::int64(), size);
+    case EDataType::kFloat_t:
+      return typeGenerator(arrow::float32(), size);
+    case EDataType::kDouble_t:
+      return typeGenerator(arrow::float64(), size);
+    default:
+      throw o2::framework::runtime_error_f("Unsupported branch type: %d", static_cast<int>(type));
+  }
+}
+
+arrow::Result<std::shared_ptr<arrow::Schema>> TTreeFileFormat::Inspect(const arrow::dataset::FileSource& source) const
+{
+  arrow::Schema schema{{}};
+  auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
+  // Actually get the TTree from the ROOT file.
+  auto treeFs = std::dynamic_pointer_cast<TTreeFileSystem>(fs->GetSubFilesystem(source));
+  if (!treeFs.get()) {
+    throw runtime_error_f("Unknown filesystem %s\n", source.filesystem()->type_name().c_str());
+  }
+  TTree* tree = treeFs->GetTree(source);
+
+  auto branches = tree->GetListOfBranches();
+  auto n = branches->GetEntries();
+
+  std::vector<BranchInfo> branchInfos;
+  for (auto i = 0; i < n; ++i) {
+    auto branch = static_cast<TBranch*>(branches->At(i));
+    auto name = std::string{branch->GetName()};
+    auto pos = name.find("_size");
+    if (pos != std::string::npos) {
+      name.erase(pos);
+      branchInfos.emplace_back(BranchInfo{name, (TBranch*)nullptr, true});
+    } else {
+      auto lookup = std::find_if(branchInfos.begin(), branchInfos.end(), [&](BranchInfo const& bi) {
+        return bi.name == name;
+      });
+      if (lookup == branchInfos.end()) {
+        branchInfos.emplace_back(BranchInfo{name, branch, false});
+      } else {
+        lookup->ptr = branch;
+      }
+    }
+  }
+
+  std::vector<std::shared_ptr<arrow::Field>> fields;
+  tree->SetCacheSize(25000000);
+  for (auto& bi : branchInfos) {
+    static TClass* cls;
+    EDataType type;
+    bi.ptr->GetExpectedType(cls, type);
+    auto listSize = -1;
+    if (!bi.mVLA) {
+      listSize = static_cast<TLeaf*>(bi.ptr->GetListOfLeaves()->At(0))->GetLenStatic();
+    }
+    auto field = std::make_shared<arrow::Field>(bi.ptr->GetName(), arrowTypeFromROOT(type, listSize));
+    fields.push_back(field);
+
+    tree->AddBranchToCache(bi.ptr);
+    if (strncmp(bi.ptr->GetName(), "fIndexArray", strlen("fIndexArray")) == 0) {
+      std::string sizeBranchName = bi.ptr->GetName();
+      sizeBranchName += "_size";
+      auto* sizeBranch = (TBranch*)tree->GetBranch(sizeBranchName.c_str());
+      if (sizeBranch) {
+        tree->AddBranchToCache(sizeBranch);
+      }
+    }
+  }
+  tree->StopCacheLearningPhase();
+
+  return std::make_shared<arrow::Schema>(fields);
+}
+
+/// \brief Create a FileFragment for a FileSource.
+arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> TTreeFileFormat::MakeFragment(
+  arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
+  std::shared_ptr<arrow::Schema> physical_schema)
+{
+  std::shared_ptr<arrow::dataset::FileFormat> format = std::make_shared<TTreeFileFormat>(mTotCompressedSize, mTotUncompressedSize);
+
+  auto fragment = std::make_shared<TTreeFileFragment>(std::move(source), std::move(format),
+                                                      std::move(partition_expression),
+                                                      std::move(physical_schema));
+  return std::dynamic_pointer_cast<arrow::dataset::FileFragment>(fragment);
+}
+
+class TTreeFileWriter : public arrow::dataset::FileWriter
+{
+  std::vector<TBranch*> branches;
+  std::vector<TBranch*> sizesBranches;
+  std::vector<std::shared_ptr<arrow::Array>> valueArrays;
+  std::vector<std::shared_ptr<arrow::Array>> sizeArrays;
+  std::vector<std::shared_ptr<arrow::DataType>> valueTypes;
+
+  std::vector<int64_t> valuesIdealBasketSize;
+  std::vector<int64_t> sizeIdealBasketSize;
+
+  std::vector<int64_t> typeSizes;
+  std::vector<int64_t> listSizes;
+  bool firstBasket = true;
+
+  // This is to create a batsket size according to the first batch.
+  void finaliseBasketSize(std::shared_ptr<arrow::RecordBatch> firstBatch)
+  {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, root_arrow_fs, this);
+    O2_SIGNPOST_START(root_arrow_fs, sid, "finaliseBasketSize", "First batch with %lli rows received and %zu columns",
+                      firstBatch->num_rows(), firstBatch->columns().size());
+    for (size_t i = 0; i < branches.size(); i++) {
+      auto* branch = branches[i];
+      auto* sizeBranch = sizesBranches[i];
+
+      int valueSize = valueTypes[i]->byte_width();
+      if (listSizes[i] == 1) {
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s exists and uses %d bytes per entry for %lli entries.",
+                               branch->GetName(), valueSize, firstBatch->num_rows());
+        assert(sizeBranch == nullptr);
+        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize);
+      } else if (listSizes[i] == -1) {
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s exists and uses %d bytes per entry.",
+                               branch->GetName(), valueSize);
+        // This should probably lookup the
+        auto column = firstBatch->GetColumnByName(schema_->field(i)->name());
+        auto list = std::static_pointer_cast<arrow::ListArray>(column);
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s needed. Associated size branch %s and there are %lli entries of size %d in that list.",
+                               branch->GetName(), sizeBranch->GetName(), list->length(), valueSize);
+        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize * list->length());
+        sizeBranch->SetBasketSize(1024 + firstBatch->num_rows() * 4);
+      } else {
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s needed. There are %lli entries per array of size %d in that list.",
+                               branch->GetName(), listSizes[i], valueSize);
+        assert(sizeBranch == nullptr);
+        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize * listSizes[i]);
+      }
+
+      auto field = firstBatch->schema()->field(i);
+      if (field->name().starts_with("fIndexArray")) {
+        // One int per array to keep track of the size
+        int idealBasketSize = 4 * firstBatch->num_rows() + 1024 + field->type()->byte_width() * firstBatch->num_rows(); // minimal additional size needed, otherwise we get 2 baskets
+        int basketSize = std::max(32000, idealBasketSize);                                                              // keep a minimum value
+        sizeBranch->SetBasketSize(basketSize);
+        branch->SetBasketSize(basketSize);
+      }
+    }
+    O2_SIGNPOST_END(root_arrow_fs, sid, "finaliseBasketSize", "Done");
+  }
+
+ public:
+  // Create the TTree based on the physical_schema, not the one in the batch.
+  // The write method will have to reconcile the two schemas.
+  TTreeFileWriter(std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options,
+                  std::shared_ptr<arrow::io::OutputStream> destination,
+                  arrow::fs::FileLocator destination_locator)
+    : FileWriter(schema, options, destination, destination_locator)
+  {
+    // Batches have the same number of entries for each column.
+    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
+    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
+
+    if (directoryStream.get()) {
+      TDirectoryFile* dir = directoryStream->GetDirectory();
+      dir->cd();
+      auto* tree = new TTree(destination_locator_.path.c_str(), "");
+      treeStream = std::make_shared<TTreeOutputStream>(tree, "");
+    } else if (treeStream.get()) {
+      // We already have a tree stream, let's derive a new one
+      // with the destination_locator_.path as prefix for the branches
+      // This way we can multiplex multiple tables in the same tree.
+      auto tree = treeStream->GetTree();
+      treeStream = std::make_shared<TTreeOutputStream>(tree, destination_locator_.path);
+    } else {
+      // I could simply set a prefix here to merge to an already existing tree.
+      throw std::runtime_error("Unsupported backend.");
+    }
+
+    for (auto i = 0u; i < schema->fields().size(); ++i) {
+      auto& field = schema->field(i);
+      listSizes.push_back(1);
+
+      int valuesIdealBasketSize = 0;
+      // Construct all the needed branches.
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          listSizes.back() = std::static_pointer_cast<arrow::FixedSizeListType>(field->type())->list_size();
+          valuesIdealBasketSize = 1024 + valueTypes.back()->byte_width() * listSizes.back();
+          valueTypes.push_back(field->type()->field(0)->type());
+          sizesBranches.push_back(nullptr);
+          std::string leafList = fmt::format("{}[{}]{}", field->name(), listSizes.back(), rootSuffixFromArrow(valueTypes.back()->id()));
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
+        } break;
+        case arrow::Type::LIST: {
+          valueTypes.push_back(field->type()->field(0)->type());
+          std::string leafList = fmt::format("{}[{}_size]{}", field->name(), field->name(), rootSuffixFromArrow(valueTypes.back()->id()));
+          listSizes.back() = -1; // VLA, we need to calculate it on the fly;
+          std::string sizeLeafList = field->name() + "_size/I";
+          sizesBranches.push_back(treeStream->CreateBranch((field->name() + "_size").c_str(), sizeLeafList.c_str()));
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
+          // Notice that this could be replaced by a better guess of the
+          // average size of the list elements, but this is not trivial.
+        } break;
+        default: {
+          valueTypes.push_back(field->type());
+          std::string leafList = field->name() + rootSuffixFromArrow(valueTypes.back()->id());
+          sizesBranches.push_back(nullptr);
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
+        } break;
+      }
+    }
+    // We create the branches from the schema
+  }
+
+  arrow::Status Write(const std::shared_ptr<arrow::RecordBatch>& batch) override
+  {
+    if (firstBasket) {
+      firstBasket = false;
+      finaliseBasketSize(batch);
+    }
+
+    // Support writing empty tables
+    if (batch->columns().empty() || batch->num_rows() == 0) {
+      return arrow::Status::OK();
+    }
+
+    // Batches have the same number of entries for each column.
+    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
+    TTree* tree = nullptr;
+    if (directoryStream.get()) {
+      TDirectoryFile* dir = directoryStream->GetDirectory();
+      tree = (TTree*)dir->Get(destination_locator_.path.c_str());
+    }
+    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
+
+    if (!tree) {
+      // I could simply set a prefix here to merge to an already existing tree.
+      throw std::runtime_error("Unsupported backend.");
+    }
+
+    for (auto i = 0u; i < batch->columns().size(); ++i) {
+      auto column = batch->column(i);
+      auto& field = batch->schema()->field(i);
+
+      valueArrays.push_back(nullptr);
+
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          auto list = std::static_pointer_cast<arrow::FixedSizeListArray>(column);
+          valueArrays.back() = list->values();
+        } break;
+        case arrow::Type::LIST: {
+          auto list = std::static_pointer_cast<arrow::ListArray>(column);
+          valueArrays.back() = list;
+        } break;
+        case arrow::Type::BOOL: {
+          // In case of arrays of booleans, we need to go back to their
+          // char based representation for ROOT to save them.
+          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(column);
+
+          int64_t length = boolArray->length();
+          arrow::UInt8Builder builder;
+          auto ok = builder.Reserve(length);
+
+          for (int64_t i = 0; i < length; ++i) {
+            if (boolArray->IsValid(i)) {
+              // Expand each boolean value (true/false) to uint8 (1/0)
+              uint8_t value = boolArray->Value(i) ? 1 : 0;
+              auto ok = builder.Append(value);
+            } else {
+              // Append null for invalid entries
+              auto ok = builder.AppendNull();
+            }
+          }
+          valueArrays.back() = *builder.Finish();
+        } break;
+        default:
+          valueArrays.back() = column;
+      }
+    }
+
+    int64_t pos = 0;
+    while (pos < batch->num_rows()) {
+      for (size_t bi = 0; bi < branches.size(); ++bi) {
+        auto* branch = branches[bi];
+        auto* sizeBranch = sizesBranches[bi];
+        auto array = batch->column(bi);
+        auto& field = batch->schema()->field(bi);
+        auto& listSize = listSizes[bi];
+        auto valueType = valueTypes[bi];
+        auto valueArray = valueArrays[bi];
+
+        switch (field->type()->id()) {
+          case arrow::Type::LIST: {
+            auto list = std::static_pointer_cast<arrow::ListArray>(array);
+            listSize = list->value_length(pos);
+            uint8_t const* buffer = std::static_pointer_cast<arrow::PrimitiveArray>(valueArray)->values()->data() + array->offset() + list->value_offset(pos) * valueType->byte_width();
+            branch->SetAddress((void*)buffer);
+            sizeBranch->SetAddress(&listSize);
+          };
+            break;
+          case arrow::Type::FIXED_SIZE_LIST:
+          default: {
+            uint8_t const* buffer = std::static_pointer_cast<arrow::PrimitiveArray>(valueArray)->values()->data() + array->offset() + pos * listSize * valueType->byte_width();
+            branch->SetAddress((void*)buffer);
+          };
+        }
+      }
+      tree->Fill();
+      ++pos;
+    }
+    return arrow::Status::OK();
+  }
+
+  arrow::Future<> FinishInternal() override
+  {
+    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
+    TTree* tree = treeStream->GetTree();
+    tree->Write("", TObject::kOverwrite);
+    tree->SetDirectory(nullptr);
+
+    return {};
+  };
+};
+arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> TTreeFileFormat::MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options, arrow::fs::FileLocator destination_locator) const
+{
+  auto writer = std::make_shared<TTreeFileWriter>(schema, options, destination, destination_locator);
+  return std::dynamic_pointer_cast<arrow::dataset::FileWriter>(writer);
+}
+
+std::shared_ptr<arrow::dataset::FileWriteOptions> TTreeFileFormat::DefaultWriteOptions()
+{
+  std::shared_ptr<TTreeFileWriteOptions> options(
+    new TTreeFileWriteOptions(shared_from_this()));
+  return options;
+}
+
+TTreeFileSystem::~TTreeFileSystem() = default;
+
+DEFINE_DPL_PLUGINS_BEGIN
+DEFINE_DPL_PLUGIN_INSTANCE(TTreeObjectReadingImplementation, RootObjectReadingImplementation);
+DEFINE_DPL_PLUGINS_END
+} // namespace o2::framework
diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index 5cdd1241ecfb0..84a4a7f8b8c69 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -159,6 +159,8 @@ o2_add_library(Framework
                                      FairMQ::FairMQ
                                      ROOT::Tree
                                      ROOT::Hist
+                                     ROOT::ROOTNTuple
+                                     ROOT::ROOTNTupleUtil
                                      O2::FrameworkFoundation
                                      O2::CommonConstants
                                      O2::Headers
@@ -299,6 +301,7 @@ add_executable(o2-test-framework-root
 target_link_libraries(o2-test-framework-root PRIVATE O2::Framework)
 target_link_libraries(o2-test-framework-root PRIVATE O2::Catch2)
 target_link_libraries(o2-test-framework-root PRIVATE ROOT::ROOTDataFrame)
+target_link_libraries(o2-test-framework-root PRIVATE ROOT::ROOTNTuple)
 set_property(TARGET o2-test-framework-root PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
 add_test(NAME framework:root COMMAND o2-test-framework-root --skip-benchmarks)
 add_test(NAME framework:crash COMMAND  sh -e -c "PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:$PATH ${CMAKE_CURRENT_LIST_DIR}/test/test_AllCrashTypes.sh")
diff --git a/Framework/Core/include/Framework/Plugins.h b/Framework/Core/include/Framework/Plugins.h
index 23d55a512e1fa..925943c6bffc3 100644
--- a/Framework/Core/include/Framework/Plugins.h
+++ b/Framework/Core/include/Framework/Plugins.h
@@ -36,6 +36,14 @@ enum struct DplPluginKind : int {
   // set, you might want to load metadata from it and attach it to the
   // configuration.
   Capability,
+  // A RootObjectReadingCapability is used to discover if there is away
+  // to read and understand an object serialised with ROOT.
+  RootObjectReadingCapability,
+
+  // A RootObjectReadingImplementation is actually used to read said object
+  // using the arrow dataset API
+  RootObjectReadingImplementation,
+
   // A plugin which was not initialised properly.
   Unknown
 };
diff --git a/Framework/Core/include/Framework/RootArrowFilesystem.h b/Framework/Core/include/Framework/RootArrowFilesystem.h
index 48d817bc9ddf2..8744656e7d55d 100644
--- a/Framework/Core/include/Framework/RootArrowFilesystem.h
+++ b/Framework/Core/include/Framework/RootArrowFilesystem.h
@@ -11,6 +11,7 @@
 #ifndef O2_FRAMEWORK_ROOT_ARROW_FILESYSTEM_H_
 #define O2_FRAMEWORK_ROOT_ARROW_FILESYSTEM_H_
 
+#include <TBufferFile.h>
 #include <arrow/dataset/type_fwd.h>
 #include <arrow/dataset/file_base.h>
 #include <arrow/filesystem/type_fwd.h>
@@ -18,23 +19,12 @@
 #include <memory>
 
 class TFile;
-class TBranch;
-class TTree;
 class TBufferFile;
 class TDirectoryFile;
 
 namespace o2::framework
 {
 
-class TTreeFileWriteOptions : public arrow::dataset::FileWriteOptions
-{
- public:
-  TTreeFileWriteOptions(std::shared_ptr<arrow::dataset::FileFormat> format)
-    : FileWriteOptions(format)
-  {
-  }
-};
-
 // This is to avoid having to implement a bunch of unimplemented methods
 // for all the possible virtual filesystem we can invent on top of ROOT
 // data structures.
@@ -79,46 +69,43 @@ class VirtualRootFileSystemBase : public arrow::fs::FileSystem
     const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override;
 };
 
-// A filesystem which allows me to get a TTree
-class TTreeFileSystem : public VirtualRootFileSystemBase
-{
- public:
-  ~TTreeFileSystem() override;
-
-  std::shared_ptr<VirtualRootFileSystemBase> GetSubFilesystem(arrow::dataset::FileSource source) override
-  {
-    return std::dynamic_pointer_cast<VirtualRootFileSystemBase>(shared_from_this());
-  };
-
-  arrow::Result<std::shared_ptr<arrow::io::OutputStream>> OpenOutputStream(
-    const std::string& path,
-    const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override;
-
-  virtual TTree* GetTree(arrow::dataset::FileSource source) = 0;
+struct RootArrowFactory final {
+  std::function<std::shared_ptr<arrow::dataset::FileWriteOptions>()> options = nullptr;
+  std::function<std::shared_ptr<arrow::dataset::FileFormat>()> format = nullptr;
+  std::function<std::shared_ptr<VirtualRootFileSystemBase>(void*)> getSubFilesystem = nullptr;
 };
 
-class SingleTreeFileSystem : public TTreeFileSystem
-{
- public:
-  SingleTreeFileSystem(TTree* tree)
-    : TTreeFileSystem(),
-      mTree(tree)
-  {
-  }
+struct RootArrowFactoryPlugin {
+  virtual RootArrowFactory* create() = 0;
+};
 
-  std::string type_name() const override
-  {
-    return "ttree";
-  }
+// A registry for all the possible ways of encoding a table in a TFile
+struct RootObjectReadingCapability {
+  // The unique name of this capability
+  std::string name = "unknown";
+  // Given a TFile, return the object which this capability support
+  // Use a void * in order not to expose the kind of object to the
+  // generic reading code. This is also where we load the plugin
+  // which will be used for the actual creation.
+  std::function<void*(TDirectoryFile* file, std::string const& path)> getHandle;
+  // Same as the above, but uses a TBufferFile as storage
+  std::function<void*(TBufferFile*, std::string const&)> getBufferHandle;
+  // This must be implemented to load the actual RootArrowFactory plugin which
+  // implements this capability. This way the detection of the file format
+  // (via get handle) does not need to know about the actual code which performs
+  // the serialization (and might depend on e.g. RNTuple).
+  std::function<RootArrowFactory&()> factory;
+};
 
-  TTree* GetTree(arrow::dataset::FileSource) override
-  {
-    // Simply return the only TTree we have
-    return mTree;
-  }
+struct RootObjectReadingCapabilityPlugin {
+  virtual RootObjectReadingCapability* create() = 0;
+};
 
- private:
-  TTree* mTree;
+// This acts as registry of all the capabilities (i.e. the ability to
+// associate a given object in a root file to the serialization plugin) and
+// the factory (i.e. the serialization plugin)
+struct RootObjectReadingFactory {
+  std::vector<RootObjectReadingCapability> capabilities;
 };
 
 class TFileFileSystem : public VirtualRootFileSystemBase
@@ -126,7 +113,7 @@ class TFileFileSystem : public VirtualRootFileSystemBase
  public:
   arrow::Result<arrow::fs::FileInfo> GetFileInfo(const std::string& path) override;
 
-  TFileFileSystem(TDirectoryFile* f, size_t readahead);
+  TFileFileSystem(TDirectoryFile* f, size_t readahead, RootObjectReadingFactory&);
 
   std::string type_name() const override
   {
@@ -147,12 +134,13 @@ class TFileFileSystem : public VirtualRootFileSystemBase
 
  private:
   TDirectoryFile* mFile;
+  RootObjectReadingFactory& mObjectFactory;
 };
 
 class TBufferFileFS : public VirtualRootFileSystemBase
 {
  public:
-  TBufferFileFS(TBufferFile* f);
+  TBufferFileFS(TBufferFile* f, RootObjectReadingFactory&);
 
   arrow::Result<arrow::fs::FileInfo> GetFileInfo(const std::string& path) override;
   std::string type_name() const override
@@ -165,68 +153,7 @@ class TBufferFileFS : public VirtualRootFileSystemBase
  private:
   TBufferFile* mBuffer;
   std::shared_ptr<VirtualRootFileSystemBase> mFilesystem;
-};
-
-class TTreeFileFragment : public arrow::dataset::FileFragment
-{
- public:
-  TTreeFileFragment(arrow::dataset::FileSource source,
-                    std::shared_ptr<arrow::dataset::FileFormat> format,
-                    arrow::compute::Expression partition_expression,
-                    std::shared_ptr<arrow::Schema> physical_schema)
-    : FileFragment(std::move(source), std::move(format), std::move(partition_expression), std::move(physical_schema))
-  {
-  }
-};
-
-class TTreeFileFormat : public arrow::dataset::FileFormat
-{
-  size_t& mTotCompressedSize;
-  size_t& mTotUncompressedSize;
-
- public:
-  TTreeFileFormat(size_t& totalCompressedSize, size_t& totalUncompressedSize)
-    : FileFormat({}),
-      mTotCompressedSize(totalCompressedSize),
-      mTotUncompressedSize(totalUncompressedSize)
-  {
-  }
-
-  ~TTreeFileFormat() override = default;
-
-  std::string type_name() const override
-  {
-    return "ttree";
-  }
-
-  bool Equals(const FileFormat& other) const override
-  {
-    return other.type_name() == this->type_name();
-  }
-
-  arrow::Result<bool> IsSupported(const arrow::dataset::FileSource& source) const override
-  {
-    auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
-    auto subFs = fs->GetSubFilesystem(source);
-    if (std::dynamic_pointer_cast<TTreeFileSystem>(subFs)) {
-      return true;
-    }
-    return false;
-  }
-
-  arrow::Result<std::shared_ptr<arrow::Schema>> Inspect(const arrow::dataset::FileSource& source) const override;
-  /// \brief Create a FileFragment for a FileSource.
-  arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> MakeFragment(
-    arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
-    std::shared_ptr<arrow::Schema> physical_schema) override;
-
-  arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options, arrow::fs::FileLocator destination_locator) const override;
-
-  std::shared_ptr<arrow::dataset::FileWriteOptions> DefaultWriteOptions() override;
-
-  arrow::Result<arrow::RecordBatchGenerator> ScanBatchesAsync(
-    const std::shared_ptr<arrow::dataset::ScanOptions>& options,
-    const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const override;
+  RootObjectReadingFactory& mObjectFactory;
 };
 
 // An arrow outputstream which allows to write to a TDirectoryFile.
@@ -255,33 +182,6 @@ class TDirectoryFileOutputStream : public arrow::io::OutputStream
   TDirectoryFile* mDirectory;
 };
 
-// An arrow outputstream which allows to write to a TTree. Eventually
-// with a prefix for the branches.
-class TTreeOutputStream : public arrow::io::OutputStream
-{
- public:
-  TTreeOutputStream(TTree*, std::string branchPrefix);
-
-  arrow::Status Close() override;
-
-  arrow::Result<int64_t> Tell() const override;
-
-  arrow::Status Write(const void* data, int64_t nbytes) override;
-
-  bool closed() const override;
-
-  TBranch* CreateBranch(char const* branchName, char const* sizeBranch);
-
-  TTree* GetTree()
-  {
-    return mTree;
-  }
-
- private:
-  TTree* mTree;
-  std::string mBranchPrefix;
-};
-
 } // namespace o2::framework
 
 #endif // O2_FRAMEWORK_ROOT_ARROW_FILESYSTEM_H_
diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx
index 0d225b81c0581..46248ab467366 100644
--- a/Framework/Core/src/Plugin.cxx
+++ b/Framework/Core/src/Plugin.cxx
@@ -11,10 +11,15 @@
 #include "Framework/Plugins.h"
 #include "Framework/ConfigParamDiscovery.h"
 #include "Framework/ConfigParamRegistry.h"
+#include "Framework/RootArrowFilesystem.h"
 #include "Framework/Logger.h"
 #include "Framework/Capability.h"
 #include "Framework/Signpost.h"
 #include "Framework/VariantJSONHelpers.h"
+#include "Framework/PluginManager.h"
+#include <TDirectory.h>
+#include <TDirectoryFile.h>
+#include <TClass.h>
 #include <cstddef>
 #include <string_view>
 
@@ -168,11 +173,75 @@ struct DiscoverAODOptionsInCommandLine : o2::framework::ConfigDiscoveryPlugin {
   }
 };
 
+struct ImplementationContext {
+  std::vector<RootArrowFactory> implementations;
+};
+
+std::function<void*(TDirectoryFile*, std::string const&)> getHandleByClass(char const* classname)
+{
+  return [classname](TDirectoryFile* file, std::string const& path) { return file->GetObjectChecked(path.c_str(), TClass::GetClass(classname)); };
+}
+
+std::function<void*(TBufferFile*, std::string const&)> getBufferHandleByClass(char const* classname)
+{
+  return [classname](TBufferFile* buffer, std::string const& path) { buffer->Reset(); return buffer->ReadObjectAny(TClass::GetClass(classname)); };
+}
+
+void lazyLoadFactory(std::vector<RootArrowFactory>& implementations, char const* specs)
+{
+  // Lazy loading of the plugin so that we do not bring in RNTuple / TTree if not needed
+  if (implementations.empty()) {
+    std::vector<LoadablePlugin> plugins;
+    auto morePlugins = PluginManager::parsePluginSpecString(specs);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+    PluginManager::loadFromPlugin<RootArrowFactory, RootArrowFactoryPlugin>(plugins, implementations);
+    if (implementations.empty()) {
+      return;
+    }
+  }
+}
+
+struct RNTupleObjectReadingCapability : o2::framework::RootObjectReadingCapabilityPlugin {
+  RootObjectReadingCapability* create() override
+  {
+    auto context = new ImplementationContext;
+
+    return new RootObjectReadingCapability{
+      .name = "rntuple",
+      .getHandle = getHandleByClass("ROOT::Experimental::RNTuple"),
+      .getBufferHandle = getBufferHandleByClass("ROOT::Experimental::RNTuple"),
+      .factory = [context]() -> RootArrowFactory& {
+        lazyLoadFactory(context->implementations, "O2FrameworkAnalysisRNTupleSupport:RNTupleObjectReadingImplementation");
+        return context->implementations.back();
+      }};
+  }
+};
+
+struct TTreeObjectReadingCapability : o2::framework::RootObjectReadingCapabilityPlugin {
+  RootObjectReadingCapability* create() override
+  {
+    auto context = new ImplementationContext;
+
+    return new RootObjectReadingCapability{
+      .name = "ttree",
+      .getHandle = getHandleByClass("TTree"),
+      .getBufferHandle = getBufferHandleByClass("TTree"),
+      .factory = [context]() -> RootArrowFactory& {
+        lazyLoadFactory(context->implementations, "O2FrameworkAnalysisTTreeSupport:TTreeObjectReadingImplementation");
+        return context->implementations.back();
+      }};
+  }
+};
+
 DEFINE_DPL_PLUGINS_BEGIN
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAODCapability, Capability);
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLineCapability, Capability);
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverAODOptionsInCommandLineCapability, Capability);
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLine, ConfigDiscovery);
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverAODOptionsInCommandLine, ConfigDiscovery);
+DEFINE_DPL_PLUGIN_INSTANCE(RNTupleObjectReadingCapability, RootObjectReadingCapability);
+DEFINE_DPL_PLUGIN_INSTANCE(TTreeObjectReadingCapability, RootObjectReadingCapability);
 DEFINE_DPL_PLUGINS_END
 } // namespace o2::framework
diff --git a/Framework/Core/src/RootArrowFilesystem.cxx b/Framework/Core/src/RootArrowFilesystem.cxx
index 5f2d21d942d37..545ba6f0afb71 100644
--- a/Framework/Core/src/RootArrowFilesystem.cxx
+++ b/Framework/Core/src/RootArrowFilesystem.cxx
@@ -9,9 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 #include "Framework/RootArrowFilesystem.h"
-#include "Framework/Endian.h"
 #include "Framework/RuntimeError.h"
-#include "Framework/Signpost.h"
 #include <Rtypes.h>
 #include <arrow/array/array_nested.h>
 #include <arrow/array/array_primitive.h>
@@ -19,93 +17,48 @@
 #include <arrow/array/builder_primitive.h>
 #include <memory>
 #include <TFile.h>
-#include <TLeaf.h>
 #include <TBufferFile.h>
-#include <TTree.h>
 #include <TDirectoryFile.h>
 #include <arrow/type.h>
 #include <arrow/type_fwd.h>
 #include <arrow/dataset/file_base.h>
 #include <arrow/result.h>
 #include <arrow/status.h>
-#include <arrow/util/key_value_metadata.h>
 #include <fmt/format.h>
 
-#include <stdexcept>
-#include <utility>
+template class
+  std::shared_ptr<arrow::Array>;
 
-O2_DECLARE_DYNAMIC_LOG(root_arrow_fs);
-
-namespace
-{
-struct BranchInfo {
-  std::string name;
-  TBranch* ptr;
-  bool mVLA;
-};
-} // namespace
-
-auto arrowTypeFromROOT(EDataType type, int size)
-{
-  auto typeGenerator = [](std::shared_ptr<arrow::DataType> const& type, int size) -> std::shared_ptr<arrow::DataType> {
-    switch (size) {
-      case -1:
-        return arrow::list(type);
-      case 1:
-        return std::move(type);
-      default:
-        return arrow::fixed_size_list(type, size);
-    }
-  };
-
-  switch (type) {
-    case EDataType::kBool_t:
-      return typeGenerator(arrow::boolean(), size);
-    case EDataType::kUChar_t:
-      return typeGenerator(arrow::uint8(), size);
-    case EDataType::kUShort_t:
-      return typeGenerator(arrow::uint16(), size);
-    case EDataType::kUInt_t:
-      return typeGenerator(arrow::uint32(), size);
-    case EDataType::kULong64_t:
-      return typeGenerator(arrow::uint64(), size);
-    case EDataType::kChar_t:
-      return typeGenerator(arrow::int8(), size);
-    case EDataType::kShort_t:
-      return typeGenerator(arrow::int16(), size);
-    case EDataType::kInt_t:
-      return typeGenerator(arrow::int32(), size);
-    case EDataType::kLong64_t:
-      return typeGenerator(arrow::int64(), size);
-    case EDataType::kFloat_t:
-      return typeGenerator(arrow::float32(), size);
-    case EDataType::kDouble_t:
-      return typeGenerator(arrow::float64(), size);
-    default:
-      throw o2::framework::runtime_error_f("Unsupported branch type: %d", static_cast<int>(type));
-  }
-}
 namespace o2::framework
 {
 using arrow::Status;
 
-TFileFileSystem::TFileFileSystem(TDirectoryFile* f, size_t readahead)
+TFileFileSystem::TFileFileSystem(TDirectoryFile* f, size_t readahead, RootObjectReadingFactory& factory)
   : VirtualRootFileSystemBase(),
-    mFile(f)
+    mFile(f),
+    mObjectFactory(factory)
 {
   ((TFile*)mFile)->SetReadaheadSize(50 * 1024 * 1024);
 }
 
 std::shared_ptr<VirtualRootFileSystemBase> TFileFileSystem::GetSubFilesystem(arrow::dataset::FileSource source)
 {
-  auto tree = (TTree*)mFile->GetObjectChecked(source.path().c_str(), TClass::GetClass<TTree>());
-  if (tree) {
-    return std::shared_ptr<VirtualRootFileSystemBase>(new SingleTreeFileSystem(tree));
+  // We use a plugin to create the actual objects inside the
+  // file, so that we can support TTree and RNTuple at the same time
+  // without having to depend on both.
+  for (auto& capability : mObjectFactory.capabilities) {
+    void* handle = capability.getHandle(mFile, source.path());
+    if (!handle) {
+      continue;
+    }
+    if (handle) {
+      return capability.factory().getSubFilesystem(handle);
+    }
   }
 
   auto directory = (TDirectoryFile*)mFile->GetObjectChecked(source.path().c_str(), TClass::GetClass<TDirectory>());
   if (directory) {
-    return std::shared_ptr<VirtualRootFileSystemBase>(new TFileFileSystem(directory, 50 * 1024 * 1024));
+    return std::shared_ptr<VirtualRootFileSystemBase>(new TFileFileSystem(directory, 50 * 1024 * 1024, mObjectFactory));
   }
   throw runtime_error_f("Unsupported file layout");
 }
@@ -120,10 +73,14 @@ arrow::Result<arrow::fs::FileInfo> TFileFileSystem::GetFileInfo(const std::strin
   auto fs = GetSubFilesystem(source);
 
   // For now we only support single trees.
-  if (std::dynamic_pointer_cast<SingleTreeFileSystem>(fs)) {
-    result.set_type(arrow::fs::FileType::File);
+  if (std::dynamic_pointer_cast<TFileFileSystem>(fs)) {
+    result.set_type(arrow::fs::FileType::Directory);
     return result;
   }
+  // Everything else is a file, if it was created.
+  if (fs.get()) {
+    result.set_type(arrow::fs::FileType::File);
+  }
   return result;
 }
 
@@ -137,7 +94,7 @@ arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TFileFileSystem::OpenOut
 
   auto* dir = dynamic_cast<TDirectoryFile*>(this->GetFile()->Get(path.c_str()));
   if (!dir) {
-    throw runtime_error_f("Unable to open directory %s in file %s", path.c_str(), GetFile()->GetName());
+    return arrow::Status::Invalid(fmt::format("Unable to open directory {} in file {} ", path.c_str(), GetFile()->GetName()));
   }
   auto stream = std::make_shared<TDirectoryFileOutputStream>(dir);
   return stream;
@@ -219,81 +176,6 @@ arrow::Result<std::shared_ptr<arrow::io::OutputStream>> VirtualRootFileSystemBas
   return arrow::Status::NotImplemented("No random access file system");
 }
 
-arrow::Result<std::shared_ptr<arrow::Schema>> TTreeFileFormat::Inspect(const arrow::dataset::FileSource& source) const
-{
-  arrow::Schema schema{{}};
-  auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
-  // Actually get the TTree from the ROOT file.
-  auto treeFs = std::dynamic_pointer_cast<TTreeFileSystem>(fs->GetSubFilesystem(source));
-  if (!treeFs.get()) {
-    throw runtime_error_f("Unknown filesystem %s\n", source.filesystem()->type_name().c_str());
-  }
-  TTree* tree = treeFs->GetTree(source);
-
-  auto branches = tree->GetListOfBranches();
-  auto n = branches->GetEntries();
-
-  std::vector<BranchInfo> branchInfos;
-  for (auto i = 0; i < n; ++i) {
-    auto branch = static_cast<TBranch*>(branches->At(i));
-    auto name = std::string{branch->GetName()};
-    auto pos = name.find("_size");
-    if (pos != std::string::npos) {
-      name.erase(pos);
-      branchInfos.emplace_back(BranchInfo{name, (TBranch*)nullptr, true});
-    } else {
-      auto lookup = std::find_if(branchInfos.begin(), branchInfos.end(), [&](BranchInfo const& bi) {
-        return bi.name == name;
-      });
-      if (lookup == branchInfos.end()) {
-        branchInfos.emplace_back(BranchInfo{name, branch, false});
-      } else {
-        lookup->ptr = branch;
-      }
-    }
-  }
-
-  std::vector<std::shared_ptr<arrow::Field>> fields;
-  tree->SetCacheSize(25000000);
-  for (auto& bi : branchInfos) {
-    static TClass* cls;
-    EDataType type;
-    bi.ptr->GetExpectedType(cls, type);
-    auto listSize = -1;
-    if (!bi.mVLA) {
-      listSize = static_cast<TLeaf*>(bi.ptr->GetListOfLeaves()->At(0))->GetLenStatic();
-    }
-    auto field = std::make_shared<arrow::Field>(bi.ptr->GetName(), arrowTypeFromROOT(type, listSize));
-    fields.push_back(field);
-
-    tree->AddBranchToCache(bi.ptr);
-    if (strncmp(bi.ptr->GetName(), "fIndexArray", strlen("fIndexArray")) == 0) {
-      std::string sizeBranchName = bi.ptr->GetName();
-      sizeBranchName += "_size";
-      auto* sizeBranch = (TBranch*)tree->GetBranch(sizeBranchName.c_str());
-      if (sizeBranch) {
-        tree->AddBranchToCache(sizeBranch);
-      }
-    }
-  }
-  tree->StopCacheLearningPhase();
-
-  return std::make_shared<arrow::Schema>(fields);
-}
-
-/// \brief Create a FileFragment for a FileSource.
-arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> TTreeFileFormat::MakeFragment(
-  arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
-  std::shared_ptr<arrow::Schema> physical_schema)
-{
-  std::shared_ptr<arrow::dataset::FileFormat> format = std::make_shared<TTreeFileFormat>(mTotCompressedSize, mTotUncompressedSize);
-
-  auto fragment = std::make_shared<TTreeFileFragment>(std::move(source), std::move(format),
-                                                      std::move(partition_expression),
-                                                      std::move(physical_schema));
-  return std::dynamic_pointer_cast<arrow::dataset::FileFragment>(fragment);
-}
-
 // An arrow outputstream which allows to write to a ttree
 TDirectoryFileOutputStream::TDirectoryFileOutputStream(TDirectoryFile* f)
   : mDirectory(f)
@@ -321,544 +203,14 @@ bool TDirectoryFileOutputStream::closed() const
   return mDirectory->GetFile()->IsOpen() == false;
 }
 
-// An arrow outputstream which allows to write to a ttree
-// @a branch prefix is to be used to identify a set of branches which all belong to
-// the same table.
-TTreeOutputStream::TTreeOutputStream(TTree* f, std::string branchPrefix)
-  : mTree(f),
-    mBranchPrefix(std::move(branchPrefix))
-{
-}
-
-arrow::Status TTreeOutputStream::Close()
-{
-  if (mTree->GetCurrentFile() == nullptr) {
-    return arrow::Status::Invalid("Cannot close a tree not attached to a file");
-  }
-  mTree->GetCurrentFile()->Close();
-  return arrow::Status::OK();
-}
-
-arrow::Result<int64_t> TTreeOutputStream::Tell() const
-{
-  return arrow::Result<int64_t>(arrow::Status::NotImplemented("Cannot move"));
-}
-
-arrow::Status TTreeOutputStream::Write(const void* data, int64_t nbytes)
-{
-  return arrow::Status::NotImplemented("Cannot write raw bytes to a TTree");
-}
-
-bool TTreeOutputStream::closed() const
-{
-  // A standalone tree is never closed.
-  if (mTree->GetCurrentFile() == nullptr) {
-    return false;
-  }
-  return mTree->GetCurrentFile()->IsOpen() == false;
-}
-
-TBranch* TTreeOutputStream::CreateBranch(char const* branchName, char const* sizeBranch)
-{
-  return mTree->Branch((mBranchPrefix + "/" + branchName).c_str(), (char*)nullptr, (mBranchPrefix + sizeBranch).c_str());
-}
-
-char const* rootSuffixFromArrow(arrow::Type::type id)
-{
-  switch (id) {
-    case arrow::Type::BOOL:
-      return "/O";
-    case arrow::Type::UINT8:
-      return "/b";
-    case arrow::Type::UINT16:
-      return "/s";
-    case arrow::Type::UINT32:
-      return "/i";
-    case arrow::Type::UINT64:
-      return "/l";
-    case arrow::Type::INT8:
-      return "/B";
-    case arrow::Type::INT16:
-      return "/S";
-    case arrow::Type::INT32:
-      return "/I";
-    case arrow::Type::INT64:
-      return "/L";
-    case arrow::Type::FLOAT:
-      return "/F";
-    case arrow::Type::DOUBLE:
-      return "/D";
-    default:
-      throw runtime_error("Unsupported arrow column type");
-  }
-}
-
-class TTreeFileWriter : public arrow::dataset::FileWriter
-{
-  std::vector<TBranch*> branches;
-  std::vector<TBranch*> sizesBranches;
-  std::vector<std::shared_ptr<arrow::Array>> valueArrays;
-  std::vector<std::shared_ptr<arrow::Array>> sizeArrays;
-  std::vector<std::shared_ptr<arrow::DataType>> valueTypes;
-
-  std::vector<int64_t> valuesIdealBasketSize;
-  std::vector<int64_t> sizeIdealBasketSize;
-
-  std::vector<int64_t> typeSizes;
-  std::vector<int64_t> listSizes;
-  bool firstBasket = true;
-
-  // This is to create a batsket size according to the first batch.
-  void finaliseBasketSize(std::shared_ptr<arrow::RecordBatch> firstBatch)
-  {
-    O2_SIGNPOST_ID_FROM_POINTER(sid, root_arrow_fs, this);
-    O2_SIGNPOST_START(root_arrow_fs, sid, "finaliseBasketSize", "First batch with %lli rows received and %zu columns",
-                      firstBatch->num_rows(), firstBatch->columns().size());
-    for (size_t i = 0; i < branches.size(); i++) {
-      auto* branch = branches[i];
-      auto* sizeBranch = sizesBranches[i];
-
-      int valueSize = valueTypes[i]->byte_width();
-      if (listSizes[i] == 1) {
-        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s exists and uses %d bytes per entry for %lli entries.",
-                               branch->GetName(), valueSize, firstBatch->num_rows());
-        assert(sizeBranch == nullptr);
-        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize);
-      } else if (listSizes[i] == -1) {
-        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s exists and uses %d bytes per entry.",
-                               branch->GetName(), valueSize);
-        // This should probably lookup the
-        auto column = firstBatch->GetColumnByName(schema_->field(i)->name());
-        auto list = std::static_pointer_cast<arrow::ListArray>(column);
-        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s needed. Associated size branch %s and there are %lli entries of size %d in that list.",
-                               branch->GetName(), sizeBranch->GetName(), list->length(), valueSize);
-        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize * list->length());
-        sizeBranch->SetBasketSize(1024 + firstBatch->num_rows() * 4);
-      } else {
-        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s needed. There are %lli entries per array of size %d in that list.",
-                               branch->GetName(), listSizes[i], valueSize);
-        assert(sizeBranch == nullptr);
-        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize * listSizes[i]);
-      }
-
-      auto field = firstBatch->schema()->field(i);
-      if (field->name().starts_with("fIndexArray")) {
-        // One int per array to keep track of the size
-        int idealBasketSize = 4 * firstBatch->num_rows() + 1024 + field->type()->byte_width() * firstBatch->num_rows(); // minimal additional size needed, otherwise we get 2 baskets
-        int basketSize = std::max(32000, idealBasketSize);                                                              // keep a minimum value
-        sizeBranch->SetBasketSize(basketSize);
-        branch->SetBasketSize(basketSize);
-      }
-    }
-    O2_SIGNPOST_END(root_arrow_fs, sid, "finaliseBasketSize", "Done");
-  }
-
- public:
-  // Create the TTree based on the physical_schema, not the one in the batch.
-  // The write method will have to reconcile the two schemas.
-  TTreeFileWriter(std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options,
-                  std::shared_ptr<arrow::io::OutputStream> destination,
-                  arrow::fs::FileLocator destination_locator)
-    : FileWriter(schema, options, destination, destination_locator)
-  {
-    // Batches have the same number of entries for each column.
-    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
-    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
-
-    if (directoryStream.get()) {
-      TDirectoryFile* dir = directoryStream->GetDirectory();
-      dir->cd();
-      auto* tree = new TTree(destination_locator_.path.c_str(), "");
-      treeStream = std::make_shared<TTreeOutputStream>(tree, "");
-    } else if (treeStream.get()) {
-      // We already have a tree stream, let's derive a new one
-      // with the destination_locator_.path as prefix for the branches
-      // This way we can multiplex multiple tables in the same tree.
-      auto tree = treeStream->GetTree();
-      treeStream = std::make_shared<TTreeOutputStream>(tree, destination_locator_.path);
-    } else {
-      // I could simply set a prefix here to merge to an already existing tree.
-      throw std::runtime_error("Unsupported backend.");
-    }
-
-    for (auto i = 0u; i < schema->fields().size(); ++i) {
-      auto& field = schema->field(i);
-      listSizes.push_back(1);
-
-      int valuesIdealBasketSize = 0;
-      // Construct all the needed branches.
-      switch (field->type()->id()) {
-        case arrow::Type::FIXED_SIZE_LIST: {
-          listSizes.back() = std::static_pointer_cast<arrow::FixedSizeListType>(field->type())->list_size();
-          valuesIdealBasketSize = 1024 + valueTypes.back()->byte_width() * listSizes.back();
-          valueTypes.push_back(field->type()->field(0)->type());
-          sizesBranches.push_back(nullptr);
-          std::string leafList = fmt::format("{}[{}]{}", field->name(), listSizes.back(), rootSuffixFromArrow(valueTypes.back()->id()));
-          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
-        } break;
-        case arrow::Type::LIST: {
-          valueTypes.push_back(field->type()->field(0)->type());
-          std::string leafList = fmt::format("{}[{}_size]{}", field->name(), field->name(), rootSuffixFromArrow(valueTypes.back()->id()));
-          listSizes.back() = -1; // VLA, we need to calculate it on the fly;
-          std::string sizeLeafList = field->name() + "_size/I";
-          sizesBranches.push_back(treeStream->CreateBranch((field->name() + "_size").c_str(), sizeLeafList.c_str()));
-          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
-          // Notice that this could be replaced by a better guess of the
-          // average size of the list elements, but this is not trivial.
-        } break;
-        default: {
-          valueTypes.push_back(field->type());
-          std::string leafList = field->name() + rootSuffixFromArrow(valueTypes.back()->id());
-          sizesBranches.push_back(nullptr);
-          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
-        } break;
-      }
-    }
-    // We create the branches from the schema
-  }
-
-  arrow::Status Write(const std::shared_ptr<arrow::RecordBatch>& batch) override
-  {
-    if (firstBasket) {
-      firstBasket = false;
-      finaliseBasketSize(batch);
-    }
-
-    // Support writing empty tables
-    if (batch->columns().empty() || batch->num_rows() == 0) {
-      return arrow::Status::OK();
-    }
-
-    // Batches have the same number of entries for each column.
-    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
-    TTree* tree = nullptr;
-    if (directoryStream.get()) {
-      TDirectoryFile* dir = directoryStream->GetDirectory();
-      tree = (TTree*)dir->Get(destination_locator_.path.c_str());
-    }
-    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
-
-    if (!tree) {
-      // I could simply set a prefix here to merge to an already existing tree.
-      throw std::runtime_error("Unsupported backend.");
-    }
-
-    for (auto i = 0u; i < batch->columns().size(); ++i) {
-      auto column = batch->column(i);
-      auto& field = batch->schema()->field(i);
-
-      valueArrays.push_back(nullptr);
-
-      switch (field->type()->id()) {
-        case arrow::Type::FIXED_SIZE_LIST: {
-          auto list = std::static_pointer_cast<arrow::FixedSizeListArray>(column);
-          valueArrays.back() = list->values();
-        } break;
-        case arrow::Type::LIST: {
-          auto list = std::static_pointer_cast<arrow::ListArray>(column);
-          valueArrays.back() = list;
-        } break;
-        case arrow::Type::BOOL: {
-          // In case of arrays of booleans, we need to go back to their
-          // char based representation for ROOT to save them.
-          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(column);
-
-          int64_t length = boolArray->length();
-          arrow::UInt8Builder builder;
-          auto ok = builder.Reserve(length);
-
-          for (int64_t i = 0; i < length; ++i) {
-            if (boolArray->IsValid(i)) {
-              // Expand each boolean value (true/false) to uint8 (1/0)
-              uint8_t value = boolArray->Value(i) ? 1 : 0;
-              auto ok = builder.Append(value);
-            } else {
-              // Append null for invalid entries
-              auto ok = builder.AppendNull();
-            }
-          }
-          valueArrays.back() = *builder.Finish();
-        } break;
-        default:
-          valueArrays.back() = column;
-      }
-    }
-
-    int64_t pos = 0;
-    while (pos < batch->num_rows()) {
-      for (size_t bi = 0; bi < branches.size(); ++bi) {
-        auto* branch = branches[bi];
-        auto* sizeBranch = sizesBranches[bi];
-        auto array = batch->column(bi);
-        auto& field = batch->schema()->field(bi);
-        auto& listSize = listSizes[bi];
-        auto valueType = valueTypes[bi];
-        auto valueArray = valueArrays[bi];
-
-        switch (field->type()->id()) {
-          case arrow::Type::LIST: {
-            auto list = std::static_pointer_cast<arrow::ListArray>(array);
-            listSize = list->value_length(pos);
-            uint8_t const* buffer = std::static_pointer_cast<arrow::PrimitiveArray>(valueArray)->values()->data() + array->offset() + list->value_offset(pos) * valueType->byte_width();
-            branch->SetAddress((void*)buffer);
-            sizeBranch->SetAddress(&listSize);
-          };
-            break;
-          case arrow::Type::FIXED_SIZE_LIST:
-          default: {
-            uint8_t const* buffer = std::static_pointer_cast<arrow::PrimitiveArray>(valueArray)->values()->data() + array->offset() + pos * listSize * valueType->byte_width();
-            branch->SetAddress((void*)buffer);
-          };
-        }
-      }
-      tree->Fill();
-      ++pos;
-    }
-    return arrow::Status::OK();
-  }
-
-  arrow::Future<> FinishInternal() override
-  {
-    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
-    TTree* tree = treeStream->GetTree();
-    tree->Write("", TObject::kOverwrite);
-    tree->SetDirectory(nullptr);
-
-    return {};
-  };
-};
-
-arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> TTreeFileFormat::MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options, arrow::fs::FileLocator destination_locator) const
-{
-  auto writer = std::make_shared<TTreeFileWriter>(schema, options, destination, destination_locator);
-  return std::dynamic_pointer_cast<arrow::dataset::FileWriter>(writer);
-}
-
-std::shared_ptr<arrow::dataset::FileWriteOptions> TTreeFileFormat::DefaultWriteOptions()
-{
-  std::shared_ptr<TTreeFileWriteOptions> options(
-    new TTreeFileWriteOptions(shared_from_this()));
-  return options;
-}
-
-arrow::Result<arrow::RecordBatchGenerator> TTreeFileFormat::ScanBatchesAsync(
-  const std::shared_ptr<arrow::dataset::ScanOptions>& options,
-  const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const
-{
-  // Get the fragment as a TTreeFragment. This might be PART of a TTree.
-  auto treeFragment = std::dynamic_pointer_cast<TTreeFileFragment>(fragment);
-  // This is the schema we want to read
-  auto dataset_schema = options->dataset_schema;
-
-  auto generator = [pool = options->pool, treeFragment, dataset_schema, &totalCompressedSize = mTotCompressedSize,
-                    &totalUncompressedSize = mTotUncompressedSize]() -> arrow::Future<std::shared_ptr<arrow::RecordBatch>> {
-    auto schema = treeFragment->format()->Inspect(treeFragment->source());
-
-    std::vector<std::shared_ptr<arrow::Array>> columns;
-    std::vector<std::shared_ptr<arrow::Field>> fields = dataset_schema->fields();
-    auto physical_schema = *treeFragment->ReadPhysicalSchema();
-
-    static TBufferFile buffer{TBuffer::EMode::kWrite, 4 * 1024 * 1024};
-    auto containerFS = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(treeFragment->source().filesystem());
-    auto fs = std::dynamic_pointer_cast<TTreeFileSystem>(containerFS->GetSubFilesystem(treeFragment->source()));
-
-    int64_t rows = -1;
-    TTree* tree = fs->GetTree(treeFragment->source());
-    for (auto& field : fields) {
-      // The field actually on disk
-      auto physicalField = physical_schema->GetFieldByName(field->name());
-      TBranch* branch = tree->GetBranch(physicalField->name().c_str());
-      assert(branch);
-      buffer.Reset();
-      auto totalEntries = branch->GetEntries();
-      if (rows == -1) {
-        rows = totalEntries;
-      }
-      if (rows != totalEntries) {
-        throw runtime_error_f("Unmatching number of rows for branch %s", branch->GetName());
-      }
-      arrow::Status status;
-      int readEntries = 0;
-      std::shared_ptr<arrow::Array> array;
-      auto listType = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type());
-      if (physicalField->type() == arrow::boolean() ||
-          (listType && physicalField->type()->field(0)->type() == arrow::boolean())) {
-        if (listType) {
-          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
-          auto status = arrow::MakeBuilder(pool, physicalField->type()->field(0)->type(), &builder);
-          if (!status.ok()) {
-            throw runtime_error("Cannot create value builder");
-          }
-          auto listBuilder = std::make_unique<arrow::FixedSizeListBuilder>(pool, std::move(builder), listType->list_size());
-          auto valueBuilder = listBuilder.get()->value_builder();
-          // boolean array special case: we need to use builder to create the bitmap
-          status = valueBuilder->Reserve(totalEntries * listType->list_size());
-          status &= listBuilder->Reserve(totalEntries);
-          if (!status.ok()) {
-            throw runtime_error("Failed to reserve memory for array builder");
-          }
-          while (readEntries < totalEntries) {
-            auto readLast = branch->GetBulkRead().GetBulkEntries(readEntries, buffer);
-            readEntries += readLast;
-            status &= static_cast<arrow::BooleanBuilder*>(valueBuilder)->AppendValues(reinterpret_cast<uint8_t const*>(buffer.GetCurrent()), readLast * listType->list_size());
-          }
-          status &= static_cast<arrow::FixedSizeListBuilder*>(listBuilder.get())->AppendValues(readEntries);
-          if (!status.ok()) {
-            throw runtime_error("Failed to append values to array");
-          }
-          status &= listBuilder->Finish(&array);
-          if (!status.ok()) {
-            throw runtime_error("Failed to create array");
-          }
-        } else if (listType == nullptr) {
-          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
-          auto status = arrow::MakeBuilder(pool, physicalField->type(), &builder);
-          if (!status.ok()) {
-            throw runtime_error("Cannot create builder");
-          }
-          auto valueBuilder = static_cast<arrow::BooleanBuilder*>(builder.get());
-          // boolean array special case: we need to use builder to create the bitmap
-          status = valueBuilder->Reserve(totalEntries);
-          if (!status.ok()) {
-            throw runtime_error("Failed to reserve memory for array builder");
-          }
-          while (readEntries < totalEntries) {
-            auto readLast = branch->GetBulkRead().GetBulkEntries(readEntries, buffer);
-            readEntries += readLast;
-            status &= valueBuilder->AppendValues(reinterpret_cast<uint8_t const*>(buffer.GetCurrent()), readLast);
-          }
-          if (!status.ok()) {
-            throw runtime_error("Failed to append values to array");
-          }
-          status &= valueBuilder->Finish(&array);
-          if (!status.ok()) {
-            throw runtime_error("Failed to create array");
-          }
-        }
-      } else {
-        // other types: use serialized read to build arrays directly.
-        auto typeSize = physicalField->type()->byte_width();
-        // This is needed for branches which have not been persisted.
-        auto bytes = branch->GetTotBytes();
-        auto branchSize = bytes ? bytes : 1000000;
-        auto&& result = arrow::AllocateResizableBuffer(branchSize, pool);
-        if (!result.ok()) {
-          throw runtime_error("Cannot allocate values buffer");
-        }
-        std::shared_ptr<arrow::Buffer> arrowValuesBuffer = std::move(result).ValueUnsafe();
-        auto ptr = arrowValuesBuffer->mutable_data();
-        if (ptr == nullptr) {
-          throw runtime_error("Invalid buffer");
-        }
-
-        std::unique_ptr<TBufferFile> offsetBuffer = nullptr;
-
-        uint32_t offset = 0;
-        int count = 0;
-        std::shared_ptr<arrow::Buffer> arrowOffsetBuffer;
-        std::span<int> offsets;
-        int size = 0;
-        uint32_t totalSize = 0;
-        TBranch* mSizeBranch = nullptr;
-        int64_t listSize = 1;
-        if (auto fixedSizeList = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type())) {
-          listSize = fixedSizeList->list_size();
-          typeSize = fixedSizeList->field(0)->type()->byte_width();
-        } else if (auto vlaListType = std::dynamic_pointer_cast<arrow::ListType>(physicalField->type())) {
-          listSize = -1;
-          typeSize = vlaListType->field(0)->type()->byte_width();
-        }
-        if (listSize == -1) {
-          mSizeBranch = branch->GetTree()->GetBranch((std::string{branch->GetName()} + "_size").c_str());
-          offsetBuffer = std::make_unique<TBufferFile>(TBuffer::EMode::kWrite, 4 * 1024 * 1024);
-          result = arrow::AllocateResizableBuffer((totalEntries + 1) * (int64_t)sizeof(int), pool);
-          if (!result.ok()) {
-            throw runtime_error("Cannot allocate offset buffer");
-          }
-          arrowOffsetBuffer = std::move(result).ValueUnsafe();
-          unsigned char* ptrOffset = arrowOffsetBuffer->mutable_data();
-          auto* tPtrOffset = reinterpret_cast<int*>(ptrOffset);
-          offsets = std::span<int>{tPtrOffset, tPtrOffset + totalEntries + 1};
-
-          // read sizes first
-          while (readEntries < totalEntries) {
-            auto readLast = mSizeBranch->GetBulkRead().GetEntriesSerialized(readEntries, *offsetBuffer);
-            readEntries += readLast;
-            for (auto i = 0; i < readLast; ++i) {
-              offsets[count++] = (int)offset;
-              offset += swap32_(reinterpret_cast<uint32_t*>(offsetBuffer->GetCurrent())[i]);
-            }
-          }
-          offsets[count] = (int)offset;
-          totalSize = offset;
-          readEntries = 0;
-        }
-
-        while (readEntries < totalEntries) {
-          auto readLast = branch->GetBulkRead().GetEntriesSerialized(readEntries, buffer);
-          if (listSize == -1) {
-            size = offsets[readEntries + readLast] - offsets[readEntries];
-          } else {
-            size = readLast * listSize;
-          }
-          readEntries += readLast;
-          swapCopy(ptr, buffer.GetCurrent(), size, typeSize);
-          ptr += (ptrdiff_t)(size * typeSize);
-        }
-        if (listSize >= 1) {
-          totalSize = readEntries * listSize;
-        }
-        std::shared_ptr<arrow::PrimitiveArray> varray;
-        switch (listSize) {
-          case -1:
-            varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
-            array = std::make_shared<arrow::ListArray>(physicalField->type(), readEntries, arrowOffsetBuffer, varray);
-            break;
-          case 1:
-            array = std::make_shared<arrow::PrimitiveArray>(physicalField->type(), readEntries, arrowValuesBuffer);
-            break;
-          default:
-            varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
-            array = std::make_shared<arrow::FixedSizeListArray>(physicalField->type(), readEntries, varray);
-        }
-      }
-
-      branch->SetStatus(false);
-      branch->DropBaskets("all");
-      branch->Reset();
-      branch->GetTransientBuffer(0)->Expand(0);
-
-      columns.push_back(array);
-    }
-    auto batch = arrow::RecordBatch::Make(dataset_schema, rows, columns);
-    totalCompressedSize += tree->GetZipBytes();
-    totalUncompressedSize += tree->GetTotBytes();
-    return batch;
-  };
-  return generator;
-}
-
-arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenOutputStream(
-  const std::string& path,
-  const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
-{
-  arrow::dataset::FileSource source{path, shared_from_this()};
-  auto prefix = metadata->Get("branch_prefix");
-  if (prefix.ok()) {
-    return std::make_shared<TTreeOutputStream>(GetTree(source), *prefix);
-  }
-  return std::make_shared<TTreeOutputStream>(GetTree(source), "");
-}
-
-TBufferFileFS::TBufferFileFS(TBufferFile* f)
+TBufferFileFS::TBufferFileFS(TBufferFile* f, RootObjectReadingFactory& factory)
   : VirtualRootFileSystemBase(),
     mBuffer(f),
-    mFilesystem(nullptr)
+    mFilesystem(nullptr),
+    mObjectFactory(factory)
 {
 }
 
-TTreeFileSystem::~TTreeFileSystem() = default;
-
 arrow::Result<arrow::fs::FileInfo> TBufferFileFS::GetFileInfo(const std::string& path)
 {
   arrow::fs::FileInfo result;
@@ -871,19 +223,26 @@ arrow::Result<arrow::fs::FileInfo> TBufferFileFS::GetFileInfo(const std::string&
     return result;
   }
 
-  // For now we only support single trees.
-  if (std::dynamic_pointer_cast<SingleTreeFileSystem>(mFilesystem)) {
-    result.set_type(arrow::fs::FileType::File);
+  auto info = mFilesystem->GetFileInfo(path);
+  if (!info.ok()) {
     return result;
   }
+
+  result.set_type(info->type());
   return result;
 }
 
 std::shared_ptr<VirtualRootFileSystemBase> TBufferFileFS::GetSubFilesystem(arrow::dataset::FileSource source)
 {
-  if (!mFilesystem.get()) {
-    auto tree = ((TTree*)mBuffer->ReadObject(TTree::Class()));
-    mFilesystem = std::make_shared<SingleTreeFileSystem>(tree);
+  // We use a plugin to create the actual objects inside the
+  // file, so that we can support TTree and RNTuple at the same time
+  // without having to depend on both.
+  for (auto& capability : mObjectFactory.capabilities) {
+    void* handle = capability.getBufferHandle(mBuffer, source.path());
+    if (handle) {
+      mFilesystem = capability.factory().getSubFilesystem(handle);
+      break;
+    }
   }
   return mFilesystem;
 }
diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx
index 8440e942903a5..8eb3a9825f0f7 100644
--- a/Framework/Core/test/test_Root2ArrowTable.cxx
+++ b/Framework/Core/test/test_Root2ArrowTable.cxx
@@ -14,6 +14,7 @@
 #include "Framework/TableBuilder.h"
 #include "Framework/RootTableBuilderHelpers.h"
 #include "Framework/ASoA.h"
+#include "Framework/PluginManager.h"
 #include "../src/ArrowDebugHelpers.h"
 
 #include <ROOT/RDataFrame.hxx>
@@ -26,6 +27,13 @@
 #include <TTree.h>
 #include <TRandom.h>
 #include <TFile.h>
+#include <ROOT/RField.hxx>
+#include <ROOT/RNTuple.hxx>
+#include <ROOT/RNTupleDescriptor.hxx>
+#include <ROOT/RNTupleModel.hxx>
+#include <ROOT/RNTupleReader.hxx>
+#include <ROOT/RNTupleUtil.hxx>
+#include <ROOT/RNTupleWriter.hxx>
 #include <memory>
 
 #include <arrow/array/array_primitive.h>
@@ -232,10 +240,31 @@ TEST_CASE("RootTree2Fragment")
   file->WriteObjectAny(&t1, t1.Class());
   auto* fileRead = new TBufferFile(TBuffer::kRead, file->BufferSize(), file->Buffer(), false, nullptr);
 
-  size_t totalSizeCompressed = 0;
-  size_t totalSizeUncompressed = 0;
-  auto format = std::make_shared<TTreeFileFormat>(totalSizeCompressed, totalSizeUncompressed);
-  auto fs = std::make_shared<TBufferFileFS>(fileRead);
+  std::vector<char const*> capabilitiesSpecs = {
+    "O2Framework:RNTupleObjectReadingCapability",
+    "O2Framework:TTreeObjectReadingCapability",
+  };
+
+  std::vector<LoadablePlugin> plugins;
+  for (auto spec : capabilitiesSpecs) {
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+  }
+  REQUIRE(plugins.size() == 2);
+
+  RootObjectReadingFactory factory;
+  std::vector<char const*> configDiscoverySpec = {};
+  PluginManager::loadFromPlugin<RootObjectReadingCapability, RootObjectReadingCapabilityPlugin>(plugins, factory.capabilities);
+  REQUIRE(factory.capabilities.size() == 2);
+  REQUIRE(factory.capabilities[0].name == "rntuple");
+  REQUIRE(factory.capabilities[1].name == "ttree");
+
+  // Plugins are hardcoded for now...
+  auto format = factory.capabilities[1].factory().format();
+
+  auto fs = std::make_shared<TBufferFileFS>(fileRead, factory);
 
   arrow::dataset::FileSource source("p", fs);
   REQUIRE(format->IsSupported(source) == true);
@@ -439,10 +468,34 @@ TEST_CASE("RootTree2Dataset")
   }
   f->Write();
 
-  size_t totalSizeCompressed = 0;
-  size_t totalSizeUncompressed = 0;
-  auto format = std::make_shared<TTreeFileFormat>(totalSizeCompressed, totalSizeUncompressed);
-  auto fs = std::make_shared<TFileFileSystem>(f, 50 * 1024 * 1024);
+  std::vector<char const*> capabilitiesSpecs = {
+    "O2Framework:RNTupleObjectReadingCapability",
+    "O2Framework:TTreeObjectReadingCapability",
+  };
+
+  RootObjectReadingFactory factory;
+
+  std::vector<LoadablePlugin> plugins;
+  for (auto spec : capabilitiesSpecs) {
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+  }
+  REQUIRE(plugins.size() == 2);
+
+  PluginManager::loadFromPlugin<RootObjectReadingCapability, RootObjectReadingCapabilityPlugin>(plugins, factory.capabilities);
+
+  REQUIRE(factory.capabilities.size() == 2);
+  REQUIRE(factory.capabilities[0].name == "rntuple");
+  REQUIRE(factory.capabilities[1].name == "ttree");
+
+  // Plugins are hardcoded for now...
+  auto rNtupleFormat = factory.capabilities[0].factory().format();
+  auto format = factory.capabilities[1].factory().format();
+
+  auto fs = std::make_shared<TFileFileSystem>(f, 50 * 1024 * 1024, factory);
+
   arrow::dataset::FileSource source("DF_2/tracks", fs);
   REQUIRE(format->IsSupported(source) == true);
   auto schemaOpt = format->Inspect(source);
@@ -464,7 +517,7 @@ TEST_CASE("RootTree2Dataset")
   validateContents(*result);
 
   auto* output = new TMemFile("foo", "RECREATE");
-  auto outFs = std::make_shared<TFileFileSystem>(output, 0);
+  auto outFs = std::make_shared<TFileFileSystem>(output, 0, factory);
 
   // Open a stream at toplevel
   auto destination = outFs->OpenOutputStream("/", {});
@@ -503,4 +556,38 @@ TEST_CASE("RootTree2Dataset")
     REQUIRE((*resultWritten)->num_rows() == 100);
     validateContents(*resultWritten);
   }
+  arrow::fs::FileLocator rnTupleLocator{outFs, "/rntuple"};
+  // We write an RNTuple in the same TMemFile, using /rntuple as a location
+  auto rntupleDestination = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(*destination);
+
+  {
+    auto rNtupleWriter = rNtupleFormat->MakeWriter(*destination, schema, {}, rnTupleLocator);
+    auto rNtupleSuccess = rNtupleWriter->get()->Write(*result);
+    REQUIRE(rNtupleSuccess.ok());
+  }
+
+  // And now we can read back the RNTuple into a RecordBatch
+  arrow::dataset::FileSource writtenRntupleSource("/rntuple", outFs);
+  auto newRNTupleFS = outFs->GetSubFilesystem(writtenRntupleSource);
+
+  REQUIRE(rNtupleFormat->IsSupported(writtenRntupleSource) == true);
+
+  auto rntupleSchemaOpt = rNtupleFormat->Inspect(writtenRntupleSource);
+  REQUIRE(rntupleSchemaOpt.ok());
+  auto rntupleSchemaWritten = *rntupleSchemaOpt;
+  REQUIRE(validateSchema(rntupleSchemaWritten));
+
+  auto rntupleFragmentWritten = rNtupleFormat->MakeFragment(writtenRntupleSource, {}, rntupleSchemaWritten);
+  REQUIRE(rntupleFragmentWritten.ok());
+  auto rntupleOptionsWritten = std::make_shared<arrow::dataset::ScanOptions>();
+  rntupleOptionsWritten->dataset_schema = rntupleSchemaWritten;
+  auto rntupleScannerWritten = rNtupleFormat->ScanBatchesAsync(rntupleOptionsWritten, *rntupleFragmentWritten);
+  REQUIRE(rntupleScannerWritten.ok());
+  auto rntupleBatchesWritten = (*rntupleScannerWritten)();
+  auto rntupleResultWritten = rntupleBatchesWritten.result();
+  REQUIRE(rntupleResultWritten.ok());
+  REQUIRE((*rntupleResultWritten)->columns().size() == 10);
+  REQUIRE(validateSchema((*rntupleResultWritten)->schema()));
+  REQUIRE((*rntupleResultWritten)->num_rows() == 100);
+  REQUIRE(validateContents(*rntupleResultWritten));
 }

From 0102838a38db2d6665a48b6e3cb0d0fee374662f Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 12 Dec 2024 13:39:40 +0100
Subject: [PATCH 725/726] DPL Analysis: add RNTuple arrow::Dataset support

As part of the changes, move the actual logic which serialises /
deserialised things to plugins so that we do not need to depend
or RNTuple in production code.

Include an initial converter to go from AO2Ds to RNTuple based files.
---
 Framework/AnalysisSupport/CMakeLists.txt      |  10 +
 .../AnalysisSupport/src/RNTuplePlugin.cxx     | 825 +++++++++++++++++
 Framework/AnalysisSupport/src/TTreePlugin.cxx | 862 ++++++++++++++++++
 Framework/Core/CMakeLists.txt                 |   5 +
 Framework/Core/include/Framework/Plugins.h    |   8 +
 .../include/Framework/RootArrowFilesystem.h   | 174 +---
 Framework/Core/src/Plugin.cxx                 |  69 ++
 Framework/Core/src/RootArrowFilesystem.cxx    | 721 +--------------
 Framework/Core/test/o2AO2DToAO3D.cxx          | 155 ++++
 Framework/Core/test/test_Root2ArrowTable.cxx  | 105 ++-
 10 files changed, 2107 insertions(+), 827 deletions(-)
 create mode 100644 Framework/AnalysisSupport/src/RNTuplePlugin.cxx
 create mode 100644 Framework/AnalysisSupport/src/TTreePlugin.cxx
 create mode 100644 Framework/Core/test/o2AO2DToAO3D.cxx

diff --git a/Framework/AnalysisSupport/CMakeLists.txt b/Framework/AnalysisSupport/CMakeLists.txt
index 5fb1282469711..05cd8075a49d8 100644
--- a/Framework/AnalysisSupport/CMakeLists.txt
+++ b/Framework/AnalysisSupport/CMakeLists.txt
@@ -24,6 +24,16 @@ o2_add_library(FrameworkAnalysisSupport
                PRIVATE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/src
                PUBLIC_LINK_LIBRARIES O2::Framework ${EXTRA_TARGETS} ROOT::TreePlayer)
 
+o2_add_library(FrameworkAnalysisRNTupleSupport
+               SOURCES src/RNTuplePlugin.cxx
+               PRIVATE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/src
+               PUBLIC_LINK_LIBRARIES O2::Framework ${EXTRA_TARGETS} ROOT::ROOTNTuple)
+
+o2_add_library(FrameworkAnalysisTTreeSupport
+               SOURCES src/TTreePlugin.cxx
+               PRIVATE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/src
+               PUBLIC_LINK_LIBRARIES O2::Framework ${EXTRA_TARGETS} ROOT::TreePlayer)
+
 o2_add_test(DataInputDirector NAME test_Framework_test_DataInputDirector
                SOURCES test/test_DataInputDirector.cxx
                COMPONENT_NAME Framework
diff --git a/Framework/AnalysisSupport/src/RNTuplePlugin.cxx b/Framework/AnalysisSupport/src/RNTuplePlugin.cxx
new file mode 100644
index 0000000000000..9f67785f1a069
--- /dev/null
+++ b/Framework/AnalysisSupport/src/RNTuplePlugin.cxx
@@ -0,0 +1,825 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "Framework/RuntimeError.h"
+#include "Framework/RootArrowFilesystem.h"
+#include "Framework/Plugins.h"
+#include <ROOT/RNTupleModel.hxx>
+#include <ROOT/RNTupleWriteOptions.hxx>
+#include <ROOT/RNTupleWriter.hxx>
+#include <ROOT/RField.hxx>
+#include <ROOT/RNTuple.hxx>
+#include <ROOT/RNTupleReader.hxx>
+#include <ROOT/RFieldVisitor.hxx>
+#include <ROOT/RNTupleInspector.hxx>
+#include <ROOT/RVec.hxx>
+#include <TBufferFile.h>
+
+#include <TDirectory.h>
+#include <arrow/array/array_nested.h>
+#include <arrow/array/array_primitive.h>
+#include <arrow/array/builder_nested.h>
+#include <arrow/array/builder_primitive.h>
+#include <arrow/dataset/file_base.h>
+
+template class
+  std::unique_ptr<ROOT::Experimental::RNTupleReader>;
+
+namespace o2::framework
+{
+
+class RNTupleFileWriteOptions : public arrow::dataset::FileWriteOptions
+{
+ public:
+  RNTupleFileWriteOptions(std::shared_ptr<arrow::dataset::FileFormat> format)
+    : FileWriteOptions(format)
+  {
+  }
+};
+
+// A filesystem which allows me to get a RNTuple
+class RNTupleFileSystem : public VirtualRootFileSystemBase
+{
+ public:
+  ~RNTupleFileSystem() override;
+
+  std::shared_ptr<VirtualRootFileSystemBase> GetSubFilesystem(arrow::dataset::FileSource source) override
+  {
+    return std::dynamic_pointer_cast<VirtualRootFileSystemBase>(shared_from_this());
+  };
+  virtual ROOT::Experimental::RNTuple* GetRNTuple(arrow::dataset::FileSource source) = 0;
+};
+
+class SingleRNTupleFileSystem : public RNTupleFileSystem
+{
+ public:
+  SingleRNTupleFileSystem(ROOT::Experimental::RNTuple* tuple)
+    : RNTupleFileSystem(),
+      mTuple(tuple)
+  {
+  }
+
+  arrow::Result<arrow::fs::FileInfo> GetFileInfo(std::string const& path) override;
+
+  std::string type_name() const override
+  {
+    return "rntuple";
+  }
+
+  ROOT::Experimental::RNTuple* GetRNTuple(arrow::dataset::FileSource) override
+  {
+    // Simply return the only TTree we have
+    return mTuple;
+  }
+
+ private:
+  ROOT::Experimental::RNTuple* mTuple;
+};
+
+arrow::Result<arrow::fs::FileInfo> SingleRNTupleFileSystem::GetFileInfo(std::string const& path)
+{
+  arrow::dataset::FileSource source(path, shared_from_this());
+  arrow::fs::FileInfo result;
+  result.set_path(path);
+  result.set_type(arrow::fs::FileType::File);
+  return result;
+}
+
+class RNTupleFileFragment : public arrow::dataset::FileFragment
+{
+ public:
+  RNTupleFileFragment(arrow::dataset::FileSource source,
+                      std::shared_ptr<arrow::dataset::FileFormat> format,
+                      arrow::compute::Expression partition_expression,
+                      std::shared_ptr<arrow::Schema> physical_schema)
+    : FileFragment(std::move(source), std::move(format), std::move(partition_expression), std::move(physical_schema))
+  {
+  }
+};
+
+class RNTupleFileFormat : public arrow::dataset::FileFormat
+{
+  size_t& mTotCompressedSize;
+  size_t& mTotUncompressedSize;
+
+ public:
+  RNTupleFileFormat(size_t& totalCompressedSize, size_t& totalUncompressedSize)
+    : FileFormat({}),
+      mTotCompressedSize(totalCompressedSize),
+      mTotUncompressedSize(totalUncompressedSize)
+  {
+  }
+
+  ~RNTupleFileFormat() override = default;
+
+  std::string type_name() const override
+  {
+    return "rntuple";
+  }
+
+  bool Equals(const FileFormat& other) const override
+  {
+    return other.type_name() == this->type_name();
+  }
+
+  arrow::Result<bool> IsSupported(const arrow::dataset::FileSource& source) const override
+  {
+    auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
+    auto subFs = fs->GetSubFilesystem(source);
+    if (std::dynamic_pointer_cast<RNTupleFileSystem>(subFs)) {
+      return true;
+    }
+    return false;
+  }
+
+  arrow::Result<std::shared_ptr<arrow::Schema>> Inspect(const arrow::dataset::FileSource& source) const override;
+
+  arrow::Result<arrow::RecordBatchGenerator> ScanBatchesAsync(
+    const std::shared_ptr<arrow::dataset::ScanOptions>& options,
+    const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const override;
+
+  std::shared_ptr<arrow::dataset::FileWriteOptions> DefaultWriteOptions() override;
+
+  arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination,
+                                                                        std::shared_ptr<arrow::Schema> schema,
+                                                                        std::shared_ptr<arrow::dataset::FileWriteOptions> options,
+                                                                        arrow::fs::FileLocator destination_locator) const override;
+  arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> MakeFragment(
+    arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
+    std::shared_ptr<arrow::Schema> physical_schema) override;
+};
+
+struct RootNTupleVisitor : public ROOT::Experimental::Detail::RFieldVisitor {
+  void VisitArrayField(const ROOT::Experimental::RArrayField& field) override
+  {
+    int size = field.GetLength();
+    RootNTupleVisitor valueVisitor{};
+    auto valueField = field.GetSubFields()[0];
+    valueField->AcceptVisitor(valueVisitor);
+    auto type = valueVisitor.datatype;
+    this->datatype = arrow::fixed_size_list(type, size);
+  }
+
+  void VisitRVecField(const ROOT::Experimental::RRVecField& field) override
+  {
+    RootNTupleVisitor valueVisitor{};
+    auto valueField = field.GetSubFields()[0];
+    valueField->AcceptVisitor(valueVisitor);
+    auto type = valueVisitor.datatype;
+    this->datatype = arrow::list(type);
+  }
+
+  void VisitField(const ROOT::Experimental::RFieldBase& field) override
+  {
+    throw o2::framework::runtime_error_f("Unknown field %s with type %s", field.GetFieldName().c_str(), field.GetTypeName().c_str());
+  }
+
+  void VisitIntField(const ROOT::Experimental::RField<int>& field) override
+  {
+    this->datatype = arrow::int32();
+  }
+
+  void VisitBoolField(const ROOT::Experimental::RField<bool>& field) override
+  {
+    this->datatype = arrow::boolean();
+  }
+
+  void VisitFloatField(const ROOT::Experimental::RField<float>& field) override
+  {
+    this->datatype = arrow::float32();
+  }
+
+  void VisitDoubleField(const ROOT::Experimental::RField<double>& field) override
+  {
+    this->datatype = arrow::float64();
+  }
+  std::shared_ptr<arrow::DataType> datatype;
+};
+} // namespace o2::framework
+
+auto arrowTypeFromRNTuple(ROOT::Experimental::RFieldBase const& field, int size)
+{
+  o2::framework::RootNTupleVisitor visitor;
+  field.AcceptVisitor(visitor);
+  return visitor.datatype;
+}
+
+namespace o2::framework
+{
+std::unique_ptr<ROOT::Experimental::RFieldBase> rootFieldFromArrow(std::shared_ptr<arrow::Field> field, std::string name)
+{
+  using namespace ROOT::Experimental;
+  switch (field->type()->id()) {
+    case arrow::Type::BOOL:
+      return std::make_unique<RField<bool>>(name);
+    case arrow::Type::UINT8:
+      return std::make_unique<RField<uint8_t>>(name);
+    case arrow::Type::UINT16:
+      return std::make_unique<RField<uint16_t>>(name);
+    case arrow::Type::UINT32:
+      return std::make_unique<RField<uint32_t>>(name);
+    case arrow::Type::UINT64:
+      return std::make_unique<RField<uint64_t>>(name);
+    case arrow::Type::INT8:
+      return std::make_unique<RField<int8_t>>(name);
+    case arrow::Type::INT16:
+      return std::make_unique<RField<int16_t>>(name);
+    case arrow::Type::INT32:
+      return std::make_unique<RField<int32_t>>(name);
+    case arrow::Type::INT64:
+      return std::make_unique<RField<int64_t>>(name);
+    case arrow::Type::FLOAT:
+      return std::make_unique<RField<float>>(name);
+    case arrow::Type::DOUBLE:
+      return std::make_unique<RField<double>>(name);
+    default:
+      throw runtime_error("Unsupported arrow column type");
+  }
+}
+
+class RNTupleFileWriter : public arrow::dataset::FileWriter
+{
+  std::shared_ptr<ROOT::Experimental::RNTupleWriter> mWriter;
+  bool firstBatch = true;
+  std::vector<std::shared_ptr<arrow::Array>> valueArrays;
+  std::vector<std::shared_ptr<arrow::DataType>> valueTypes;
+  std::vector<size_t> valueCount;
+
+ public:
+  RNTupleFileWriter(std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options,
+                    std::shared_ptr<arrow::io::OutputStream> destination,
+                    arrow::fs::FileLocator destination_locator)
+    : FileWriter(schema, options, destination, destination_locator)
+  {
+    using namespace ROOT::Experimental;
+
+    auto model = RNTupleModel::CreateBare();
+    // Let's create a model from the physical schema
+    for (auto i = 0u; i < schema->fields().size(); ++i) {
+      auto& field = schema->field(i);
+
+      // Construct all the needed branches.
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          auto list = std::static_pointer_cast<arrow::FixedSizeListType>(field->type());
+          auto valueField = field->type()->field(0);
+          model->AddField(std::make_unique<RArrayField>(field->name(), rootFieldFromArrow(valueField, "_0"), list->list_size()));
+        } break;
+        case arrow::Type::LIST: {
+          auto valueField = field->type()->field(0);
+          model->AddField(std::make_unique<RRVecField>(field->name(), rootFieldFromArrow(valueField, "_0")));
+        } break;
+        default: {
+          model->AddField(rootFieldFromArrow(field, field->name()));
+        } break;
+      }
+    }
+    auto fileStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
+    auto* file = dynamic_cast<TFile*>(fileStream->GetDirectory());
+    mWriter = RNTupleWriter::Append(std::move(model), destination_locator_.path, *file, {});
+  }
+
+  arrow::Status Write(const std::shared_ptr<arrow::RecordBatch>& batch) override
+  {
+    if (firstBatch) {
+      firstBatch = false;
+    }
+
+    // Support writing empty tables
+    if (batch->columns().empty() || batch->num_rows() == 0) {
+      return arrow::Status::OK();
+    }
+
+    for (auto i = 0u; i < batch->columns().size(); ++i) {
+      auto column = batch->column(i);
+      auto& field = batch->schema()->field(i);
+
+      valueArrays.push_back(nullptr);
+      valueTypes.push_back(nullptr);
+      valueCount.push_back(1);
+
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          auto list = std::static_pointer_cast<arrow::FixedSizeListArray>(column);
+          auto listType = std::static_pointer_cast<arrow::FixedSizeListType>(field->type());
+          if (field->type()->field(0)->type()->id() == arrow::Type::BOOL) {
+            auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(list->values());
+            int64_t length = boolArray->length();
+            arrow::UInt8Builder builder;
+            auto ok = builder.Reserve(length);
+
+            for (int64_t i = 0; i < length; ++i) {
+              if (boolArray->IsValid(i)) {
+                // Expand each boolean value (true/false) to uint8 (1/0)
+                uint8_t value = boolArray->Value(i) ? 1 : 0;
+                auto ok = builder.Append(value);
+              } else {
+                // Append null for invalid entries
+                auto ok = builder.AppendNull();
+              }
+            }
+            valueArrays.back() = *builder.Finish();
+            valueTypes.back() = valueArrays.back()->type();
+          } else {
+            valueArrays.back() = list->values();
+            valueTypes.back() = field->type()->field(0)->type();
+          }
+          valueCount.back() = listType->list_size();
+        } break;
+        case arrow::Type::LIST: {
+          auto list = std::static_pointer_cast<arrow::ListArray>(column);
+          valueArrays.back() = list;
+          valueTypes.back() = field->type()->field(0)->type();
+          valueCount.back() = -1;
+        } break;
+        case arrow::Type::BOOL: {
+          // We unpack the array
+          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(column);
+          int64_t length = boolArray->length();
+          arrow::UInt8Builder builder;
+          auto ok = builder.Reserve(length);
+
+          for (int64_t i = 0; i < length; ++i) {
+            if (boolArray->IsValid(i)) {
+              // Expand each boolean value (true/false) to uint8 (1/0)
+              uint8_t value = boolArray->Value(i) ? 1 : 0;
+              auto ok = builder.Append(value);
+            } else {
+              // Append null for invalid entries
+              auto ok = builder.AppendNull();
+            }
+          }
+          valueArrays.back() = *builder.Finish();
+          valueTypes.back() = valueArrays.back()->type();
+        } break;
+        default:
+          valueArrays.back() = column;
+          valueTypes.back() = field->type();
+          break;
+      }
+    }
+
+    int64_t pos = 0;
+
+    auto entry = mWriter->CreateEntry();
+    std::vector<ROOT::Experimental::REntry::RFieldToken> tokens;
+    tokens.reserve(batch->num_columns());
+    std::vector<size_t> typeIds;
+    typeIds.reserve(batch->num_columns());
+
+    for (size_t ci = 0; ci < batch->num_columns(); ++ci) {
+      auto& field = batch->schema()->field(ci);
+      typeIds.push_back(batch->column(ci)->type()->id());
+      tokens.push_back(entry->GetToken(field->name()));
+    }
+
+    while (pos < batch->num_rows()) {
+      for (size_t ci = 0; ci < batch->num_columns(); ++ci) {
+        auto typeId = typeIds[ci];
+        auto token = tokens[ci];
+
+        switch (typeId) {
+          case arrow::Type::LIST: {
+            auto list = std::static_pointer_cast<arrow::ListArray>(valueArrays[ci]);
+            auto value_slice = list->value_slice(pos);
+
+            valueCount[ci] = value_slice->length();
+            auto bindValue = [&vc = valueCount, ci, token](auto array, std::unique_ptr<ROOT::Experimental::REntry>& entry) -> void {
+              using value_type = std::decay_t<decltype(*array.get())>::value_type;
+              auto v = std::make_shared<ROOT::VecOps::RVec<value_type>>((value_type*)array->raw_values(), vc[ci]);
+              entry->BindValue(token, v);
+            };
+            switch (valueTypes[ci]->id()) {
+              case arrow::Type::FLOAT: {
+                bindValue(std::static_pointer_cast<arrow::FloatArray>(value_slice), entry);
+              } break;
+              case arrow::Type::DOUBLE: {
+                bindValue(std::static_pointer_cast<arrow::DoubleArray>(value_slice), entry);
+              } break;
+              case arrow::Type::INT8: {
+                bindValue(std::static_pointer_cast<arrow::Int8Array>(value_slice), entry);
+              } break;
+              case arrow::Type::INT16: {
+                bindValue(std::static_pointer_cast<arrow::Int16Array>(value_slice), entry);
+              } break;
+              case arrow::Type::INT32: {
+                bindValue(std::static_pointer_cast<arrow::Int32Array>(value_slice), entry);
+              } break;
+              case arrow::Type::INT64: {
+                bindValue(std::static_pointer_cast<arrow::Int64Array>(value_slice), entry);
+              } break;
+              case arrow::Type::UINT8: {
+                bindValue(std::static_pointer_cast<arrow::UInt8Array>(value_slice), entry);
+              } break;
+              case arrow::Type::UINT16: {
+                bindValue(std::static_pointer_cast<arrow::UInt16Array>(value_slice), entry);
+              } break;
+              case arrow::Type::UINT32: {
+                bindValue(std::static_pointer_cast<arrow::UInt32Array>(value_slice), entry);
+              } break;
+              case arrow::Type::UINT64: {
+                bindValue(std::static_pointer_cast<arrow::UInt64Array>(value_slice), entry);
+              } break;
+              default: {
+                throw runtime_error("Unsupported kind of VLA");
+              } break;
+            }
+          } break;
+          case arrow::Type::FIXED_SIZE_LIST: {
+            entry->BindRawPtr<void>(token, (void*)(valueArrays[ci]->data()->buffers[1]->data() + pos * valueCount[ci] * valueTypes[ci]->byte_width()));
+          } break;
+          case arrow::Type::BOOL: {
+            // Not sure we actually need this
+            entry->BindRawPtr<bool>(token, (bool*)(valueArrays[ci]->data()->buffers[1]->data() + pos * 1));
+          } break;
+          default:
+            // By default we consider things scalars.
+            entry->BindRawPtr<void>(token, (void*)(valueArrays[ci]->data()->buffers[1]->data() + pos * valueTypes[ci]->byte_width()));
+            break;
+        }
+      }
+      mWriter->Fill(*entry);
+      ++pos;
+    }
+    // mWriter->CommitCluster();
+
+    return arrow::Status::OK();
+  }
+
+  arrow::Future<>
+    FinishInternal() override
+  {
+    return {};
+  };
+};
+
+arrow::Result<std::shared_ptr<arrow::Schema>> RNTupleFileFormat::Inspect(const arrow::dataset::FileSource& source) const
+{
+
+  auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
+  // Actually get the TTree from the ROOT file.
+  auto ntupleFs = std::dynamic_pointer_cast<RNTupleFileSystem>(fs->GetSubFilesystem(source));
+  if (!ntupleFs.get()) {
+    throw runtime_error_f("Unknown filesystem %s\n", source.filesystem()->type_name().c_str());
+  }
+  ROOT::Experimental::RNTuple* rntuple = ntupleFs->GetRNTuple(source);
+
+  auto inspector = ROOT::Experimental::RNTupleInspector::Create(rntuple);
+
+  auto reader = ROOT::Experimental::RNTupleReader::Open(rntuple);
+
+  auto& tupleField0 = reader->GetModel().GetFieldZero();
+  std::vector<std::shared_ptr<arrow::Field>> fields;
+  for (auto& tupleField : tupleField0.GetSubFields()) {
+    auto field = std::make_shared<arrow::Field>(tupleField->GetFieldName(), arrowTypeFromRNTuple(*tupleField, tupleField->GetValueSize()));
+    fields.push_back(field);
+  }
+
+  return std::make_shared<arrow::Schema>(fields);
+}
+
+arrow::Result<arrow::RecordBatchGenerator> RNTupleFileFormat::ScanBatchesAsync(
+  const std::shared_ptr<arrow::dataset::ScanOptions>& options,
+  const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const
+{
+  auto dataset_schema = options->dataset_schema;
+  auto ntupleFragment = std::dynamic_pointer_cast<RNTupleFileFragment>(fragment);
+
+  auto generator = [pool = options->pool, ntupleFragment, dataset_schema, &totalCompressedSize = mTotCompressedSize,
+                    &totalUncompressedSize = mTotUncompressedSize]() -> arrow::Future<std::shared_ptr<arrow::RecordBatch>> {
+    using namespace ROOT::Experimental;
+    std::vector<std::shared_ptr<arrow::Array>> columns;
+    std::vector<std::shared_ptr<arrow::Field>> fields = dataset_schema->fields();
+
+    auto containerFS = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(ntupleFragment->source().filesystem());
+    auto fs = std::dynamic_pointer_cast<RNTupleFileSystem>(containerFS->GetSubFilesystem(ntupleFragment->source()));
+
+    int64_t rows = -1;
+    ROOT::Experimental::RNTuple* rntuple = fs->GetRNTuple(ntupleFragment->source());
+    auto reader = ROOT::Experimental::RNTupleReader::Open(rntuple);
+    auto& model = reader->GetModel();
+    for (auto& physicalField : fields) {
+      auto bulk = model.CreateBulk(physicalField->name());
+
+      auto listType = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type());
+
+      auto& descriptor = reader->GetDescriptor();
+      auto totalEntries = reader->GetNEntries();
+
+      if (rows == -1) {
+        rows = totalEntries;
+      }
+      if (rows != totalEntries) {
+        throw runtime_error_f("Unmatching number of rows for branch %s", physicalField->name().c_str());
+      }
+      arrow::Status status;
+      int readEntries = 0;
+      std::shared_ptr<arrow::Array> array;
+      if (physicalField->type() == arrow::boolean() ||
+          (listType && physicalField->type()->field(0)->type() == arrow::boolean())) {
+        if (listType) {
+          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
+          auto status = arrow::MakeBuilder(pool, physicalField->type()->field(0)->type(), &builder);
+          if (!status.ok()) {
+            throw runtime_error("Cannot create value builder");
+          }
+          auto listBuilder = std::make_unique<arrow::FixedSizeListBuilder>(pool, std::move(builder), listType->list_size());
+          auto valueBuilder = listBuilder.get()->value_builder();
+          // boolean array special case: we need to use builder to create the bitmap
+          status = valueBuilder->Reserve(totalEntries * listType->list_size());
+          status &= listBuilder->Reserve(totalEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to reserve memory for array builder");
+          }
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          // No adoption for now...
+          // bulk.AdoptBuffer(buffer, totalEntries)
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            void* ptr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries());
+            int readLast = index.GetNEntries();
+            readEntries += readLast;
+            status &= static_cast<arrow::BooleanBuilder*>(valueBuilder)->AppendValues(reinterpret_cast<uint8_t const*>(ptr), readLast * listType->list_size());
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+          status &= static_cast<arrow::FixedSizeListBuilder*>(listBuilder.get())->AppendValues(readEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to append values to array");
+          }
+          status &= listBuilder->Finish(&array);
+          if (!status.ok()) {
+            throw runtime_error("Failed to create array");
+          }
+        } else if (listType == nullptr) {
+          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
+          auto status = arrow::MakeBuilder(pool, physicalField->type(), &builder);
+          if (!status.ok()) {
+            throw runtime_error("Cannot create builder");
+          }
+          auto valueBuilder = static_cast<arrow::BooleanBuilder*>(builder.get());
+          // boolean array special case: we need to use builder to create the bitmap
+          status = valueBuilder->Reserve(totalEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to reserve memory for array builder");
+          }
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            void* ptr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries());
+            int readLast = index.GetNEntries();
+            readEntries += readLast;
+            status &= valueBuilder->AppendValues(reinterpret_cast<uint8_t const*>(ptr), readLast);
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+          if (!status.ok()) {
+            throw runtime_error("Failed to append values to array");
+          }
+          status &= valueBuilder->Finish(&array);
+          if (!status.ok()) {
+            throw runtime_error("Failed to create array");
+          }
+        }
+      } else {
+        // other types: use serialized read to build arrays directly.
+        auto typeSize = physicalField->type()->byte_width();
+        // FIXME: for now...
+        auto bytes = 0;
+        auto branchSize = bytes ? bytes : 1000000;
+        auto&& result = arrow::AllocateResizableBuffer(branchSize, pool);
+        if (!result.ok()) {
+          throw runtime_error("Cannot allocate values buffer");
+        }
+        std::shared_ptr<arrow::Buffer> arrowValuesBuffer = std::move(result).ValueUnsafe();
+        auto ptr = arrowValuesBuffer->mutable_data();
+        if (ptr == nullptr) {
+          throw runtime_error("Invalid buffer");
+        }
+
+        std::unique_ptr<TBufferFile> offsetBuffer = nullptr;
+
+        std::shared_ptr<arrow::Buffer> arrowOffsetBuffer;
+        std::span<int> offsets;
+        int size = 0;
+        uint32_t totalSize = 0;
+        int64_t listSize = 1;
+        if (auto fixedSizeList = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type())) {
+          listSize = fixedSizeList->list_size();
+          typeSize = fixedSizeList->field(0)->type()->byte_width();
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            void* inPtr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries());
+
+            int readLast = index.GetNEntries();
+            if (listSize == -1) {
+              size = offsets[readEntries + readLast] - offsets[readEntries];
+            } else {
+              size = readLast * listSize;
+            }
+            readEntries += readLast;
+            memcpy(ptr, inPtr, size * typeSize);
+            ptr += (ptrdiff_t)(size * typeSize);
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+        } else if (auto vlaListType = std::dynamic_pointer_cast<arrow::ListType>(physicalField->type())) {
+          listSize = -1;
+          typeSize = vlaListType->field(0)->type()->byte_width();
+          offsetBuffer = std::make_unique<TBufferFile>(TBuffer::EMode::kWrite, 4 * 1024 * 1024);
+          result = arrow::AllocateResizableBuffer((totalEntries + 1) * (int64_t)sizeof(int), pool);
+          if (!result.ok()) {
+            throw runtime_error("Cannot allocate offset buffer");
+          }
+          arrowOffsetBuffer = std::move(result).ValueUnsafe();
+
+          // Offset bulk
+          auto offsetBulk = model.CreateBulk(physicalField->name());
+          // Actual values are in a different place...
+          bulk = model.CreateBulk(physicalField->name());
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          auto* ptrOffset = reinterpret_cast<int*>(arrowOffsetBuffer->mutable_data());
+          auto* tPtrOffset = reinterpret_cast<int*>(ptrOffset);
+          offsets = std::span<int>{tPtrOffset, tPtrOffset + totalEntries + 1};
+
+          auto copyOffsets = [&arrowValuesBuffer, &pool, &ptrOffset, &ptr, &totalSize](auto inPtr, size_t total) {
+            using value_type = typename std::decay_t<decltype(*inPtr)>::value_type;
+            for (size_t i = 0; i < total; i++) {
+              *ptrOffset++ = totalSize;
+              totalSize += inPtr[i].size();
+            }
+            *ptrOffset = totalSize;
+            auto&& result = arrow::AllocateResizableBuffer(totalSize * sizeof(value_type), pool);
+            if (!result.ok()) {
+              throw runtime_error("Cannot allocate values buffer");
+            }
+            arrowValuesBuffer = std::move(result).ValueUnsafe();
+            ptr = (uint8_t*)(arrowValuesBuffer->mutable_data());
+            // Calculate the size of the buffer here.
+            for (size_t i = 0; i < total; i++) {
+              int vlaSizeInBytes = inPtr[i].size() * sizeof(value_type);
+              if (vlaSizeInBytes == 0) {
+                continue;
+              }
+              memcpy(ptr, inPtr[i].data(), vlaSizeInBytes);
+              ptr += vlaSizeInBytes;
+            }
+          };
+
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            int readLast = index.GetNEntries();
+            switch (vlaListType->field(0)->type()->id()) {
+              case arrow::Type::FLOAT: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<float>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::DOUBLE: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<double>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::INT8: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<int8_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::INT16: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<int16_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::INT32: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<int32_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::INT64: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<int64_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::UINT8: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<uint8_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::UINT16: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<uint16_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::UINT32: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<uint32_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::UINT64: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<uint64_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              default: {
+                throw runtime_error("Unsupported kind of VLA");
+              } break;
+            }
+
+            readEntries += readLast;
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+        } else {
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            void* inPtr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries());
+
+            int readLast = index.GetNEntries();
+            if (listSize == -1) {
+              size = offsets[readEntries + readLast] - offsets[readEntries];
+            } else {
+              size = readLast * listSize;
+            }
+            readEntries += readLast;
+            memcpy(ptr, inPtr, size * typeSize);
+            ptr += (ptrdiff_t)(size * typeSize);
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+        }
+        switch (listSize) {
+          case -1: {
+            auto varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
+            array = std::make_shared<arrow::ListArray>(physicalField->type(), readEntries, arrowOffsetBuffer, varray);
+          } break;
+          case 1: {
+            totalSize = readEntries * listSize;
+            array = std::make_shared<arrow::PrimitiveArray>(physicalField->type(), readEntries, arrowValuesBuffer);
+
+          } break;
+          default: {
+            totalSize = readEntries * listSize;
+            auto varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
+            array = std::make_shared<arrow::FixedSizeListArray>(physicalField->type(), readEntries, varray);
+          }
+        }
+      }
+      columns.push_back(array);
+    }
+
+    auto batch = arrow::RecordBatch::Make(dataset_schema, rows, columns);
+    return batch;
+  };
+
+  return generator;
+}
+
+arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> RNTupleFileFormat::MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination,
+                                                                                         std::shared_ptr<arrow::Schema> schema,
+                                                                                         std::shared_ptr<arrow::dataset::FileWriteOptions> options,
+                                                                                         arrow::fs::FileLocator destination_locator) const
+{
+  auto writer = std::make_shared<RNTupleFileWriter>(schema, options, destination, destination_locator);
+  return std::dynamic_pointer_cast<arrow::dataset::FileWriter>(writer);
+}
+
+arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> RNTupleFileFormat::MakeFragment(
+  arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
+  std::shared_ptr<arrow::Schema> physical_schema)
+{
+  std::shared_ptr<arrow::dataset::FileFormat> format = std::make_shared<RNTupleFileFormat>(mTotCompressedSize, mTotUncompressedSize);
+
+  auto fragment = std::make_shared<RNTupleFileFragment>(std::move(source), std::move(format),
+                                                        std::move(partition_expression),
+                                                        std::move(physical_schema));
+  return std::dynamic_pointer_cast<arrow::dataset::FileFragment>(fragment);
+}
+
+RNTupleFileSystem::~RNTupleFileSystem() = default;
+
+std::shared_ptr<arrow::dataset::FileWriteOptions>
+  RNTupleFileFormat::DefaultWriteOptions()
+{
+  return std::make_shared<RNTupleFileWriteOptions>(shared_from_this());
+}
+
+struct RNTuplePluginContext {
+  size_t totalCompressedSize = 0;
+  size_t totalUncompressedSize = 0;
+  std::shared_ptr<o2::framework::RNTupleFileFormat> format = nullptr;
+};
+
+struct RNTupleObjectReadingImplementation : public RootArrowFactoryPlugin {
+  RootArrowFactory* create() override
+  {
+    auto context = new RNTuplePluginContext;
+    context->format = std::make_shared<o2::framework::RNTupleFileFormat>(context->totalCompressedSize, context->totalUncompressedSize);
+    return new RootArrowFactory{
+      .options = [context]() { return context->format->DefaultWriteOptions(); },
+      .format = [context]() { return context->format; },
+      .getSubFilesystem = [](void* handle) {
+        auto rntuple = (ROOT::Experimental::RNTuple*)handle;
+        return std::shared_ptr<VirtualRootFileSystemBase>(new SingleRNTupleFileSystem(rntuple)); },
+    };
+  }
+};
+
+DEFINE_DPL_PLUGINS_BEGIN
+DEFINE_DPL_PLUGIN_INSTANCE(RNTupleObjectReadingImplementation, RootObjectReadingImplementation);
+DEFINE_DPL_PLUGINS_END
+} // namespace o2::framework
diff --git a/Framework/AnalysisSupport/src/TTreePlugin.cxx b/Framework/AnalysisSupport/src/TTreePlugin.cxx
new file mode 100644
index 0000000000000..e376ed8b96268
--- /dev/null
+++ b/Framework/AnalysisSupport/src/TTreePlugin.cxx
@@ -0,0 +1,862 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "Framework/RootArrowFilesystem.h"
+#include "Framework/Plugins.h"
+#include "Framework/Signpost.h"
+#include "Framework/Endian.h"
+#include <arrow/dataset/file_base.h>
+#include <arrow/util/key_value_metadata.h>
+#include <arrow/array/array_nested.h>
+#include <arrow/array/array_primitive.h>
+#include <arrow/array/builder_nested.h>
+#include <arrow/array/builder_primitive.h>
+#include <TTree.h>
+#include <TBranch.h>
+#include <TFile.h>
+#include <TLeaf.h>
+
+O2_DECLARE_DYNAMIC_LOG(root_arrow_fs);
+
+namespace o2::framework
+{
+
+class TTreeFileWriteOptions : public arrow::dataset::FileWriteOptions
+{
+ public:
+  TTreeFileWriteOptions(std::shared_ptr<arrow::dataset::FileFormat> format)
+    : FileWriteOptions(format)
+  {
+  }
+};
+
+// A filesystem which allows me to get a TTree
+class TTreeFileSystem : public VirtualRootFileSystemBase
+{
+ public:
+  ~TTreeFileSystem() override;
+
+  std::shared_ptr<VirtualRootFileSystemBase> GetSubFilesystem(arrow::dataset::FileSource source) override
+  {
+    return std::dynamic_pointer_cast<VirtualRootFileSystemBase>(shared_from_this());
+  };
+
+  arrow::Result<std::shared_ptr<arrow::io::OutputStream>> OpenOutputStream(
+    const std::string& path,
+    const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override;
+
+  virtual TTree* GetTree(arrow::dataset::FileSource source) = 0;
+};
+
+class SingleTreeFileSystem : public TTreeFileSystem
+{
+ public:
+  SingleTreeFileSystem(TTree* tree)
+    : TTreeFileSystem(),
+      mTree(tree)
+  {
+  }
+
+  arrow::Result<arrow::fs::FileInfo> GetFileInfo(std::string const& path) override;
+
+  std::string type_name() const override
+  {
+    return "ttree";
+  }
+
+  TTree* GetTree(arrow::dataset::FileSource) override
+  {
+    // Simply return the only TTree we have
+    return mTree;
+  }
+
+ private:
+  TTree* mTree;
+};
+
+arrow::Result<arrow::fs::FileInfo> SingleTreeFileSystem::GetFileInfo(std::string const& path)
+{
+  arrow::dataset::FileSource source(path, shared_from_this());
+  arrow::fs::FileInfo result;
+  result.set_path(path);
+  result.set_type(arrow::fs::FileType::File);
+  return result;
+}
+
+class TTreeFileFragment : public arrow::dataset::FileFragment
+{
+ public:
+  TTreeFileFragment(arrow::dataset::FileSource source,
+                    std::shared_ptr<arrow::dataset::FileFormat> format,
+                    arrow::compute::Expression partition_expression,
+                    std::shared_ptr<arrow::Schema> physical_schema)
+    : FileFragment(std::move(source), std::move(format), std::move(partition_expression), std::move(physical_schema))
+  {
+  }
+};
+
+class TTreeFileFormat : public arrow::dataset::FileFormat
+{
+  size_t& mTotCompressedSize;
+  size_t& mTotUncompressedSize;
+
+ public:
+  TTreeFileFormat(size_t& totalCompressedSize, size_t& totalUncompressedSize)
+    : FileFormat({}),
+      mTotCompressedSize(totalCompressedSize),
+      mTotUncompressedSize(totalUncompressedSize)
+  {
+  }
+
+  ~TTreeFileFormat() override = default;
+
+  std::string type_name() const override
+  {
+    return "ttree";
+  }
+
+  bool Equals(const FileFormat& other) const override
+  {
+    return other.type_name() == this->type_name();
+  }
+
+  arrow::Result<bool> IsSupported(const arrow::dataset::FileSource& source) const override
+  {
+    auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
+    auto subFs = fs->GetSubFilesystem(source);
+    if (std::dynamic_pointer_cast<TTreeFileSystem>(subFs)) {
+      return true;
+    }
+    return false;
+  }
+
+  arrow::Result<std::shared_ptr<arrow::Schema>> Inspect(const arrow::dataset::FileSource& source) const override;
+  /// \brief Create a FileFragment for a FileSource.
+  arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> MakeFragment(
+    arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
+    std::shared_ptr<arrow::Schema> physical_schema) override;
+
+  arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options, arrow::fs::FileLocator destination_locator) const override;
+
+  std::shared_ptr<arrow::dataset::FileWriteOptions> DefaultWriteOptions() override;
+
+  arrow::Result<arrow::RecordBatchGenerator> ScanBatchesAsync(
+    const std::shared_ptr<arrow::dataset::ScanOptions>& options,
+    const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const override;
+};
+
+// An arrow outputstream which allows to write to a TTree. Eventually
+// with a prefix for the branches.
+class TTreeOutputStream : public arrow::io::OutputStream
+{
+ public:
+  TTreeOutputStream(TTree*, std::string branchPrefix);
+
+  arrow::Status Close() override;
+
+  arrow::Result<int64_t> Tell() const override;
+
+  arrow::Status Write(const void* data, int64_t nbytes) override;
+
+  bool closed() const override;
+
+  TBranch* CreateBranch(char const* branchName, char const* sizeBranch);
+
+  TTree* GetTree()
+  {
+    return mTree;
+  }
+
+ private:
+  TTree* mTree;
+  std::string mBranchPrefix;
+};
+
+// An arrow outputstream which allows to write to a ttree
+// @a branch prefix is to be used to identify a set of branches which all belong to
+// the same table.
+TTreeOutputStream::TTreeOutputStream(TTree* f, std::string branchPrefix)
+  : mTree(f),
+    mBranchPrefix(std::move(branchPrefix))
+{
+}
+
+arrow::Status TTreeOutputStream::Close()
+{
+  if (mTree->GetCurrentFile() == nullptr) {
+    return arrow::Status::Invalid("Cannot close a tree not attached to a file");
+  }
+  mTree->GetCurrentFile()->Close();
+  return arrow::Status::OK();
+}
+
+arrow::Result<int64_t> TTreeOutputStream::Tell() const
+{
+  return arrow::Result<int64_t>(arrow::Status::NotImplemented("Cannot move"));
+}
+
+arrow::Status TTreeOutputStream::Write(const void* data, int64_t nbytes)
+{
+  return arrow::Status::NotImplemented("Cannot write raw bytes to a TTree");
+}
+
+bool TTreeOutputStream::closed() const
+{
+  // A standalone tree is never closed.
+  if (mTree->GetCurrentFile() == nullptr) {
+    return false;
+  }
+  return mTree->GetCurrentFile()->IsOpen() == false;
+}
+
+TBranch* TTreeOutputStream::CreateBranch(char const* branchName, char const* sizeBranch)
+{
+  return mTree->Branch((mBranchPrefix + "/" + branchName).c_str(), (char*)nullptr, (mBranchPrefix + sizeBranch).c_str());
+}
+
+struct TTreePluginContext {
+  size_t totalCompressedSize = 0;
+  size_t totalUncompressedSize = 0;
+  std::shared_ptr<o2::framework::TTreeFileFormat> format = nullptr;
+};
+
+struct TTreeObjectReadingImplementation : public RootArrowFactoryPlugin {
+  RootArrowFactory* create() override
+  {
+    auto context = new TTreePluginContext;
+    context->format = std::make_shared<o2::framework::TTreeFileFormat>(context->totalCompressedSize, context->totalUncompressedSize);
+    return new RootArrowFactory{
+      .options = [context]() { return context->format->DefaultWriteOptions(); },
+      .format = [context]() { return context->format; },
+      .getSubFilesystem = [](void* handle) {
+        auto tree = (TTree*)handle;
+        return std::shared_ptr<VirtualRootFileSystemBase>(new SingleTreeFileSystem(tree)); },
+    };
+  }
+};
+
+arrow::Result<arrow::RecordBatchGenerator> TTreeFileFormat::ScanBatchesAsync(
+  const std::shared_ptr<arrow::dataset::ScanOptions>& options,
+  const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const
+{
+  // Get the fragment as a TTreeFragment. This might be PART of a TTree.
+  auto treeFragment = std::dynamic_pointer_cast<TTreeFileFragment>(fragment);
+  // This is the schema we want to read
+  auto dataset_schema = options->dataset_schema;
+
+  auto generator = [pool = options->pool, treeFragment, dataset_schema, &totalCompressedSize = mTotCompressedSize,
+                    &totalUncompressedSize = mTotUncompressedSize]() -> arrow::Future<std::shared_ptr<arrow::RecordBatch>> {
+    auto schema = treeFragment->format()->Inspect(treeFragment->source());
+
+    std::vector<std::shared_ptr<arrow::Array>> columns;
+    std::vector<std::shared_ptr<arrow::Field>> fields = dataset_schema->fields();
+    auto physical_schema = *treeFragment->ReadPhysicalSchema();
+
+    static TBufferFile buffer{TBuffer::EMode::kWrite, 4 * 1024 * 1024};
+    auto containerFS = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(treeFragment->source().filesystem());
+    auto fs = std::dynamic_pointer_cast<TTreeFileSystem>(containerFS->GetSubFilesystem(treeFragment->source()));
+
+    int64_t rows = -1;
+    TTree* tree = fs->GetTree(treeFragment->source());
+    for (auto& field : fields) {
+      // The field actually on disk
+      auto physicalField = physical_schema->GetFieldByName(field->name());
+      TBranch* branch = tree->GetBranch(physicalField->name().c_str());
+      assert(branch);
+      buffer.Reset();
+      auto totalEntries = branch->GetEntries();
+      if (rows == -1) {
+        rows = totalEntries;
+      }
+      if (rows != totalEntries) {
+        throw runtime_error_f("Unmatching number of rows for branch %s", branch->GetName());
+      }
+      arrow::Status status;
+      int readEntries = 0;
+      std::shared_ptr<arrow::Array> array;
+      auto listType = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type());
+      if (physicalField->type() == arrow::boolean() ||
+          (listType && physicalField->type()->field(0)->type() == arrow::boolean())) {
+        if (listType) {
+          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
+          auto status = arrow::MakeBuilder(pool, physicalField->type()->field(0)->type(), &builder);
+          if (!status.ok()) {
+            throw runtime_error("Cannot create value builder");
+          }
+          auto listBuilder = std::make_unique<arrow::FixedSizeListBuilder>(pool, std::move(builder), listType->list_size());
+          auto valueBuilder = listBuilder.get()->value_builder();
+          // boolean array special case: we need to use builder to create the bitmap
+          status = valueBuilder->Reserve(totalEntries * listType->list_size());
+          status &= listBuilder->Reserve(totalEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to reserve memory for array builder");
+          }
+          while (readEntries < totalEntries) {
+            auto readLast = branch->GetBulkRead().GetBulkEntries(readEntries, buffer);
+            readEntries += readLast;
+            status &= static_cast<arrow::BooleanBuilder*>(valueBuilder)->AppendValues(reinterpret_cast<uint8_t const*>(buffer.GetCurrent()), readLast * listType->list_size());
+          }
+          status &= static_cast<arrow::FixedSizeListBuilder*>(listBuilder.get())->AppendValues(readEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to append values to array");
+          }
+          status &= listBuilder->Finish(&array);
+          if (!status.ok()) {
+            throw runtime_error("Failed to create array");
+          }
+        } else if (listType == nullptr) {
+          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
+          auto status = arrow::MakeBuilder(pool, physicalField->type(), &builder);
+          if (!status.ok()) {
+            throw runtime_error("Cannot create builder");
+          }
+          auto valueBuilder = static_cast<arrow::BooleanBuilder*>(builder.get());
+          // boolean array special case: we need to use builder to create the bitmap
+          status = valueBuilder->Reserve(totalEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to reserve memory for array builder");
+          }
+          while (readEntries < totalEntries) {
+            auto readLast = branch->GetBulkRead().GetBulkEntries(readEntries, buffer);
+            readEntries += readLast;
+            status &= valueBuilder->AppendValues(reinterpret_cast<uint8_t const*>(buffer.GetCurrent()), readLast);
+          }
+          if (!status.ok()) {
+            throw runtime_error("Failed to append values to array");
+          }
+          status &= valueBuilder->Finish(&array);
+          if (!status.ok()) {
+            throw runtime_error("Failed to create array");
+          }
+        }
+      } else {
+        // other types: use serialized read to build arrays directly.
+        auto typeSize = physicalField->type()->byte_width();
+        // This is needed for branches which have not been persisted.
+        auto bytes = branch->GetTotBytes();
+        auto branchSize = bytes ? bytes : 1000000;
+        auto&& result = arrow::AllocateResizableBuffer(branchSize, pool);
+        if (!result.ok()) {
+          throw runtime_error("Cannot allocate values buffer");
+        }
+        std::shared_ptr<arrow::Buffer> arrowValuesBuffer = std::move(result).ValueUnsafe();
+        auto ptr = arrowValuesBuffer->mutable_data();
+        if (ptr == nullptr) {
+          throw runtime_error("Invalid buffer");
+        }
+
+        std::unique_ptr<TBufferFile> offsetBuffer = nullptr;
+
+        uint32_t offset = 0;
+        int count = 0;
+        std::shared_ptr<arrow::Buffer> arrowOffsetBuffer;
+        std::span<int> offsets;
+        int size = 0;
+        uint32_t totalSize = 0;
+        TBranch* mSizeBranch = nullptr;
+        int64_t listSize = 1;
+        if (auto fixedSizeList = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type())) {
+          listSize = fixedSizeList->list_size();
+          typeSize = fixedSizeList->field(0)->type()->byte_width();
+        } else if (auto vlaListType = std::dynamic_pointer_cast<arrow::ListType>(physicalField->type())) {
+          listSize = -1;
+          typeSize = vlaListType->field(0)->type()->byte_width();
+        }
+        if (listSize == -1) {
+          mSizeBranch = branch->GetTree()->GetBranch((std::string{branch->GetName()} + "_size").c_str());
+          offsetBuffer = std::make_unique<TBufferFile>(TBuffer::EMode::kWrite, 4 * 1024 * 1024);
+          result = arrow::AllocateResizableBuffer((totalEntries + 1) * (int64_t)sizeof(int), pool);
+          if (!result.ok()) {
+            throw runtime_error("Cannot allocate offset buffer");
+          }
+          arrowOffsetBuffer = std::move(result).ValueUnsafe();
+          unsigned char* ptrOffset = arrowOffsetBuffer->mutable_data();
+          auto* tPtrOffset = reinterpret_cast<int*>(ptrOffset);
+          offsets = std::span<int>{tPtrOffset, tPtrOffset + totalEntries + 1};
+
+          // read sizes first
+          while (readEntries < totalEntries) {
+            auto readLast = mSizeBranch->GetBulkRead().GetEntriesSerialized(readEntries, *offsetBuffer);
+            readEntries += readLast;
+            for (auto i = 0; i < readLast; ++i) {
+              offsets[count++] = (int)offset;
+              offset += swap32_(reinterpret_cast<uint32_t*>(offsetBuffer->GetCurrent())[i]);
+            }
+          }
+          offsets[count] = (int)offset;
+          totalSize = offset;
+          readEntries = 0;
+        }
+
+        while (readEntries < totalEntries) {
+          auto readLast = branch->GetBulkRead().GetEntriesSerialized(readEntries, buffer);
+          if (listSize == -1) {
+            size = offsets[readEntries + readLast] - offsets[readEntries];
+          } else {
+            size = readLast * listSize;
+          }
+          readEntries += readLast;
+          swapCopy(ptr, buffer.GetCurrent(), size, typeSize);
+          ptr += (ptrdiff_t)(size * typeSize);
+        }
+        if (listSize >= 1) {
+          totalSize = readEntries * listSize;
+        }
+        std::shared_ptr<arrow::PrimitiveArray> varray;
+        switch (listSize) {
+          case -1:
+            varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
+            array = std::make_shared<arrow::ListArray>(physicalField->type(), readEntries, arrowOffsetBuffer, varray);
+            break;
+          case 1:
+            array = std::make_shared<arrow::PrimitiveArray>(physicalField->type(), readEntries, arrowValuesBuffer);
+            break;
+          default:
+            varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
+            array = std::make_shared<arrow::FixedSizeListArray>(physicalField->type(), readEntries, varray);
+        }
+      }
+
+      branch->SetStatus(false);
+      branch->DropBaskets("all");
+      branch->Reset();
+      branch->GetTransientBuffer(0)->Expand(0);
+
+      columns.push_back(array);
+    }
+    auto batch = arrow::RecordBatch::Make(dataset_schema, rows, columns);
+    totalCompressedSize += tree->GetZipBytes();
+    totalUncompressedSize += tree->GetTotBytes();
+    return batch;
+  };
+  return generator;
+}
+
+char const* rootSuffixFromArrow(arrow::Type::type id)
+{
+  switch (id) {
+    case arrow::Type::BOOL:
+      return "/O";
+    case arrow::Type::UINT8:
+      return "/b";
+    case arrow::Type::UINT16:
+      return "/s";
+    case arrow::Type::UINT32:
+      return "/i";
+    case arrow::Type::UINT64:
+      return "/l";
+    case arrow::Type::INT8:
+      return "/B";
+    case arrow::Type::INT16:
+      return "/S";
+    case arrow::Type::INT32:
+      return "/I";
+    case arrow::Type::INT64:
+      return "/L";
+    case arrow::Type::FLOAT:
+      return "/F";
+    case arrow::Type::DOUBLE:
+      return "/D";
+    default:
+      throw runtime_error("Unsupported arrow column type");
+  }
+}
+
+arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenOutputStream(
+  const std::string& path,
+  const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
+{
+  arrow::dataset::FileSource source{path, shared_from_this()};
+  auto prefix = metadata->Get("branch_prefix");
+  if (prefix.ok()) {
+    return std::make_shared<TTreeOutputStream>(GetTree(source), *prefix);
+  }
+  return std::make_shared<TTreeOutputStream>(GetTree(source), "");
+}
+
+namespace
+{
+struct BranchInfo {
+  std::string name;
+  TBranch* ptr;
+  bool mVLA;
+};
+} // namespace
+
+auto arrowTypeFromROOT(EDataType type, int size)
+{
+  auto typeGenerator = [](std::shared_ptr<arrow::DataType> const& type, int size) -> std::shared_ptr<arrow::DataType> {
+    switch (size) {
+      case -1:
+        return arrow::list(type);
+      case 1:
+        return std::move(type);
+      default:
+        return arrow::fixed_size_list(type, size);
+    }
+  };
+
+  switch (type) {
+    case EDataType::kBool_t:
+      return typeGenerator(arrow::boolean(), size);
+    case EDataType::kUChar_t:
+      return typeGenerator(arrow::uint8(), size);
+    case EDataType::kUShort_t:
+      return typeGenerator(arrow::uint16(), size);
+    case EDataType::kUInt_t:
+      return typeGenerator(arrow::uint32(), size);
+    case EDataType::kULong64_t:
+      return typeGenerator(arrow::uint64(), size);
+    case EDataType::kChar_t:
+      return typeGenerator(arrow::int8(), size);
+    case EDataType::kShort_t:
+      return typeGenerator(arrow::int16(), size);
+    case EDataType::kInt_t:
+      return typeGenerator(arrow::int32(), size);
+    case EDataType::kLong64_t:
+      return typeGenerator(arrow::int64(), size);
+    case EDataType::kFloat_t:
+      return typeGenerator(arrow::float32(), size);
+    case EDataType::kDouble_t:
+      return typeGenerator(arrow::float64(), size);
+    default:
+      throw o2::framework::runtime_error_f("Unsupported branch type: %d", static_cast<int>(type));
+  }
+}
+
+arrow::Result<std::shared_ptr<arrow::Schema>> TTreeFileFormat::Inspect(const arrow::dataset::FileSource& source) const
+{
+  arrow::Schema schema{{}};
+  auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
+  // Actually get the TTree from the ROOT file.
+  auto treeFs = std::dynamic_pointer_cast<TTreeFileSystem>(fs->GetSubFilesystem(source));
+  if (!treeFs.get()) {
+    throw runtime_error_f("Unknown filesystem %s\n", source.filesystem()->type_name().c_str());
+  }
+  TTree* tree = treeFs->GetTree(source);
+
+  auto branches = tree->GetListOfBranches();
+  auto n = branches->GetEntries();
+
+  std::vector<BranchInfo> branchInfos;
+  for (auto i = 0; i < n; ++i) {
+    auto branch = static_cast<TBranch*>(branches->At(i));
+    auto name = std::string{branch->GetName()};
+    auto pos = name.find("_size");
+    if (pos != std::string::npos) {
+      name.erase(pos);
+      branchInfos.emplace_back(BranchInfo{name, (TBranch*)nullptr, true});
+    } else {
+      auto lookup = std::find_if(branchInfos.begin(), branchInfos.end(), [&](BranchInfo const& bi) {
+        return bi.name == name;
+      });
+      if (lookup == branchInfos.end()) {
+        branchInfos.emplace_back(BranchInfo{name, branch, false});
+      } else {
+        lookup->ptr = branch;
+      }
+    }
+  }
+
+  std::vector<std::shared_ptr<arrow::Field>> fields;
+  tree->SetCacheSize(25000000);
+  for (auto& bi : branchInfos) {
+    static TClass* cls;
+    EDataType type;
+    bi.ptr->GetExpectedType(cls, type);
+    auto listSize = -1;
+    if (!bi.mVLA) {
+      listSize = static_cast<TLeaf*>(bi.ptr->GetListOfLeaves()->At(0))->GetLenStatic();
+    }
+    auto field = std::make_shared<arrow::Field>(bi.ptr->GetName(), arrowTypeFromROOT(type, listSize));
+    fields.push_back(field);
+
+    tree->AddBranchToCache(bi.ptr);
+    if (strncmp(bi.ptr->GetName(), "fIndexArray", strlen("fIndexArray")) == 0) {
+      std::string sizeBranchName = bi.ptr->GetName();
+      sizeBranchName += "_size";
+      auto* sizeBranch = (TBranch*)tree->GetBranch(sizeBranchName.c_str());
+      if (sizeBranch) {
+        tree->AddBranchToCache(sizeBranch);
+      }
+    }
+  }
+  tree->StopCacheLearningPhase();
+
+  return std::make_shared<arrow::Schema>(fields);
+}
+
+/// \brief Create a FileFragment for a FileSource.
+arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> TTreeFileFormat::MakeFragment(
+  arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
+  std::shared_ptr<arrow::Schema> physical_schema)
+{
+  std::shared_ptr<arrow::dataset::FileFormat> format = std::make_shared<TTreeFileFormat>(mTotCompressedSize, mTotUncompressedSize);
+
+  auto fragment = std::make_shared<TTreeFileFragment>(std::move(source), std::move(format),
+                                                      std::move(partition_expression),
+                                                      std::move(physical_schema));
+  return std::dynamic_pointer_cast<arrow::dataset::FileFragment>(fragment);
+}
+
+class TTreeFileWriter : public arrow::dataset::FileWriter
+{
+  std::vector<TBranch*> branches;
+  std::vector<TBranch*> sizesBranches;
+  std::vector<std::shared_ptr<arrow::Array>> valueArrays;
+  std::vector<std::shared_ptr<arrow::Array>> sizeArrays;
+  std::vector<std::shared_ptr<arrow::DataType>> valueTypes;
+
+  std::vector<int64_t> valuesIdealBasketSize;
+  std::vector<int64_t> sizeIdealBasketSize;
+
+  std::vector<int64_t> typeSizes;
+  std::vector<int64_t> listSizes;
+  bool firstBasket = true;
+
+  // This is to create a batsket size according to the first batch.
+  void finaliseBasketSize(std::shared_ptr<arrow::RecordBatch> firstBatch)
+  {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, root_arrow_fs, this);
+    O2_SIGNPOST_START(root_arrow_fs, sid, "finaliseBasketSize", "First batch with %lli rows received and %zu columns",
+                      firstBatch->num_rows(), firstBatch->columns().size());
+    for (size_t i = 0; i < branches.size(); i++) {
+      auto* branch = branches[i];
+      auto* sizeBranch = sizesBranches[i];
+
+      int valueSize = valueTypes[i]->byte_width();
+      if (listSizes[i] == 1) {
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s exists and uses %d bytes per entry for %lli entries.",
+                               branch->GetName(), valueSize, firstBatch->num_rows());
+        assert(sizeBranch == nullptr);
+        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize);
+      } else if (listSizes[i] == -1) {
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s exists and uses %d bytes per entry.",
+                               branch->GetName(), valueSize);
+        // This should probably lookup the
+        auto column = firstBatch->GetColumnByName(schema_->field(i)->name());
+        auto list = std::static_pointer_cast<arrow::ListArray>(column);
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s needed. Associated size branch %s and there are %lli entries of size %d in that list.",
+                               branch->GetName(), sizeBranch->GetName(), list->length(), valueSize);
+        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize * list->length());
+        sizeBranch->SetBasketSize(1024 + firstBatch->num_rows() * 4);
+      } else {
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s needed. There are %lli entries per array of size %d in that list.",
+                               branch->GetName(), listSizes[i], valueSize);
+        assert(sizeBranch == nullptr);
+        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize * listSizes[i]);
+      }
+
+      auto field = firstBatch->schema()->field(i);
+      if (field->name().starts_with("fIndexArray")) {
+        // One int per array to keep track of the size
+        int idealBasketSize = 4 * firstBatch->num_rows() + 1024 + field->type()->byte_width() * firstBatch->num_rows(); // minimal additional size needed, otherwise we get 2 baskets
+        int basketSize = std::max(32000, idealBasketSize);                                                              // keep a minimum value
+        sizeBranch->SetBasketSize(basketSize);
+        branch->SetBasketSize(basketSize);
+      }
+    }
+    O2_SIGNPOST_END(root_arrow_fs, sid, "finaliseBasketSize", "Done");
+  }
+
+ public:
+  // Create the TTree based on the physical_schema, not the one in the batch.
+  // The write method will have to reconcile the two schemas.
+  TTreeFileWriter(std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options,
+                  std::shared_ptr<arrow::io::OutputStream> destination,
+                  arrow::fs::FileLocator destination_locator)
+    : FileWriter(schema, options, destination, destination_locator)
+  {
+    // Batches have the same number of entries for each column.
+    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
+    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
+
+    if (directoryStream.get()) {
+      TDirectoryFile* dir = directoryStream->GetDirectory();
+      dir->cd();
+      auto* tree = new TTree(destination_locator_.path.c_str(), "");
+      treeStream = std::make_shared<TTreeOutputStream>(tree, "");
+    } else if (treeStream.get()) {
+      // We already have a tree stream, let's derive a new one
+      // with the destination_locator_.path as prefix for the branches
+      // This way we can multiplex multiple tables in the same tree.
+      auto tree = treeStream->GetTree();
+      treeStream = std::make_shared<TTreeOutputStream>(tree, destination_locator_.path);
+    } else {
+      // I could simply set a prefix here to merge to an already existing tree.
+      throw std::runtime_error("Unsupported backend.");
+    }
+
+    for (auto i = 0u; i < schema->fields().size(); ++i) {
+      auto& field = schema->field(i);
+      listSizes.push_back(1);
+
+      int valuesIdealBasketSize = 0;
+      // Construct all the needed branches.
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          listSizes.back() = std::static_pointer_cast<arrow::FixedSizeListType>(field->type())->list_size();
+          valuesIdealBasketSize = 1024 + valueTypes.back()->byte_width() * listSizes.back();
+          valueTypes.push_back(field->type()->field(0)->type());
+          sizesBranches.push_back(nullptr);
+          std::string leafList = fmt::format("{}[{}]{}", field->name(), listSizes.back(), rootSuffixFromArrow(valueTypes.back()->id()));
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
+        } break;
+        case arrow::Type::LIST: {
+          valueTypes.push_back(field->type()->field(0)->type());
+          std::string leafList = fmt::format("{}[{}_size]{}", field->name(), field->name(), rootSuffixFromArrow(valueTypes.back()->id()));
+          listSizes.back() = -1; // VLA, we need to calculate it on the fly;
+          std::string sizeLeafList = field->name() + "_size/I";
+          sizesBranches.push_back(treeStream->CreateBranch((field->name() + "_size").c_str(), sizeLeafList.c_str()));
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
+          // Notice that this could be replaced by a better guess of the
+          // average size of the list elements, but this is not trivial.
+        } break;
+        default: {
+          valueTypes.push_back(field->type());
+          std::string leafList = field->name() + rootSuffixFromArrow(valueTypes.back()->id());
+          sizesBranches.push_back(nullptr);
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
+        } break;
+      }
+    }
+    // We create the branches from the schema
+  }
+
+  arrow::Status Write(const std::shared_ptr<arrow::RecordBatch>& batch) override
+  {
+    if (firstBasket) {
+      firstBasket = false;
+      finaliseBasketSize(batch);
+    }
+
+    // Support writing empty tables
+    if (batch->columns().empty() || batch->num_rows() == 0) {
+      return arrow::Status::OK();
+    }
+
+    // Batches have the same number of entries for each column.
+    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
+    TTree* tree = nullptr;
+    if (directoryStream.get()) {
+      TDirectoryFile* dir = directoryStream->GetDirectory();
+      tree = (TTree*)dir->Get(destination_locator_.path.c_str());
+    }
+    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
+
+    if (!tree) {
+      // I could simply set a prefix here to merge to an already existing tree.
+      throw std::runtime_error("Unsupported backend.");
+    }
+
+    for (auto i = 0u; i < batch->columns().size(); ++i) {
+      auto column = batch->column(i);
+      auto& field = batch->schema()->field(i);
+
+      valueArrays.push_back(nullptr);
+
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          auto list = std::static_pointer_cast<arrow::FixedSizeListArray>(column);
+          valueArrays.back() = list->values();
+        } break;
+        case arrow::Type::LIST: {
+          auto list = std::static_pointer_cast<arrow::ListArray>(column);
+          valueArrays.back() = list;
+        } break;
+        case arrow::Type::BOOL: {
+          // In case of arrays of booleans, we need to go back to their
+          // char based representation for ROOT to save them.
+          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(column);
+
+          int64_t length = boolArray->length();
+          arrow::UInt8Builder builder;
+          auto ok = builder.Reserve(length);
+
+          for (int64_t i = 0; i < length; ++i) {
+            if (boolArray->IsValid(i)) {
+              // Expand each boolean value (true/false) to uint8 (1/0)
+              uint8_t value = boolArray->Value(i) ? 1 : 0;
+              auto ok = builder.Append(value);
+            } else {
+              // Append null for invalid entries
+              auto ok = builder.AppendNull();
+            }
+          }
+          valueArrays.back() = *builder.Finish();
+        } break;
+        default:
+          valueArrays.back() = column;
+      }
+    }
+
+    int64_t pos = 0;
+    while (pos < batch->num_rows()) {
+      for (size_t bi = 0; bi < branches.size(); ++bi) {
+        auto* branch = branches[bi];
+        auto* sizeBranch = sizesBranches[bi];
+        auto array = batch->column(bi);
+        auto& field = batch->schema()->field(bi);
+        auto& listSize = listSizes[bi];
+        auto valueType = valueTypes[bi];
+        auto valueArray = valueArrays[bi];
+
+        switch (field->type()->id()) {
+          case arrow::Type::LIST: {
+            auto list = std::static_pointer_cast<arrow::ListArray>(array);
+            listSize = list->value_length(pos);
+            uint8_t const* buffer = std::static_pointer_cast<arrow::PrimitiveArray>(valueArray)->values()->data() + array->offset() + list->value_offset(pos) * valueType->byte_width();
+            branch->SetAddress((void*)buffer);
+            sizeBranch->SetAddress(&listSize);
+          };
+            break;
+          case arrow::Type::FIXED_SIZE_LIST:
+          default: {
+            uint8_t const* buffer = std::static_pointer_cast<arrow::PrimitiveArray>(valueArray)->values()->data() + array->offset() + pos * listSize * valueType->byte_width();
+            branch->SetAddress((void*)buffer);
+          };
+        }
+      }
+      tree->Fill();
+      ++pos;
+    }
+    return arrow::Status::OK();
+  }
+
+  arrow::Future<> FinishInternal() override
+  {
+    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
+    TTree* tree = treeStream->GetTree();
+    tree->Write("", TObject::kOverwrite);
+    tree->SetDirectory(nullptr);
+
+    return {};
+  };
+};
+arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> TTreeFileFormat::MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options, arrow::fs::FileLocator destination_locator) const
+{
+  auto writer = std::make_shared<TTreeFileWriter>(schema, options, destination, destination_locator);
+  return std::dynamic_pointer_cast<arrow::dataset::FileWriter>(writer);
+}
+
+std::shared_ptr<arrow::dataset::FileWriteOptions> TTreeFileFormat::DefaultWriteOptions()
+{
+  std::shared_ptr<TTreeFileWriteOptions> options(
+    new TTreeFileWriteOptions(shared_from_this()));
+  return options;
+}
+
+TTreeFileSystem::~TTreeFileSystem() = default;
+
+DEFINE_DPL_PLUGINS_BEGIN
+DEFINE_DPL_PLUGIN_INSTANCE(TTreeObjectReadingImplementation, RootObjectReadingImplementation);
+DEFINE_DPL_PLUGINS_END
+} // namespace o2::framework
diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index 5cdd1241ecfb0..c1214a8f56beb 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -270,6 +270,10 @@ o2_add_test(Timers NAME test_Framework_test_Timers
             LABELS framework
             PUBLIC_LINK_LIBRARIES O2::Framework)
 
+o2_add_executable(framework-ao2d-to-ao3d
+            SOURCES test/o2AO2DToAO3D.cxx
+            PUBLIC_LINK_LIBRARIES O2::Framework)
+
 # FIXME: make this a proper test, when it actually does not hang.
 o2_add_executable(test-framework-ConsumeWhenAllOrdered
             SOURCES test/test_ConsumeWhenAllOrdered.cxx
@@ -299,6 +303,7 @@ add_executable(o2-test-framework-root
 target_link_libraries(o2-test-framework-root PRIVATE O2::Framework)
 target_link_libraries(o2-test-framework-root PRIVATE O2::Catch2)
 target_link_libraries(o2-test-framework-root PRIVATE ROOT::ROOTDataFrame)
+target_link_libraries(o2-test-framework-root PRIVATE ROOT::ROOTNTuple)
 set_property(TARGET o2-test-framework-root PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
 add_test(NAME framework:root COMMAND o2-test-framework-root --skip-benchmarks)
 add_test(NAME framework:crash COMMAND  sh -e -c "PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:$PATH ${CMAKE_CURRENT_LIST_DIR}/test/test_AllCrashTypes.sh")
diff --git a/Framework/Core/include/Framework/Plugins.h b/Framework/Core/include/Framework/Plugins.h
index 23d55a512e1fa..925943c6bffc3 100644
--- a/Framework/Core/include/Framework/Plugins.h
+++ b/Framework/Core/include/Framework/Plugins.h
@@ -36,6 +36,14 @@ enum struct DplPluginKind : int {
   // set, you might want to load metadata from it and attach it to the
   // configuration.
   Capability,
+  // A RootObjectReadingCapability is used to discover if there is away
+  // to read and understand an object serialised with ROOT.
+  RootObjectReadingCapability,
+
+  // A RootObjectReadingImplementation is actually used to read said object
+  // using the arrow dataset API
+  RootObjectReadingImplementation,
+
   // A plugin which was not initialised properly.
   Unknown
 };
diff --git a/Framework/Core/include/Framework/RootArrowFilesystem.h b/Framework/Core/include/Framework/RootArrowFilesystem.h
index 48d817bc9ddf2..8744656e7d55d 100644
--- a/Framework/Core/include/Framework/RootArrowFilesystem.h
+++ b/Framework/Core/include/Framework/RootArrowFilesystem.h
@@ -11,6 +11,7 @@
 #ifndef O2_FRAMEWORK_ROOT_ARROW_FILESYSTEM_H_
 #define O2_FRAMEWORK_ROOT_ARROW_FILESYSTEM_H_
 
+#include <TBufferFile.h>
 #include <arrow/dataset/type_fwd.h>
 #include <arrow/dataset/file_base.h>
 #include <arrow/filesystem/type_fwd.h>
@@ -18,23 +19,12 @@
 #include <memory>
 
 class TFile;
-class TBranch;
-class TTree;
 class TBufferFile;
 class TDirectoryFile;
 
 namespace o2::framework
 {
 
-class TTreeFileWriteOptions : public arrow::dataset::FileWriteOptions
-{
- public:
-  TTreeFileWriteOptions(std::shared_ptr<arrow::dataset::FileFormat> format)
-    : FileWriteOptions(format)
-  {
-  }
-};
-
 // This is to avoid having to implement a bunch of unimplemented methods
 // for all the possible virtual filesystem we can invent on top of ROOT
 // data structures.
@@ -79,46 +69,43 @@ class VirtualRootFileSystemBase : public arrow::fs::FileSystem
     const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override;
 };
 
-// A filesystem which allows me to get a TTree
-class TTreeFileSystem : public VirtualRootFileSystemBase
-{
- public:
-  ~TTreeFileSystem() override;
-
-  std::shared_ptr<VirtualRootFileSystemBase> GetSubFilesystem(arrow::dataset::FileSource source) override
-  {
-    return std::dynamic_pointer_cast<VirtualRootFileSystemBase>(shared_from_this());
-  };
-
-  arrow::Result<std::shared_ptr<arrow::io::OutputStream>> OpenOutputStream(
-    const std::string& path,
-    const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override;
-
-  virtual TTree* GetTree(arrow::dataset::FileSource source) = 0;
+struct RootArrowFactory final {
+  std::function<std::shared_ptr<arrow::dataset::FileWriteOptions>()> options = nullptr;
+  std::function<std::shared_ptr<arrow::dataset::FileFormat>()> format = nullptr;
+  std::function<std::shared_ptr<VirtualRootFileSystemBase>(void*)> getSubFilesystem = nullptr;
 };
 
-class SingleTreeFileSystem : public TTreeFileSystem
-{
- public:
-  SingleTreeFileSystem(TTree* tree)
-    : TTreeFileSystem(),
-      mTree(tree)
-  {
-  }
+struct RootArrowFactoryPlugin {
+  virtual RootArrowFactory* create() = 0;
+};
 
-  std::string type_name() const override
-  {
-    return "ttree";
-  }
+// A registry for all the possible ways of encoding a table in a TFile
+struct RootObjectReadingCapability {
+  // The unique name of this capability
+  std::string name = "unknown";
+  // Given a TFile, return the object which this capability support
+  // Use a void * in order not to expose the kind of object to the
+  // generic reading code. This is also where we load the plugin
+  // which will be used for the actual creation.
+  std::function<void*(TDirectoryFile* file, std::string const& path)> getHandle;
+  // Same as the above, but uses a TBufferFile as storage
+  std::function<void*(TBufferFile*, std::string const&)> getBufferHandle;
+  // This must be implemented to load the actual RootArrowFactory plugin which
+  // implements this capability. This way the detection of the file format
+  // (via get handle) does not need to know about the actual code which performs
+  // the serialization (and might depend on e.g. RNTuple).
+  std::function<RootArrowFactory&()> factory;
+};
 
-  TTree* GetTree(arrow::dataset::FileSource) override
-  {
-    // Simply return the only TTree we have
-    return mTree;
-  }
+struct RootObjectReadingCapabilityPlugin {
+  virtual RootObjectReadingCapability* create() = 0;
+};
 
- private:
-  TTree* mTree;
+// This acts as registry of all the capabilities (i.e. the ability to
+// associate a given object in a root file to the serialization plugin) and
+// the factory (i.e. the serialization plugin)
+struct RootObjectReadingFactory {
+  std::vector<RootObjectReadingCapability> capabilities;
 };
 
 class TFileFileSystem : public VirtualRootFileSystemBase
@@ -126,7 +113,7 @@ class TFileFileSystem : public VirtualRootFileSystemBase
  public:
   arrow::Result<arrow::fs::FileInfo> GetFileInfo(const std::string& path) override;
 
-  TFileFileSystem(TDirectoryFile* f, size_t readahead);
+  TFileFileSystem(TDirectoryFile* f, size_t readahead, RootObjectReadingFactory&);
 
   std::string type_name() const override
   {
@@ -147,12 +134,13 @@ class TFileFileSystem : public VirtualRootFileSystemBase
 
  private:
   TDirectoryFile* mFile;
+  RootObjectReadingFactory& mObjectFactory;
 };
 
 class TBufferFileFS : public VirtualRootFileSystemBase
 {
  public:
-  TBufferFileFS(TBufferFile* f);
+  TBufferFileFS(TBufferFile* f, RootObjectReadingFactory&);
 
   arrow::Result<arrow::fs::FileInfo> GetFileInfo(const std::string& path) override;
   std::string type_name() const override
@@ -165,68 +153,7 @@ class TBufferFileFS : public VirtualRootFileSystemBase
  private:
   TBufferFile* mBuffer;
   std::shared_ptr<VirtualRootFileSystemBase> mFilesystem;
-};
-
-class TTreeFileFragment : public arrow::dataset::FileFragment
-{
- public:
-  TTreeFileFragment(arrow::dataset::FileSource source,
-                    std::shared_ptr<arrow::dataset::FileFormat> format,
-                    arrow::compute::Expression partition_expression,
-                    std::shared_ptr<arrow::Schema> physical_schema)
-    : FileFragment(std::move(source), std::move(format), std::move(partition_expression), std::move(physical_schema))
-  {
-  }
-};
-
-class TTreeFileFormat : public arrow::dataset::FileFormat
-{
-  size_t& mTotCompressedSize;
-  size_t& mTotUncompressedSize;
-
- public:
-  TTreeFileFormat(size_t& totalCompressedSize, size_t& totalUncompressedSize)
-    : FileFormat({}),
-      mTotCompressedSize(totalCompressedSize),
-      mTotUncompressedSize(totalUncompressedSize)
-  {
-  }
-
-  ~TTreeFileFormat() override = default;
-
-  std::string type_name() const override
-  {
-    return "ttree";
-  }
-
-  bool Equals(const FileFormat& other) const override
-  {
-    return other.type_name() == this->type_name();
-  }
-
-  arrow::Result<bool> IsSupported(const arrow::dataset::FileSource& source) const override
-  {
-    auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
-    auto subFs = fs->GetSubFilesystem(source);
-    if (std::dynamic_pointer_cast<TTreeFileSystem>(subFs)) {
-      return true;
-    }
-    return false;
-  }
-
-  arrow::Result<std::shared_ptr<arrow::Schema>> Inspect(const arrow::dataset::FileSource& source) const override;
-  /// \brief Create a FileFragment for a FileSource.
-  arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> MakeFragment(
-    arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
-    std::shared_ptr<arrow::Schema> physical_schema) override;
-
-  arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options, arrow::fs::FileLocator destination_locator) const override;
-
-  std::shared_ptr<arrow::dataset::FileWriteOptions> DefaultWriteOptions() override;
-
-  arrow::Result<arrow::RecordBatchGenerator> ScanBatchesAsync(
-    const std::shared_ptr<arrow::dataset::ScanOptions>& options,
-    const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const override;
+  RootObjectReadingFactory& mObjectFactory;
 };
 
 // An arrow outputstream which allows to write to a TDirectoryFile.
@@ -255,33 +182,6 @@ class TDirectoryFileOutputStream : public arrow::io::OutputStream
   TDirectoryFile* mDirectory;
 };
 
-// An arrow outputstream which allows to write to a TTree. Eventually
-// with a prefix for the branches.
-class TTreeOutputStream : public arrow::io::OutputStream
-{
- public:
-  TTreeOutputStream(TTree*, std::string branchPrefix);
-
-  arrow::Status Close() override;
-
-  arrow::Result<int64_t> Tell() const override;
-
-  arrow::Status Write(const void* data, int64_t nbytes) override;
-
-  bool closed() const override;
-
-  TBranch* CreateBranch(char const* branchName, char const* sizeBranch);
-
-  TTree* GetTree()
-  {
-    return mTree;
-  }
-
- private:
-  TTree* mTree;
-  std::string mBranchPrefix;
-};
-
 } // namespace o2::framework
 
 #endif // O2_FRAMEWORK_ROOT_ARROW_FILESYSTEM_H_
diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx
index 0d225b81c0581..46248ab467366 100644
--- a/Framework/Core/src/Plugin.cxx
+++ b/Framework/Core/src/Plugin.cxx
@@ -11,10 +11,15 @@
 #include "Framework/Plugins.h"
 #include "Framework/ConfigParamDiscovery.h"
 #include "Framework/ConfigParamRegistry.h"
+#include "Framework/RootArrowFilesystem.h"
 #include "Framework/Logger.h"
 #include "Framework/Capability.h"
 #include "Framework/Signpost.h"
 #include "Framework/VariantJSONHelpers.h"
+#include "Framework/PluginManager.h"
+#include <TDirectory.h>
+#include <TDirectoryFile.h>
+#include <TClass.h>
 #include <cstddef>
 #include <string_view>
 
@@ -168,11 +173,75 @@ struct DiscoverAODOptionsInCommandLine : o2::framework::ConfigDiscoveryPlugin {
   }
 };
 
+struct ImplementationContext {
+  std::vector<RootArrowFactory> implementations;
+};
+
+std::function<void*(TDirectoryFile*, std::string const&)> getHandleByClass(char const* classname)
+{
+  return [classname](TDirectoryFile* file, std::string const& path) { return file->GetObjectChecked(path.c_str(), TClass::GetClass(classname)); };
+}
+
+std::function<void*(TBufferFile*, std::string const&)> getBufferHandleByClass(char const* classname)
+{
+  return [classname](TBufferFile* buffer, std::string const& path) { buffer->Reset(); return buffer->ReadObjectAny(TClass::GetClass(classname)); };
+}
+
+void lazyLoadFactory(std::vector<RootArrowFactory>& implementations, char const* specs)
+{
+  // Lazy loading of the plugin so that we do not bring in RNTuple / TTree if not needed
+  if (implementations.empty()) {
+    std::vector<LoadablePlugin> plugins;
+    auto morePlugins = PluginManager::parsePluginSpecString(specs);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+    PluginManager::loadFromPlugin<RootArrowFactory, RootArrowFactoryPlugin>(plugins, implementations);
+    if (implementations.empty()) {
+      return;
+    }
+  }
+}
+
+struct RNTupleObjectReadingCapability : o2::framework::RootObjectReadingCapabilityPlugin {
+  RootObjectReadingCapability* create() override
+  {
+    auto context = new ImplementationContext;
+
+    return new RootObjectReadingCapability{
+      .name = "rntuple",
+      .getHandle = getHandleByClass("ROOT::Experimental::RNTuple"),
+      .getBufferHandle = getBufferHandleByClass("ROOT::Experimental::RNTuple"),
+      .factory = [context]() -> RootArrowFactory& {
+        lazyLoadFactory(context->implementations, "O2FrameworkAnalysisRNTupleSupport:RNTupleObjectReadingImplementation");
+        return context->implementations.back();
+      }};
+  }
+};
+
+struct TTreeObjectReadingCapability : o2::framework::RootObjectReadingCapabilityPlugin {
+  RootObjectReadingCapability* create() override
+  {
+    auto context = new ImplementationContext;
+
+    return new RootObjectReadingCapability{
+      .name = "ttree",
+      .getHandle = getHandleByClass("TTree"),
+      .getBufferHandle = getBufferHandleByClass("TTree"),
+      .factory = [context]() -> RootArrowFactory& {
+        lazyLoadFactory(context->implementations, "O2FrameworkAnalysisTTreeSupport:TTreeObjectReadingImplementation");
+        return context->implementations.back();
+      }};
+  }
+};
+
 DEFINE_DPL_PLUGINS_BEGIN
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAODCapability, Capability);
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLineCapability, Capability);
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverAODOptionsInCommandLineCapability, Capability);
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLine, ConfigDiscovery);
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverAODOptionsInCommandLine, ConfigDiscovery);
+DEFINE_DPL_PLUGIN_INSTANCE(RNTupleObjectReadingCapability, RootObjectReadingCapability);
+DEFINE_DPL_PLUGIN_INSTANCE(TTreeObjectReadingCapability, RootObjectReadingCapability);
 DEFINE_DPL_PLUGINS_END
 } // namespace o2::framework
diff --git a/Framework/Core/src/RootArrowFilesystem.cxx b/Framework/Core/src/RootArrowFilesystem.cxx
index 5f2d21d942d37..545ba6f0afb71 100644
--- a/Framework/Core/src/RootArrowFilesystem.cxx
+++ b/Framework/Core/src/RootArrowFilesystem.cxx
@@ -9,9 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 #include "Framework/RootArrowFilesystem.h"
-#include "Framework/Endian.h"
 #include "Framework/RuntimeError.h"
-#include "Framework/Signpost.h"
 #include <Rtypes.h>
 #include <arrow/array/array_nested.h>
 #include <arrow/array/array_primitive.h>
@@ -19,93 +17,48 @@
 #include <arrow/array/builder_primitive.h>
 #include <memory>
 #include <TFile.h>
-#include <TLeaf.h>
 #include <TBufferFile.h>
-#include <TTree.h>
 #include <TDirectoryFile.h>
 #include <arrow/type.h>
 #include <arrow/type_fwd.h>
 #include <arrow/dataset/file_base.h>
 #include <arrow/result.h>
 #include <arrow/status.h>
-#include <arrow/util/key_value_metadata.h>
 #include <fmt/format.h>
 
-#include <stdexcept>
-#include <utility>
+template class
+  std::shared_ptr<arrow::Array>;
 
-O2_DECLARE_DYNAMIC_LOG(root_arrow_fs);
-
-namespace
-{
-struct BranchInfo {
-  std::string name;
-  TBranch* ptr;
-  bool mVLA;
-};
-} // namespace
-
-auto arrowTypeFromROOT(EDataType type, int size)
-{
-  auto typeGenerator = [](std::shared_ptr<arrow::DataType> const& type, int size) -> std::shared_ptr<arrow::DataType> {
-    switch (size) {
-      case -1:
-        return arrow::list(type);
-      case 1:
-        return std::move(type);
-      default:
-        return arrow::fixed_size_list(type, size);
-    }
-  };
-
-  switch (type) {
-    case EDataType::kBool_t:
-      return typeGenerator(arrow::boolean(), size);
-    case EDataType::kUChar_t:
-      return typeGenerator(arrow::uint8(), size);
-    case EDataType::kUShort_t:
-      return typeGenerator(arrow::uint16(), size);
-    case EDataType::kUInt_t:
-      return typeGenerator(arrow::uint32(), size);
-    case EDataType::kULong64_t:
-      return typeGenerator(arrow::uint64(), size);
-    case EDataType::kChar_t:
-      return typeGenerator(arrow::int8(), size);
-    case EDataType::kShort_t:
-      return typeGenerator(arrow::int16(), size);
-    case EDataType::kInt_t:
-      return typeGenerator(arrow::int32(), size);
-    case EDataType::kLong64_t:
-      return typeGenerator(arrow::int64(), size);
-    case EDataType::kFloat_t:
-      return typeGenerator(arrow::float32(), size);
-    case EDataType::kDouble_t:
-      return typeGenerator(arrow::float64(), size);
-    default:
-      throw o2::framework::runtime_error_f("Unsupported branch type: %d", static_cast<int>(type));
-  }
-}
 namespace o2::framework
 {
 using arrow::Status;
 
-TFileFileSystem::TFileFileSystem(TDirectoryFile* f, size_t readahead)
+TFileFileSystem::TFileFileSystem(TDirectoryFile* f, size_t readahead, RootObjectReadingFactory& factory)
   : VirtualRootFileSystemBase(),
-    mFile(f)
+    mFile(f),
+    mObjectFactory(factory)
 {
   ((TFile*)mFile)->SetReadaheadSize(50 * 1024 * 1024);
 }
 
 std::shared_ptr<VirtualRootFileSystemBase> TFileFileSystem::GetSubFilesystem(arrow::dataset::FileSource source)
 {
-  auto tree = (TTree*)mFile->GetObjectChecked(source.path().c_str(), TClass::GetClass<TTree>());
-  if (tree) {
-    return std::shared_ptr<VirtualRootFileSystemBase>(new SingleTreeFileSystem(tree));
+  // We use a plugin to create the actual objects inside the
+  // file, so that we can support TTree and RNTuple at the same time
+  // without having to depend on both.
+  for (auto& capability : mObjectFactory.capabilities) {
+    void* handle = capability.getHandle(mFile, source.path());
+    if (!handle) {
+      continue;
+    }
+    if (handle) {
+      return capability.factory().getSubFilesystem(handle);
+    }
   }
 
   auto directory = (TDirectoryFile*)mFile->GetObjectChecked(source.path().c_str(), TClass::GetClass<TDirectory>());
   if (directory) {
-    return std::shared_ptr<VirtualRootFileSystemBase>(new TFileFileSystem(directory, 50 * 1024 * 1024));
+    return std::shared_ptr<VirtualRootFileSystemBase>(new TFileFileSystem(directory, 50 * 1024 * 1024, mObjectFactory));
   }
   throw runtime_error_f("Unsupported file layout");
 }
@@ -120,10 +73,14 @@ arrow::Result<arrow::fs::FileInfo> TFileFileSystem::GetFileInfo(const std::strin
   auto fs = GetSubFilesystem(source);
 
   // For now we only support single trees.
-  if (std::dynamic_pointer_cast<SingleTreeFileSystem>(fs)) {
-    result.set_type(arrow::fs::FileType::File);
+  if (std::dynamic_pointer_cast<TFileFileSystem>(fs)) {
+    result.set_type(arrow::fs::FileType::Directory);
     return result;
   }
+  // Everything else is a file, if it was created.
+  if (fs.get()) {
+    result.set_type(arrow::fs::FileType::File);
+  }
   return result;
 }
 
@@ -137,7 +94,7 @@ arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TFileFileSystem::OpenOut
 
   auto* dir = dynamic_cast<TDirectoryFile*>(this->GetFile()->Get(path.c_str()));
   if (!dir) {
-    throw runtime_error_f("Unable to open directory %s in file %s", path.c_str(), GetFile()->GetName());
+    return arrow::Status::Invalid(fmt::format("Unable to open directory {} in file {} ", path.c_str(), GetFile()->GetName()));
   }
   auto stream = std::make_shared<TDirectoryFileOutputStream>(dir);
   return stream;
@@ -219,81 +176,6 @@ arrow::Result<std::shared_ptr<arrow::io::OutputStream>> VirtualRootFileSystemBas
   return arrow::Status::NotImplemented("No random access file system");
 }
 
-arrow::Result<std::shared_ptr<arrow::Schema>> TTreeFileFormat::Inspect(const arrow::dataset::FileSource& source) const
-{
-  arrow::Schema schema{{}};
-  auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
-  // Actually get the TTree from the ROOT file.
-  auto treeFs = std::dynamic_pointer_cast<TTreeFileSystem>(fs->GetSubFilesystem(source));
-  if (!treeFs.get()) {
-    throw runtime_error_f("Unknown filesystem %s\n", source.filesystem()->type_name().c_str());
-  }
-  TTree* tree = treeFs->GetTree(source);
-
-  auto branches = tree->GetListOfBranches();
-  auto n = branches->GetEntries();
-
-  std::vector<BranchInfo> branchInfos;
-  for (auto i = 0; i < n; ++i) {
-    auto branch = static_cast<TBranch*>(branches->At(i));
-    auto name = std::string{branch->GetName()};
-    auto pos = name.find("_size");
-    if (pos != std::string::npos) {
-      name.erase(pos);
-      branchInfos.emplace_back(BranchInfo{name, (TBranch*)nullptr, true});
-    } else {
-      auto lookup = std::find_if(branchInfos.begin(), branchInfos.end(), [&](BranchInfo const& bi) {
-        return bi.name == name;
-      });
-      if (lookup == branchInfos.end()) {
-        branchInfos.emplace_back(BranchInfo{name, branch, false});
-      } else {
-        lookup->ptr = branch;
-      }
-    }
-  }
-
-  std::vector<std::shared_ptr<arrow::Field>> fields;
-  tree->SetCacheSize(25000000);
-  for (auto& bi : branchInfos) {
-    static TClass* cls;
-    EDataType type;
-    bi.ptr->GetExpectedType(cls, type);
-    auto listSize = -1;
-    if (!bi.mVLA) {
-      listSize = static_cast<TLeaf*>(bi.ptr->GetListOfLeaves()->At(0))->GetLenStatic();
-    }
-    auto field = std::make_shared<arrow::Field>(bi.ptr->GetName(), arrowTypeFromROOT(type, listSize));
-    fields.push_back(field);
-
-    tree->AddBranchToCache(bi.ptr);
-    if (strncmp(bi.ptr->GetName(), "fIndexArray", strlen("fIndexArray")) == 0) {
-      std::string sizeBranchName = bi.ptr->GetName();
-      sizeBranchName += "_size";
-      auto* sizeBranch = (TBranch*)tree->GetBranch(sizeBranchName.c_str());
-      if (sizeBranch) {
-        tree->AddBranchToCache(sizeBranch);
-      }
-    }
-  }
-  tree->StopCacheLearningPhase();
-
-  return std::make_shared<arrow::Schema>(fields);
-}
-
-/// \brief Create a FileFragment for a FileSource.
-arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> TTreeFileFormat::MakeFragment(
-  arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
-  std::shared_ptr<arrow::Schema> physical_schema)
-{
-  std::shared_ptr<arrow::dataset::FileFormat> format = std::make_shared<TTreeFileFormat>(mTotCompressedSize, mTotUncompressedSize);
-
-  auto fragment = std::make_shared<TTreeFileFragment>(std::move(source), std::move(format),
-                                                      std::move(partition_expression),
-                                                      std::move(physical_schema));
-  return std::dynamic_pointer_cast<arrow::dataset::FileFragment>(fragment);
-}
-
 // An arrow outputstream which allows to write to a ttree
 TDirectoryFileOutputStream::TDirectoryFileOutputStream(TDirectoryFile* f)
   : mDirectory(f)
@@ -321,544 +203,14 @@ bool TDirectoryFileOutputStream::closed() const
   return mDirectory->GetFile()->IsOpen() == false;
 }
 
-// An arrow outputstream which allows to write to a ttree
-// @a branch prefix is to be used to identify a set of branches which all belong to
-// the same table.
-TTreeOutputStream::TTreeOutputStream(TTree* f, std::string branchPrefix)
-  : mTree(f),
-    mBranchPrefix(std::move(branchPrefix))
-{
-}
-
-arrow::Status TTreeOutputStream::Close()
-{
-  if (mTree->GetCurrentFile() == nullptr) {
-    return arrow::Status::Invalid("Cannot close a tree not attached to a file");
-  }
-  mTree->GetCurrentFile()->Close();
-  return arrow::Status::OK();
-}
-
-arrow::Result<int64_t> TTreeOutputStream::Tell() const
-{
-  return arrow::Result<int64_t>(arrow::Status::NotImplemented("Cannot move"));
-}
-
-arrow::Status TTreeOutputStream::Write(const void* data, int64_t nbytes)
-{
-  return arrow::Status::NotImplemented("Cannot write raw bytes to a TTree");
-}
-
-bool TTreeOutputStream::closed() const
-{
-  // A standalone tree is never closed.
-  if (mTree->GetCurrentFile() == nullptr) {
-    return false;
-  }
-  return mTree->GetCurrentFile()->IsOpen() == false;
-}
-
-TBranch* TTreeOutputStream::CreateBranch(char const* branchName, char const* sizeBranch)
-{
-  return mTree->Branch((mBranchPrefix + "/" + branchName).c_str(), (char*)nullptr, (mBranchPrefix + sizeBranch).c_str());
-}
-
-char const* rootSuffixFromArrow(arrow::Type::type id)
-{
-  switch (id) {
-    case arrow::Type::BOOL:
-      return "/O";
-    case arrow::Type::UINT8:
-      return "/b";
-    case arrow::Type::UINT16:
-      return "/s";
-    case arrow::Type::UINT32:
-      return "/i";
-    case arrow::Type::UINT64:
-      return "/l";
-    case arrow::Type::INT8:
-      return "/B";
-    case arrow::Type::INT16:
-      return "/S";
-    case arrow::Type::INT32:
-      return "/I";
-    case arrow::Type::INT64:
-      return "/L";
-    case arrow::Type::FLOAT:
-      return "/F";
-    case arrow::Type::DOUBLE:
-      return "/D";
-    default:
-      throw runtime_error("Unsupported arrow column type");
-  }
-}
-
-class TTreeFileWriter : public arrow::dataset::FileWriter
-{
-  std::vector<TBranch*> branches;
-  std::vector<TBranch*> sizesBranches;
-  std::vector<std::shared_ptr<arrow::Array>> valueArrays;
-  std::vector<std::shared_ptr<arrow::Array>> sizeArrays;
-  std::vector<std::shared_ptr<arrow::DataType>> valueTypes;
-
-  std::vector<int64_t> valuesIdealBasketSize;
-  std::vector<int64_t> sizeIdealBasketSize;
-
-  std::vector<int64_t> typeSizes;
-  std::vector<int64_t> listSizes;
-  bool firstBasket = true;
-
-  // This is to create a batsket size according to the first batch.
-  void finaliseBasketSize(std::shared_ptr<arrow::RecordBatch> firstBatch)
-  {
-    O2_SIGNPOST_ID_FROM_POINTER(sid, root_arrow_fs, this);
-    O2_SIGNPOST_START(root_arrow_fs, sid, "finaliseBasketSize", "First batch with %lli rows received and %zu columns",
-                      firstBatch->num_rows(), firstBatch->columns().size());
-    for (size_t i = 0; i < branches.size(); i++) {
-      auto* branch = branches[i];
-      auto* sizeBranch = sizesBranches[i];
-
-      int valueSize = valueTypes[i]->byte_width();
-      if (listSizes[i] == 1) {
-        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s exists and uses %d bytes per entry for %lli entries.",
-                               branch->GetName(), valueSize, firstBatch->num_rows());
-        assert(sizeBranch == nullptr);
-        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize);
-      } else if (listSizes[i] == -1) {
-        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s exists and uses %d bytes per entry.",
-                               branch->GetName(), valueSize);
-        // This should probably lookup the
-        auto column = firstBatch->GetColumnByName(schema_->field(i)->name());
-        auto list = std::static_pointer_cast<arrow::ListArray>(column);
-        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s needed. Associated size branch %s and there are %lli entries of size %d in that list.",
-                               branch->GetName(), sizeBranch->GetName(), list->length(), valueSize);
-        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize * list->length());
-        sizeBranch->SetBasketSize(1024 + firstBatch->num_rows() * 4);
-      } else {
-        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s needed. There are %lli entries per array of size %d in that list.",
-                               branch->GetName(), listSizes[i], valueSize);
-        assert(sizeBranch == nullptr);
-        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize * listSizes[i]);
-      }
-
-      auto field = firstBatch->schema()->field(i);
-      if (field->name().starts_with("fIndexArray")) {
-        // One int per array to keep track of the size
-        int idealBasketSize = 4 * firstBatch->num_rows() + 1024 + field->type()->byte_width() * firstBatch->num_rows(); // minimal additional size needed, otherwise we get 2 baskets
-        int basketSize = std::max(32000, idealBasketSize);                                                              // keep a minimum value
-        sizeBranch->SetBasketSize(basketSize);
-        branch->SetBasketSize(basketSize);
-      }
-    }
-    O2_SIGNPOST_END(root_arrow_fs, sid, "finaliseBasketSize", "Done");
-  }
-
- public:
-  // Create the TTree based on the physical_schema, not the one in the batch.
-  // The write method will have to reconcile the two schemas.
-  TTreeFileWriter(std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options,
-                  std::shared_ptr<arrow::io::OutputStream> destination,
-                  arrow::fs::FileLocator destination_locator)
-    : FileWriter(schema, options, destination, destination_locator)
-  {
-    // Batches have the same number of entries for each column.
-    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
-    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
-
-    if (directoryStream.get()) {
-      TDirectoryFile* dir = directoryStream->GetDirectory();
-      dir->cd();
-      auto* tree = new TTree(destination_locator_.path.c_str(), "");
-      treeStream = std::make_shared<TTreeOutputStream>(tree, "");
-    } else if (treeStream.get()) {
-      // We already have a tree stream, let's derive a new one
-      // with the destination_locator_.path as prefix for the branches
-      // This way we can multiplex multiple tables in the same tree.
-      auto tree = treeStream->GetTree();
-      treeStream = std::make_shared<TTreeOutputStream>(tree, destination_locator_.path);
-    } else {
-      // I could simply set a prefix here to merge to an already existing tree.
-      throw std::runtime_error("Unsupported backend.");
-    }
-
-    for (auto i = 0u; i < schema->fields().size(); ++i) {
-      auto& field = schema->field(i);
-      listSizes.push_back(1);
-
-      int valuesIdealBasketSize = 0;
-      // Construct all the needed branches.
-      switch (field->type()->id()) {
-        case arrow::Type::FIXED_SIZE_LIST: {
-          listSizes.back() = std::static_pointer_cast<arrow::FixedSizeListType>(field->type())->list_size();
-          valuesIdealBasketSize = 1024 + valueTypes.back()->byte_width() * listSizes.back();
-          valueTypes.push_back(field->type()->field(0)->type());
-          sizesBranches.push_back(nullptr);
-          std::string leafList = fmt::format("{}[{}]{}", field->name(), listSizes.back(), rootSuffixFromArrow(valueTypes.back()->id()));
-          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
-        } break;
-        case arrow::Type::LIST: {
-          valueTypes.push_back(field->type()->field(0)->type());
-          std::string leafList = fmt::format("{}[{}_size]{}", field->name(), field->name(), rootSuffixFromArrow(valueTypes.back()->id()));
-          listSizes.back() = -1; // VLA, we need to calculate it on the fly;
-          std::string sizeLeafList = field->name() + "_size/I";
-          sizesBranches.push_back(treeStream->CreateBranch((field->name() + "_size").c_str(), sizeLeafList.c_str()));
-          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
-          // Notice that this could be replaced by a better guess of the
-          // average size of the list elements, but this is not trivial.
-        } break;
-        default: {
-          valueTypes.push_back(field->type());
-          std::string leafList = field->name() + rootSuffixFromArrow(valueTypes.back()->id());
-          sizesBranches.push_back(nullptr);
-          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
-        } break;
-      }
-    }
-    // We create the branches from the schema
-  }
-
-  arrow::Status Write(const std::shared_ptr<arrow::RecordBatch>& batch) override
-  {
-    if (firstBasket) {
-      firstBasket = false;
-      finaliseBasketSize(batch);
-    }
-
-    // Support writing empty tables
-    if (batch->columns().empty() || batch->num_rows() == 0) {
-      return arrow::Status::OK();
-    }
-
-    // Batches have the same number of entries for each column.
-    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
-    TTree* tree = nullptr;
-    if (directoryStream.get()) {
-      TDirectoryFile* dir = directoryStream->GetDirectory();
-      tree = (TTree*)dir->Get(destination_locator_.path.c_str());
-    }
-    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
-
-    if (!tree) {
-      // I could simply set a prefix here to merge to an already existing tree.
-      throw std::runtime_error("Unsupported backend.");
-    }
-
-    for (auto i = 0u; i < batch->columns().size(); ++i) {
-      auto column = batch->column(i);
-      auto& field = batch->schema()->field(i);
-
-      valueArrays.push_back(nullptr);
-
-      switch (field->type()->id()) {
-        case arrow::Type::FIXED_SIZE_LIST: {
-          auto list = std::static_pointer_cast<arrow::FixedSizeListArray>(column);
-          valueArrays.back() = list->values();
-        } break;
-        case arrow::Type::LIST: {
-          auto list = std::static_pointer_cast<arrow::ListArray>(column);
-          valueArrays.back() = list;
-        } break;
-        case arrow::Type::BOOL: {
-          // In case of arrays of booleans, we need to go back to their
-          // char based representation for ROOT to save them.
-          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(column);
-
-          int64_t length = boolArray->length();
-          arrow::UInt8Builder builder;
-          auto ok = builder.Reserve(length);
-
-          for (int64_t i = 0; i < length; ++i) {
-            if (boolArray->IsValid(i)) {
-              // Expand each boolean value (true/false) to uint8 (1/0)
-              uint8_t value = boolArray->Value(i) ? 1 : 0;
-              auto ok = builder.Append(value);
-            } else {
-              // Append null for invalid entries
-              auto ok = builder.AppendNull();
-            }
-          }
-          valueArrays.back() = *builder.Finish();
-        } break;
-        default:
-          valueArrays.back() = column;
-      }
-    }
-
-    int64_t pos = 0;
-    while (pos < batch->num_rows()) {
-      for (size_t bi = 0; bi < branches.size(); ++bi) {
-        auto* branch = branches[bi];
-        auto* sizeBranch = sizesBranches[bi];
-        auto array = batch->column(bi);
-        auto& field = batch->schema()->field(bi);
-        auto& listSize = listSizes[bi];
-        auto valueType = valueTypes[bi];
-        auto valueArray = valueArrays[bi];
-
-        switch (field->type()->id()) {
-          case arrow::Type::LIST: {
-            auto list = std::static_pointer_cast<arrow::ListArray>(array);
-            listSize = list->value_length(pos);
-            uint8_t const* buffer = std::static_pointer_cast<arrow::PrimitiveArray>(valueArray)->values()->data() + array->offset() + list->value_offset(pos) * valueType->byte_width();
-            branch->SetAddress((void*)buffer);
-            sizeBranch->SetAddress(&listSize);
-          };
-            break;
-          case arrow::Type::FIXED_SIZE_LIST:
-          default: {
-            uint8_t const* buffer = std::static_pointer_cast<arrow::PrimitiveArray>(valueArray)->values()->data() + array->offset() + pos * listSize * valueType->byte_width();
-            branch->SetAddress((void*)buffer);
-          };
-        }
-      }
-      tree->Fill();
-      ++pos;
-    }
-    return arrow::Status::OK();
-  }
-
-  arrow::Future<> FinishInternal() override
-  {
-    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
-    TTree* tree = treeStream->GetTree();
-    tree->Write("", TObject::kOverwrite);
-    tree->SetDirectory(nullptr);
-
-    return {};
-  };
-};
-
-arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> TTreeFileFormat::MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options, arrow::fs::FileLocator destination_locator) const
-{
-  auto writer = std::make_shared<TTreeFileWriter>(schema, options, destination, destination_locator);
-  return std::dynamic_pointer_cast<arrow::dataset::FileWriter>(writer);
-}
-
-std::shared_ptr<arrow::dataset::FileWriteOptions> TTreeFileFormat::DefaultWriteOptions()
-{
-  std::shared_ptr<TTreeFileWriteOptions> options(
-    new TTreeFileWriteOptions(shared_from_this()));
-  return options;
-}
-
-arrow::Result<arrow::RecordBatchGenerator> TTreeFileFormat::ScanBatchesAsync(
-  const std::shared_ptr<arrow::dataset::ScanOptions>& options,
-  const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const
-{
-  // Get the fragment as a TTreeFragment. This might be PART of a TTree.
-  auto treeFragment = std::dynamic_pointer_cast<TTreeFileFragment>(fragment);
-  // This is the schema we want to read
-  auto dataset_schema = options->dataset_schema;
-
-  auto generator = [pool = options->pool, treeFragment, dataset_schema, &totalCompressedSize = mTotCompressedSize,
-                    &totalUncompressedSize = mTotUncompressedSize]() -> arrow::Future<std::shared_ptr<arrow::RecordBatch>> {
-    auto schema = treeFragment->format()->Inspect(treeFragment->source());
-
-    std::vector<std::shared_ptr<arrow::Array>> columns;
-    std::vector<std::shared_ptr<arrow::Field>> fields = dataset_schema->fields();
-    auto physical_schema = *treeFragment->ReadPhysicalSchema();
-
-    static TBufferFile buffer{TBuffer::EMode::kWrite, 4 * 1024 * 1024};
-    auto containerFS = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(treeFragment->source().filesystem());
-    auto fs = std::dynamic_pointer_cast<TTreeFileSystem>(containerFS->GetSubFilesystem(treeFragment->source()));
-
-    int64_t rows = -1;
-    TTree* tree = fs->GetTree(treeFragment->source());
-    for (auto& field : fields) {
-      // The field actually on disk
-      auto physicalField = physical_schema->GetFieldByName(field->name());
-      TBranch* branch = tree->GetBranch(physicalField->name().c_str());
-      assert(branch);
-      buffer.Reset();
-      auto totalEntries = branch->GetEntries();
-      if (rows == -1) {
-        rows = totalEntries;
-      }
-      if (rows != totalEntries) {
-        throw runtime_error_f("Unmatching number of rows for branch %s", branch->GetName());
-      }
-      arrow::Status status;
-      int readEntries = 0;
-      std::shared_ptr<arrow::Array> array;
-      auto listType = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type());
-      if (physicalField->type() == arrow::boolean() ||
-          (listType && physicalField->type()->field(0)->type() == arrow::boolean())) {
-        if (listType) {
-          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
-          auto status = arrow::MakeBuilder(pool, physicalField->type()->field(0)->type(), &builder);
-          if (!status.ok()) {
-            throw runtime_error("Cannot create value builder");
-          }
-          auto listBuilder = std::make_unique<arrow::FixedSizeListBuilder>(pool, std::move(builder), listType->list_size());
-          auto valueBuilder = listBuilder.get()->value_builder();
-          // boolean array special case: we need to use builder to create the bitmap
-          status = valueBuilder->Reserve(totalEntries * listType->list_size());
-          status &= listBuilder->Reserve(totalEntries);
-          if (!status.ok()) {
-            throw runtime_error("Failed to reserve memory for array builder");
-          }
-          while (readEntries < totalEntries) {
-            auto readLast = branch->GetBulkRead().GetBulkEntries(readEntries, buffer);
-            readEntries += readLast;
-            status &= static_cast<arrow::BooleanBuilder*>(valueBuilder)->AppendValues(reinterpret_cast<uint8_t const*>(buffer.GetCurrent()), readLast * listType->list_size());
-          }
-          status &= static_cast<arrow::FixedSizeListBuilder*>(listBuilder.get())->AppendValues(readEntries);
-          if (!status.ok()) {
-            throw runtime_error("Failed to append values to array");
-          }
-          status &= listBuilder->Finish(&array);
-          if (!status.ok()) {
-            throw runtime_error("Failed to create array");
-          }
-        } else if (listType == nullptr) {
-          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
-          auto status = arrow::MakeBuilder(pool, physicalField->type(), &builder);
-          if (!status.ok()) {
-            throw runtime_error("Cannot create builder");
-          }
-          auto valueBuilder = static_cast<arrow::BooleanBuilder*>(builder.get());
-          // boolean array special case: we need to use builder to create the bitmap
-          status = valueBuilder->Reserve(totalEntries);
-          if (!status.ok()) {
-            throw runtime_error("Failed to reserve memory for array builder");
-          }
-          while (readEntries < totalEntries) {
-            auto readLast = branch->GetBulkRead().GetBulkEntries(readEntries, buffer);
-            readEntries += readLast;
-            status &= valueBuilder->AppendValues(reinterpret_cast<uint8_t const*>(buffer.GetCurrent()), readLast);
-          }
-          if (!status.ok()) {
-            throw runtime_error("Failed to append values to array");
-          }
-          status &= valueBuilder->Finish(&array);
-          if (!status.ok()) {
-            throw runtime_error("Failed to create array");
-          }
-        }
-      } else {
-        // other types: use serialized read to build arrays directly.
-        auto typeSize = physicalField->type()->byte_width();
-        // This is needed for branches which have not been persisted.
-        auto bytes = branch->GetTotBytes();
-        auto branchSize = bytes ? bytes : 1000000;
-        auto&& result = arrow::AllocateResizableBuffer(branchSize, pool);
-        if (!result.ok()) {
-          throw runtime_error("Cannot allocate values buffer");
-        }
-        std::shared_ptr<arrow::Buffer> arrowValuesBuffer = std::move(result).ValueUnsafe();
-        auto ptr = arrowValuesBuffer->mutable_data();
-        if (ptr == nullptr) {
-          throw runtime_error("Invalid buffer");
-        }
-
-        std::unique_ptr<TBufferFile> offsetBuffer = nullptr;
-
-        uint32_t offset = 0;
-        int count = 0;
-        std::shared_ptr<arrow::Buffer> arrowOffsetBuffer;
-        std::span<int> offsets;
-        int size = 0;
-        uint32_t totalSize = 0;
-        TBranch* mSizeBranch = nullptr;
-        int64_t listSize = 1;
-        if (auto fixedSizeList = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type())) {
-          listSize = fixedSizeList->list_size();
-          typeSize = fixedSizeList->field(0)->type()->byte_width();
-        } else if (auto vlaListType = std::dynamic_pointer_cast<arrow::ListType>(physicalField->type())) {
-          listSize = -1;
-          typeSize = vlaListType->field(0)->type()->byte_width();
-        }
-        if (listSize == -1) {
-          mSizeBranch = branch->GetTree()->GetBranch((std::string{branch->GetName()} + "_size").c_str());
-          offsetBuffer = std::make_unique<TBufferFile>(TBuffer::EMode::kWrite, 4 * 1024 * 1024);
-          result = arrow::AllocateResizableBuffer((totalEntries + 1) * (int64_t)sizeof(int), pool);
-          if (!result.ok()) {
-            throw runtime_error("Cannot allocate offset buffer");
-          }
-          arrowOffsetBuffer = std::move(result).ValueUnsafe();
-          unsigned char* ptrOffset = arrowOffsetBuffer->mutable_data();
-          auto* tPtrOffset = reinterpret_cast<int*>(ptrOffset);
-          offsets = std::span<int>{tPtrOffset, tPtrOffset + totalEntries + 1};
-
-          // read sizes first
-          while (readEntries < totalEntries) {
-            auto readLast = mSizeBranch->GetBulkRead().GetEntriesSerialized(readEntries, *offsetBuffer);
-            readEntries += readLast;
-            for (auto i = 0; i < readLast; ++i) {
-              offsets[count++] = (int)offset;
-              offset += swap32_(reinterpret_cast<uint32_t*>(offsetBuffer->GetCurrent())[i]);
-            }
-          }
-          offsets[count] = (int)offset;
-          totalSize = offset;
-          readEntries = 0;
-        }
-
-        while (readEntries < totalEntries) {
-          auto readLast = branch->GetBulkRead().GetEntriesSerialized(readEntries, buffer);
-          if (listSize == -1) {
-            size = offsets[readEntries + readLast] - offsets[readEntries];
-          } else {
-            size = readLast * listSize;
-          }
-          readEntries += readLast;
-          swapCopy(ptr, buffer.GetCurrent(), size, typeSize);
-          ptr += (ptrdiff_t)(size * typeSize);
-        }
-        if (listSize >= 1) {
-          totalSize = readEntries * listSize;
-        }
-        std::shared_ptr<arrow::PrimitiveArray> varray;
-        switch (listSize) {
-          case -1:
-            varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
-            array = std::make_shared<arrow::ListArray>(physicalField->type(), readEntries, arrowOffsetBuffer, varray);
-            break;
-          case 1:
-            array = std::make_shared<arrow::PrimitiveArray>(physicalField->type(), readEntries, arrowValuesBuffer);
-            break;
-          default:
-            varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
-            array = std::make_shared<arrow::FixedSizeListArray>(physicalField->type(), readEntries, varray);
-        }
-      }
-
-      branch->SetStatus(false);
-      branch->DropBaskets("all");
-      branch->Reset();
-      branch->GetTransientBuffer(0)->Expand(0);
-
-      columns.push_back(array);
-    }
-    auto batch = arrow::RecordBatch::Make(dataset_schema, rows, columns);
-    totalCompressedSize += tree->GetZipBytes();
-    totalUncompressedSize += tree->GetTotBytes();
-    return batch;
-  };
-  return generator;
-}
-
-arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenOutputStream(
-  const std::string& path,
-  const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
-{
-  arrow::dataset::FileSource source{path, shared_from_this()};
-  auto prefix = metadata->Get("branch_prefix");
-  if (prefix.ok()) {
-    return std::make_shared<TTreeOutputStream>(GetTree(source), *prefix);
-  }
-  return std::make_shared<TTreeOutputStream>(GetTree(source), "");
-}
-
-TBufferFileFS::TBufferFileFS(TBufferFile* f)
+TBufferFileFS::TBufferFileFS(TBufferFile* f, RootObjectReadingFactory& factory)
   : VirtualRootFileSystemBase(),
     mBuffer(f),
-    mFilesystem(nullptr)
+    mFilesystem(nullptr),
+    mObjectFactory(factory)
 {
 }
 
-TTreeFileSystem::~TTreeFileSystem() = default;
-
 arrow::Result<arrow::fs::FileInfo> TBufferFileFS::GetFileInfo(const std::string& path)
 {
   arrow::fs::FileInfo result;
@@ -871,19 +223,26 @@ arrow::Result<arrow::fs::FileInfo> TBufferFileFS::GetFileInfo(const std::string&
     return result;
   }
 
-  // For now we only support single trees.
-  if (std::dynamic_pointer_cast<SingleTreeFileSystem>(mFilesystem)) {
-    result.set_type(arrow::fs::FileType::File);
+  auto info = mFilesystem->GetFileInfo(path);
+  if (!info.ok()) {
     return result;
   }
+
+  result.set_type(info->type());
   return result;
 }
 
 std::shared_ptr<VirtualRootFileSystemBase> TBufferFileFS::GetSubFilesystem(arrow::dataset::FileSource source)
 {
-  if (!mFilesystem.get()) {
-    auto tree = ((TTree*)mBuffer->ReadObject(TTree::Class()));
-    mFilesystem = std::make_shared<SingleTreeFileSystem>(tree);
+  // We use a plugin to create the actual objects inside the
+  // file, so that we can support TTree and RNTuple at the same time
+  // without having to depend on both.
+  for (auto& capability : mObjectFactory.capabilities) {
+    void* handle = capability.getBufferHandle(mBuffer, source.path());
+    if (handle) {
+      mFilesystem = capability.factory().getSubFilesystem(handle);
+      break;
+    }
   }
   return mFilesystem;
 }
diff --git a/Framework/Core/test/o2AO2DToAO3D.cxx b/Framework/Core/test/o2AO2DToAO3D.cxx
new file mode 100644
index 0000000000000..664f25a852745
--- /dev/null
+++ b/Framework/Core/test/o2AO2DToAO3D.cxx
@@ -0,0 +1,155 @@
+#include "Framework/RootArrowFilesystem.h"
+#include "Framework/PluginManager.h"
+#include <TDirectory.h>
+#include <TDirectoryFile.h>
+#include <getopt.h>
+#include <TFile.h>
+#include <iostream>
+#include <TMap.h>
+#include <TTree.h>
+#include <fmt/format.h>
+
+int main(int argc, char** argv)
+{
+
+  char* input_file = nullptr;
+  char* output_file = nullptr;
+
+  // Define long options
+  static struct option long_options[] = {
+    {"input", required_argument, nullptr, 'i'},
+    {"output", required_argument, nullptr, 'o'},
+    {nullptr, 0, nullptr, 0} // End of options
+  };
+
+  int option_index = 0;
+  int c;
+
+  // Parse options
+  while ((c = getopt_long(argc, argv, "i:o:", long_options, &option_index)) != -1) {
+    switch (c) {
+      case 'i':
+        input_file = optarg;
+        break;
+      case 'o':
+        output_file = optarg;
+        break;
+      case '?':
+        // Unknown option
+        printf("Unknown option. Use --input <file> and --output <file>\n");
+        return 1;
+      default:
+        break;
+    }
+  }
+
+  // Check if input and output files are provided
+  if (input_file && output_file) {
+    printf("Input file: %s\n", input_file);
+    printf("Output file: %s\n", output_file);
+  } else {
+    fprintf(stderr, "Usage: %s --input <file> --output <file>\n", argv[0]);
+    return 1;
+  }
+
+  // Plugins which understand
+  std::vector<char const*> capabilitiesSpecs = {
+    "O2Framework:RNTupleObjectReadingCapability",
+    "O2Framework:TTreeObjectReadingCapability",
+  };
+
+  o2::framework::RootObjectReadingFactory factory;
+
+  std::vector<LoadablePlugin> plugins;
+  for (auto spec : capabilitiesSpecs) {
+    auto morePlugins = o2::framework::PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+  }
+
+  auto in = TFile::Open(input_file, "READ");
+  auto out = TFile::Open(output_file, "RECREATE");
+
+  auto fs = std::make_shared<o2::framework::TFileFileSystem>(in, 50 * 1024 * 1024, factory);
+  auto outFs = std::make_shared<o2::framework::TFileFileSystem>(out, 0, factory);
+
+  o2::framework::PluginManager::loadFromPlugin<o2::framework::RootObjectReadingCapability, o2::framework::RootObjectReadingCapabilityPlugin>(plugins, factory.capabilities);
+
+  // Plugins are hardcoded for now...
+  auto rNtupleFormat = factory.capabilities[0].factory().format();
+  auto format = factory.capabilities[1].factory().format();
+
+  for (TObject* dk : *in->GetListOfKeys()) {
+    if (dk->GetName() == std::string("metaData")) {
+      TMap* m = dynamic_cast<TMap*>(in->Get(dk->GetName()));
+      m->Print();
+      auto* copy = m->Clone("metaData");
+      out->WriteTObject(copy);
+      continue;
+    }
+    auto* d = (TDirectory*)in->Get(dk->GetName());
+    std::cout << "Processing: " << dk->GetName() << std::endl;
+    // For the moment RNTuple does not support TDirectory, so
+    // we write everything at toplevel.
+    auto destination = outFs->OpenOutputStream("/", {});
+    if (!destination.ok()) {
+      std::cerr << "Could not open destination folder " << output_file << std::endl;
+      exit(1);
+    }
+
+    for (TObject* tk : *d->GetListOfKeys()) {
+      auto sourceUrl = fmt::format("{}/{}", dk->GetName(), tk->GetName());
+      // FIXME: there is no support for TDirectory yet. Let's write everything
+      // at the same level.
+      auto destUrl = fmt::format("/{}-{}", dk->GetName(), tk->GetName());
+      arrow::dataset::FileSource source(sourceUrl, fs);
+      if (!format->IsSupported(source).ok()) {
+        std::cout << "Source " << source.path() << " is not supported" << std::endl;
+        continue;
+      }
+      std::cout << "  Processing tree: " << tk->GetName() << std::endl;
+      auto schemaOpt = format->Inspect(source);
+      if (!schemaOpt.ok()) {
+        std::cout << "Could not inspect source " << source.path() << std::endl;
+      }
+      auto schema = *schemaOpt;
+      auto fragment = format->MakeFragment(source, {}, schema);
+      if (!fragment.ok()) {
+        std::cout << "Could not make fragment from " << source.path() << "with schema:" << schema->ToString() << std::endl;
+        continue;
+      }
+      auto options = std::make_shared<arrow::dataset::ScanOptions>();
+      options->dataset_schema = schema;
+      auto scanner = format->ScanBatchesAsync(options, *fragment);
+      if (!scanner.ok()) {
+        std::cout << "Scanner not ok" << std::endl;
+        continue;
+      }
+      auto batches = (*scanner)();
+      auto result = batches.result();
+      if (!result.ok()) {
+        std::cout << "Could not get batches." << std::endl;
+        continue;
+      }
+      std::cout << "   Found a table with " << (*result)->columns().size() << " columns " << (*result)->num_rows() << " rows." << std::endl;
+
+      if ((*result)->num_rows() == 0) {
+        std::cout << "Empty table, skipping for now" << std::endl;
+        continue;
+      }
+      arrow::fs::FileLocator locator{outFs, destUrl};
+      std::cout << schema->ToString() << std::endl;
+      auto writer = rNtupleFormat->MakeWriter(*destination, schema, {}, locator);
+      auto success = writer->get()->Write(*result);
+      if (!success.ok()) {
+        std::cout << "Error while writing" << std::endl;
+        continue;
+      }
+    }
+    out->ls();
+    auto rootDestination = std::dynamic_pointer_cast<o2::framework::TDirectoryFileOutputStream>(*destination);
+  }
+  in->Close();
+  out->Close();
+}
diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx
index 8440e942903a5..8eb3a9825f0f7 100644
--- a/Framework/Core/test/test_Root2ArrowTable.cxx
+++ b/Framework/Core/test/test_Root2ArrowTable.cxx
@@ -14,6 +14,7 @@
 #include "Framework/TableBuilder.h"
 #include "Framework/RootTableBuilderHelpers.h"
 #include "Framework/ASoA.h"
+#include "Framework/PluginManager.h"
 #include "../src/ArrowDebugHelpers.h"
 
 #include <ROOT/RDataFrame.hxx>
@@ -26,6 +27,13 @@
 #include <TTree.h>
 #include <TRandom.h>
 #include <TFile.h>
+#include <ROOT/RField.hxx>
+#include <ROOT/RNTuple.hxx>
+#include <ROOT/RNTupleDescriptor.hxx>
+#include <ROOT/RNTupleModel.hxx>
+#include <ROOT/RNTupleReader.hxx>
+#include <ROOT/RNTupleUtil.hxx>
+#include <ROOT/RNTupleWriter.hxx>
 #include <memory>
 
 #include <arrow/array/array_primitive.h>
@@ -232,10 +240,31 @@ TEST_CASE("RootTree2Fragment")
   file->WriteObjectAny(&t1, t1.Class());
   auto* fileRead = new TBufferFile(TBuffer::kRead, file->BufferSize(), file->Buffer(), false, nullptr);
 
-  size_t totalSizeCompressed = 0;
-  size_t totalSizeUncompressed = 0;
-  auto format = std::make_shared<TTreeFileFormat>(totalSizeCompressed, totalSizeUncompressed);
-  auto fs = std::make_shared<TBufferFileFS>(fileRead);
+  std::vector<char const*> capabilitiesSpecs = {
+    "O2Framework:RNTupleObjectReadingCapability",
+    "O2Framework:TTreeObjectReadingCapability",
+  };
+
+  std::vector<LoadablePlugin> plugins;
+  for (auto spec : capabilitiesSpecs) {
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+  }
+  REQUIRE(plugins.size() == 2);
+
+  RootObjectReadingFactory factory;
+  std::vector<char const*> configDiscoverySpec = {};
+  PluginManager::loadFromPlugin<RootObjectReadingCapability, RootObjectReadingCapabilityPlugin>(plugins, factory.capabilities);
+  REQUIRE(factory.capabilities.size() == 2);
+  REQUIRE(factory.capabilities[0].name == "rntuple");
+  REQUIRE(factory.capabilities[1].name == "ttree");
+
+  // Plugins are hardcoded for now...
+  auto format = factory.capabilities[1].factory().format();
+
+  auto fs = std::make_shared<TBufferFileFS>(fileRead, factory);
 
   arrow::dataset::FileSource source("p", fs);
   REQUIRE(format->IsSupported(source) == true);
@@ -439,10 +468,34 @@ TEST_CASE("RootTree2Dataset")
   }
   f->Write();
 
-  size_t totalSizeCompressed = 0;
-  size_t totalSizeUncompressed = 0;
-  auto format = std::make_shared<TTreeFileFormat>(totalSizeCompressed, totalSizeUncompressed);
-  auto fs = std::make_shared<TFileFileSystem>(f, 50 * 1024 * 1024);
+  std::vector<char const*> capabilitiesSpecs = {
+    "O2Framework:RNTupleObjectReadingCapability",
+    "O2Framework:TTreeObjectReadingCapability",
+  };
+
+  RootObjectReadingFactory factory;
+
+  std::vector<LoadablePlugin> plugins;
+  for (auto spec : capabilitiesSpecs) {
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+  }
+  REQUIRE(plugins.size() == 2);
+
+  PluginManager::loadFromPlugin<RootObjectReadingCapability, RootObjectReadingCapabilityPlugin>(plugins, factory.capabilities);
+
+  REQUIRE(factory.capabilities.size() == 2);
+  REQUIRE(factory.capabilities[0].name == "rntuple");
+  REQUIRE(factory.capabilities[1].name == "ttree");
+
+  // Plugins are hardcoded for now...
+  auto rNtupleFormat = factory.capabilities[0].factory().format();
+  auto format = factory.capabilities[1].factory().format();
+
+  auto fs = std::make_shared<TFileFileSystem>(f, 50 * 1024 * 1024, factory);
+
   arrow::dataset::FileSource source("DF_2/tracks", fs);
   REQUIRE(format->IsSupported(source) == true);
   auto schemaOpt = format->Inspect(source);
@@ -464,7 +517,7 @@ TEST_CASE("RootTree2Dataset")
   validateContents(*result);
 
   auto* output = new TMemFile("foo", "RECREATE");
-  auto outFs = std::make_shared<TFileFileSystem>(output, 0);
+  auto outFs = std::make_shared<TFileFileSystem>(output, 0, factory);
 
   // Open a stream at toplevel
   auto destination = outFs->OpenOutputStream("/", {});
@@ -503,4 +556,38 @@ TEST_CASE("RootTree2Dataset")
     REQUIRE((*resultWritten)->num_rows() == 100);
     validateContents(*resultWritten);
   }
+  arrow::fs::FileLocator rnTupleLocator{outFs, "/rntuple"};
+  // We write an RNTuple in the same TMemFile, using /rntuple as a location
+  auto rntupleDestination = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(*destination);
+
+  {
+    auto rNtupleWriter = rNtupleFormat->MakeWriter(*destination, schema, {}, rnTupleLocator);
+    auto rNtupleSuccess = rNtupleWriter->get()->Write(*result);
+    REQUIRE(rNtupleSuccess.ok());
+  }
+
+  // And now we can read back the RNTuple into a RecordBatch
+  arrow::dataset::FileSource writtenRntupleSource("/rntuple", outFs);
+  auto newRNTupleFS = outFs->GetSubFilesystem(writtenRntupleSource);
+
+  REQUIRE(rNtupleFormat->IsSupported(writtenRntupleSource) == true);
+
+  auto rntupleSchemaOpt = rNtupleFormat->Inspect(writtenRntupleSource);
+  REQUIRE(rntupleSchemaOpt.ok());
+  auto rntupleSchemaWritten = *rntupleSchemaOpt;
+  REQUIRE(validateSchema(rntupleSchemaWritten));
+
+  auto rntupleFragmentWritten = rNtupleFormat->MakeFragment(writtenRntupleSource, {}, rntupleSchemaWritten);
+  REQUIRE(rntupleFragmentWritten.ok());
+  auto rntupleOptionsWritten = std::make_shared<arrow::dataset::ScanOptions>();
+  rntupleOptionsWritten->dataset_schema = rntupleSchemaWritten;
+  auto rntupleScannerWritten = rNtupleFormat->ScanBatchesAsync(rntupleOptionsWritten, *rntupleFragmentWritten);
+  REQUIRE(rntupleScannerWritten.ok());
+  auto rntupleBatchesWritten = (*rntupleScannerWritten)();
+  auto rntupleResultWritten = rntupleBatchesWritten.result();
+  REQUIRE(rntupleResultWritten.ok());
+  REQUIRE((*rntupleResultWritten)->columns().size() == 10);
+  REQUIRE(validateSchema((*rntupleResultWritten)->schema()));
+  REQUIRE((*rntupleResultWritten)->num_rows() == 100);
+  REQUIRE(validateContents(*rntupleResultWritten));
 }

From 8a0cef36211e08ddb87daa94cedb087876470bdf Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+ktf@users.noreply.github.com>
Date: Thu, 12 Dec 2024 13:44:02 +0100
Subject: [PATCH 726/726] DPL Analysis: add RNTuple arrow::Dataset support

As part of the changes, move the actual logic which serialises /
deserialised things to plugins so that we do not need to depend
or RNTuple in production code.

Include an initial converter to go from AO2Ds to RNTuple based files.
---
 Framework/AnalysisSupport/CMakeLists.txt      |  10 +
 .../AnalysisSupport/src/RNTuplePlugin.cxx     | 825 +++++++++++++++++
 Framework/AnalysisSupport/src/TTreePlugin.cxx | 862 ++++++++++++++++++
 Framework/Core/CMakeLists.txt                 |   5 +
 Framework/Core/include/Framework/Plugins.h    |   8 +
 .../include/Framework/RootArrowFilesystem.h   | 174 +---
 Framework/Core/src/Plugin.cxx                 |  69 ++
 Framework/Core/src/RootArrowFilesystem.cxx    | 721 +--------------
 Framework/Core/test/o2AO2DToAO3D.cxx          | 165 ++++
 Framework/Core/test/test_Root2ArrowTable.cxx  | 105 ++-
 10 files changed, 2117 insertions(+), 827 deletions(-)
 create mode 100644 Framework/AnalysisSupport/src/RNTuplePlugin.cxx
 create mode 100644 Framework/AnalysisSupport/src/TTreePlugin.cxx
 create mode 100644 Framework/Core/test/o2AO2DToAO3D.cxx

diff --git a/Framework/AnalysisSupport/CMakeLists.txt b/Framework/AnalysisSupport/CMakeLists.txt
index 5fb1282469711..05cd8075a49d8 100644
--- a/Framework/AnalysisSupport/CMakeLists.txt
+++ b/Framework/AnalysisSupport/CMakeLists.txt
@@ -24,6 +24,16 @@ o2_add_library(FrameworkAnalysisSupport
                PRIVATE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/src
                PUBLIC_LINK_LIBRARIES O2::Framework ${EXTRA_TARGETS} ROOT::TreePlayer)
 
+o2_add_library(FrameworkAnalysisRNTupleSupport
+               SOURCES src/RNTuplePlugin.cxx
+               PRIVATE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/src
+               PUBLIC_LINK_LIBRARIES O2::Framework ${EXTRA_TARGETS} ROOT::ROOTNTuple)
+
+o2_add_library(FrameworkAnalysisTTreeSupport
+               SOURCES src/TTreePlugin.cxx
+               PRIVATE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/src
+               PUBLIC_LINK_LIBRARIES O2::Framework ${EXTRA_TARGETS} ROOT::TreePlayer)
+
 o2_add_test(DataInputDirector NAME test_Framework_test_DataInputDirector
                SOURCES test/test_DataInputDirector.cxx
                COMPONENT_NAME Framework
diff --git a/Framework/AnalysisSupport/src/RNTuplePlugin.cxx b/Framework/AnalysisSupport/src/RNTuplePlugin.cxx
new file mode 100644
index 0000000000000..9f67785f1a069
--- /dev/null
+++ b/Framework/AnalysisSupport/src/RNTuplePlugin.cxx
@@ -0,0 +1,825 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "Framework/RuntimeError.h"
+#include "Framework/RootArrowFilesystem.h"
+#include "Framework/Plugins.h"
+#include <ROOT/RNTupleModel.hxx>
+#include <ROOT/RNTupleWriteOptions.hxx>
+#include <ROOT/RNTupleWriter.hxx>
+#include <ROOT/RField.hxx>
+#include <ROOT/RNTuple.hxx>
+#include <ROOT/RNTupleReader.hxx>
+#include <ROOT/RFieldVisitor.hxx>
+#include <ROOT/RNTupleInspector.hxx>
+#include <ROOT/RVec.hxx>
+#include <TBufferFile.h>
+
+#include <TDirectory.h>
+#include <arrow/array/array_nested.h>
+#include <arrow/array/array_primitive.h>
+#include <arrow/array/builder_nested.h>
+#include <arrow/array/builder_primitive.h>
+#include <arrow/dataset/file_base.h>
+
+template class
+  std::unique_ptr<ROOT::Experimental::RNTupleReader>;
+
+namespace o2::framework
+{
+
+class RNTupleFileWriteOptions : public arrow::dataset::FileWriteOptions
+{
+ public:
+  RNTupleFileWriteOptions(std::shared_ptr<arrow::dataset::FileFormat> format)
+    : FileWriteOptions(format)
+  {
+  }
+};
+
+// A filesystem which allows me to get a RNTuple
+class RNTupleFileSystem : public VirtualRootFileSystemBase
+{
+ public:
+  ~RNTupleFileSystem() override;
+
+  std::shared_ptr<VirtualRootFileSystemBase> GetSubFilesystem(arrow::dataset::FileSource source) override
+  {
+    return std::dynamic_pointer_cast<VirtualRootFileSystemBase>(shared_from_this());
+  };
+  virtual ROOT::Experimental::RNTuple* GetRNTuple(arrow::dataset::FileSource source) = 0;
+};
+
+class SingleRNTupleFileSystem : public RNTupleFileSystem
+{
+ public:
+  SingleRNTupleFileSystem(ROOT::Experimental::RNTuple* tuple)
+    : RNTupleFileSystem(),
+      mTuple(tuple)
+  {
+  }
+
+  arrow::Result<arrow::fs::FileInfo> GetFileInfo(std::string const& path) override;
+
+  std::string type_name() const override
+  {
+    return "rntuple";
+  }
+
+  ROOT::Experimental::RNTuple* GetRNTuple(arrow::dataset::FileSource) override
+  {
+    // Simply return the only TTree we have
+    return mTuple;
+  }
+
+ private:
+  ROOT::Experimental::RNTuple* mTuple;
+};
+
+arrow::Result<arrow::fs::FileInfo> SingleRNTupleFileSystem::GetFileInfo(std::string const& path)
+{
+  arrow::dataset::FileSource source(path, shared_from_this());
+  arrow::fs::FileInfo result;
+  result.set_path(path);
+  result.set_type(arrow::fs::FileType::File);
+  return result;
+}
+
+class RNTupleFileFragment : public arrow::dataset::FileFragment
+{
+ public:
+  RNTupleFileFragment(arrow::dataset::FileSource source,
+                      std::shared_ptr<arrow::dataset::FileFormat> format,
+                      arrow::compute::Expression partition_expression,
+                      std::shared_ptr<arrow::Schema> physical_schema)
+    : FileFragment(std::move(source), std::move(format), std::move(partition_expression), std::move(physical_schema))
+  {
+  }
+};
+
+class RNTupleFileFormat : public arrow::dataset::FileFormat
+{
+  size_t& mTotCompressedSize;
+  size_t& mTotUncompressedSize;
+
+ public:
+  RNTupleFileFormat(size_t& totalCompressedSize, size_t& totalUncompressedSize)
+    : FileFormat({}),
+      mTotCompressedSize(totalCompressedSize),
+      mTotUncompressedSize(totalUncompressedSize)
+  {
+  }
+
+  ~RNTupleFileFormat() override = default;
+
+  std::string type_name() const override
+  {
+    return "rntuple";
+  }
+
+  bool Equals(const FileFormat& other) const override
+  {
+    return other.type_name() == this->type_name();
+  }
+
+  arrow::Result<bool> IsSupported(const arrow::dataset::FileSource& source) const override
+  {
+    auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
+    auto subFs = fs->GetSubFilesystem(source);
+    if (std::dynamic_pointer_cast<RNTupleFileSystem>(subFs)) {
+      return true;
+    }
+    return false;
+  }
+
+  arrow::Result<std::shared_ptr<arrow::Schema>> Inspect(const arrow::dataset::FileSource& source) const override;
+
+  arrow::Result<arrow::RecordBatchGenerator> ScanBatchesAsync(
+    const std::shared_ptr<arrow::dataset::ScanOptions>& options,
+    const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const override;
+
+  std::shared_ptr<arrow::dataset::FileWriteOptions> DefaultWriteOptions() override;
+
+  arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination,
+                                                                        std::shared_ptr<arrow::Schema> schema,
+                                                                        std::shared_ptr<arrow::dataset::FileWriteOptions> options,
+                                                                        arrow::fs::FileLocator destination_locator) const override;
+  arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> MakeFragment(
+    arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
+    std::shared_ptr<arrow::Schema> physical_schema) override;
+};
+
+struct RootNTupleVisitor : public ROOT::Experimental::Detail::RFieldVisitor {
+  void VisitArrayField(const ROOT::Experimental::RArrayField& field) override
+  {
+    int size = field.GetLength();
+    RootNTupleVisitor valueVisitor{};
+    auto valueField = field.GetSubFields()[0];
+    valueField->AcceptVisitor(valueVisitor);
+    auto type = valueVisitor.datatype;
+    this->datatype = arrow::fixed_size_list(type, size);
+  }
+
+  void VisitRVecField(const ROOT::Experimental::RRVecField& field) override
+  {
+    RootNTupleVisitor valueVisitor{};
+    auto valueField = field.GetSubFields()[0];
+    valueField->AcceptVisitor(valueVisitor);
+    auto type = valueVisitor.datatype;
+    this->datatype = arrow::list(type);
+  }
+
+  void VisitField(const ROOT::Experimental::RFieldBase& field) override
+  {
+    throw o2::framework::runtime_error_f("Unknown field %s with type %s", field.GetFieldName().c_str(), field.GetTypeName().c_str());
+  }
+
+  void VisitIntField(const ROOT::Experimental::RField<int>& field) override
+  {
+    this->datatype = arrow::int32();
+  }
+
+  void VisitBoolField(const ROOT::Experimental::RField<bool>& field) override
+  {
+    this->datatype = arrow::boolean();
+  }
+
+  void VisitFloatField(const ROOT::Experimental::RField<float>& field) override
+  {
+    this->datatype = arrow::float32();
+  }
+
+  void VisitDoubleField(const ROOT::Experimental::RField<double>& field) override
+  {
+    this->datatype = arrow::float64();
+  }
+  std::shared_ptr<arrow::DataType> datatype;
+};
+} // namespace o2::framework
+
+auto arrowTypeFromRNTuple(ROOT::Experimental::RFieldBase const& field, int size)
+{
+  o2::framework::RootNTupleVisitor visitor;
+  field.AcceptVisitor(visitor);
+  return visitor.datatype;
+}
+
+namespace o2::framework
+{
+std::unique_ptr<ROOT::Experimental::RFieldBase> rootFieldFromArrow(std::shared_ptr<arrow::Field> field, std::string name)
+{
+  using namespace ROOT::Experimental;
+  switch (field->type()->id()) {
+    case arrow::Type::BOOL:
+      return std::make_unique<RField<bool>>(name);
+    case arrow::Type::UINT8:
+      return std::make_unique<RField<uint8_t>>(name);
+    case arrow::Type::UINT16:
+      return std::make_unique<RField<uint16_t>>(name);
+    case arrow::Type::UINT32:
+      return std::make_unique<RField<uint32_t>>(name);
+    case arrow::Type::UINT64:
+      return std::make_unique<RField<uint64_t>>(name);
+    case arrow::Type::INT8:
+      return std::make_unique<RField<int8_t>>(name);
+    case arrow::Type::INT16:
+      return std::make_unique<RField<int16_t>>(name);
+    case arrow::Type::INT32:
+      return std::make_unique<RField<int32_t>>(name);
+    case arrow::Type::INT64:
+      return std::make_unique<RField<int64_t>>(name);
+    case arrow::Type::FLOAT:
+      return std::make_unique<RField<float>>(name);
+    case arrow::Type::DOUBLE:
+      return std::make_unique<RField<double>>(name);
+    default:
+      throw runtime_error("Unsupported arrow column type");
+  }
+}
+
+class RNTupleFileWriter : public arrow::dataset::FileWriter
+{
+  std::shared_ptr<ROOT::Experimental::RNTupleWriter> mWriter;
+  bool firstBatch = true;
+  std::vector<std::shared_ptr<arrow::Array>> valueArrays;
+  std::vector<std::shared_ptr<arrow::DataType>> valueTypes;
+  std::vector<size_t> valueCount;
+
+ public:
+  RNTupleFileWriter(std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options,
+                    std::shared_ptr<arrow::io::OutputStream> destination,
+                    arrow::fs::FileLocator destination_locator)
+    : FileWriter(schema, options, destination, destination_locator)
+  {
+    using namespace ROOT::Experimental;
+
+    auto model = RNTupleModel::CreateBare();
+    // Let's create a model from the physical schema
+    for (auto i = 0u; i < schema->fields().size(); ++i) {
+      auto& field = schema->field(i);
+
+      // Construct all the needed branches.
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          auto list = std::static_pointer_cast<arrow::FixedSizeListType>(field->type());
+          auto valueField = field->type()->field(0);
+          model->AddField(std::make_unique<RArrayField>(field->name(), rootFieldFromArrow(valueField, "_0"), list->list_size()));
+        } break;
+        case arrow::Type::LIST: {
+          auto valueField = field->type()->field(0);
+          model->AddField(std::make_unique<RRVecField>(field->name(), rootFieldFromArrow(valueField, "_0")));
+        } break;
+        default: {
+          model->AddField(rootFieldFromArrow(field, field->name()));
+        } break;
+      }
+    }
+    auto fileStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
+    auto* file = dynamic_cast<TFile*>(fileStream->GetDirectory());
+    mWriter = RNTupleWriter::Append(std::move(model), destination_locator_.path, *file, {});
+  }
+
+  arrow::Status Write(const std::shared_ptr<arrow::RecordBatch>& batch) override
+  {
+    if (firstBatch) {
+      firstBatch = false;
+    }
+
+    // Support writing empty tables
+    if (batch->columns().empty() || batch->num_rows() == 0) {
+      return arrow::Status::OK();
+    }
+
+    for (auto i = 0u; i < batch->columns().size(); ++i) {
+      auto column = batch->column(i);
+      auto& field = batch->schema()->field(i);
+
+      valueArrays.push_back(nullptr);
+      valueTypes.push_back(nullptr);
+      valueCount.push_back(1);
+
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          auto list = std::static_pointer_cast<arrow::FixedSizeListArray>(column);
+          auto listType = std::static_pointer_cast<arrow::FixedSizeListType>(field->type());
+          if (field->type()->field(0)->type()->id() == arrow::Type::BOOL) {
+            auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(list->values());
+            int64_t length = boolArray->length();
+            arrow::UInt8Builder builder;
+            auto ok = builder.Reserve(length);
+
+            for (int64_t i = 0; i < length; ++i) {
+              if (boolArray->IsValid(i)) {
+                // Expand each boolean value (true/false) to uint8 (1/0)
+                uint8_t value = boolArray->Value(i) ? 1 : 0;
+                auto ok = builder.Append(value);
+              } else {
+                // Append null for invalid entries
+                auto ok = builder.AppendNull();
+              }
+            }
+            valueArrays.back() = *builder.Finish();
+            valueTypes.back() = valueArrays.back()->type();
+          } else {
+            valueArrays.back() = list->values();
+            valueTypes.back() = field->type()->field(0)->type();
+          }
+          valueCount.back() = listType->list_size();
+        } break;
+        case arrow::Type::LIST: {
+          auto list = std::static_pointer_cast<arrow::ListArray>(column);
+          valueArrays.back() = list;
+          valueTypes.back() = field->type()->field(0)->type();
+          valueCount.back() = -1;
+        } break;
+        case arrow::Type::BOOL: {
+          // We unpack the array
+          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(column);
+          int64_t length = boolArray->length();
+          arrow::UInt8Builder builder;
+          auto ok = builder.Reserve(length);
+
+          for (int64_t i = 0; i < length; ++i) {
+            if (boolArray->IsValid(i)) {
+              // Expand each boolean value (true/false) to uint8 (1/0)
+              uint8_t value = boolArray->Value(i) ? 1 : 0;
+              auto ok = builder.Append(value);
+            } else {
+              // Append null for invalid entries
+              auto ok = builder.AppendNull();
+            }
+          }
+          valueArrays.back() = *builder.Finish();
+          valueTypes.back() = valueArrays.back()->type();
+        } break;
+        default:
+          valueArrays.back() = column;
+          valueTypes.back() = field->type();
+          break;
+      }
+    }
+
+    int64_t pos = 0;
+
+    auto entry = mWriter->CreateEntry();
+    std::vector<ROOT::Experimental::REntry::RFieldToken> tokens;
+    tokens.reserve(batch->num_columns());
+    std::vector<size_t> typeIds;
+    typeIds.reserve(batch->num_columns());
+
+    for (size_t ci = 0; ci < batch->num_columns(); ++ci) {
+      auto& field = batch->schema()->field(ci);
+      typeIds.push_back(batch->column(ci)->type()->id());
+      tokens.push_back(entry->GetToken(field->name()));
+    }
+
+    while (pos < batch->num_rows()) {
+      for (size_t ci = 0; ci < batch->num_columns(); ++ci) {
+        auto typeId = typeIds[ci];
+        auto token = tokens[ci];
+
+        switch (typeId) {
+          case arrow::Type::LIST: {
+            auto list = std::static_pointer_cast<arrow::ListArray>(valueArrays[ci]);
+            auto value_slice = list->value_slice(pos);
+
+            valueCount[ci] = value_slice->length();
+            auto bindValue = [&vc = valueCount, ci, token](auto array, std::unique_ptr<ROOT::Experimental::REntry>& entry) -> void {
+              using value_type = std::decay_t<decltype(*array.get())>::value_type;
+              auto v = std::make_shared<ROOT::VecOps::RVec<value_type>>((value_type*)array->raw_values(), vc[ci]);
+              entry->BindValue(token, v);
+            };
+            switch (valueTypes[ci]->id()) {
+              case arrow::Type::FLOAT: {
+                bindValue(std::static_pointer_cast<arrow::FloatArray>(value_slice), entry);
+              } break;
+              case arrow::Type::DOUBLE: {
+                bindValue(std::static_pointer_cast<arrow::DoubleArray>(value_slice), entry);
+              } break;
+              case arrow::Type::INT8: {
+                bindValue(std::static_pointer_cast<arrow::Int8Array>(value_slice), entry);
+              } break;
+              case arrow::Type::INT16: {
+                bindValue(std::static_pointer_cast<arrow::Int16Array>(value_slice), entry);
+              } break;
+              case arrow::Type::INT32: {
+                bindValue(std::static_pointer_cast<arrow::Int32Array>(value_slice), entry);
+              } break;
+              case arrow::Type::INT64: {
+                bindValue(std::static_pointer_cast<arrow::Int64Array>(value_slice), entry);
+              } break;
+              case arrow::Type::UINT8: {
+                bindValue(std::static_pointer_cast<arrow::UInt8Array>(value_slice), entry);
+              } break;
+              case arrow::Type::UINT16: {
+                bindValue(std::static_pointer_cast<arrow::UInt16Array>(value_slice), entry);
+              } break;
+              case arrow::Type::UINT32: {
+                bindValue(std::static_pointer_cast<arrow::UInt32Array>(value_slice), entry);
+              } break;
+              case arrow::Type::UINT64: {
+                bindValue(std::static_pointer_cast<arrow::UInt64Array>(value_slice), entry);
+              } break;
+              default: {
+                throw runtime_error("Unsupported kind of VLA");
+              } break;
+            }
+          } break;
+          case arrow::Type::FIXED_SIZE_LIST: {
+            entry->BindRawPtr<void>(token, (void*)(valueArrays[ci]->data()->buffers[1]->data() + pos * valueCount[ci] * valueTypes[ci]->byte_width()));
+          } break;
+          case arrow::Type::BOOL: {
+            // Not sure we actually need this
+            entry->BindRawPtr<bool>(token, (bool*)(valueArrays[ci]->data()->buffers[1]->data() + pos * 1));
+          } break;
+          default:
+            // By default we consider things scalars.
+            entry->BindRawPtr<void>(token, (void*)(valueArrays[ci]->data()->buffers[1]->data() + pos * valueTypes[ci]->byte_width()));
+            break;
+        }
+      }
+      mWriter->Fill(*entry);
+      ++pos;
+    }
+    // mWriter->CommitCluster();
+
+    return arrow::Status::OK();
+  }
+
+  arrow::Future<>
+    FinishInternal() override
+  {
+    return {};
+  };
+};
+
+arrow::Result<std::shared_ptr<arrow::Schema>> RNTupleFileFormat::Inspect(const arrow::dataset::FileSource& source) const
+{
+
+  auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
+  // Actually get the TTree from the ROOT file.
+  auto ntupleFs = std::dynamic_pointer_cast<RNTupleFileSystem>(fs->GetSubFilesystem(source));
+  if (!ntupleFs.get()) {
+    throw runtime_error_f("Unknown filesystem %s\n", source.filesystem()->type_name().c_str());
+  }
+  ROOT::Experimental::RNTuple* rntuple = ntupleFs->GetRNTuple(source);
+
+  auto inspector = ROOT::Experimental::RNTupleInspector::Create(rntuple);
+
+  auto reader = ROOT::Experimental::RNTupleReader::Open(rntuple);
+
+  auto& tupleField0 = reader->GetModel().GetFieldZero();
+  std::vector<std::shared_ptr<arrow::Field>> fields;
+  for (auto& tupleField : tupleField0.GetSubFields()) {
+    auto field = std::make_shared<arrow::Field>(tupleField->GetFieldName(), arrowTypeFromRNTuple(*tupleField, tupleField->GetValueSize()));
+    fields.push_back(field);
+  }
+
+  return std::make_shared<arrow::Schema>(fields);
+}
+
+arrow::Result<arrow::RecordBatchGenerator> RNTupleFileFormat::ScanBatchesAsync(
+  const std::shared_ptr<arrow::dataset::ScanOptions>& options,
+  const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const
+{
+  auto dataset_schema = options->dataset_schema;
+  auto ntupleFragment = std::dynamic_pointer_cast<RNTupleFileFragment>(fragment);
+
+  auto generator = [pool = options->pool, ntupleFragment, dataset_schema, &totalCompressedSize = mTotCompressedSize,
+                    &totalUncompressedSize = mTotUncompressedSize]() -> arrow::Future<std::shared_ptr<arrow::RecordBatch>> {
+    using namespace ROOT::Experimental;
+    std::vector<std::shared_ptr<arrow::Array>> columns;
+    std::vector<std::shared_ptr<arrow::Field>> fields = dataset_schema->fields();
+
+    auto containerFS = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(ntupleFragment->source().filesystem());
+    auto fs = std::dynamic_pointer_cast<RNTupleFileSystem>(containerFS->GetSubFilesystem(ntupleFragment->source()));
+
+    int64_t rows = -1;
+    ROOT::Experimental::RNTuple* rntuple = fs->GetRNTuple(ntupleFragment->source());
+    auto reader = ROOT::Experimental::RNTupleReader::Open(rntuple);
+    auto& model = reader->GetModel();
+    for (auto& physicalField : fields) {
+      auto bulk = model.CreateBulk(physicalField->name());
+
+      auto listType = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type());
+
+      auto& descriptor = reader->GetDescriptor();
+      auto totalEntries = reader->GetNEntries();
+
+      if (rows == -1) {
+        rows = totalEntries;
+      }
+      if (rows != totalEntries) {
+        throw runtime_error_f("Unmatching number of rows for branch %s", physicalField->name().c_str());
+      }
+      arrow::Status status;
+      int readEntries = 0;
+      std::shared_ptr<arrow::Array> array;
+      if (physicalField->type() == arrow::boolean() ||
+          (listType && physicalField->type()->field(0)->type() == arrow::boolean())) {
+        if (listType) {
+          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
+          auto status = arrow::MakeBuilder(pool, physicalField->type()->field(0)->type(), &builder);
+          if (!status.ok()) {
+            throw runtime_error("Cannot create value builder");
+          }
+          auto listBuilder = std::make_unique<arrow::FixedSizeListBuilder>(pool, std::move(builder), listType->list_size());
+          auto valueBuilder = listBuilder.get()->value_builder();
+          // boolean array special case: we need to use builder to create the bitmap
+          status = valueBuilder->Reserve(totalEntries * listType->list_size());
+          status &= listBuilder->Reserve(totalEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to reserve memory for array builder");
+          }
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          // No adoption for now...
+          // bulk.AdoptBuffer(buffer, totalEntries)
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            void* ptr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries());
+            int readLast = index.GetNEntries();
+            readEntries += readLast;
+            status &= static_cast<arrow::BooleanBuilder*>(valueBuilder)->AppendValues(reinterpret_cast<uint8_t const*>(ptr), readLast * listType->list_size());
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+          status &= static_cast<arrow::FixedSizeListBuilder*>(listBuilder.get())->AppendValues(readEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to append values to array");
+          }
+          status &= listBuilder->Finish(&array);
+          if (!status.ok()) {
+            throw runtime_error("Failed to create array");
+          }
+        } else if (listType == nullptr) {
+          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
+          auto status = arrow::MakeBuilder(pool, physicalField->type(), &builder);
+          if (!status.ok()) {
+            throw runtime_error("Cannot create builder");
+          }
+          auto valueBuilder = static_cast<arrow::BooleanBuilder*>(builder.get());
+          // boolean array special case: we need to use builder to create the bitmap
+          status = valueBuilder->Reserve(totalEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to reserve memory for array builder");
+          }
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            void* ptr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries());
+            int readLast = index.GetNEntries();
+            readEntries += readLast;
+            status &= valueBuilder->AppendValues(reinterpret_cast<uint8_t const*>(ptr), readLast);
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+          if (!status.ok()) {
+            throw runtime_error("Failed to append values to array");
+          }
+          status &= valueBuilder->Finish(&array);
+          if (!status.ok()) {
+            throw runtime_error("Failed to create array");
+          }
+        }
+      } else {
+        // other types: use serialized read to build arrays directly.
+        auto typeSize = physicalField->type()->byte_width();
+        // FIXME: for now...
+        auto bytes = 0;
+        auto branchSize = bytes ? bytes : 1000000;
+        auto&& result = arrow::AllocateResizableBuffer(branchSize, pool);
+        if (!result.ok()) {
+          throw runtime_error("Cannot allocate values buffer");
+        }
+        std::shared_ptr<arrow::Buffer> arrowValuesBuffer = std::move(result).ValueUnsafe();
+        auto ptr = arrowValuesBuffer->mutable_data();
+        if (ptr == nullptr) {
+          throw runtime_error("Invalid buffer");
+        }
+
+        std::unique_ptr<TBufferFile> offsetBuffer = nullptr;
+
+        std::shared_ptr<arrow::Buffer> arrowOffsetBuffer;
+        std::span<int> offsets;
+        int size = 0;
+        uint32_t totalSize = 0;
+        int64_t listSize = 1;
+        if (auto fixedSizeList = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type())) {
+          listSize = fixedSizeList->list_size();
+          typeSize = fixedSizeList->field(0)->type()->byte_width();
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            void* inPtr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries());
+
+            int readLast = index.GetNEntries();
+            if (listSize == -1) {
+              size = offsets[readEntries + readLast] - offsets[readEntries];
+            } else {
+              size = readLast * listSize;
+            }
+            readEntries += readLast;
+            memcpy(ptr, inPtr, size * typeSize);
+            ptr += (ptrdiff_t)(size * typeSize);
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+        } else if (auto vlaListType = std::dynamic_pointer_cast<arrow::ListType>(physicalField->type())) {
+          listSize = -1;
+          typeSize = vlaListType->field(0)->type()->byte_width();
+          offsetBuffer = std::make_unique<TBufferFile>(TBuffer::EMode::kWrite, 4 * 1024 * 1024);
+          result = arrow::AllocateResizableBuffer((totalEntries + 1) * (int64_t)sizeof(int), pool);
+          if (!result.ok()) {
+            throw runtime_error("Cannot allocate offset buffer");
+          }
+          arrowOffsetBuffer = std::move(result).ValueUnsafe();
+
+          // Offset bulk
+          auto offsetBulk = model.CreateBulk(physicalField->name());
+          // Actual values are in a different place...
+          bulk = model.CreateBulk(physicalField->name());
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          auto* ptrOffset = reinterpret_cast<int*>(arrowOffsetBuffer->mutable_data());
+          auto* tPtrOffset = reinterpret_cast<int*>(ptrOffset);
+          offsets = std::span<int>{tPtrOffset, tPtrOffset + totalEntries + 1};
+
+          auto copyOffsets = [&arrowValuesBuffer, &pool, &ptrOffset, &ptr, &totalSize](auto inPtr, size_t total) {
+            using value_type = typename std::decay_t<decltype(*inPtr)>::value_type;
+            for (size_t i = 0; i < total; i++) {
+              *ptrOffset++ = totalSize;
+              totalSize += inPtr[i].size();
+            }
+            *ptrOffset = totalSize;
+            auto&& result = arrow::AllocateResizableBuffer(totalSize * sizeof(value_type), pool);
+            if (!result.ok()) {
+              throw runtime_error("Cannot allocate values buffer");
+            }
+            arrowValuesBuffer = std::move(result).ValueUnsafe();
+            ptr = (uint8_t*)(arrowValuesBuffer->mutable_data());
+            // Calculate the size of the buffer here.
+            for (size_t i = 0; i < total; i++) {
+              int vlaSizeInBytes = inPtr[i].size() * sizeof(value_type);
+              if (vlaSizeInBytes == 0) {
+                continue;
+              }
+              memcpy(ptr, inPtr[i].data(), vlaSizeInBytes);
+              ptr += vlaSizeInBytes;
+            }
+          };
+
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            int readLast = index.GetNEntries();
+            switch (vlaListType->field(0)->type()->id()) {
+              case arrow::Type::FLOAT: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<float>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::DOUBLE: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<double>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::INT8: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<int8_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::INT16: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<int16_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::INT32: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<int32_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::INT64: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<int64_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::UINT8: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<uint8_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::UINT16: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<uint16_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::UINT32: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<uint32_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              case arrow::Type::UINT64: {
+                copyOffsets((ROOT::Internal::VecOps::RVec<uint64_t>*)offsetBulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries()), readLast);
+              } break;
+              default: {
+                throw runtime_error("Unsupported kind of VLA");
+              } break;
+            }
+
+            readEntries += readLast;
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+        } else {
+          auto clusterIt = descriptor.FindClusterId(0, 0);
+          while (clusterIt != kInvalidDescriptorId) {
+            auto& index = descriptor.GetClusterDescriptor(clusterIt);
+            auto mask = std::make_unique<bool[]>(index.GetNEntries());
+            std::fill(mask.get(), mask.get() + index.GetNEntries(), true);
+            void* inPtr = bulk.ReadBulk(RClusterIndex(clusterIt, index.GetFirstEntryIndex()), mask.get(), index.GetNEntries());
+
+            int readLast = index.GetNEntries();
+            if (listSize == -1) {
+              size = offsets[readEntries + readLast] - offsets[readEntries];
+            } else {
+              size = readLast * listSize;
+            }
+            readEntries += readLast;
+            memcpy(ptr, inPtr, size * typeSize);
+            ptr += (ptrdiff_t)(size * typeSize);
+            clusterIt = descriptor.FindNextClusterId(clusterIt);
+          }
+        }
+        switch (listSize) {
+          case -1: {
+            auto varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
+            array = std::make_shared<arrow::ListArray>(physicalField->type(), readEntries, arrowOffsetBuffer, varray);
+          } break;
+          case 1: {
+            totalSize = readEntries * listSize;
+            array = std::make_shared<arrow::PrimitiveArray>(physicalField->type(), readEntries, arrowValuesBuffer);
+
+          } break;
+          default: {
+            totalSize = readEntries * listSize;
+            auto varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
+            array = std::make_shared<arrow::FixedSizeListArray>(physicalField->type(), readEntries, varray);
+          }
+        }
+      }
+      columns.push_back(array);
+    }
+
+    auto batch = arrow::RecordBatch::Make(dataset_schema, rows, columns);
+    return batch;
+  };
+
+  return generator;
+}
+
+arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> RNTupleFileFormat::MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination,
+                                                                                         std::shared_ptr<arrow::Schema> schema,
+                                                                                         std::shared_ptr<arrow::dataset::FileWriteOptions> options,
+                                                                                         arrow::fs::FileLocator destination_locator) const
+{
+  auto writer = std::make_shared<RNTupleFileWriter>(schema, options, destination, destination_locator);
+  return std::dynamic_pointer_cast<arrow::dataset::FileWriter>(writer);
+}
+
+arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> RNTupleFileFormat::MakeFragment(
+  arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
+  std::shared_ptr<arrow::Schema> physical_schema)
+{
+  std::shared_ptr<arrow::dataset::FileFormat> format = std::make_shared<RNTupleFileFormat>(mTotCompressedSize, mTotUncompressedSize);
+
+  auto fragment = std::make_shared<RNTupleFileFragment>(std::move(source), std::move(format),
+                                                        std::move(partition_expression),
+                                                        std::move(physical_schema));
+  return std::dynamic_pointer_cast<arrow::dataset::FileFragment>(fragment);
+}
+
+RNTupleFileSystem::~RNTupleFileSystem() = default;
+
+std::shared_ptr<arrow::dataset::FileWriteOptions>
+  RNTupleFileFormat::DefaultWriteOptions()
+{
+  return std::make_shared<RNTupleFileWriteOptions>(shared_from_this());
+}
+
+struct RNTuplePluginContext {
+  size_t totalCompressedSize = 0;
+  size_t totalUncompressedSize = 0;
+  std::shared_ptr<o2::framework::RNTupleFileFormat> format = nullptr;
+};
+
+struct RNTupleObjectReadingImplementation : public RootArrowFactoryPlugin {
+  RootArrowFactory* create() override
+  {
+    auto context = new RNTuplePluginContext;
+    context->format = std::make_shared<o2::framework::RNTupleFileFormat>(context->totalCompressedSize, context->totalUncompressedSize);
+    return new RootArrowFactory{
+      .options = [context]() { return context->format->DefaultWriteOptions(); },
+      .format = [context]() { return context->format; },
+      .getSubFilesystem = [](void* handle) {
+        auto rntuple = (ROOT::Experimental::RNTuple*)handle;
+        return std::shared_ptr<VirtualRootFileSystemBase>(new SingleRNTupleFileSystem(rntuple)); },
+    };
+  }
+};
+
+DEFINE_DPL_PLUGINS_BEGIN
+DEFINE_DPL_PLUGIN_INSTANCE(RNTupleObjectReadingImplementation, RootObjectReadingImplementation);
+DEFINE_DPL_PLUGINS_END
+} // namespace o2::framework
diff --git a/Framework/AnalysisSupport/src/TTreePlugin.cxx b/Framework/AnalysisSupport/src/TTreePlugin.cxx
new file mode 100644
index 0000000000000..e376ed8b96268
--- /dev/null
+++ b/Framework/AnalysisSupport/src/TTreePlugin.cxx
@@ -0,0 +1,862 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "Framework/RootArrowFilesystem.h"
+#include "Framework/Plugins.h"
+#include "Framework/Signpost.h"
+#include "Framework/Endian.h"
+#include <arrow/dataset/file_base.h>
+#include <arrow/util/key_value_metadata.h>
+#include <arrow/array/array_nested.h>
+#include <arrow/array/array_primitive.h>
+#include <arrow/array/builder_nested.h>
+#include <arrow/array/builder_primitive.h>
+#include <TTree.h>
+#include <TBranch.h>
+#include <TFile.h>
+#include <TLeaf.h>
+
+O2_DECLARE_DYNAMIC_LOG(root_arrow_fs);
+
+namespace o2::framework
+{
+
+class TTreeFileWriteOptions : public arrow::dataset::FileWriteOptions
+{
+ public:
+  TTreeFileWriteOptions(std::shared_ptr<arrow::dataset::FileFormat> format)
+    : FileWriteOptions(format)
+  {
+  }
+};
+
+// A filesystem which allows me to get a TTree
+class TTreeFileSystem : public VirtualRootFileSystemBase
+{
+ public:
+  ~TTreeFileSystem() override;
+
+  std::shared_ptr<VirtualRootFileSystemBase> GetSubFilesystem(arrow::dataset::FileSource source) override
+  {
+    return std::dynamic_pointer_cast<VirtualRootFileSystemBase>(shared_from_this());
+  };
+
+  arrow::Result<std::shared_ptr<arrow::io::OutputStream>> OpenOutputStream(
+    const std::string& path,
+    const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override;
+
+  virtual TTree* GetTree(arrow::dataset::FileSource source) = 0;
+};
+
+class SingleTreeFileSystem : public TTreeFileSystem
+{
+ public:
+  SingleTreeFileSystem(TTree* tree)
+    : TTreeFileSystem(),
+      mTree(tree)
+  {
+  }
+
+  arrow::Result<arrow::fs::FileInfo> GetFileInfo(std::string const& path) override;
+
+  std::string type_name() const override
+  {
+    return "ttree";
+  }
+
+  TTree* GetTree(arrow::dataset::FileSource) override
+  {
+    // Simply return the only TTree we have
+    return mTree;
+  }
+
+ private:
+  TTree* mTree;
+};
+
+arrow::Result<arrow::fs::FileInfo> SingleTreeFileSystem::GetFileInfo(std::string const& path)
+{
+  arrow::dataset::FileSource source(path, shared_from_this());
+  arrow::fs::FileInfo result;
+  result.set_path(path);
+  result.set_type(arrow::fs::FileType::File);
+  return result;
+}
+
+class TTreeFileFragment : public arrow::dataset::FileFragment
+{
+ public:
+  TTreeFileFragment(arrow::dataset::FileSource source,
+                    std::shared_ptr<arrow::dataset::FileFormat> format,
+                    arrow::compute::Expression partition_expression,
+                    std::shared_ptr<arrow::Schema> physical_schema)
+    : FileFragment(std::move(source), std::move(format), std::move(partition_expression), std::move(physical_schema))
+  {
+  }
+};
+
+class TTreeFileFormat : public arrow::dataset::FileFormat
+{
+  size_t& mTotCompressedSize;
+  size_t& mTotUncompressedSize;
+
+ public:
+  TTreeFileFormat(size_t& totalCompressedSize, size_t& totalUncompressedSize)
+    : FileFormat({}),
+      mTotCompressedSize(totalCompressedSize),
+      mTotUncompressedSize(totalUncompressedSize)
+  {
+  }
+
+  ~TTreeFileFormat() override = default;
+
+  std::string type_name() const override
+  {
+    return "ttree";
+  }
+
+  bool Equals(const FileFormat& other) const override
+  {
+    return other.type_name() == this->type_name();
+  }
+
+  arrow::Result<bool> IsSupported(const arrow::dataset::FileSource& source) const override
+  {
+    auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
+    auto subFs = fs->GetSubFilesystem(source);
+    if (std::dynamic_pointer_cast<TTreeFileSystem>(subFs)) {
+      return true;
+    }
+    return false;
+  }
+
+  arrow::Result<std::shared_ptr<arrow::Schema>> Inspect(const arrow::dataset::FileSource& source) const override;
+  /// \brief Create a FileFragment for a FileSource.
+  arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> MakeFragment(
+    arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
+    std::shared_ptr<arrow::Schema> physical_schema) override;
+
+  arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options, arrow::fs::FileLocator destination_locator) const override;
+
+  std::shared_ptr<arrow::dataset::FileWriteOptions> DefaultWriteOptions() override;
+
+  arrow::Result<arrow::RecordBatchGenerator> ScanBatchesAsync(
+    const std::shared_ptr<arrow::dataset::ScanOptions>& options,
+    const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const override;
+};
+
+// An arrow outputstream which allows to write to a TTree. Eventually
+// with a prefix for the branches.
+class TTreeOutputStream : public arrow::io::OutputStream
+{
+ public:
+  TTreeOutputStream(TTree*, std::string branchPrefix);
+
+  arrow::Status Close() override;
+
+  arrow::Result<int64_t> Tell() const override;
+
+  arrow::Status Write(const void* data, int64_t nbytes) override;
+
+  bool closed() const override;
+
+  TBranch* CreateBranch(char const* branchName, char const* sizeBranch);
+
+  TTree* GetTree()
+  {
+    return mTree;
+  }
+
+ private:
+  TTree* mTree;
+  std::string mBranchPrefix;
+};
+
+// An arrow outputstream which allows to write to a ttree
+// @a branch prefix is to be used to identify a set of branches which all belong to
+// the same table.
+TTreeOutputStream::TTreeOutputStream(TTree* f, std::string branchPrefix)
+  : mTree(f),
+    mBranchPrefix(std::move(branchPrefix))
+{
+}
+
+arrow::Status TTreeOutputStream::Close()
+{
+  if (mTree->GetCurrentFile() == nullptr) {
+    return arrow::Status::Invalid("Cannot close a tree not attached to a file");
+  }
+  mTree->GetCurrentFile()->Close();
+  return arrow::Status::OK();
+}
+
+arrow::Result<int64_t> TTreeOutputStream::Tell() const
+{
+  return arrow::Result<int64_t>(arrow::Status::NotImplemented("Cannot move"));
+}
+
+arrow::Status TTreeOutputStream::Write(const void* data, int64_t nbytes)
+{
+  return arrow::Status::NotImplemented("Cannot write raw bytes to a TTree");
+}
+
+bool TTreeOutputStream::closed() const
+{
+  // A standalone tree is never closed.
+  if (mTree->GetCurrentFile() == nullptr) {
+    return false;
+  }
+  return mTree->GetCurrentFile()->IsOpen() == false;
+}
+
+TBranch* TTreeOutputStream::CreateBranch(char const* branchName, char const* sizeBranch)
+{
+  return mTree->Branch((mBranchPrefix + "/" + branchName).c_str(), (char*)nullptr, (mBranchPrefix + sizeBranch).c_str());
+}
+
+struct TTreePluginContext {
+  size_t totalCompressedSize = 0;
+  size_t totalUncompressedSize = 0;
+  std::shared_ptr<o2::framework::TTreeFileFormat> format = nullptr;
+};
+
+struct TTreeObjectReadingImplementation : public RootArrowFactoryPlugin {
+  RootArrowFactory* create() override
+  {
+    auto context = new TTreePluginContext;
+    context->format = std::make_shared<o2::framework::TTreeFileFormat>(context->totalCompressedSize, context->totalUncompressedSize);
+    return new RootArrowFactory{
+      .options = [context]() { return context->format->DefaultWriteOptions(); },
+      .format = [context]() { return context->format; },
+      .getSubFilesystem = [](void* handle) {
+        auto tree = (TTree*)handle;
+        return std::shared_ptr<VirtualRootFileSystemBase>(new SingleTreeFileSystem(tree)); },
+    };
+  }
+};
+
+arrow::Result<arrow::RecordBatchGenerator> TTreeFileFormat::ScanBatchesAsync(
+  const std::shared_ptr<arrow::dataset::ScanOptions>& options,
+  const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const
+{
+  // Get the fragment as a TTreeFragment. This might be PART of a TTree.
+  auto treeFragment = std::dynamic_pointer_cast<TTreeFileFragment>(fragment);
+  // This is the schema we want to read
+  auto dataset_schema = options->dataset_schema;
+
+  auto generator = [pool = options->pool, treeFragment, dataset_schema, &totalCompressedSize = mTotCompressedSize,
+                    &totalUncompressedSize = mTotUncompressedSize]() -> arrow::Future<std::shared_ptr<arrow::RecordBatch>> {
+    auto schema = treeFragment->format()->Inspect(treeFragment->source());
+
+    std::vector<std::shared_ptr<arrow::Array>> columns;
+    std::vector<std::shared_ptr<arrow::Field>> fields = dataset_schema->fields();
+    auto physical_schema = *treeFragment->ReadPhysicalSchema();
+
+    static TBufferFile buffer{TBuffer::EMode::kWrite, 4 * 1024 * 1024};
+    auto containerFS = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(treeFragment->source().filesystem());
+    auto fs = std::dynamic_pointer_cast<TTreeFileSystem>(containerFS->GetSubFilesystem(treeFragment->source()));
+
+    int64_t rows = -1;
+    TTree* tree = fs->GetTree(treeFragment->source());
+    for (auto& field : fields) {
+      // The field actually on disk
+      auto physicalField = physical_schema->GetFieldByName(field->name());
+      TBranch* branch = tree->GetBranch(physicalField->name().c_str());
+      assert(branch);
+      buffer.Reset();
+      auto totalEntries = branch->GetEntries();
+      if (rows == -1) {
+        rows = totalEntries;
+      }
+      if (rows != totalEntries) {
+        throw runtime_error_f("Unmatching number of rows for branch %s", branch->GetName());
+      }
+      arrow::Status status;
+      int readEntries = 0;
+      std::shared_ptr<arrow::Array> array;
+      auto listType = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type());
+      if (physicalField->type() == arrow::boolean() ||
+          (listType && physicalField->type()->field(0)->type() == arrow::boolean())) {
+        if (listType) {
+          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
+          auto status = arrow::MakeBuilder(pool, physicalField->type()->field(0)->type(), &builder);
+          if (!status.ok()) {
+            throw runtime_error("Cannot create value builder");
+          }
+          auto listBuilder = std::make_unique<arrow::FixedSizeListBuilder>(pool, std::move(builder), listType->list_size());
+          auto valueBuilder = listBuilder.get()->value_builder();
+          // boolean array special case: we need to use builder to create the bitmap
+          status = valueBuilder->Reserve(totalEntries * listType->list_size());
+          status &= listBuilder->Reserve(totalEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to reserve memory for array builder");
+          }
+          while (readEntries < totalEntries) {
+            auto readLast = branch->GetBulkRead().GetBulkEntries(readEntries, buffer);
+            readEntries += readLast;
+            status &= static_cast<arrow::BooleanBuilder*>(valueBuilder)->AppendValues(reinterpret_cast<uint8_t const*>(buffer.GetCurrent()), readLast * listType->list_size());
+          }
+          status &= static_cast<arrow::FixedSizeListBuilder*>(listBuilder.get())->AppendValues(readEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to append values to array");
+          }
+          status &= listBuilder->Finish(&array);
+          if (!status.ok()) {
+            throw runtime_error("Failed to create array");
+          }
+        } else if (listType == nullptr) {
+          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
+          auto status = arrow::MakeBuilder(pool, physicalField->type(), &builder);
+          if (!status.ok()) {
+            throw runtime_error("Cannot create builder");
+          }
+          auto valueBuilder = static_cast<arrow::BooleanBuilder*>(builder.get());
+          // boolean array special case: we need to use builder to create the bitmap
+          status = valueBuilder->Reserve(totalEntries);
+          if (!status.ok()) {
+            throw runtime_error("Failed to reserve memory for array builder");
+          }
+          while (readEntries < totalEntries) {
+            auto readLast = branch->GetBulkRead().GetBulkEntries(readEntries, buffer);
+            readEntries += readLast;
+            status &= valueBuilder->AppendValues(reinterpret_cast<uint8_t const*>(buffer.GetCurrent()), readLast);
+          }
+          if (!status.ok()) {
+            throw runtime_error("Failed to append values to array");
+          }
+          status &= valueBuilder->Finish(&array);
+          if (!status.ok()) {
+            throw runtime_error("Failed to create array");
+          }
+        }
+      } else {
+        // other types: use serialized read to build arrays directly.
+        auto typeSize = physicalField->type()->byte_width();
+        // This is needed for branches which have not been persisted.
+        auto bytes = branch->GetTotBytes();
+        auto branchSize = bytes ? bytes : 1000000;
+        auto&& result = arrow::AllocateResizableBuffer(branchSize, pool);
+        if (!result.ok()) {
+          throw runtime_error("Cannot allocate values buffer");
+        }
+        std::shared_ptr<arrow::Buffer> arrowValuesBuffer = std::move(result).ValueUnsafe();
+        auto ptr = arrowValuesBuffer->mutable_data();
+        if (ptr == nullptr) {
+          throw runtime_error("Invalid buffer");
+        }
+
+        std::unique_ptr<TBufferFile> offsetBuffer = nullptr;
+
+        uint32_t offset = 0;
+        int count = 0;
+        std::shared_ptr<arrow::Buffer> arrowOffsetBuffer;
+        std::span<int> offsets;
+        int size = 0;
+        uint32_t totalSize = 0;
+        TBranch* mSizeBranch = nullptr;
+        int64_t listSize = 1;
+        if (auto fixedSizeList = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type())) {
+          listSize = fixedSizeList->list_size();
+          typeSize = fixedSizeList->field(0)->type()->byte_width();
+        } else if (auto vlaListType = std::dynamic_pointer_cast<arrow::ListType>(physicalField->type())) {
+          listSize = -1;
+          typeSize = vlaListType->field(0)->type()->byte_width();
+        }
+        if (listSize == -1) {
+          mSizeBranch = branch->GetTree()->GetBranch((std::string{branch->GetName()} + "_size").c_str());
+          offsetBuffer = std::make_unique<TBufferFile>(TBuffer::EMode::kWrite, 4 * 1024 * 1024);
+          result = arrow::AllocateResizableBuffer((totalEntries + 1) * (int64_t)sizeof(int), pool);
+          if (!result.ok()) {
+            throw runtime_error("Cannot allocate offset buffer");
+          }
+          arrowOffsetBuffer = std::move(result).ValueUnsafe();
+          unsigned char* ptrOffset = arrowOffsetBuffer->mutable_data();
+          auto* tPtrOffset = reinterpret_cast<int*>(ptrOffset);
+          offsets = std::span<int>{tPtrOffset, tPtrOffset + totalEntries + 1};
+
+          // read sizes first
+          while (readEntries < totalEntries) {
+            auto readLast = mSizeBranch->GetBulkRead().GetEntriesSerialized(readEntries, *offsetBuffer);
+            readEntries += readLast;
+            for (auto i = 0; i < readLast; ++i) {
+              offsets[count++] = (int)offset;
+              offset += swap32_(reinterpret_cast<uint32_t*>(offsetBuffer->GetCurrent())[i]);
+            }
+          }
+          offsets[count] = (int)offset;
+          totalSize = offset;
+          readEntries = 0;
+        }
+
+        while (readEntries < totalEntries) {
+          auto readLast = branch->GetBulkRead().GetEntriesSerialized(readEntries, buffer);
+          if (listSize == -1) {
+            size = offsets[readEntries + readLast] - offsets[readEntries];
+          } else {
+            size = readLast * listSize;
+          }
+          readEntries += readLast;
+          swapCopy(ptr, buffer.GetCurrent(), size, typeSize);
+          ptr += (ptrdiff_t)(size * typeSize);
+        }
+        if (listSize >= 1) {
+          totalSize = readEntries * listSize;
+        }
+        std::shared_ptr<arrow::PrimitiveArray> varray;
+        switch (listSize) {
+          case -1:
+            varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
+            array = std::make_shared<arrow::ListArray>(physicalField->type(), readEntries, arrowOffsetBuffer, varray);
+            break;
+          case 1:
+            array = std::make_shared<arrow::PrimitiveArray>(physicalField->type(), readEntries, arrowValuesBuffer);
+            break;
+          default:
+            varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
+            array = std::make_shared<arrow::FixedSizeListArray>(physicalField->type(), readEntries, varray);
+        }
+      }
+
+      branch->SetStatus(false);
+      branch->DropBaskets("all");
+      branch->Reset();
+      branch->GetTransientBuffer(0)->Expand(0);
+
+      columns.push_back(array);
+    }
+    auto batch = arrow::RecordBatch::Make(dataset_schema, rows, columns);
+    totalCompressedSize += tree->GetZipBytes();
+    totalUncompressedSize += tree->GetTotBytes();
+    return batch;
+  };
+  return generator;
+}
+
+char const* rootSuffixFromArrow(arrow::Type::type id)
+{
+  switch (id) {
+    case arrow::Type::BOOL:
+      return "/O";
+    case arrow::Type::UINT8:
+      return "/b";
+    case arrow::Type::UINT16:
+      return "/s";
+    case arrow::Type::UINT32:
+      return "/i";
+    case arrow::Type::UINT64:
+      return "/l";
+    case arrow::Type::INT8:
+      return "/B";
+    case arrow::Type::INT16:
+      return "/S";
+    case arrow::Type::INT32:
+      return "/I";
+    case arrow::Type::INT64:
+      return "/L";
+    case arrow::Type::FLOAT:
+      return "/F";
+    case arrow::Type::DOUBLE:
+      return "/D";
+    default:
+      throw runtime_error("Unsupported arrow column type");
+  }
+}
+
+arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenOutputStream(
+  const std::string& path,
+  const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
+{
+  arrow::dataset::FileSource source{path, shared_from_this()};
+  auto prefix = metadata->Get("branch_prefix");
+  if (prefix.ok()) {
+    return std::make_shared<TTreeOutputStream>(GetTree(source), *prefix);
+  }
+  return std::make_shared<TTreeOutputStream>(GetTree(source), "");
+}
+
+namespace
+{
+struct BranchInfo {
+  std::string name;
+  TBranch* ptr;
+  bool mVLA;
+};
+} // namespace
+
+auto arrowTypeFromROOT(EDataType type, int size)
+{
+  auto typeGenerator = [](std::shared_ptr<arrow::DataType> const& type, int size) -> std::shared_ptr<arrow::DataType> {
+    switch (size) {
+      case -1:
+        return arrow::list(type);
+      case 1:
+        return std::move(type);
+      default:
+        return arrow::fixed_size_list(type, size);
+    }
+  };
+
+  switch (type) {
+    case EDataType::kBool_t:
+      return typeGenerator(arrow::boolean(), size);
+    case EDataType::kUChar_t:
+      return typeGenerator(arrow::uint8(), size);
+    case EDataType::kUShort_t:
+      return typeGenerator(arrow::uint16(), size);
+    case EDataType::kUInt_t:
+      return typeGenerator(arrow::uint32(), size);
+    case EDataType::kULong64_t:
+      return typeGenerator(arrow::uint64(), size);
+    case EDataType::kChar_t:
+      return typeGenerator(arrow::int8(), size);
+    case EDataType::kShort_t:
+      return typeGenerator(arrow::int16(), size);
+    case EDataType::kInt_t:
+      return typeGenerator(arrow::int32(), size);
+    case EDataType::kLong64_t:
+      return typeGenerator(arrow::int64(), size);
+    case EDataType::kFloat_t:
+      return typeGenerator(arrow::float32(), size);
+    case EDataType::kDouble_t:
+      return typeGenerator(arrow::float64(), size);
+    default:
+      throw o2::framework::runtime_error_f("Unsupported branch type: %d", static_cast<int>(type));
+  }
+}
+
+arrow::Result<std::shared_ptr<arrow::Schema>> TTreeFileFormat::Inspect(const arrow::dataset::FileSource& source) const
+{
+  arrow::Schema schema{{}};
+  auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
+  // Actually get the TTree from the ROOT file.
+  auto treeFs = std::dynamic_pointer_cast<TTreeFileSystem>(fs->GetSubFilesystem(source));
+  if (!treeFs.get()) {
+    throw runtime_error_f("Unknown filesystem %s\n", source.filesystem()->type_name().c_str());
+  }
+  TTree* tree = treeFs->GetTree(source);
+
+  auto branches = tree->GetListOfBranches();
+  auto n = branches->GetEntries();
+
+  std::vector<BranchInfo> branchInfos;
+  for (auto i = 0; i < n; ++i) {
+    auto branch = static_cast<TBranch*>(branches->At(i));
+    auto name = std::string{branch->GetName()};
+    auto pos = name.find("_size");
+    if (pos != std::string::npos) {
+      name.erase(pos);
+      branchInfos.emplace_back(BranchInfo{name, (TBranch*)nullptr, true});
+    } else {
+      auto lookup = std::find_if(branchInfos.begin(), branchInfos.end(), [&](BranchInfo const& bi) {
+        return bi.name == name;
+      });
+      if (lookup == branchInfos.end()) {
+        branchInfos.emplace_back(BranchInfo{name, branch, false});
+      } else {
+        lookup->ptr = branch;
+      }
+    }
+  }
+
+  std::vector<std::shared_ptr<arrow::Field>> fields;
+  tree->SetCacheSize(25000000);
+  for (auto& bi : branchInfos) {
+    static TClass* cls;
+    EDataType type;
+    bi.ptr->GetExpectedType(cls, type);
+    auto listSize = -1;
+    if (!bi.mVLA) {
+      listSize = static_cast<TLeaf*>(bi.ptr->GetListOfLeaves()->At(0))->GetLenStatic();
+    }
+    auto field = std::make_shared<arrow::Field>(bi.ptr->GetName(), arrowTypeFromROOT(type, listSize));
+    fields.push_back(field);
+
+    tree->AddBranchToCache(bi.ptr);
+    if (strncmp(bi.ptr->GetName(), "fIndexArray", strlen("fIndexArray")) == 0) {
+      std::string sizeBranchName = bi.ptr->GetName();
+      sizeBranchName += "_size";
+      auto* sizeBranch = (TBranch*)tree->GetBranch(sizeBranchName.c_str());
+      if (sizeBranch) {
+        tree->AddBranchToCache(sizeBranch);
+      }
+    }
+  }
+  tree->StopCacheLearningPhase();
+
+  return std::make_shared<arrow::Schema>(fields);
+}
+
+/// \brief Create a FileFragment for a FileSource.
+arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> TTreeFileFormat::MakeFragment(
+  arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
+  std::shared_ptr<arrow::Schema> physical_schema)
+{
+  std::shared_ptr<arrow::dataset::FileFormat> format = std::make_shared<TTreeFileFormat>(mTotCompressedSize, mTotUncompressedSize);
+
+  auto fragment = std::make_shared<TTreeFileFragment>(std::move(source), std::move(format),
+                                                      std::move(partition_expression),
+                                                      std::move(physical_schema));
+  return std::dynamic_pointer_cast<arrow::dataset::FileFragment>(fragment);
+}
+
+class TTreeFileWriter : public arrow::dataset::FileWriter
+{
+  std::vector<TBranch*> branches;
+  std::vector<TBranch*> sizesBranches;
+  std::vector<std::shared_ptr<arrow::Array>> valueArrays;
+  std::vector<std::shared_ptr<arrow::Array>> sizeArrays;
+  std::vector<std::shared_ptr<arrow::DataType>> valueTypes;
+
+  std::vector<int64_t> valuesIdealBasketSize;
+  std::vector<int64_t> sizeIdealBasketSize;
+
+  std::vector<int64_t> typeSizes;
+  std::vector<int64_t> listSizes;
+  bool firstBasket = true;
+
+  // This is to create a batsket size according to the first batch.
+  void finaliseBasketSize(std::shared_ptr<arrow::RecordBatch> firstBatch)
+  {
+    O2_SIGNPOST_ID_FROM_POINTER(sid, root_arrow_fs, this);
+    O2_SIGNPOST_START(root_arrow_fs, sid, "finaliseBasketSize", "First batch with %lli rows received and %zu columns",
+                      firstBatch->num_rows(), firstBatch->columns().size());
+    for (size_t i = 0; i < branches.size(); i++) {
+      auto* branch = branches[i];
+      auto* sizeBranch = sizesBranches[i];
+
+      int valueSize = valueTypes[i]->byte_width();
+      if (listSizes[i] == 1) {
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s exists and uses %d bytes per entry for %lli entries.",
+                               branch->GetName(), valueSize, firstBatch->num_rows());
+        assert(sizeBranch == nullptr);
+        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize);
+      } else if (listSizes[i] == -1) {
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s exists and uses %d bytes per entry.",
+                               branch->GetName(), valueSize);
+        // This should probably lookup the
+        auto column = firstBatch->GetColumnByName(schema_->field(i)->name());
+        auto list = std::static_pointer_cast<arrow::ListArray>(column);
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s needed. Associated size branch %s and there are %lli entries of size %d in that list.",
+                               branch->GetName(), sizeBranch->GetName(), list->length(), valueSize);
+        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize * list->length());
+        sizeBranch->SetBasketSize(1024 + firstBatch->num_rows() * 4);
+      } else {
+        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s needed. There are %lli entries per array of size %d in that list.",
+                               branch->GetName(), listSizes[i], valueSize);
+        assert(sizeBranch == nullptr);
+        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize * listSizes[i]);
+      }
+
+      auto field = firstBatch->schema()->field(i);
+      if (field->name().starts_with("fIndexArray")) {
+        // One int per array to keep track of the size
+        int idealBasketSize = 4 * firstBatch->num_rows() + 1024 + field->type()->byte_width() * firstBatch->num_rows(); // minimal additional size needed, otherwise we get 2 baskets
+        int basketSize = std::max(32000, idealBasketSize);                                                              // keep a minimum value
+        sizeBranch->SetBasketSize(basketSize);
+        branch->SetBasketSize(basketSize);
+      }
+    }
+    O2_SIGNPOST_END(root_arrow_fs, sid, "finaliseBasketSize", "Done");
+  }
+
+ public:
+  // Create the TTree based on the physical_schema, not the one in the batch.
+  // The write method will have to reconcile the two schemas.
+  TTreeFileWriter(std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options,
+                  std::shared_ptr<arrow::io::OutputStream> destination,
+                  arrow::fs::FileLocator destination_locator)
+    : FileWriter(schema, options, destination, destination_locator)
+  {
+    // Batches have the same number of entries for each column.
+    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
+    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
+
+    if (directoryStream.get()) {
+      TDirectoryFile* dir = directoryStream->GetDirectory();
+      dir->cd();
+      auto* tree = new TTree(destination_locator_.path.c_str(), "");
+      treeStream = std::make_shared<TTreeOutputStream>(tree, "");
+    } else if (treeStream.get()) {
+      // We already have a tree stream, let's derive a new one
+      // with the destination_locator_.path as prefix for the branches
+      // This way we can multiplex multiple tables in the same tree.
+      auto tree = treeStream->GetTree();
+      treeStream = std::make_shared<TTreeOutputStream>(tree, destination_locator_.path);
+    } else {
+      // I could simply set a prefix here to merge to an already existing tree.
+      throw std::runtime_error("Unsupported backend.");
+    }
+
+    for (auto i = 0u; i < schema->fields().size(); ++i) {
+      auto& field = schema->field(i);
+      listSizes.push_back(1);
+
+      int valuesIdealBasketSize = 0;
+      // Construct all the needed branches.
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          listSizes.back() = std::static_pointer_cast<arrow::FixedSizeListType>(field->type())->list_size();
+          valuesIdealBasketSize = 1024 + valueTypes.back()->byte_width() * listSizes.back();
+          valueTypes.push_back(field->type()->field(0)->type());
+          sizesBranches.push_back(nullptr);
+          std::string leafList = fmt::format("{}[{}]{}", field->name(), listSizes.back(), rootSuffixFromArrow(valueTypes.back()->id()));
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
+        } break;
+        case arrow::Type::LIST: {
+          valueTypes.push_back(field->type()->field(0)->type());
+          std::string leafList = fmt::format("{}[{}_size]{}", field->name(), field->name(), rootSuffixFromArrow(valueTypes.back()->id()));
+          listSizes.back() = -1; // VLA, we need to calculate it on the fly;
+          std::string sizeLeafList = field->name() + "_size/I";
+          sizesBranches.push_back(treeStream->CreateBranch((field->name() + "_size").c_str(), sizeLeafList.c_str()));
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
+          // Notice that this could be replaced by a better guess of the
+          // average size of the list elements, but this is not trivial.
+        } break;
+        default: {
+          valueTypes.push_back(field->type());
+          std::string leafList = field->name() + rootSuffixFromArrow(valueTypes.back()->id());
+          sizesBranches.push_back(nullptr);
+          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
+        } break;
+      }
+    }
+    // We create the branches from the schema
+  }
+
+  arrow::Status Write(const std::shared_ptr<arrow::RecordBatch>& batch) override
+  {
+    if (firstBasket) {
+      firstBasket = false;
+      finaliseBasketSize(batch);
+    }
+
+    // Support writing empty tables
+    if (batch->columns().empty() || batch->num_rows() == 0) {
+      return arrow::Status::OK();
+    }
+
+    // Batches have the same number of entries for each column.
+    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
+    TTree* tree = nullptr;
+    if (directoryStream.get()) {
+      TDirectoryFile* dir = directoryStream->GetDirectory();
+      tree = (TTree*)dir->Get(destination_locator_.path.c_str());
+    }
+    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
+
+    if (!tree) {
+      // I could simply set a prefix here to merge to an already existing tree.
+      throw std::runtime_error("Unsupported backend.");
+    }
+
+    for (auto i = 0u; i < batch->columns().size(); ++i) {
+      auto column = batch->column(i);
+      auto& field = batch->schema()->field(i);
+
+      valueArrays.push_back(nullptr);
+
+      switch (field->type()->id()) {
+        case arrow::Type::FIXED_SIZE_LIST: {
+          auto list = std::static_pointer_cast<arrow::FixedSizeListArray>(column);
+          valueArrays.back() = list->values();
+        } break;
+        case arrow::Type::LIST: {
+          auto list = std::static_pointer_cast<arrow::ListArray>(column);
+          valueArrays.back() = list;
+        } break;
+        case arrow::Type::BOOL: {
+          // In case of arrays of booleans, we need to go back to their
+          // char based representation for ROOT to save them.
+          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(column);
+
+          int64_t length = boolArray->length();
+          arrow::UInt8Builder builder;
+          auto ok = builder.Reserve(length);
+
+          for (int64_t i = 0; i < length; ++i) {
+            if (boolArray->IsValid(i)) {
+              // Expand each boolean value (true/false) to uint8 (1/0)
+              uint8_t value = boolArray->Value(i) ? 1 : 0;
+              auto ok = builder.Append(value);
+            } else {
+              // Append null for invalid entries
+              auto ok = builder.AppendNull();
+            }
+          }
+          valueArrays.back() = *builder.Finish();
+        } break;
+        default:
+          valueArrays.back() = column;
+      }
+    }
+
+    int64_t pos = 0;
+    while (pos < batch->num_rows()) {
+      for (size_t bi = 0; bi < branches.size(); ++bi) {
+        auto* branch = branches[bi];
+        auto* sizeBranch = sizesBranches[bi];
+        auto array = batch->column(bi);
+        auto& field = batch->schema()->field(bi);
+        auto& listSize = listSizes[bi];
+        auto valueType = valueTypes[bi];
+        auto valueArray = valueArrays[bi];
+
+        switch (field->type()->id()) {
+          case arrow::Type::LIST: {
+            auto list = std::static_pointer_cast<arrow::ListArray>(array);
+            listSize = list->value_length(pos);
+            uint8_t const* buffer = std::static_pointer_cast<arrow::PrimitiveArray>(valueArray)->values()->data() + array->offset() + list->value_offset(pos) * valueType->byte_width();
+            branch->SetAddress((void*)buffer);
+            sizeBranch->SetAddress(&listSize);
+          };
+            break;
+          case arrow::Type::FIXED_SIZE_LIST:
+          default: {
+            uint8_t const* buffer = std::static_pointer_cast<arrow::PrimitiveArray>(valueArray)->values()->data() + array->offset() + pos * listSize * valueType->byte_width();
+            branch->SetAddress((void*)buffer);
+          };
+        }
+      }
+      tree->Fill();
+      ++pos;
+    }
+    return arrow::Status::OK();
+  }
+
+  arrow::Future<> FinishInternal() override
+  {
+    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
+    TTree* tree = treeStream->GetTree();
+    tree->Write("", TObject::kOverwrite);
+    tree->SetDirectory(nullptr);
+
+    return {};
+  };
+};
+arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> TTreeFileFormat::MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options, arrow::fs::FileLocator destination_locator) const
+{
+  auto writer = std::make_shared<TTreeFileWriter>(schema, options, destination, destination_locator);
+  return std::dynamic_pointer_cast<arrow::dataset::FileWriter>(writer);
+}
+
+std::shared_ptr<arrow::dataset::FileWriteOptions> TTreeFileFormat::DefaultWriteOptions()
+{
+  std::shared_ptr<TTreeFileWriteOptions> options(
+    new TTreeFileWriteOptions(shared_from_this()));
+  return options;
+}
+
+TTreeFileSystem::~TTreeFileSystem() = default;
+
+DEFINE_DPL_PLUGINS_BEGIN
+DEFINE_DPL_PLUGIN_INSTANCE(TTreeObjectReadingImplementation, RootObjectReadingImplementation);
+DEFINE_DPL_PLUGINS_END
+} // namespace o2::framework
diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt
index 5cdd1241ecfb0..c1214a8f56beb 100644
--- a/Framework/Core/CMakeLists.txt
+++ b/Framework/Core/CMakeLists.txt
@@ -270,6 +270,10 @@ o2_add_test(Timers NAME test_Framework_test_Timers
             LABELS framework
             PUBLIC_LINK_LIBRARIES O2::Framework)
 
+o2_add_executable(framework-ao2d-to-ao3d
+            SOURCES test/o2AO2DToAO3D.cxx
+            PUBLIC_LINK_LIBRARIES O2::Framework)
+
 # FIXME: make this a proper test, when it actually does not hang.
 o2_add_executable(test-framework-ConsumeWhenAllOrdered
             SOURCES test/test_ConsumeWhenAllOrdered.cxx
@@ -299,6 +303,7 @@ add_executable(o2-test-framework-root
 target_link_libraries(o2-test-framework-root PRIVATE O2::Framework)
 target_link_libraries(o2-test-framework-root PRIVATE O2::Catch2)
 target_link_libraries(o2-test-framework-root PRIVATE ROOT::ROOTDataFrame)
+target_link_libraries(o2-test-framework-root PRIVATE ROOT::ROOTNTuple)
 set_property(TARGET o2-test-framework-root PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
 add_test(NAME framework:root COMMAND o2-test-framework-root --skip-benchmarks)
 add_test(NAME framework:crash COMMAND  sh -e -c "PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:$PATH ${CMAKE_CURRENT_LIST_DIR}/test/test_AllCrashTypes.sh")
diff --git a/Framework/Core/include/Framework/Plugins.h b/Framework/Core/include/Framework/Plugins.h
index 23d55a512e1fa..925943c6bffc3 100644
--- a/Framework/Core/include/Framework/Plugins.h
+++ b/Framework/Core/include/Framework/Plugins.h
@@ -36,6 +36,14 @@ enum struct DplPluginKind : int {
   // set, you might want to load metadata from it and attach it to the
   // configuration.
   Capability,
+  // A RootObjectReadingCapability is used to discover if there is away
+  // to read and understand an object serialised with ROOT.
+  RootObjectReadingCapability,
+
+  // A RootObjectReadingImplementation is actually used to read said object
+  // using the arrow dataset API
+  RootObjectReadingImplementation,
+
   // A plugin which was not initialised properly.
   Unknown
 };
diff --git a/Framework/Core/include/Framework/RootArrowFilesystem.h b/Framework/Core/include/Framework/RootArrowFilesystem.h
index 48d817bc9ddf2..8744656e7d55d 100644
--- a/Framework/Core/include/Framework/RootArrowFilesystem.h
+++ b/Framework/Core/include/Framework/RootArrowFilesystem.h
@@ -11,6 +11,7 @@
 #ifndef O2_FRAMEWORK_ROOT_ARROW_FILESYSTEM_H_
 #define O2_FRAMEWORK_ROOT_ARROW_FILESYSTEM_H_
 
+#include <TBufferFile.h>
 #include <arrow/dataset/type_fwd.h>
 #include <arrow/dataset/file_base.h>
 #include <arrow/filesystem/type_fwd.h>
@@ -18,23 +19,12 @@
 #include <memory>
 
 class TFile;
-class TBranch;
-class TTree;
 class TBufferFile;
 class TDirectoryFile;
 
 namespace o2::framework
 {
 
-class TTreeFileWriteOptions : public arrow::dataset::FileWriteOptions
-{
- public:
-  TTreeFileWriteOptions(std::shared_ptr<arrow::dataset::FileFormat> format)
-    : FileWriteOptions(format)
-  {
-  }
-};
-
 // This is to avoid having to implement a bunch of unimplemented methods
 // for all the possible virtual filesystem we can invent on top of ROOT
 // data structures.
@@ -79,46 +69,43 @@ class VirtualRootFileSystemBase : public arrow::fs::FileSystem
     const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override;
 };
 
-// A filesystem which allows me to get a TTree
-class TTreeFileSystem : public VirtualRootFileSystemBase
-{
- public:
-  ~TTreeFileSystem() override;
-
-  std::shared_ptr<VirtualRootFileSystemBase> GetSubFilesystem(arrow::dataset::FileSource source) override
-  {
-    return std::dynamic_pointer_cast<VirtualRootFileSystemBase>(shared_from_this());
-  };
-
-  arrow::Result<std::shared_ptr<arrow::io::OutputStream>> OpenOutputStream(
-    const std::string& path,
-    const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override;
-
-  virtual TTree* GetTree(arrow::dataset::FileSource source) = 0;
+struct RootArrowFactory final {
+  std::function<std::shared_ptr<arrow::dataset::FileWriteOptions>()> options = nullptr;
+  std::function<std::shared_ptr<arrow::dataset::FileFormat>()> format = nullptr;
+  std::function<std::shared_ptr<VirtualRootFileSystemBase>(void*)> getSubFilesystem = nullptr;
 };
 
-class SingleTreeFileSystem : public TTreeFileSystem
-{
- public:
-  SingleTreeFileSystem(TTree* tree)
-    : TTreeFileSystem(),
-      mTree(tree)
-  {
-  }
+struct RootArrowFactoryPlugin {
+  virtual RootArrowFactory* create() = 0;
+};
 
-  std::string type_name() const override
-  {
-    return "ttree";
-  }
+// A registry for all the possible ways of encoding a table in a TFile
+struct RootObjectReadingCapability {
+  // The unique name of this capability
+  std::string name = "unknown";
+  // Given a TFile, return the object which this capability support
+  // Use a void * in order not to expose the kind of object to the
+  // generic reading code. This is also where we load the plugin
+  // which will be used for the actual creation.
+  std::function<void*(TDirectoryFile* file, std::string const& path)> getHandle;
+  // Same as the above, but uses a TBufferFile as storage
+  std::function<void*(TBufferFile*, std::string const&)> getBufferHandle;
+  // This must be implemented to load the actual RootArrowFactory plugin which
+  // implements this capability. This way the detection of the file format
+  // (via get handle) does not need to know about the actual code which performs
+  // the serialization (and might depend on e.g. RNTuple).
+  std::function<RootArrowFactory&()> factory;
+};
 
-  TTree* GetTree(arrow::dataset::FileSource) override
-  {
-    // Simply return the only TTree we have
-    return mTree;
-  }
+struct RootObjectReadingCapabilityPlugin {
+  virtual RootObjectReadingCapability* create() = 0;
+};
 
- private:
-  TTree* mTree;
+// This acts as registry of all the capabilities (i.e. the ability to
+// associate a given object in a root file to the serialization plugin) and
+// the factory (i.e. the serialization plugin)
+struct RootObjectReadingFactory {
+  std::vector<RootObjectReadingCapability> capabilities;
 };
 
 class TFileFileSystem : public VirtualRootFileSystemBase
@@ -126,7 +113,7 @@ class TFileFileSystem : public VirtualRootFileSystemBase
  public:
   arrow::Result<arrow::fs::FileInfo> GetFileInfo(const std::string& path) override;
 
-  TFileFileSystem(TDirectoryFile* f, size_t readahead);
+  TFileFileSystem(TDirectoryFile* f, size_t readahead, RootObjectReadingFactory&);
 
   std::string type_name() const override
   {
@@ -147,12 +134,13 @@ class TFileFileSystem : public VirtualRootFileSystemBase
 
  private:
   TDirectoryFile* mFile;
+  RootObjectReadingFactory& mObjectFactory;
 };
 
 class TBufferFileFS : public VirtualRootFileSystemBase
 {
  public:
-  TBufferFileFS(TBufferFile* f);
+  TBufferFileFS(TBufferFile* f, RootObjectReadingFactory&);
 
   arrow::Result<arrow::fs::FileInfo> GetFileInfo(const std::string& path) override;
   std::string type_name() const override
@@ -165,68 +153,7 @@ class TBufferFileFS : public VirtualRootFileSystemBase
  private:
   TBufferFile* mBuffer;
   std::shared_ptr<VirtualRootFileSystemBase> mFilesystem;
-};
-
-class TTreeFileFragment : public arrow::dataset::FileFragment
-{
- public:
-  TTreeFileFragment(arrow::dataset::FileSource source,
-                    std::shared_ptr<arrow::dataset::FileFormat> format,
-                    arrow::compute::Expression partition_expression,
-                    std::shared_ptr<arrow::Schema> physical_schema)
-    : FileFragment(std::move(source), std::move(format), std::move(partition_expression), std::move(physical_schema))
-  {
-  }
-};
-
-class TTreeFileFormat : public arrow::dataset::FileFormat
-{
-  size_t& mTotCompressedSize;
-  size_t& mTotUncompressedSize;
-
- public:
-  TTreeFileFormat(size_t& totalCompressedSize, size_t& totalUncompressedSize)
-    : FileFormat({}),
-      mTotCompressedSize(totalCompressedSize),
-      mTotUncompressedSize(totalUncompressedSize)
-  {
-  }
-
-  ~TTreeFileFormat() override = default;
-
-  std::string type_name() const override
-  {
-    return "ttree";
-  }
-
-  bool Equals(const FileFormat& other) const override
-  {
-    return other.type_name() == this->type_name();
-  }
-
-  arrow::Result<bool> IsSupported(const arrow::dataset::FileSource& source) const override
-  {
-    auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
-    auto subFs = fs->GetSubFilesystem(source);
-    if (std::dynamic_pointer_cast<TTreeFileSystem>(subFs)) {
-      return true;
-    }
-    return false;
-  }
-
-  arrow::Result<std::shared_ptr<arrow::Schema>> Inspect(const arrow::dataset::FileSource& source) const override;
-  /// \brief Create a FileFragment for a FileSource.
-  arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> MakeFragment(
-    arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
-    std::shared_ptr<arrow::Schema> physical_schema) override;
-
-  arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options, arrow::fs::FileLocator destination_locator) const override;
-
-  std::shared_ptr<arrow::dataset::FileWriteOptions> DefaultWriteOptions() override;
-
-  arrow::Result<arrow::RecordBatchGenerator> ScanBatchesAsync(
-    const std::shared_ptr<arrow::dataset::ScanOptions>& options,
-    const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const override;
+  RootObjectReadingFactory& mObjectFactory;
 };
 
 // An arrow outputstream which allows to write to a TDirectoryFile.
@@ -255,33 +182,6 @@ class TDirectoryFileOutputStream : public arrow::io::OutputStream
   TDirectoryFile* mDirectory;
 };
 
-// An arrow outputstream which allows to write to a TTree. Eventually
-// with a prefix for the branches.
-class TTreeOutputStream : public arrow::io::OutputStream
-{
- public:
-  TTreeOutputStream(TTree*, std::string branchPrefix);
-
-  arrow::Status Close() override;
-
-  arrow::Result<int64_t> Tell() const override;
-
-  arrow::Status Write(const void* data, int64_t nbytes) override;
-
-  bool closed() const override;
-
-  TBranch* CreateBranch(char const* branchName, char const* sizeBranch);
-
-  TTree* GetTree()
-  {
-    return mTree;
-  }
-
- private:
-  TTree* mTree;
-  std::string mBranchPrefix;
-};
-
 } // namespace o2::framework
 
 #endif // O2_FRAMEWORK_ROOT_ARROW_FILESYSTEM_H_
diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx
index 0d225b81c0581..46248ab467366 100644
--- a/Framework/Core/src/Plugin.cxx
+++ b/Framework/Core/src/Plugin.cxx
@@ -11,10 +11,15 @@
 #include "Framework/Plugins.h"
 #include "Framework/ConfigParamDiscovery.h"
 #include "Framework/ConfigParamRegistry.h"
+#include "Framework/RootArrowFilesystem.h"
 #include "Framework/Logger.h"
 #include "Framework/Capability.h"
 #include "Framework/Signpost.h"
 #include "Framework/VariantJSONHelpers.h"
+#include "Framework/PluginManager.h"
+#include <TDirectory.h>
+#include <TDirectoryFile.h>
+#include <TClass.h>
 #include <cstddef>
 #include <string_view>
 
@@ -168,11 +173,75 @@ struct DiscoverAODOptionsInCommandLine : o2::framework::ConfigDiscoveryPlugin {
   }
 };
 
+struct ImplementationContext {
+  std::vector<RootArrowFactory> implementations;
+};
+
+std::function<void*(TDirectoryFile*, std::string const&)> getHandleByClass(char const* classname)
+{
+  return [classname](TDirectoryFile* file, std::string const& path) { return file->GetObjectChecked(path.c_str(), TClass::GetClass(classname)); };
+}
+
+std::function<void*(TBufferFile*, std::string const&)> getBufferHandleByClass(char const* classname)
+{
+  return [classname](TBufferFile* buffer, std::string const& path) { buffer->Reset(); return buffer->ReadObjectAny(TClass::GetClass(classname)); };
+}
+
+void lazyLoadFactory(std::vector<RootArrowFactory>& implementations, char const* specs)
+{
+  // Lazy loading of the plugin so that we do not bring in RNTuple / TTree if not needed
+  if (implementations.empty()) {
+    std::vector<LoadablePlugin> plugins;
+    auto morePlugins = PluginManager::parsePluginSpecString(specs);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+    PluginManager::loadFromPlugin<RootArrowFactory, RootArrowFactoryPlugin>(plugins, implementations);
+    if (implementations.empty()) {
+      return;
+    }
+  }
+}
+
+struct RNTupleObjectReadingCapability : o2::framework::RootObjectReadingCapabilityPlugin {
+  RootObjectReadingCapability* create() override
+  {
+    auto context = new ImplementationContext;
+
+    return new RootObjectReadingCapability{
+      .name = "rntuple",
+      .getHandle = getHandleByClass("ROOT::Experimental::RNTuple"),
+      .getBufferHandle = getBufferHandleByClass("ROOT::Experimental::RNTuple"),
+      .factory = [context]() -> RootArrowFactory& {
+        lazyLoadFactory(context->implementations, "O2FrameworkAnalysisRNTupleSupport:RNTupleObjectReadingImplementation");
+        return context->implementations.back();
+      }};
+  }
+};
+
+struct TTreeObjectReadingCapability : o2::framework::RootObjectReadingCapabilityPlugin {
+  RootObjectReadingCapability* create() override
+  {
+    auto context = new ImplementationContext;
+
+    return new RootObjectReadingCapability{
+      .name = "ttree",
+      .getHandle = getHandleByClass("TTree"),
+      .getBufferHandle = getBufferHandleByClass("TTree"),
+      .factory = [context]() -> RootArrowFactory& {
+        lazyLoadFactory(context->implementations, "O2FrameworkAnalysisTTreeSupport:TTreeObjectReadingImplementation");
+        return context->implementations.back();
+      }};
+  }
+};
+
 DEFINE_DPL_PLUGINS_BEGIN
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAODCapability, Capability);
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLineCapability, Capability);
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverAODOptionsInCommandLineCapability, Capability);
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInCommandLine, ConfigDiscovery);
 DEFINE_DPL_PLUGIN_INSTANCE(DiscoverAODOptionsInCommandLine, ConfigDiscovery);
+DEFINE_DPL_PLUGIN_INSTANCE(RNTupleObjectReadingCapability, RootObjectReadingCapability);
+DEFINE_DPL_PLUGIN_INSTANCE(TTreeObjectReadingCapability, RootObjectReadingCapability);
 DEFINE_DPL_PLUGINS_END
 } // namespace o2::framework
diff --git a/Framework/Core/src/RootArrowFilesystem.cxx b/Framework/Core/src/RootArrowFilesystem.cxx
index 5f2d21d942d37..545ba6f0afb71 100644
--- a/Framework/Core/src/RootArrowFilesystem.cxx
+++ b/Framework/Core/src/RootArrowFilesystem.cxx
@@ -9,9 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 #include "Framework/RootArrowFilesystem.h"
-#include "Framework/Endian.h"
 #include "Framework/RuntimeError.h"
-#include "Framework/Signpost.h"
 #include <Rtypes.h>
 #include <arrow/array/array_nested.h>
 #include <arrow/array/array_primitive.h>
@@ -19,93 +17,48 @@
 #include <arrow/array/builder_primitive.h>
 #include <memory>
 #include <TFile.h>
-#include <TLeaf.h>
 #include <TBufferFile.h>
-#include <TTree.h>
 #include <TDirectoryFile.h>
 #include <arrow/type.h>
 #include <arrow/type_fwd.h>
 #include <arrow/dataset/file_base.h>
 #include <arrow/result.h>
 #include <arrow/status.h>
-#include <arrow/util/key_value_metadata.h>
 #include <fmt/format.h>
 
-#include <stdexcept>
-#include <utility>
+template class
+  std::shared_ptr<arrow::Array>;
 
-O2_DECLARE_DYNAMIC_LOG(root_arrow_fs);
-
-namespace
-{
-struct BranchInfo {
-  std::string name;
-  TBranch* ptr;
-  bool mVLA;
-};
-} // namespace
-
-auto arrowTypeFromROOT(EDataType type, int size)
-{
-  auto typeGenerator = [](std::shared_ptr<arrow::DataType> const& type, int size) -> std::shared_ptr<arrow::DataType> {
-    switch (size) {
-      case -1:
-        return arrow::list(type);
-      case 1:
-        return std::move(type);
-      default:
-        return arrow::fixed_size_list(type, size);
-    }
-  };
-
-  switch (type) {
-    case EDataType::kBool_t:
-      return typeGenerator(arrow::boolean(), size);
-    case EDataType::kUChar_t:
-      return typeGenerator(arrow::uint8(), size);
-    case EDataType::kUShort_t:
-      return typeGenerator(arrow::uint16(), size);
-    case EDataType::kUInt_t:
-      return typeGenerator(arrow::uint32(), size);
-    case EDataType::kULong64_t:
-      return typeGenerator(arrow::uint64(), size);
-    case EDataType::kChar_t:
-      return typeGenerator(arrow::int8(), size);
-    case EDataType::kShort_t:
-      return typeGenerator(arrow::int16(), size);
-    case EDataType::kInt_t:
-      return typeGenerator(arrow::int32(), size);
-    case EDataType::kLong64_t:
-      return typeGenerator(arrow::int64(), size);
-    case EDataType::kFloat_t:
-      return typeGenerator(arrow::float32(), size);
-    case EDataType::kDouble_t:
-      return typeGenerator(arrow::float64(), size);
-    default:
-      throw o2::framework::runtime_error_f("Unsupported branch type: %d", static_cast<int>(type));
-  }
-}
 namespace o2::framework
 {
 using arrow::Status;
 
-TFileFileSystem::TFileFileSystem(TDirectoryFile* f, size_t readahead)
+TFileFileSystem::TFileFileSystem(TDirectoryFile* f, size_t readahead, RootObjectReadingFactory& factory)
   : VirtualRootFileSystemBase(),
-    mFile(f)
+    mFile(f),
+    mObjectFactory(factory)
 {
   ((TFile*)mFile)->SetReadaheadSize(50 * 1024 * 1024);
 }
 
 std::shared_ptr<VirtualRootFileSystemBase> TFileFileSystem::GetSubFilesystem(arrow::dataset::FileSource source)
 {
-  auto tree = (TTree*)mFile->GetObjectChecked(source.path().c_str(), TClass::GetClass<TTree>());
-  if (tree) {
-    return std::shared_ptr<VirtualRootFileSystemBase>(new SingleTreeFileSystem(tree));
+  // We use a plugin to create the actual objects inside the
+  // file, so that we can support TTree and RNTuple at the same time
+  // without having to depend on both.
+  for (auto& capability : mObjectFactory.capabilities) {
+    void* handle = capability.getHandle(mFile, source.path());
+    if (!handle) {
+      continue;
+    }
+    if (handle) {
+      return capability.factory().getSubFilesystem(handle);
+    }
   }
 
   auto directory = (TDirectoryFile*)mFile->GetObjectChecked(source.path().c_str(), TClass::GetClass<TDirectory>());
   if (directory) {
-    return std::shared_ptr<VirtualRootFileSystemBase>(new TFileFileSystem(directory, 50 * 1024 * 1024));
+    return std::shared_ptr<VirtualRootFileSystemBase>(new TFileFileSystem(directory, 50 * 1024 * 1024, mObjectFactory));
   }
   throw runtime_error_f("Unsupported file layout");
 }
@@ -120,10 +73,14 @@ arrow::Result<arrow::fs::FileInfo> TFileFileSystem::GetFileInfo(const std::strin
   auto fs = GetSubFilesystem(source);
 
   // For now we only support single trees.
-  if (std::dynamic_pointer_cast<SingleTreeFileSystem>(fs)) {
-    result.set_type(arrow::fs::FileType::File);
+  if (std::dynamic_pointer_cast<TFileFileSystem>(fs)) {
+    result.set_type(arrow::fs::FileType::Directory);
     return result;
   }
+  // Everything else is a file, if it was created.
+  if (fs.get()) {
+    result.set_type(arrow::fs::FileType::File);
+  }
   return result;
 }
 
@@ -137,7 +94,7 @@ arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TFileFileSystem::OpenOut
 
   auto* dir = dynamic_cast<TDirectoryFile*>(this->GetFile()->Get(path.c_str()));
   if (!dir) {
-    throw runtime_error_f("Unable to open directory %s in file %s", path.c_str(), GetFile()->GetName());
+    return arrow::Status::Invalid(fmt::format("Unable to open directory {} in file {} ", path.c_str(), GetFile()->GetName()));
   }
   auto stream = std::make_shared<TDirectoryFileOutputStream>(dir);
   return stream;
@@ -219,81 +176,6 @@ arrow::Result<std::shared_ptr<arrow::io::OutputStream>> VirtualRootFileSystemBas
   return arrow::Status::NotImplemented("No random access file system");
 }
 
-arrow::Result<std::shared_ptr<arrow::Schema>> TTreeFileFormat::Inspect(const arrow::dataset::FileSource& source) const
-{
-  arrow::Schema schema{{}};
-  auto fs = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(source.filesystem());
-  // Actually get the TTree from the ROOT file.
-  auto treeFs = std::dynamic_pointer_cast<TTreeFileSystem>(fs->GetSubFilesystem(source));
-  if (!treeFs.get()) {
-    throw runtime_error_f("Unknown filesystem %s\n", source.filesystem()->type_name().c_str());
-  }
-  TTree* tree = treeFs->GetTree(source);
-
-  auto branches = tree->GetListOfBranches();
-  auto n = branches->GetEntries();
-
-  std::vector<BranchInfo> branchInfos;
-  for (auto i = 0; i < n; ++i) {
-    auto branch = static_cast<TBranch*>(branches->At(i));
-    auto name = std::string{branch->GetName()};
-    auto pos = name.find("_size");
-    if (pos != std::string::npos) {
-      name.erase(pos);
-      branchInfos.emplace_back(BranchInfo{name, (TBranch*)nullptr, true});
-    } else {
-      auto lookup = std::find_if(branchInfos.begin(), branchInfos.end(), [&](BranchInfo const& bi) {
-        return bi.name == name;
-      });
-      if (lookup == branchInfos.end()) {
-        branchInfos.emplace_back(BranchInfo{name, branch, false});
-      } else {
-        lookup->ptr = branch;
-      }
-    }
-  }
-
-  std::vector<std::shared_ptr<arrow::Field>> fields;
-  tree->SetCacheSize(25000000);
-  for (auto& bi : branchInfos) {
-    static TClass* cls;
-    EDataType type;
-    bi.ptr->GetExpectedType(cls, type);
-    auto listSize = -1;
-    if (!bi.mVLA) {
-      listSize = static_cast<TLeaf*>(bi.ptr->GetListOfLeaves()->At(0))->GetLenStatic();
-    }
-    auto field = std::make_shared<arrow::Field>(bi.ptr->GetName(), arrowTypeFromROOT(type, listSize));
-    fields.push_back(field);
-
-    tree->AddBranchToCache(bi.ptr);
-    if (strncmp(bi.ptr->GetName(), "fIndexArray", strlen("fIndexArray")) == 0) {
-      std::string sizeBranchName = bi.ptr->GetName();
-      sizeBranchName += "_size";
-      auto* sizeBranch = (TBranch*)tree->GetBranch(sizeBranchName.c_str());
-      if (sizeBranch) {
-        tree->AddBranchToCache(sizeBranch);
-      }
-    }
-  }
-  tree->StopCacheLearningPhase();
-
-  return std::make_shared<arrow::Schema>(fields);
-}
-
-/// \brief Create a FileFragment for a FileSource.
-arrow::Result<std::shared_ptr<arrow::dataset::FileFragment>> TTreeFileFormat::MakeFragment(
-  arrow::dataset::FileSource source, arrow::compute::Expression partition_expression,
-  std::shared_ptr<arrow::Schema> physical_schema)
-{
-  std::shared_ptr<arrow::dataset::FileFormat> format = std::make_shared<TTreeFileFormat>(mTotCompressedSize, mTotUncompressedSize);
-
-  auto fragment = std::make_shared<TTreeFileFragment>(std::move(source), std::move(format),
-                                                      std::move(partition_expression),
-                                                      std::move(physical_schema));
-  return std::dynamic_pointer_cast<arrow::dataset::FileFragment>(fragment);
-}
-
 // An arrow outputstream which allows to write to a ttree
 TDirectoryFileOutputStream::TDirectoryFileOutputStream(TDirectoryFile* f)
   : mDirectory(f)
@@ -321,544 +203,14 @@ bool TDirectoryFileOutputStream::closed() const
   return mDirectory->GetFile()->IsOpen() == false;
 }
 
-// An arrow outputstream which allows to write to a ttree
-// @a branch prefix is to be used to identify a set of branches which all belong to
-// the same table.
-TTreeOutputStream::TTreeOutputStream(TTree* f, std::string branchPrefix)
-  : mTree(f),
-    mBranchPrefix(std::move(branchPrefix))
-{
-}
-
-arrow::Status TTreeOutputStream::Close()
-{
-  if (mTree->GetCurrentFile() == nullptr) {
-    return arrow::Status::Invalid("Cannot close a tree not attached to a file");
-  }
-  mTree->GetCurrentFile()->Close();
-  return arrow::Status::OK();
-}
-
-arrow::Result<int64_t> TTreeOutputStream::Tell() const
-{
-  return arrow::Result<int64_t>(arrow::Status::NotImplemented("Cannot move"));
-}
-
-arrow::Status TTreeOutputStream::Write(const void* data, int64_t nbytes)
-{
-  return arrow::Status::NotImplemented("Cannot write raw bytes to a TTree");
-}
-
-bool TTreeOutputStream::closed() const
-{
-  // A standalone tree is never closed.
-  if (mTree->GetCurrentFile() == nullptr) {
-    return false;
-  }
-  return mTree->GetCurrentFile()->IsOpen() == false;
-}
-
-TBranch* TTreeOutputStream::CreateBranch(char const* branchName, char const* sizeBranch)
-{
-  return mTree->Branch((mBranchPrefix + "/" + branchName).c_str(), (char*)nullptr, (mBranchPrefix + sizeBranch).c_str());
-}
-
-char const* rootSuffixFromArrow(arrow::Type::type id)
-{
-  switch (id) {
-    case arrow::Type::BOOL:
-      return "/O";
-    case arrow::Type::UINT8:
-      return "/b";
-    case arrow::Type::UINT16:
-      return "/s";
-    case arrow::Type::UINT32:
-      return "/i";
-    case arrow::Type::UINT64:
-      return "/l";
-    case arrow::Type::INT8:
-      return "/B";
-    case arrow::Type::INT16:
-      return "/S";
-    case arrow::Type::INT32:
-      return "/I";
-    case arrow::Type::INT64:
-      return "/L";
-    case arrow::Type::FLOAT:
-      return "/F";
-    case arrow::Type::DOUBLE:
-      return "/D";
-    default:
-      throw runtime_error("Unsupported arrow column type");
-  }
-}
-
-class TTreeFileWriter : public arrow::dataset::FileWriter
-{
-  std::vector<TBranch*> branches;
-  std::vector<TBranch*> sizesBranches;
-  std::vector<std::shared_ptr<arrow::Array>> valueArrays;
-  std::vector<std::shared_ptr<arrow::Array>> sizeArrays;
-  std::vector<std::shared_ptr<arrow::DataType>> valueTypes;
-
-  std::vector<int64_t> valuesIdealBasketSize;
-  std::vector<int64_t> sizeIdealBasketSize;
-
-  std::vector<int64_t> typeSizes;
-  std::vector<int64_t> listSizes;
-  bool firstBasket = true;
-
-  // This is to create a batsket size according to the first batch.
-  void finaliseBasketSize(std::shared_ptr<arrow::RecordBatch> firstBatch)
-  {
-    O2_SIGNPOST_ID_FROM_POINTER(sid, root_arrow_fs, this);
-    O2_SIGNPOST_START(root_arrow_fs, sid, "finaliseBasketSize", "First batch with %lli rows received and %zu columns",
-                      firstBatch->num_rows(), firstBatch->columns().size());
-    for (size_t i = 0; i < branches.size(); i++) {
-      auto* branch = branches[i];
-      auto* sizeBranch = sizesBranches[i];
-
-      int valueSize = valueTypes[i]->byte_width();
-      if (listSizes[i] == 1) {
-        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s exists and uses %d bytes per entry for %lli entries.",
-                               branch->GetName(), valueSize, firstBatch->num_rows());
-        assert(sizeBranch == nullptr);
-        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize);
-      } else if (listSizes[i] == -1) {
-        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s exists and uses %d bytes per entry.",
-                               branch->GetName(), valueSize);
-        // This should probably lookup the
-        auto column = firstBatch->GetColumnByName(schema_->field(i)->name());
-        auto list = std::static_pointer_cast<arrow::ListArray>(column);
-        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s needed. Associated size branch %s and there are %lli entries of size %d in that list.",
-                               branch->GetName(), sizeBranch->GetName(), list->length(), valueSize);
-        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize * list->length());
-        sizeBranch->SetBasketSize(1024 + firstBatch->num_rows() * 4);
-      } else {
-        O2_SIGNPOST_EVENT_EMIT(root_arrow_fs, sid, "finaliseBasketSize", "Branch %s needed. There are %lli entries per array of size %d in that list.",
-                               branch->GetName(), listSizes[i], valueSize);
-        assert(sizeBranch == nullptr);
-        branch->SetBasketSize(1024 + firstBatch->num_rows() * valueSize * listSizes[i]);
-      }
-
-      auto field = firstBatch->schema()->field(i);
-      if (field->name().starts_with("fIndexArray")) {
-        // One int per array to keep track of the size
-        int idealBasketSize = 4 * firstBatch->num_rows() + 1024 + field->type()->byte_width() * firstBatch->num_rows(); // minimal additional size needed, otherwise we get 2 baskets
-        int basketSize = std::max(32000, idealBasketSize);                                                              // keep a minimum value
-        sizeBranch->SetBasketSize(basketSize);
-        branch->SetBasketSize(basketSize);
-      }
-    }
-    O2_SIGNPOST_END(root_arrow_fs, sid, "finaliseBasketSize", "Done");
-  }
-
- public:
-  // Create the TTree based on the physical_schema, not the one in the batch.
-  // The write method will have to reconcile the two schemas.
-  TTreeFileWriter(std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options,
-                  std::shared_ptr<arrow::io::OutputStream> destination,
-                  arrow::fs::FileLocator destination_locator)
-    : FileWriter(schema, options, destination, destination_locator)
-  {
-    // Batches have the same number of entries for each column.
-    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
-    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
-
-    if (directoryStream.get()) {
-      TDirectoryFile* dir = directoryStream->GetDirectory();
-      dir->cd();
-      auto* tree = new TTree(destination_locator_.path.c_str(), "");
-      treeStream = std::make_shared<TTreeOutputStream>(tree, "");
-    } else if (treeStream.get()) {
-      // We already have a tree stream, let's derive a new one
-      // with the destination_locator_.path as prefix for the branches
-      // This way we can multiplex multiple tables in the same tree.
-      auto tree = treeStream->GetTree();
-      treeStream = std::make_shared<TTreeOutputStream>(tree, destination_locator_.path);
-    } else {
-      // I could simply set a prefix here to merge to an already existing tree.
-      throw std::runtime_error("Unsupported backend.");
-    }
-
-    for (auto i = 0u; i < schema->fields().size(); ++i) {
-      auto& field = schema->field(i);
-      listSizes.push_back(1);
-
-      int valuesIdealBasketSize = 0;
-      // Construct all the needed branches.
-      switch (field->type()->id()) {
-        case arrow::Type::FIXED_SIZE_LIST: {
-          listSizes.back() = std::static_pointer_cast<arrow::FixedSizeListType>(field->type())->list_size();
-          valuesIdealBasketSize = 1024 + valueTypes.back()->byte_width() * listSizes.back();
-          valueTypes.push_back(field->type()->field(0)->type());
-          sizesBranches.push_back(nullptr);
-          std::string leafList = fmt::format("{}[{}]{}", field->name(), listSizes.back(), rootSuffixFromArrow(valueTypes.back()->id()));
-          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
-        } break;
-        case arrow::Type::LIST: {
-          valueTypes.push_back(field->type()->field(0)->type());
-          std::string leafList = fmt::format("{}[{}_size]{}", field->name(), field->name(), rootSuffixFromArrow(valueTypes.back()->id()));
-          listSizes.back() = -1; // VLA, we need to calculate it on the fly;
-          std::string sizeLeafList = field->name() + "_size/I";
-          sizesBranches.push_back(treeStream->CreateBranch((field->name() + "_size").c_str(), sizeLeafList.c_str()));
-          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
-          // Notice that this could be replaced by a better guess of the
-          // average size of the list elements, but this is not trivial.
-        } break;
-        default: {
-          valueTypes.push_back(field->type());
-          std::string leafList = field->name() + rootSuffixFromArrow(valueTypes.back()->id());
-          sizesBranches.push_back(nullptr);
-          branches.push_back(treeStream->CreateBranch(field->name().c_str(), leafList.c_str()));
-        } break;
-      }
-    }
-    // We create the branches from the schema
-  }
-
-  arrow::Status Write(const std::shared_ptr<arrow::RecordBatch>& batch) override
-  {
-    if (firstBasket) {
-      firstBasket = false;
-      finaliseBasketSize(batch);
-    }
-
-    // Support writing empty tables
-    if (batch->columns().empty() || batch->num_rows() == 0) {
-      return arrow::Status::OK();
-    }
-
-    // Batches have the same number of entries for each column.
-    auto directoryStream = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(destination_);
-    TTree* tree = nullptr;
-    if (directoryStream.get()) {
-      TDirectoryFile* dir = directoryStream->GetDirectory();
-      tree = (TTree*)dir->Get(destination_locator_.path.c_str());
-    }
-    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
-
-    if (!tree) {
-      // I could simply set a prefix here to merge to an already existing tree.
-      throw std::runtime_error("Unsupported backend.");
-    }
-
-    for (auto i = 0u; i < batch->columns().size(); ++i) {
-      auto column = batch->column(i);
-      auto& field = batch->schema()->field(i);
-
-      valueArrays.push_back(nullptr);
-
-      switch (field->type()->id()) {
-        case arrow::Type::FIXED_SIZE_LIST: {
-          auto list = std::static_pointer_cast<arrow::FixedSizeListArray>(column);
-          valueArrays.back() = list->values();
-        } break;
-        case arrow::Type::LIST: {
-          auto list = std::static_pointer_cast<arrow::ListArray>(column);
-          valueArrays.back() = list;
-        } break;
-        case arrow::Type::BOOL: {
-          // In case of arrays of booleans, we need to go back to their
-          // char based representation for ROOT to save them.
-          auto boolArray = std::static_pointer_cast<arrow::BooleanArray>(column);
-
-          int64_t length = boolArray->length();
-          arrow::UInt8Builder builder;
-          auto ok = builder.Reserve(length);
-
-          for (int64_t i = 0; i < length; ++i) {
-            if (boolArray->IsValid(i)) {
-              // Expand each boolean value (true/false) to uint8 (1/0)
-              uint8_t value = boolArray->Value(i) ? 1 : 0;
-              auto ok = builder.Append(value);
-            } else {
-              // Append null for invalid entries
-              auto ok = builder.AppendNull();
-            }
-          }
-          valueArrays.back() = *builder.Finish();
-        } break;
-        default:
-          valueArrays.back() = column;
-      }
-    }
-
-    int64_t pos = 0;
-    while (pos < batch->num_rows()) {
-      for (size_t bi = 0; bi < branches.size(); ++bi) {
-        auto* branch = branches[bi];
-        auto* sizeBranch = sizesBranches[bi];
-        auto array = batch->column(bi);
-        auto& field = batch->schema()->field(bi);
-        auto& listSize = listSizes[bi];
-        auto valueType = valueTypes[bi];
-        auto valueArray = valueArrays[bi];
-
-        switch (field->type()->id()) {
-          case arrow::Type::LIST: {
-            auto list = std::static_pointer_cast<arrow::ListArray>(array);
-            listSize = list->value_length(pos);
-            uint8_t const* buffer = std::static_pointer_cast<arrow::PrimitiveArray>(valueArray)->values()->data() + array->offset() + list->value_offset(pos) * valueType->byte_width();
-            branch->SetAddress((void*)buffer);
-            sizeBranch->SetAddress(&listSize);
-          };
-            break;
-          case arrow::Type::FIXED_SIZE_LIST:
-          default: {
-            uint8_t const* buffer = std::static_pointer_cast<arrow::PrimitiveArray>(valueArray)->values()->data() + array->offset() + pos * listSize * valueType->byte_width();
-            branch->SetAddress((void*)buffer);
-          };
-        }
-      }
-      tree->Fill();
-      ++pos;
-    }
-    return arrow::Status::OK();
-  }
-
-  arrow::Future<> FinishInternal() override
-  {
-    auto treeStream = std::dynamic_pointer_cast<TTreeOutputStream>(destination_);
-    TTree* tree = treeStream->GetTree();
-    tree->Write("", TObject::kOverwrite);
-    tree->SetDirectory(nullptr);
-
-    return {};
-  };
-};
-
-arrow::Result<std::shared_ptr<arrow::dataset::FileWriter>> TTreeFileFormat::MakeWriter(std::shared_ptr<arrow::io::OutputStream> destination, std::shared_ptr<arrow::Schema> schema, std::shared_ptr<arrow::dataset::FileWriteOptions> options, arrow::fs::FileLocator destination_locator) const
-{
-  auto writer = std::make_shared<TTreeFileWriter>(schema, options, destination, destination_locator);
-  return std::dynamic_pointer_cast<arrow::dataset::FileWriter>(writer);
-}
-
-std::shared_ptr<arrow::dataset::FileWriteOptions> TTreeFileFormat::DefaultWriteOptions()
-{
-  std::shared_ptr<TTreeFileWriteOptions> options(
-    new TTreeFileWriteOptions(shared_from_this()));
-  return options;
-}
-
-arrow::Result<arrow::RecordBatchGenerator> TTreeFileFormat::ScanBatchesAsync(
-  const std::shared_ptr<arrow::dataset::ScanOptions>& options,
-  const std::shared_ptr<arrow::dataset::FileFragment>& fragment) const
-{
-  // Get the fragment as a TTreeFragment. This might be PART of a TTree.
-  auto treeFragment = std::dynamic_pointer_cast<TTreeFileFragment>(fragment);
-  // This is the schema we want to read
-  auto dataset_schema = options->dataset_schema;
-
-  auto generator = [pool = options->pool, treeFragment, dataset_schema, &totalCompressedSize = mTotCompressedSize,
-                    &totalUncompressedSize = mTotUncompressedSize]() -> arrow::Future<std::shared_ptr<arrow::RecordBatch>> {
-    auto schema = treeFragment->format()->Inspect(treeFragment->source());
-
-    std::vector<std::shared_ptr<arrow::Array>> columns;
-    std::vector<std::shared_ptr<arrow::Field>> fields = dataset_schema->fields();
-    auto physical_schema = *treeFragment->ReadPhysicalSchema();
-
-    static TBufferFile buffer{TBuffer::EMode::kWrite, 4 * 1024 * 1024};
-    auto containerFS = std::dynamic_pointer_cast<VirtualRootFileSystemBase>(treeFragment->source().filesystem());
-    auto fs = std::dynamic_pointer_cast<TTreeFileSystem>(containerFS->GetSubFilesystem(treeFragment->source()));
-
-    int64_t rows = -1;
-    TTree* tree = fs->GetTree(treeFragment->source());
-    for (auto& field : fields) {
-      // The field actually on disk
-      auto physicalField = physical_schema->GetFieldByName(field->name());
-      TBranch* branch = tree->GetBranch(physicalField->name().c_str());
-      assert(branch);
-      buffer.Reset();
-      auto totalEntries = branch->GetEntries();
-      if (rows == -1) {
-        rows = totalEntries;
-      }
-      if (rows != totalEntries) {
-        throw runtime_error_f("Unmatching number of rows for branch %s", branch->GetName());
-      }
-      arrow::Status status;
-      int readEntries = 0;
-      std::shared_ptr<arrow::Array> array;
-      auto listType = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type());
-      if (physicalField->type() == arrow::boolean() ||
-          (listType && physicalField->type()->field(0)->type() == arrow::boolean())) {
-        if (listType) {
-          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
-          auto status = arrow::MakeBuilder(pool, physicalField->type()->field(0)->type(), &builder);
-          if (!status.ok()) {
-            throw runtime_error("Cannot create value builder");
-          }
-          auto listBuilder = std::make_unique<arrow::FixedSizeListBuilder>(pool, std::move(builder), listType->list_size());
-          auto valueBuilder = listBuilder.get()->value_builder();
-          // boolean array special case: we need to use builder to create the bitmap
-          status = valueBuilder->Reserve(totalEntries * listType->list_size());
-          status &= listBuilder->Reserve(totalEntries);
-          if (!status.ok()) {
-            throw runtime_error("Failed to reserve memory for array builder");
-          }
-          while (readEntries < totalEntries) {
-            auto readLast = branch->GetBulkRead().GetBulkEntries(readEntries, buffer);
-            readEntries += readLast;
-            status &= static_cast<arrow::BooleanBuilder*>(valueBuilder)->AppendValues(reinterpret_cast<uint8_t const*>(buffer.GetCurrent()), readLast * listType->list_size());
-          }
-          status &= static_cast<arrow::FixedSizeListBuilder*>(listBuilder.get())->AppendValues(readEntries);
-          if (!status.ok()) {
-            throw runtime_error("Failed to append values to array");
-          }
-          status &= listBuilder->Finish(&array);
-          if (!status.ok()) {
-            throw runtime_error("Failed to create array");
-          }
-        } else if (listType == nullptr) {
-          std::unique_ptr<arrow::ArrayBuilder> builder = nullptr;
-          auto status = arrow::MakeBuilder(pool, physicalField->type(), &builder);
-          if (!status.ok()) {
-            throw runtime_error("Cannot create builder");
-          }
-          auto valueBuilder = static_cast<arrow::BooleanBuilder*>(builder.get());
-          // boolean array special case: we need to use builder to create the bitmap
-          status = valueBuilder->Reserve(totalEntries);
-          if (!status.ok()) {
-            throw runtime_error("Failed to reserve memory for array builder");
-          }
-          while (readEntries < totalEntries) {
-            auto readLast = branch->GetBulkRead().GetBulkEntries(readEntries, buffer);
-            readEntries += readLast;
-            status &= valueBuilder->AppendValues(reinterpret_cast<uint8_t const*>(buffer.GetCurrent()), readLast);
-          }
-          if (!status.ok()) {
-            throw runtime_error("Failed to append values to array");
-          }
-          status &= valueBuilder->Finish(&array);
-          if (!status.ok()) {
-            throw runtime_error("Failed to create array");
-          }
-        }
-      } else {
-        // other types: use serialized read to build arrays directly.
-        auto typeSize = physicalField->type()->byte_width();
-        // This is needed for branches which have not been persisted.
-        auto bytes = branch->GetTotBytes();
-        auto branchSize = bytes ? bytes : 1000000;
-        auto&& result = arrow::AllocateResizableBuffer(branchSize, pool);
-        if (!result.ok()) {
-          throw runtime_error("Cannot allocate values buffer");
-        }
-        std::shared_ptr<arrow::Buffer> arrowValuesBuffer = std::move(result).ValueUnsafe();
-        auto ptr = arrowValuesBuffer->mutable_data();
-        if (ptr == nullptr) {
-          throw runtime_error("Invalid buffer");
-        }
-
-        std::unique_ptr<TBufferFile> offsetBuffer = nullptr;
-
-        uint32_t offset = 0;
-        int count = 0;
-        std::shared_ptr<arrow::Buffer> arrowOffsetBuffer;
-        std::span<int> offsets;
-        int size = 0;
-        uint32_t totalSize = 0;
-        TBranch* mSizeBranch = nullptr;
-        int64_t listSize = 1;
-        if (auto fixedSizeList = std::dynamic_pointer_cast<arrow::FixedSizeListType>(physicalField->type())) {
-          listSize = fixedSizeList->list_size();
-          typeSize = fixedSizeList->field(0)->type()->byte_width();
-        } else if (auto vlaListType = std::dynamic_pointer_cast<arrow::ListType>(physicalField->type())) {
-          listSize = -1;
-          typeSize = vlaListType->field(0)->type()->byte_width();
-        }
-        if (listSize == -1) {
-          mSizeBranch = branch->GetTree()->GetBranch((std::string{branch->GetName()} + "_size").c_str());
-          offsetBuffer = std::make_unique<TBufferFile>(TBuffer::EMode::kWrite, 4 * 1024 * 1024);
-          result = arrow::AllocateResizableBuffer((totalEntries + 1) * (int64_t)sizeof(int), pool);
-          if (!result.ok()) {
-            throw runtime_error("Cannot allocate offset buffer");
-          }
-          arrowOffsetBuffer = std::move(result).ValueUnsafe();
-          unsigned char* ptrOffset = arrowOffsetBuffer->mutable_data();
-          auto* tPtrOffset = reinterpret_cast<int*>(ptrOffset);
-          offsets = std::span<int>{tPtrOffset, tPtrOffset + totalEntries + 1};
-
-          // read sizes first
-          while (readEntries < totalEntries) {
-            auto readLast = mSizeBranch->GetBulkRead().GetEntriesSerialized(readEntries, *offsetBuffer);
-            readEntries += readLast;
-            for (auto i = 0; i < readLast; ++i) {
-              offsets[count++] = (int)offset;
-              offset += swap32_(reinterpret_cast<uint32_t*>(offsetBuffer->GetCurrent())[i]);
-            }
-          }
-          offsets[count] = (int)offset;
-          totalSize = offset;
-          readEntries = 0;
-        }
-
-        while (readEntries < totalEntries) {
-          auto readLast = branch->GetBulkRead().GetEntriesSerialized(readEntries, buffer);
-          if (listSize == -1) {
-            size = offsets[readEntries + readLast] - offsets[readEntries];
-          } else {
-            size = readLast * listSize;
-          }
-          readEntries += readLast;
-          swapCopy(ptr, buffer.GetCurrent(), size, typeSize);
-          ptr += (ptrdiff_t)(size * typeSize);
-        }
-        if (listSize >= 1) {
-          totalSize = readEntries * listSize;
-        }
-        std::shared_ptr<arrow::PrimitiveArray> varray;
-        switch (listSize) {
-          case -1:
-            varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
-            array = std::make_shared<arrow::ListArray>(physicalField->type(), readEntries, arrowOffsetBuffer, varray);
-            break;
-          case 1:
-            array = std::make_shared<arrow::PrimitiveArray>(physicalField->type(), readEntries, arrowValuesBuffer);
-            break;
-          default:
-            varray = std::make_shared<arrow::PrimitiveArray>(physicalField->type()->field(0)->type(), totalSize, arrowValuesBuffer);
-            array = std::make_shared<arrow::FixedSizeListArray>(physicalField->type(), readEntries, varray);
-        }
-      }
-
-      branch->SetStatus(false);
-      branch->DropBaskets("all");
-      branch->Reset();
-      branch->GetTransientBuffer(0)->Expand(0);
-
-      columns.push_back(array);
-    }
-    auto batch = arrow::RecordBatch::Make(dataset_schema, rows, columns);
-    totalCompressedSize += tree->GetZipBytes();
-    totalUncompressedSize += tree->GetTotBytes();
-    return batch;
-  };
-  return generator;
-}
-
-arrow::Result<std::shared_ptr<arrow::io::OutputStream>> TTreeFileSystem::OpenOutputStream(
-  const std::string& path,
-  const std::shared_ptr<const arrow::KeyValueMetadata>& metadata)
-{
-  arrow::dataset::FileSource source{path, shared_from_this()};
-  auto prefix = metadata->Get("branch_prefix");
-  if (prefix.ok()) {
-    return std::make_shared<TTreeOutputStream>(GetTree(source), *prefix);
-  }
-  return std::make_shared<TTreeOutputStream>(GetTree(source), "");
-}
-
-TBufferFileFS::TBufferFileFS(TBufferFile* f)
+TBufferFileFS::TBufferFileFS(TBufferFile* f, RootObjectReadingFactory& factory)
   : VirtualRootFileSystemBase(),
     mBuffer(f),
-    mFilesystem(nullptr)
+    mFilesystem(nullptr),
+    mObjectFactory(factory)
 {
 }
 
-TTreeFileSystem::~TTreeFileSystem() = default;
-
 arrow::Result<arrow::fs::FileInfo> TBufferFileFS::GetFileInfo(const std::string& path)
 {
   arrow::fs::FileInfo result;
@@ -871,19 +223,26 @@ arrow::Result<arrow::fs::FileInfo> TBufferFileFS::GetFileInfo(const std::string&
     return result;
   }
 
-  // For now we only support single trees.
-  if (std::dynamic_pointer_cast<SingleTreeFileSystem>(mFilesystem)) {
-    result.set_type(arrow::fs::FileType::File);
+  auto info = mFilesystem->GetFileInfo(path);
+  if (!info.ok()) {
     return result;
   }
+
+  result.set_type(info->type());
   return result;
 }
 
 std::shared_ptr<VirtualRootFileSystemBase> TBufferFileFS::GetSubFilesystem(arrow::dataset::FileSource source)
 {
-  if (!mFilesystem.get()) {
-    auto tree = ((TTree*)mBuffer->ReadObject(TTree::Class()));
-    mFilesystem = std::make_shared<SingleTreeFileSystem>(tree);
+  // We use a plugin to create the actual objects inside the
+  // file, so that we can support TTree and RNTuple at the same time
+  // without having to depend on both.
+  for (auto& capability : mObjectFactory.capabilities) {
+    void* handle = capability.getBufferHandle(mBuffer, source.path());
+    if (handle) {
+      mFilesystem = capability.factory().getSubFilesystem(handle);
+      break;
+    }
   }
   return mFilesystem;
 }
diff --git a/Framework/Core/test/o2AO2DToAO3D.cxx b/Framework/Core/test/o2AO2DToAO3D.cxx
new file mode 100644
index 0000000000000..25fa292d66ed9
--- /dev/null
+++ b/Framework/Core/test/o2AO2DToAO3D.cxx
@@ -0,0 +1,165 @@
+// Copyright 2019-2024 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/RootArrowFilesystem.h"
+#include "Framework/PluginManager.h"
+#include <TDirectory.h>
+#include <TDirectoryFile.h>
+#include <getopt.h>
+#include <TFile.h>
+#include <iostream>
+#include <TMap.h>
+#include <TTree.h>
+#include <fmt/format.h>
+
+int main(int argc, char** argv)
+{
+
+  char* input_file = nullptr;
+  char* output_file = nullptr;
+
+  // Define long options
+  static struct option long_options[] = {
+    {"input", required_argument, nullptr, 'i'},
+    {"output", required_argument, nullptr, 'o'},
+    {nullptr, 0, nullptr, 0} // End of options
+  };
+
+  int option_index = 0;
+  int c;
+
+  // Parse options
+  while ((c = getopt_long(argc, argv, "i:o:", long_options, &option_index)) != -1) {
+    switch (c) {
+      case 'i':
+        input_file = optarg;
+        break;
+      case 'o':
+        output_file = optarg;
+        break;
+      case '?':
+        // Unknown option
+        printf("Unknown option. Use --input <file> and --output <file>\n");
+        return 1;
+      default:
+        break;
+    }
+  }
+
+  // Check if input and output files are provided
+  if (input_file && output_file) {
+    printf("Input file: %s\n", input_file);
+    printf("Output file: %s\n", output_file);
+  } else {
+    fprintf(stderr, "Usage: %s --input <file> --output <file>\n", argv[0]);
+    return 1;
+  }
+
+  // Plugins which understand
+  std::vector<char const*> capabilitiesSpecs = {
+    "O2Framework:RNTupleObjectReadingCapability",
+    "O2Framework:TTreeObjectReadingCapability",
+  };
+
+  o2::framework::RootObjectReadingFactory factory;
+
+  std::vector<LoadablePlugin> plugins;
+  for (auto spec : capabilitiesSpecs) {
+    auto morePlugins = o2::framework::PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+  }
+
+  auto in = TFile::Open(input_file, "READ");
+  auto out = TFile::Open(output_file, "RECREATE");
+
+  auto fs = std::make_shared<o2::framework::TFileFileSystem>(in, 50 * 1024 * 1024, factory);
+  auto outFs = std::make_shared<o2::framework::TFileFileSystem>(out, 0, factory);
+
+  o2::framework::PluginManager::loadFromPlugin<o2::framework::RootObjectReadingCapability, o2::framework::RootObjectReadingCapabilityPlugin>(plugins, factory.capabilities);
+
+  // Plugins are hardcoded for now...
+  auto rNtupleFormat = factory.capabilities[0].factory().format();
+  auto format = factory.capabilities[1].factory().format();
+
+  for (TObject* dk : *in->GetListOfKeys()) {
+    if (dk->GetName() == std::string("metaData")) {
+      TMap* m = dynamic_cast<TMap*>(in->Get(dk->GetName()));
+      m->Print();
+      auto* copy = m->Clone("metaData");
+      out->WriteTObject(copy);
+      continue;
+    }
+    auto* d = (TDirectory*)in->Get(dk->GetName());
+    std::cout << "Processing: " << dk->GetName() << std::endl;
+    // For the moment RNTuple does not support TDirectory, so
+    // we write everything at toplevel.
+    auto destination = outFs->OpenOutputStream("/", {});
+    if (!destination.ok()) {
+      std::cerr << "Could not open destination folder " << output_file << std::endl;
+      exit(1);
+    }
+
+    for (TObject* tk : *d->GetListOfKeys()) {
+      auto sourceUrl = fmt::format("{}/{}", dk->GetName(), tk->GetName());
+      // FIXME: there is no support for TDirectory yet. Let's write everything
+      // at the same level.
+      auto destUrl = fmt::format("/{}-{}", dk->GetName(), tk->GetName());
+      arrow::dataset::FileSource source(sourceUrl, fs);
+      if (!format->IsSupported(source).ok()) {
+        std::cout << "Source " << source.path() << " is not supported" << std::endl;
+        continue;
+      }
+      std::cout << "  Processing tree: " << tk->GetName() << std::endl;
+      auto schemaOpt = format->Inspect(source);
+      if (!schemaOpt.ok()) {
+        std::cout << "Could not inspect source " << source.path() << std::endl;
+      }
+      auto schema = *schemaOpt;
+      auto fragment = format->MakeFragment(source, {}, schema);
+      if (!fragment.ok()) {
+        std::cout << "Could not make fragment from " << source.path() << "with schema:" << schema->ToString() << std::endl;
+        continue;
+      }
+      auto options = std::make_shared<arrow::dataset::ScanOptions>();
+      options->dataset_schema = schema;
+      auto scanner = format->ScanBatchesAsync(options, *fragment);
+      if (!scanner.ok()) {
+        std::cout << "Scanner not ok" << std::endl;
+        continue;
+      }
+      auto batches = (*scanner)();
+      auto result = batches.result();
+      if (!result.ok()) {
+        std::cout << "Could not get batches." << std::endl;
+        continue;
+      }
+      std::cout << "   Found a table with " << (*result)->columns().size() << " columns " << (*result)->num_rows() << " rows." << std::endl;
+
+      if ((*result)->num_rows() == 0) {
+        std::cout << "Empty table, skipping for now" << std::endl;
+        continue;
+      }
+      arrow::fs::FileLocator locator{outFs, destUrl};
+      std::cout << schema->ToString() << std::endl;
+      auto writer = rNtupleFormat->MakeWriter(*destination, schema, {}, locator);
+      auto success = writer->get()->Write(*result);
+      if (!success.ok()) {
+        std::cout << "Error while writing" << std::endl;
+        continue;
+      }
+    }
+    out->ls();
+    auto rootDestination = std::dynamic_pointer_cast<o2::framework::TDirectoryFileOutputStream>(*destination);
+  }
+  in->Close();
+  out->Close();
+}
diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx
index 8440e942903a5..8eb3a9825f0f7 100644
--- a/Framework/Core/test/test_Root2ArrowTable.cxx
+++ b/Framework/Core/test/test_Root2ArrowTable.cxx
@@ -14,6 +14,7 @@
 #include "Framework/TableBuilder.h"
 #include "Framework/RootTableBuilderHelpers.h"
 #include "Framework/ASoA.h"
+#include "Framework/PluginManager.h"
 #include "../src/ArrowDebugHelpers.h"
 
 #include <ROOT/RDataFrame.hxx>
@@ -26,6 +27,13 @@
 #include <TTree.h>
 #include <TRandom.h>
 #include <TFile.h>
+#include <ROOT/RField.hxx>
+#include <ROOT/RNTuple.hxx>
+#include <ROOT/RNTupleDescriptor.hxx>
+#include <ROOT/RNTupleModel.hxx>
+#include <ROOT/RNTupleReader.hxx>
+#include <ROOT/RNTupleUtil.hxx>
+#include <ROOT/RNTupleWriter.hxx>
 #include <memory>
 
 #include <arrow/array/array_primitive.h>
@@ -232,10 +240,31 @@ TEST_CASE("RootTree2Fragment")
   file->WriteObjectAny(&t1, t1.Class());
   auto* fileRead = new TBufferFile(TBuffer::kRead, file->BufferSize(), file->Buffer(), false, nullptr);
 
-  size_t totalSizeCompressed = 0;
-  size_t totalSizeUncompressed = 0;
-  auto format = std::make_shared<TTreeFileFormat>(totalSizeCompressed, totalSizeUncompressed);
-  auto fs = std::make_shared<TBufferFileFS>(fileRead);
+  std::vector<char const*> capabilitiesSpecs = {
+    "O2Framework:RNTupleObjectReadingCapability",
+    "O2Framework:TTreeObjectReadingCapability",
+  };
+
+  std::vector<LoadablePlugin> plugins;
+  for (auto spec : capabilitiesSpecs) {
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+  }
+  REQUIRE(plugins.size() == 2);
+
+  RootObjectReadingFactory factory;
+  std::vector<char const*> configDiscoverySpec = {};
+  PluginManager::loadFromPlugin<RootObjectReadingCapability, RootObjectReadingCapabilityPlugin>(plugins, factory.capabilities);
+  REQUIRE(factory.capabilities.size() == 2);
+  REQUIRE(factory.capabilities[0].name == "rntuple");
+  REQUIRE(factory.capabilities[1].name == "ttree");
+
+  // Plugins are hardcoded for now...
+  auto format = factory.capabilities[1].factory().format();
+
+  auto fs = std::make_shared<TBufferFileFS>(fileRead, factory);
 
   arrow::dataset::FileSource source("p", fs);
   REQUIRE(format->IsSupported(source) == true);
@@ -439,10 +468,34 @@ TEST_CASE("RootTree2Dataset")
   }
   f->Write();
 
-  size_t totalSizeCompressed = 0;
-  size_t totalSizeUncompressed = 0;
-  auto format = std::make_shared<TTreeFileFormat>(totalSizeCompressed, totalSizeUncompressed);
-  auto fs = std::make_shared<TFileFileSystem>(f, 50 * 1024 * 1024);
+  std::vector<char const*> capabilitiesSpecs = {
+    "O2Framework:RNTupleObjectReadingCapability",
+    "O2Framework:TTreeObjectReadingCapability",
+  };
+
+  RootObjectReadingFactory factory;
+
+  std::vector<LoadablePlugin> plugins;
+  for (auto spec : capabilitiesSpecs) {
+    auto morePlugins = PluginManager::parsePluginSpecString(spec);
+    for (auto& extra : morePlugins) {
+      plugins.push_back(extra);
+    }
+  }
+  REQUIRE(plugins.size() == 2);
+
+  PluginManager::loadFromPlugin<RootObjectReadingCapability, RootObjectReadingCapabilityPlugin>(plugins, factory.capabilities);
+
+  REQUIRE(factory.capabilities.size() == 2);
+  REQUIRE(factory.capabilities[0].name == "rntuple");
+  REQUIRE(factory.capabilities[1].name == "ttree");
+
+  // Plugins are hardcoded for now...
+  auto rNtupleFormat = factory.capabilities[0].factory().format();
+  auto format = factory.capabilities[1].factory().format();
+
+  auto fs = std::make_shared<TFileFileSystem>(f, 50 * 1024 * 1024, factory);
+
   arrow::dataset::FileSource source("DF_2/tracks", fs);
   REQUIRE(format->IsSupported(source) == true);
   auto schemaOpt = format->Inspect(source);
@@ -464,7 +517,7 @@ TEST_CASE("RootTree2Dataset")
   validateContents(*result);
 
   auto* output = new TMemFile("foo", "RECREATE");
-  auto outFs = std::make_shared<TFileFileSystem>(output, 0);
+  auto outFs = std::make_shared<TFileFileSystem>(output, 0, factory);
 
   // Open a stream at toplevel
   auto destination = outFs->OpenOutputStream("/", {});
@@ -503,4 +556,38 @@ TEST_CASE("RootTree2Dataset")
     REQUIRE((*resultWritten)->num_rows() == 100);
     validateContents(*resultWritten);
   }
+  arrow::fs::FileLocator rnTupleLocator{outFs, "/rntuple"};
+  // We write an RNTuple in the same TMemFile, using /rntuple as a location
+  auto rntupleDestination = std::dynamic_pointer_cast<TDirectoryFileOutputStream>(*destination);
+
+  {
+    auto rNtupleWriter = rNtupleFormat->MakeWriter(*destination, schema, {}, rnTupleLocator);
+    auto rNtupleSuccess = rNtupleWriter->get()->Write(*result);
+    REQUIRE(rNtupleSuccess.ok());
+  }
+
+  // And now we can read back the RNTuple into a RecordBatch
+  arrow::dataset::FileSource writtenRntupleSource("/rntuple", outFs);
+  auto newRNTupleFS = outFs->GetSubFilesystem(writtenRntupleSource);
+
+  REQUIRE(rNtupleFormat->IsSupported(writtenRntupleSource) == true);
+
+  auto rntupleSchemaOpt = rNtupleFormat->Inspect(writtenRntupleSource);
+  REQUIRE(rntupleSchemaOpt.ok());
+  auto rntupleSchemaWritten = *rntupleSchemaOpt;
+  REQUIRE(validateSchema(rntupleSchemaWritten));
+
+  auto rntupleFragmentWritten = rNtupleFormat->MakeFragment(writtenRntupleSource, {}, rntupleSchemaWritten);
+  REQUIRE(rntupleFragmentWritten.ok());
+  auto rntupleOptionsWritten = std::make_shared<arrow::dataset::ScanOptions>();
+  rntupleOptionsWritten->dataset_schema = rntupleSchemaWritten;
+  auto rntupleScannerWritten = rNtupleFormat->ScanBatchesAsync(rntupleOptionsWritten, *rntupleFragmentWritten);
+  REQUIRE(rntupleScannerWritten.ok());
+  auto rntupleBatchesWritten = (*rntupleScannerWritten)();
+  auto rntupleResultWritten = rntupleBatchesWritten.result();
+  REQUIRE(rntupleResultWritten.ok());
+  REQUIRE((*rntupleResultWritten)->columns().size() == 10);
+  REQUIRE(validateSchema((*rntupleResultWritten)->schema()));
+  REQUIRE((*rntupleResultWritten)->num_rows() == 100);
+  REQUIRE(validateContents(*rntupleResultWritten));
 }