Patch missing collections on the fly

StandardConfig/production/ILDReconstruction.py

@@ -15,10 +15,7 @@
 )
 from Gaudi.Configuration import INFO
 
-try:
- from k4FWCore.utils import SequenceLoader, import_from
-except ImportError:
- from py_utils import import_from, SequenceLoader
+from py_utils import import_from, SequenceLoader, parse_collection_patch_file
 
 from k4FWCore.parseArgs import parser
 from k4MarlinWrapper.parseConstants import parseConstants
@@ -54,6 +51,8 @@
  "ILD_l5_v11",
 )
 
+REC_COLLECTION_CONTENTS_FILE = "collections_rec_level.txt"
+
 parser.add_argument(
  "--inputFiles",
  action="extend",
@@ -301,16 +300,26 @@ def create_reader(input_files):
  "Printing": ["0"],
  "RootFile": [f"{reco_args.outputFileBase}_PfoAnalysis.root"],
 }
-
 algList.append(MyPfoAnalysis)
 
 if reco_args.lcioOutput != "only":
+ # Attach the LCIO -> EDM4hep conversion to the last processor that is run
+ # before the output
  lcioToEDM4hepOutput = Lcio2EDM4hepTool("OutputConversion")
  # Take care of the different naming conventions
  lcioToEDM4hepOutput.collNameMapping = {"MCParticle": "MCParticles"}
  lcioToEDM4hepOutput.OutputLevel = INFO
- # Attach the conversion to the last non-output processor that is always run
- MyPfoAnalysis.Lcio2EDM4hepTool = lcioToEDM4hepOutput
+
+ # Make sure that all collections are always available by patching in missing
+ # ones on-the-fly
+ collPatcherRec = MarlinProcessorWrapper(
+ "CollPacherREC", OutputLevel=INFO, ProcessorType="PatchCollections"
+ )
+ collPatcherRec.Parameters = {
+ "PatchCollections": parse_collection_patch_file(REC_COLLECTION_CONTENTS_FILE)
+ }
+ collPatcherRec.Lcio2EDM4hepTool = lcioToEDM4hepOutput
+ algList.append(collPatcherRec)
 
  edm4hepOutput = PodioOutput("EDM4hepOutput")
  edm4hepOutput.filename = f"{reco_args.outputFileBase}_REC.edm4hep.root"

StandardConfig/production/collections_rec_level.txt

@@ -0,0 +1,124 @@
+ YokeEndcapsCollection SimCalorimeterHit 
+ VXDCollection SimTrackerHit 
+ V0RecoParticles ReconstructedParticle 
+ TPCTrackerHitRelations LCRelation[TrackerHit,SimTrackerHit] 
+ TPCCollection SimTrackerHit 
+ SITTrackerHitRelations LCRelation[TrackerHitPlane,SimTrackerHit] 
+ SITCollection SimTrackerHit 
+ SETSpacePoints TrackerHit 
+ SETSpacePointRelations LCRelation[TrackerHit,SimTrackerHit] 
+ RecoMCTruthLink LCRelation[ReconstructedParticle,MCParticle] 
+ ProngVertices Vertex 
+ ProngRecoParticles ReconstructedParticle 
+ PrimaryVertex Vertex 
+ PandoraPFOs ReconstructedParticle 
+ RelationBCalHit LCRelation[CalorimeterHit,SimCalorimeterHit] 
+ PandoraPFANewStartVertices Vertex 
+ MarlinTrkTracksProtonMCP LCRelation[Track,MCParticle] 
+ SubsetTracks Track 
+ MarlinTrkTracksProton Track 
+ TPCTrackerHits TrackerHit 
+ MarlinTrkTracks Track 
+ SETTrackerHitRelations LCRelation[TrackerHitPlane,SimTrackerHit] 
+ MUON CalorimeterHit 
+ VXDTrackerHits TrackerHitPlane 
+ MCTruthMarlinTrkTracksLink LCRelation[MCParticle,Track] 
+ MCParticle MCParticle 
+ LumiCalCollection SimCalorimeterHit 
+ LHCalCollection SimCalorimeterHit 
+ LHCAL CalorimeterHit 
+ YokeBarrelCollection SimCalorimeterHit 
+ LCAL CalorimeterHit 
+ KinkVertices Vertex 
+ KinkRecoParticles ReconstructedParticle 
+ MCTruthClusterLink LCRelation[MCParticle,Cluster] 
+ EcalEndcapsCollectionDigi CalorimeterHit 
+ BCAL CalorimeterHit 
+ FTDSpacePointRelations LCRelation[TrackerHit,SimTrackerHit] 
+ PrimaryVertex_RP ReconstructedParticle 
+ HcalEndcapRingCollectionRec CalorimeterHit 
+ HcalEndcapsCollectionDigi CalorimeterHit 
+ EcalEndcapRingRelationsSimRec LCRelation[CalorimeterHit,SimCalorimeterHit] 
+ EcalEndcapRingCollection SimCalorimeterHit 
+ TPCLowPtCollection SimTrackerHit 
+ HcalEndcapRingCollection SimCalorimeterHit 
+ EcalEndcapsCollectionGapHits CalorimeterHit 
+ GammaGammaCandidateEtaPrimes ReconstructedParticle 
+ EcalBarrelRelationsSimRec LCRelation[CalorimeterHit,SimCalorimeterHit] 
+ EcalBarrelRelationsSimDigi LCRelation[CalorimeterHit,SimCalorimeterHit] 
+ EcalBarrelCollectionDigi CalorimeterHit 
+ SITTrackerHits TrackerHitPlane 
+ EcalBarrelCollection SimCalorimeterHit 
+ BuildUpVertex_V0 Vertex 
+ EcalBarrelCollectionRec CalorimeterHit 
+ SplitRecoParticles ReconstructedParticle 
+ PandoraClusters Cluster 
+ EcalEndcapsRelationsSimRec LCRelation[CalorimeterHit,SimCalorimeterHit] 
+ HcalEndcapRingRelationsSimDigi LCRelation[CalorimeterHit,SimCalorimeterHit] 
+ ECalEndcapSiHitsOdd SimCalorimeterHit 
+ BuildUpVertex_V0_RP ReconstructedParticle 
+ GammaGammaCandidatePi0s ReconstructedParticle 
+ MarlinTrkTracksKaonMCP LCRelation[Track,MCParticle] 
+ EcalEndcapRingCollectionDigi CalorimeterHit 
+ ClusterMCTruthLink LCRelation[Cluster,MCParticle] 
+ V0Vertices Vertex 
+ HcalEndcapsCollection SimCalorimeterHit 
+ ECalBarrelSiHitsOdd SimCalorimeterHit 
+ ClupatraTracks Track 
+ SETCollection SimTrackerHit 
+ EcalEndcapsCollection SimCalorimeterHit 
+ EcalEndcapRingRelationsSimDigi LCRelation[CalorimeterHit,SimCalorimeterHit] 
+ FTDPixelTrackerHitRelations LCRelation[TrackerHitPlane,SimTrackerHit] 
+ RelationMuonHit LCRelation[CalorimeterHit,SimCalorimeterHit] 
+ EcalBarrelCollectionGapHits CalorimeterHit 
+ MarlinTrkTracksMCTruthLink LCRelation[Track,MCParticle] 
+ ClupatraTrackSegments Track 
+ SplitVertices Vertex 
+ HcalEndcapRingRelationsSimRec LCRelation[CalorimeterHit,SimCalorimeterHit] 
+ MCParticlesSkimmed MCParticle 
+ ECalEndcapSiHitsEven SimCalorimeterHit 
+ MarlinTrkTracksKaon Track 
+ EcalEndcapsCollectionRec CalorimeterHit 
+ HcalEndcapRingCollectionDigi CalorimeterHit 
+ GammaGammaCandidateEtas ReconstructedParticle 
+ GammaGammaParticles ReconstructedParticle 
+ SiTracks Track 
+ RelationLHcalHit LCRelation[CalorimeterHit,SimCalorimeterHit] 
+ MCTruthRecoLink LCRelation[MCParticle,ReconstructedParticle] 
+ DistilledPFOs ReconstructedParticle 
+ BuildUpVertex_RP ReconstructedParticle 
+ EcalEndcapsRelationsSimDigi LCRelation[CalorimeterHit,SimCalorimeterHit] 
+ VXDTrackerHitRelations LCRelation[TrackerHitPlane,SimTrackerHit] 
+ FTDCollection SimTrackerHit 
+ BuildUpVertex Vertex 
+ FTDPixelTrackerHits TrackerHitPlane 
+ TPCSpacePointCollection SimTrackerHit 
+ FTDStripTrackerHits TrackerHitPlane 
+ ForwardTracks Track 
+ ECalBarrelSiHitsEven SimCalorimeterHit 
+ FTDSpacePoints TrackerHit 
+ FTDStripTrackerHitRelations LCRelation[TrackerHitPlane,SimTrackerHit] 
+ BeamCalCollection SimCalorimeterHit 
+ HcalEndcapsCollectionRec CalorimeterHit 
+ HcalBarrelRelationsSimDigi LCRelation[CalorimeterHit,SimCalorimeterHit] 
+ FTD_STRIPCollection SimTrackerHit 
+ FTD_PIXELCollection SimTrackerHit 
+ HcalBarrelCollectionDigi CalorimeterHit 
+ SETTrackerHits TrackerHitPlane 
+ EcalEndcapRingCollectionRec CalorimeterHit 
+ HcalBarrelCollectionRec CalorimeterHit 
+ HcalBarrelRelationsSimRec LCRelation[CalorimeterHit,SimCalorimeterHit] 
+ HcalBarrelRegCollection SimCalorimeterHit 
+ RelationLcalHit LCRelation[CalorimeterHit,SimCalorimeterHit] 
+ HcalEndcapsRelationsSimRec LCRelation[CalorimeterHit,SimCalorimeterHit] 
+ HcalEndcapsRelationsSimDigi LCRelation[CalorimeterHit,SimCalorimeterHit] 
+BasicVariablePID PandoraPFOs|electronLikelihood,muonLikelihood,pionLikelihood,kaonLikelihood,protonLikelihood,hadronLikelihood,MVAOutput_mupiSeparation,electronProbability,muonProbability,pionProbability,kaonProbability,protonProbability,hadronProbability,electron_dEdxdistance,muon_dEdxdistance,pion_dEdxdistance,kaon_dEdxdistance,proton_dEdxdistance
+LikelihoodPID PandoraPFOs|electronLikelihood,muonLikelihood,pionLikelihood,kaonLikelihood,protonLikelihood,hadronLikelihood,MVAOutput_mupiSeparation,electronProbability,muonProbability,pionProbability,kaonProbability,protonProbability,hadronProbability,electron_dEdxdistance,muon_dEdxdistance,pion_dEdxdistance,kaon_dEdxdistance,proton_dEdxdistance
+LowMomMuID PandoraPFOs|electronLikelihood,muonLikelihood,pionLikelihood,kaonLikelihood,protonLikelihood,hadronLikelihood,MVAOutput_mupiSeparation,electronProbability,muonProbability,pionProbability,kaonProbability,protonProbability,hadronProbability,electron_dEdxdistance,muon_dEdxdistance,pion_dEdxdistance,kaon_dEdxdistance,proton_dEdxdistance
+ShowerShapesPID PandoraPFOs|electronLikelihood,muonLikelihood,pionLikelihood,kaonLikelihood,protonLikelihood,hadronLikelihood,MVAOutput_mupiSeparation,electronProbability,muonProbability,pionProbability,kaonProbability,protonProbability,hadronProbability,electron_dEdxdistance,muon_dEdxdistance,pion_dEdxdistance,kaon_dEdxdistance,proton_dEdxdistance
+TOFEstimators0ps PandoraPFOs|timeOfFlight
+TOFEstimators100ps PandoraPFOs|timeOfFlight
+TOFEstimators10ps PandoraPFOs|timeOfFlight
+TOFEstimators50ps PandoraPFOs|timeOfFlight
+TrackLengthProcessor PandoraPFOs|trackLengthToSET,trackLengthToEcal,momentumHMToSET,momentumHMToEcal
+dEdxPID PandoraPFOs|electronLikelihood,muonLikelihood,pionLikelihood,kaonLikelihood,protonLikelihood,hadronLikelihood,MVAOutput_mupiSeparation,electronProbability,muonProbability,pionProbability,kaonProbability,protonProbability,hadronProbability,electron_dEdxdistance,muon_dEdxdistance,pion_dEdxdistance,kaon_dEdxdistance,proton_dEdxdistance

StandardConfig/production/py_utils.py

@@ -1,6 +1,6 @@
 import os
 from io import TextIOWrapper
-from typing import Union, Optional, Dict, Any
+from typing import Union, Optional, Dict, Any, List
 import importlib.util
 import importlib.abc
 from importlib.machinery import SourceFileLoader
@@ -140,3 +140,30 @@ def load(self, sequence: str) -> None:
 
  seq = getattr(seq_module, seq_name)
  self.alg_list.extend(seq)
+
+
+def parse_collection_patch_file(patch_file: Union[str, os.PathLike]) -> List[str]:
+ """Parse a collection patch file such that it can be used by the
+ PatchCollections processor.
+
+ This function reads the file that has been passed in and effectively
+ flattens its contents into one list of strings. The main assumption is that
+ the file has been produced via `check_missing_colls --minimal <...>` in
+ which case it can be directly consumed. Note that no real error checking is
+ done to detect malformed inputs in which case something else at a later
+ stage will most likely break.
+
+ Args:
+ patch_file (Union[str, os.PathLike]): The path to the file that should
+ be parsed
+
+ Returns:
+ List[str]: A list of strings (pairs of "names" and "types") that can be
+ consumed by the PatchCollections processor
+
+ """
+ with open(patch_file, "r") as pfile:
+ patch_colls = [l.split() for l in pfile.readlines()]
+
+ # Flatten the list of lists into one large list
+ return [s for strings in patch_colls for s in strings]