femc_*,insert_*: enable caching

benchmarks/calo_pid/config.yml

@@ -18,7 +18,7 @@ sim:calo_pid:
         ]
   script:
     - |
-      snakemake --cache --cores 5 \
+      snakemake $SNAKEMAKE_FLAGS --cores 5 \
         $(seq --format="sim_output/calo_pid/epic_inner_detector/${PARTICLE}/100MeVto20GeV/130to177deg/${PARTICLE}_100MeVto20GeV_130to177deg.%04.f.eicrecon.tree.edm4eic.root" ${INDEX_RANGE})
 
 bench:calo_pid:
@@ -30,7 +30,7 @@ bench:calo_pid:
   script:
     - export PYTHONUSERBASE=$LOCAL_DATA_PATH/deps
     - pip install -r benchmarks/calo_pid/requirements.txt
-    - snakemake --cache --cores 1 results/epic_inner_detector/calo_pid
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 results/epic_inner_detector/calo_pid
 
 collect_results:calo_pid:
   allow_failure: true # until inference merged into EICrecon
@@ -41,5 +41,5 @@ collect_results:calo_pid:
   script:
     - ls -lrht
     - mv results{,_save}/ # move results directory out of the way to preserve it
-    - snakemake --cores 1 --delete-all-output results/epic_inner_detector/calo_pid
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 --delete-all-output results/epic_inner_detector/calo_pid
     - mv results{_save,}/

benchmarks/ecal_gaps/config.yml

@@ -4,7 +4,7 @@ sim:ecal_gaps:
   script:
     - mkdir -p $LOCAL_DATA_PATH/input
     - ln -s $LOCAL_DATA_PATH/input input
-    - snakemake $SNAKEMAKE_FLAGS --cache --cores 10 results/epic_inner_detector/ecal_gaps --omit-from ecal_gaps
+    - snakemake $SNAKEMAKE_FLAGS --cores 10 results/epic_inner_detector/ecal_gaps --omit-from ecal_gaps
 
 bench:ecal_gaps:
   extends: .det_benchmark

benchmarks/femc_electron/Snakefile

@@ -8,12 +8,12 @@ def get_n_events(wildcards):
 rule femc_electron_generate:
     input:
         script="benchmarks/femc_electron/analysis/gen_particles.cxx",
+    output:
+        GEN_FILE="sim_output/femc_electron/e-_{P}GeV.hepmc",
     params:
         N_EVENTS=get_n_events,
         th_max=28,
-        th_min=2.0
-    output:
-        GEN_FILE="sim_output/femc_electron/e-_{P}GeV.hepmc"
+        th_min=2.0,
     shell:
         """
 mkdir -p sim_output/femc_electron
@@ -22,17 +22,24 @@ root -l -b -q '{input.script}({params.N_EVENTS},"{output.GEN_FILE}", "e-", {para
 
 rule femc_electron_simulate:
     input:
-        GEN_FILE="sim_output/femc_electron/e-_{P}GeV.hepmc"
+        GEN_FILE="sim_output/femc_electron/e-_{P}GeV.hepmc",
+        warmup="warmup/{DETECTOR_CONFIG}.edm4hep.root",
+        geometry_lib=find_epic_libraries(),
+    output:
+        SIM_FILE="sim_output/femc_electron/{DETECTOR_CONFIG}_sim_e-_{P}GeV.edm4hep.root",
     params:
         N_EVENTS=get_n_events,
-        PHYSICS_LIST="FTFP_BERT"
-    output:
-        SIM_FILE="sim_output/femc_electron/{DETECTOR_CONFIG}_sim_e-_{P}GeV.edm4hep.root"
+        PHYSICS_LIST="FTFP_BERT",
+        DETECTOR_PATH=os.environ["DETECTOR_PATH"],
+        DETECTOR_CONFIG=lambda wildcards: wildcards.DETECTOR_CONFIG,
+        DD4HEP_HASH=get_spack_package_hash("dd4hep"),
+        NPSIM_HASH=get_spack_package_hash("npsim"),
+    cache: True
     shell:
         """
 # Running simulation
 npsim \
-   --compactFile $DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml \
+   --compactFile {params.DETECTOR_PATH}/{params.DETECTOR_CONFIG}.xml \
    --numberOfEvents {params.N_EVENTS} \
    --physicsList {params.PHYSICS_LIST} \
    --inputFiles {input.GEN_FILE} \
@@ -41,14 +48,18 @@ npsim \
 
 rule femc_electron_recon:
     input:
-        SIM_FILE="sim_output/femc_electron/{DETECTOR_CONFIG}_sim_e-_{P}GeV.edm4hep.root"
+        SIM_FILE="sim_output/femc_electron/{DETECTOR_CONFIG}_sim_e-_{P}GeV.edm4hep.root",
     output:
-        REC_FILE="sim_output/femc_electron/{DETECTOR_CONFIG}_rec_e-_{P}GeV.edm4eic.root"
+        REC_FILE="sim_output/femc_electron/{DETECTOR_CONFIG}_rec_e-_{P}GeV.edm4eic.root",
     params:
         N_EVENTS=get_n_events,
+        DETECTOR_CONFIG=lambda wildcards: wildcards.DETECTOR_CONFIG,
+        EICRECON_HASH=get_spack_package_hash("eicrecon"),
+    cache: True
     shell:
         """
-eicrecon {input.SIM_FILE} -Ppodio:output_file={output.REC_FILE} -Pdd4hep:xml_files=$DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml -Ppodio:output_collections=MCParticles,HcalEndcapPInsertRecHits,HcalEndcapPInsertClusters,HcalEndcapPInsertSubcellHits,EcalEndcapPInsertRecHits,EcalEndcapPInsertClusters,EcalEndcapPInsertRecHits,EcalEndcapPClusters -Pjana:nevents={params.N_EVENTS}
+env DETECTOR_CONFIG={params.DETECTOR_CONFIG} \
+  eicrecon {input.SIM_FILE} -Ppodio:output_file={output.REC_FILE} -Ppodio:output_collections=MCParticles,HcalEndcapPInsertRecHits,HcalEndcapPInsertClusters,HcalEndcapPInsertSubcellHits,EcalEndcapPInsertRecHits,EcalEndcapPInsertClusters,EcalEndcapPInsertRecHits,EcalEndcapPClusters -Pjana:nevents={params.N_EVENTS}
 """
 
 rule femc_electron_analysis:

benchmarks/femc_photon/Snakefile

@@ -8,12 +8,12 @@ def get_n_events(wildcards):
 rule femc_photon_generate:
     input:
         script="benchmarks/femc_photon/analysis/gen_particles.cxx",
+    output:
+        GEN_FILE="sim_output/femc_photon/photon_{P}GeV.hepmc",
     params:
         N_EVENTS=get_n_events,
         th_max=28,
-        th_min=2.0
-    output:
-        GEN_FILE="sim_output/femc_photon/photon_{P}GeV.hepmc"
+        th_min=2.0,
     shell:
         """
 mkdir -p sim_output/femc_photon
@@ -22,17 +22,24 @@ root -l -b -q '{input.script}({params.N_EVENTS},"{output.GEN_FILE}", "gamma", {p
 
 rule femc_photon_simulate:
     input:
-        GEN_FILE="sim_output/femc_photon/photon_{P}GeV.hepmc"
+        GEN_FILE="sim_output/femc_photon/photon_{P}GeV.hepmc",
+        warmup="warmup/{DETECTOR_CONFIG}.edm4hep.root",
+        geometry_lib=find_epic_libraries(),
+    output:
+        SIM_FILE="sim_output/femc_photon/{DETECTOR_CONFIG}_sim_photon_{P}GeV.edm4hep.root",
     params:
         N_EVENTS=get_n_events,
-        PHYSICS_LIST="FTFP_BERT"
-    output:
-        SIM_FILE="sim_output/femc_photon/{DETECTOR_CONFIG}_sim_photon_{P}GeV.edm4hep.root"
+        PHYSICS_LIST="FTFP_BERT",
+        DETECTOR_PATH=os.environ["DETECTOR_PATH"],
+        DETECTOR_CONFIG=lambda wildcards: wildcards.DETECTOR_CONFIG,
+        DD4HEP_HASH=get_spack_package_hash("dd4hep"),
+        NPSIM_HASH=get_spack_package_hash("npsim"),
+    cache: True
     shell:
         """
 # Running simulation
 npsim \
-   --compactFile $DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml \
+   --compactFile {params.DETECTOR_PATH}/{params.DETECTOR_CONFIG}.xml \
    --numberOfEvents {params.N_EVENTS} \
    --physicsList {params.PHYSICS_LIST} \
    --inputFiles {input.GEN_FILE} \
@@ -46,9 +53,13 @@ rule femc_photon_recon:
         REC_FILE="sim_output/femc_photon/{DETECTOR_CONFIG}_rec_photon_{P}GeV.edm4eic.root"
     params:
         N_EVENTS=get_n_events,
+        DETECTOR_CONFIG=lambda wildcards: wildcards.DETECTOR_CONFIG,
+        EICRECON_HASH=get_spack_package_hash("eicrecon"),
+    cache: True
     shell:
         """
-eicrecon {input.SIM_FILE} -Ppodio:output_file={output.REC_FILE} -Pdd4hep:xml_files=$DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml -Ppodio:output_collections=MCParticles,HcalEndcapPInsertRecHits,HcalEndcapPInsertClusters,HcalEndcapPInsertSubcellHits,EcalEndcapPInsertRecHits,EcalEndcapPInsertClusters,EcalEndcapPInsertRecHits,EcalEndcapPClusters -Pjana:nevents={params.N_EVENTS}
+env DETECTOR_CONFIG={params.DETECTOR_CONFIG} \
+  eicrecon {input.SIM_FILE} -Ppodio:output_file={output.REC_FILE} -Ppodio:output_collections=MCParticles,HcalEndcapPInsertRecHits,HcalEndcapPInsertClusters,HcalEndcapPInsertSubcellHits,EcalEndcapPInsertRecHits,EcalEndcapPInsertClusters,EcalEndcapPInsertRecHits,EcalEndcapPClusters -Pjana:nevents={params.N_EVENTS}
 """
 
 rule femc_photon_analysis:

benchmarks/femc_pi0/Snakefile

@@ -8,12 +8,12 @@ def get_n_events(wildcards):
 rule femc_pi0_generate:
     input:
         script="benchmarks/femc_pi0/analysis/gen_particles.cxx",
+    output:
+        GEN_FILE="sim_output/femc_pi0/pi0_{P}GeV.hepmc",
     params:
         N_EVENTS=get_n_events,
         th_max=28,
-        th_min=2.0
-    output:
-        GEN_FILE="sim_output/femc_pi0/pi0_{P}GeV.hepmc"
+        th_min=2.0,
     shell:
         """
 mkdir -p sim_output/femc_pi0
@@ -22,17 +22,24 @@ root -l -b -q '{input.script}({params.N_EVENTS},"{output.GEN_FILE}", "pi0", {par
 
 rule femc_pi0_simulate:
     input:
-        GEN_FILE="sim_output/femc_pi0/pi0_{P}GeV.hepmc"
+        GEN_FILE="sim_output/femc_pi0/pi0_{P}GeV.hepmc",
+        warmup="warmup/{DETECTOR_CONFIG}.edm4hep.root",
+        geometry_lib=find_epic_libraries(),
     params:
         N_EVENTS=get_n_events,
-        PHYSICS_LIST="FTFP_BERT"
+        PHYSICS_LIST="FTFP_BERT",
+        DETECTOR_PATH=os.environ["DETECTOR_PATH"],
+        DETECTOR_CONFIG=lambda wildcards: wildcards.DETECTOR_CONFIG,
+        DD4HEP_HASH=get_spack_package_hash("dd4hep"),
+        NPSIM_HASH=get_spack_package_hash("npsim"),
+    cache: True
     output:
         SIM_FILE="sim_output/femc_pi0/{DETECTOR_CONFIG}_sim_pi0_{P}GeV.edm4hep.root"
     shell:
         """
 # Running simulation
 npsim \
-   --compactFile $DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml \
+   --compactFile {params.DETECTOR_PATH}/{params.DETECTOR_CONFIG}.xml \
    --numberOfEvents {params.N_EVENTS} \
    --physicsList {params.PHYSICS_LIST} \
    --inputFiles {input.GEN_FILE} \
@@ -46,9 +53,13 @@ rule femc_pi0_recon:
         REC_FILE="sim_output/femc_pi0/{DETECTOR_CONFIG}_rec_pi0_{P}GeV.edm4eic.root"
     params:
         N_EVENTS=get_n_events,
+        DETECTOR_CONFIG=lambda wildcards: wildcards.DETECTOR_CONFIG,
+        EICRECON_HASH=get_spack_package_hash("eicrecon"),
+    cache: True
     shell:
         """
-eicrecon {input.SIM_FILE} -Ppodio:output_file={output.REC_FILE} -Pdd4hep:xml_files=$DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml -Ppodio:output_collections=MCParticles,HcalEndcapPInsertRecHits,HcalEndcapPInsertClusters,HcalEndcapPInsertSubcellHits,EcalEndcapPInsertRecHits,EcalEndcapPInsertClusters,EcalEndcapPInsertRecHits,EcalEndcapPClusters -Pjana:nevents={params.N_EVENTS}
+env DETECTOR_CONFIG={params.DETECTOR_CONFIG} \
+  eicrecon {input.SIM_FILE} -Ppodio:output_file={output.REC_FILE} -Ppodio:output_collections=MCParticles,HcalEndcapPInsertRecHits,HcalEndcapPInsertClusters,HcalEndcapPInsertSubcellHits,EcalEndcapPInsertRecHits,EcalEndcapPInsertClusters,EcalEndcapPInsertRecHits,EcalEndcapPClusters -Pjana:nevents={params.N_EVENTS}
 """
 
 rule femc_pi0_analysis:

benchmarks/insert_muon/Snakefile

@@ -1,31 +1,37 @@
 rule insert_muon_generate:
-	input:
-                script="benchmarks/insert_muon/analysis/gen_particles.cxx",
-	params:
-		NEVENTS_GEN=5000,
-		th_max=7.0,
-		th_min=1.7
-	output:
-		GEN_FILE="sim_output/insert_muon/mu-_{P}GeV.hepmc"
-	shell:
-		"""
+    input:
+        script="benchmarks/insert_muon/analysis/gen_particles.cxx",
+    output:
+        GEN_FILE="sim_output/insert_muon/mu-_{P}GeV.hepmc",
+    params:
+        NEVENTS_GEN=5000,
+        th_max=7.0,
+        th_min=1.7,
+    shell:
+        """
 root -l -b -q '{input.script}({params.NEVENTS_GEN},"{output.GEN_FILE}", "mu-", {params.th_min}, {params.th_max}, 0., 360., {wildcards.P})'
 """
 
 rule insert_muon_simulate:
-	input:
-                GEN_FILE="sim_output/insert_muon/mu-_{P}GeV.hepmc",
-                warmup="warmup/{DETECTOR_CONFIG}.edm4hep.root",
-	params:
-		PHYSICS_LIST="FTFP_BERT"
-	output:
-		SIM_FILE="sim_output/insert_muon/{DETECTOR_CONFIG}_sim_mu-_{P}GeV_{INDEX}.edm4hep.root"
-	shell:
-		"""
+    input:
+        GEN_FILE="sim_output/insert_muon/mu-_{P}GeV.hepmc",
+        warmup="warmup/{DETECTOR_CONFIG}.edm4hep.root",
+        geometry_lib=find_epic_libraries(),
+    output:
+        SIM_FILE="sim_output/insert_muon/{DETECTOR_CONFIG}_sim_mu-_{P}GeV_{INDEX}.edm4hep.root",
+    params:
+        PHYSICS_LIST="FTFP_BERT",
+        DETECTOR_PATH=os.environ["DETECTOR_PATH"],
+        DETECTOR_CONFIG=lambda wildcards: wildcards.DETECTOR_CONFIG,
+        DD4HEP_HASH=get_spack_package_hash("dd4hep"),
+        NPSIM_HASH=get_spack_package_hash("npsim"),
+    cache: True
+    shell:
+        """
 NEVENTS_SIM=1000
 # Running simulation
 npsim \
-   --compactFile $DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml \
+   --compactFile {params.DETECTOR_PATH}/{params.DETECTOR_CONFIG}.xml \
    --skipNEvents $(( $NEVENTS_SIM * {wildcards.INDEX} )) \
    --numberOfEvents $NEVENTS_SIM \
    --physicsList {params.PHYSICS_LIST} \
@@ -34,28 +40,33 @@ npsim \
 """
 
 rule insert_muon_recon:
-        input:
-                SIM_FILE="sim_output/insert_muon/{DETECTOR_CONFIG}_sim_mu-_{P}GeV_{INDEX}.edm4hep.root"
-        output:
-                REC_FILE="sim_output/insert_muon/{DETECTOR_CONFIG}_rec_mu-_{P}GeV_{INDEX}.edm4hep.root"
-        shell:
-                """
+    input:
+        SIM_FILE="sim_output/insert_muon/{DETECTOR_CONFIG}_sim_mu-_{P}GeV_{INDEX}.edm4hep.root",
+    output:
+        REC_FILE="sim_output/insert_muon/{DETECTOR_CONFIG}_rec_mu-_{P}GeV_{INDEX}.edm4hep.root",
+    params:
+        DETECTOR_CONFIG=lambda wildcards: wildcards.DETECTOR_CONFIG,
+        EICRECON_HASH=get_spack_package_hash("eicrecon"),
+    cache: True
+    shell:
+        """
 NEVENTS_REC=1000
-eicrecon {input.SIM_FILE} -Ppodio:output_file={output.REC_FILE} -Pdd4hep:xml_files=$DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml -Ppodio:output_collections=MCParticles,HcalEndcapPInsertRecHits,HcalEndcapPInsertClusters,HcalEndcapPInsertSubcellHits,EcalEndcapPInsertRecHits,EcalEndcapPInsertClusters  -Pjana:nevents=$NEVENTS_REC
+env DETECTOR_CONFIG={params.DETECTOR_CONFIG} \
+  eicrecon {input.SIM_FILE} -Ppodio:output_file={output.REC_FILE} -Ppodio:output_collections=MCParticles,HcalEndcapPInsertRecHits,HcalEndcapPInsertClusters,HcalEndcapPInsertSubcellHits,EcalEndcapPInsertRecHits,EcalEndcapPInsertClusters  -Pjana:nevents=$NEVENTS_REC
 """
 
 rule insert_muon_analysis:
-	input:
-                expand("sim_output/insert_muon/{DETECTOR_CONFIG}_sim_mu-_{P}GeV_{INDEX}.edm4hep.root",
-		    P=[50],
-		    DETECTOR_CONFIG=["{DETECTOR_CONFIG}"],
-                    INDEX=range(5),
-                ),
-                script="benchmarks/insert_muon/analysis/muon_plots.py",
-	output:
-		results_dir=directory("results/{DETECTOR_CONFIG}/insert_muon"),
-	shell:
-		"""
+    input:
+        expand("sim_output/insert_muon/{DETECTOR_CONFIG}_sim_mu-_{P}GeV_{INDEX}.edm4hep.root",
+            P=[50],
+            DETECTOR_CONFIG=["{DETECTOR_CONFIG}"],
+            INDEX=range(5),
+        ),
+        script="benchmarks/insert_muon/analysis/muon_plots.py",
+    output:
+        results_dir=directory("results/{DETECTOR_CONFIG}/insert_muon"),
+    shell:
+        """
 mkdir -p {output.results_dir}
 python {input.script} {output.results_dir}
 """