.gitlab-ci.yml: introduce global SNAKEMAKE_FLAGS, don't re-run based …

…on mtime (#92)

.gitlab-ci.yml

@@ -5,6 +5,7 @@ variables:
   DETECTOR_CONFIG: epic_craterlake
   GITHUB_SHA: ''
   GITHUB_REPOSITORY: ''
+  SNAKEMAKE_FLAGS: '--cache --rerun-triggers code input params software-env'
 
 workflow:
   name: '$PIPELINE_NAME'

benchmarks/backgrounds/config.yml

@@ -5,7 +5,7 @@ sim:backgrounds:
     - mkdir -p $LOCAL_DATA_PATH/input
     - ln -s $LOCAL_DATA_PATH/input input
     - |
-      snakemake --cache --cores 2 \
+      snakemake $SNAKEMAKE_FLAGS --cores 2 \
         sim_output/$DETECTOR_CONFIG/backgrounds/EPIC/EVGEN/BACKGROUNDS/BEAMGAS/electron/GETaLM1.0.0-1.0/10GeV/GETaLM1.0.0-1.0_ElectronBeamGas_10GeV_foam_emin10keV_run001.edm4hep.root \
         sim_output/$DETECTOR_CONFIG/backgrounds/EPIC/EVGEN/DIS/NC/10x100/minQ2=1/pythia8NCDIS_10x100_minQ2=1_beamEffects_xAngle=-0.025_hiDiv_1.edm4hep.root \
         sim_output/$DETECTOR_CONFIG/backgrounds/EPIC/EVGEN/BACKGROUNDS/BEAMGAS/proton/pythia8.306-1.0/100GeV/pythia8.306-1.0_ProtonBeamGas_100GeV_run001.edm4hep.root
@@ -19,7 +19,7 @@ bench:backgrounds_emcal_backwards:
     - ln -s $LOCAL_DATA_PATH/input input
     - export PYTHONUSERBASE=$LOCAL_DATA_PATH/deps
     - pip install -r benchmarks/backgrounds/requirements.txt
-    - snakemake --cores 8 backgrounds_ecal_backwards
+    - snakemake $SNAKEMAKE_FLAGS --cores 8 backgrounds_ecal_backwards
 
 collect_results:backgrounds:
   extends: .det_benchmark
@@ -29,5 +29,5 @@ collect_results:backgrounds:
   script:
     - ls -lrht
     - mv results{,_save}/ # move results directory out of the way to preserve it
-    - snakemake --cores 1 --delete-all-output backgrounds_ecal_backwards
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 --delete-all-output backgrounds_ecal_backwards
     - mv results{_save,}/

benchmarks/backwards_ecal/config.yml

@@ -16,7 +16,7 @@ sim:backwards_ecal:
         ]
   script:
     - |
-      snakemake --cache --cores 5 sim_output/backwards_ecal/${DETECTOR_CONFIG}/${PARTICLE}/${MOMENTUM}/130to177deg/flag
+      snakemake $SNAKEMAKE_FLAGS --cores 5 sim_output/backwards_ecal/${DETECTOR_CONFIG}/${PARTICLE}/${MOMENTUM}/130to177deg/flag
 
 bench:backwards_ecal:
   extends: .det_benchmark
@@ -26,7 +26,7 @@ bench:backwards_ecal:
   script:
     - export PYTHONUSERBASE=$LOCAL_DATA_PATH/deps
     - pip install -r benchmarks/backwards_ecal/requirements.txt
-    - snakemake --cores 1 backwards_ecal
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 backwards_ecal
 
 collect_results:backwards_ecal:
   extends: .det_benchmark
@@ -36,5 +36,5 @@ collect_results:backwards_ecal:
   script:
     - ls -lrht
     - mv results{,_save}/ # move results directory out of the way to preserve it
-    - snakemake --cores 1 --delete-all-output backwards_ecal
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 --delete-all-output backwards_ecal
     - mv results{_save,}/

benchmarks/barrel_ecal/config.yml

@@ -2,34 +2,34 @@ sim:emcal_barrel_pions:
   extends: .det_benchmark
   stage: simulate
   script:
-    - snakemake --cores 1 $DETECTOR_CONFIG/sim_output/sim_emcal_barrel_{piplus,piminus}_energies5.0_5.0.edm4hep.root
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 $DETECTOR_CONFIG/sim_output/sim_emcal_barrel_{piplus,piminus}_energies5.0_5.0.edm4hep.root
 
 sim:emcal_barrel_pi0:
   extends: .det_benchmark
   stage: simulate
   script:
-    - snakemake --cores 1 $DETECTOR_CONFIG/sim_output/sim_emcal_barrel_pi0_energies5.0_5.0.edm4hep.root
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 $DETECTOR_CONFIG/sim_output/sim_emcal_barrel_pi0_energies5.0_5.0.edm4hep.root
 
 sim:emcal_barrel_electrons:
   extends: .det_benchmark
   stage: simulate
   script:
     - if [[ "$RUN_EXTENDED_BENCHMARK" == "true" ]] ; then snakemake --cores $DETECTOR_CONFIG/results/energy_scan/emcal_barrel_electron_fsam_scan.png; fi
     - export JUGGLER_N_EVENTS=400
-    - snakemake --cores 1 $DETECTOR_CONFIG/sim_output/sim_emcal_barrel_electron_energies5.0_5.0.edm4hep.root
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 $DETECTOR_CONFIG/sim_output/sim_emcal_barrel_electron_energies5.0_5.0.edm4hep.root
 
 sim:emcal_barrel_photons:
   extends: .det_benchmark
   stage: simulate
   script:
     - if [[ "$RUN_EXTENDED_BENCHMARK" == "true" ]] ; then snakemake --cores $DETECTOR_CONFIG/results/energy_scan/emcal_barrel_proton_fsam_scan.png; fi
-    - snakemake --cores 1 $DETECTOR_CONFIG/sim_output/sim_emcal_barrel_photon_energies5.0_5.0.edm4hep.root
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 $DETECTOR_CONFIG/sim_output/sim_emcal_barrel_photon_energies5.0_5.0.edm4hep.root
 
 sim:emcal_barrel_pion_rejection:
   extends: .det_benchmark
   stage: simulate
   script:
-    - snakemake --cores 1 $DETECTOR_CONFIG/sim_output/sim_emcal_barrel_{piminus,electron}_energies1.0_18.0.edm4hep.root
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 $DETECTOR_CONFIG/sim_output/sim_emcal_barrel_{piminus,electron}_energies1.0_18.0.edm4hep.root
 
 calib:emcal_barrel_electrons:
   extends: .det_benchmark
@@ -39,7 +39,7 @@ calib:emcal_barrel_electrons:
   script:
     - ls -lhtR sim_output/
     - rootls -t sim_output/sim_emcal_barrel_electron_energies5.0_5.0.edm4hep.root
-    - snakemake --cores 1 $DETECTOR_CONFIG/results/emcal_barrel_electron_calibration.json
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 $DETECTOR_CONFIG/results/emcal_barrel_electron_calibration.json
     - mv sim_output/sim_emcal_barrel_electron.edm4hep.root results/.
     - echo "JSON file(s) from analysis:" ; cat results/*.json
 
@@ -49,7 +49,7 @@ bench:emcal_barrel_pions:
   needs:
     - ["sim:emcal_barrel_pions"]
   script:
-    - snakemake --cores 1 $DETECTOR_CONFIG/results/emcal_barrel_pions_Ethr.png
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 $DETECTOR_CONFIG/results/emcal_barrel_pions_Ethr.png
 
 bench:emcal_barrel_electrons_scan:
   extends: .det_benchmark
@@ -66,7 +66,7 @@ bench:emcal_barrel_pi0:
     - ["sim:emcal_barrel_pi0", "calib:emcal_barrel_electrons"]
   script:
     - echo "JSON file(s) from analysis:" ; cat results/*.json
-    - snakemake --cores 1 epic_craterlake/results/Barrel_emcal_pi0.json
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 epic_craterlake/results/Barrel_emcal_pi0.json
 
 bench:emcal_barrel_photons:
   extends: .det_benchmark
@@ -76,7 +76,7 @@ bench:emcal_barrel_photons:
   script:
     - ls -lhtR sim_output/
     - rootls -t sim_output/sim_emcal_barrel_photon_energies5.0_5.0.edm4hep.root
-    - snakemake --cores 1 $DETECTOR_CONFIG/results/emcal_barrel_photon_calibration.json
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 $DETECTOR_CONFIG/results/emcal_barrel_photon_calibration.json
     - mv sim_output/sim_emcal_barrel_photon.edm4hep.root results/.
     - if [[ "$RUN_EXTENDED_BENCHMARK" == "true" ]] ; then snakemake --cores $DETECTOR_CONFIG/results/energy_scan/emcal_barrel_proton_fsam_scan.png; fi
 
@@ -89,7 +89,7 @@ bench:emcal_barrel_pion_rejection:
     - ls -lhtR sim_output/
     - rootls -t $DETECTOR_CONFIG/sim_output/sim_emcal_barrel_piminus_energies1.0_18.0.edm4hep.root
     - rootls -t $DETECTOR_CONFIG/sim_output/sim_emcal_barrel_electron_energies1.0_18.0.edm4hep.root
-    - snakemake --cores 1 $DETECTOR_CONFIG/results/Barrel_emcal_pion_rej.json
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 $DETECTOR_CONFIG/results/Barrel_emcal_pion_rej.json
 
 collect_results:barrel_ecal:
   extends: .det_benchmark

benchmarks/ecal_gaps/config.yml

@@ -4,7 +4,7 @@ sim:ecal_gaps:
   script:
     - mkdir -p $LOCAL_DATA_PATH/input
     - ln -s $LOCAL_DATA_PATH/input input
-    - snakemake --cache --cores 10 ecal_gaps --omit-from ecal_gaps
+    - snakemake $SNAKEMAKE_FLAGS --cache --cores 10 ecal_gaps --omit-from ecal_gaps
 
 bench:ecal_gaps:
   extends: .det_benchmark
@@ -15,7 +15,7 @@ bench:ecal_gaps:
     - ln -s $LOCAL_DATA_PATH/input input
     - export PYTHONUSERBASE=$LOCAL_DATA_PATH/deps
     - pip install -r benchmarks/ecal_gaps/requirements.txt
-    - snakemake --cores 8 ecal_gaps
+    - snakemake $SNAKEMAKE_FLAGS --cores 8 ecal_gaps
 
 collect_results:ecal_gaps:
   extends: .det_benchmark
@@ -25,5 +25,5 @@ collect_results:ecal_gaps:
   script:
     - ls -lrht
     - mv results{,_save}/ # move results directory out of the way to preserve it
-    - snakemake --cores 1 --delete-all-output ecal_gaps
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 --delete-all-output ecal_gaps
     - mv results{_save,}/

benchmarks/femc_electron/config.yml

@@ -13,7 +13,7 @@ sim:femc_electron:
       - P: 80
   timeout: 1 hours
   script:
-    - snakemake --cores 1 sim_output/femc_electron/epic_craterlake_rec_e-_${P}GeV.edm4eic.root
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 sim_output/femc_electron/epic_craterlake_rec_e-_${P}GeV.edm4eic.root
   retry:
     max: 2
     when:
@@ -24,7 +24,7 @@ bench:femc_electron:
   stage: benchmarks
   needs: ["sim:femc_electron"]
   script:
-    - snakemake --cores 1 results/epic_craterlake/femc_electron
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 results/epic_craterlake/femc_electron
 
 collect_results:femc_electron:
   extends: .det_benchmark
@@ -33,5 +33,5 @@ collect_results:femc_electron:
   script:
     - ls -al
     - mv results{,_save}/ # move results directory out of the way to preserve it
-    - snakemake --cores 1 --delete-all-output results/epic_craterlake/femc_electron
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 --delete-all-output results/epic_craterlake/femc_electron
     - mv results{_save,}/

benchmarks/femc_photon/config.yml

@@ -13,7 +13,7 @@ sim:femc_photon:
       - P: 80
   timeout: 1 hours
   script:
-    - snakemake --cores 1 sim_output/femc_photon/epic_craterlake_rec_photon_${P}GeV.edm4eic.root
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 sim_output/femc_photon/epic_craterlake_rec_photon_${P}GeV.edm4eic.root
   retry:
     max: 2
     when:
@@ -24,7 +24,7 @@ bench:femc_photon:
   stage: benchmarks
   needs: ["sim:femc_photon"]
   script:
-    - snakemake --cores 1 results/epic_craterlake/femc_photon
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 results/epic_craterlake/femc_photon
 
 collect_results:femc_photon:
   extends: .det_benchmark
@@ -33,5 +33,5 @@ collect_results:femc_photon:
   script:
     - ls -al
     - mv results{,_save}/ # move results directory out of the way to preserve it
-    - snakemake --cores 1 --delete-all-output results/epic_craterlake/femc_photon
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 --delete-all-output results/epic_craterlake/femc_photon
     - mv results{_save,}/

benchmarks/femc_pi0/config.yml

@@ -12,7 +12,7 @@ sim:femc_pi0:
       - P: 70
       - P: 80
   script:
-    - snakemake --cores 1 sim_output/femc_pi0/epic_craterlake_rec_pi0_${P}GeV.edm4eic.root
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 sim_output/femc_pi0/epic_craterlake_rec_pi0_${P}GeV.edm4eic.root
   retry:
     max: 2
     when:
@@ -23,7 +23,7 @@ bench:femc_pi0:
   stage: benchmarks
   needs: ["sim:femc_pi0"]
   script:
-    - snakemake --cores 1 results/epic_craterlake/femc_pi0
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 results/epic_craterlake/femc_pi0
 
 collect_results:femc_pi0:
   extends: .det_benchmark
@@ -32,5 +32,5 @@ collect_results:femc_pi0:
   script:
     - ls -al
     - mv results{,_save}/ # move results directory out of the way to preserve it
-    - snakemake --cores 1 --delete-all-output results/epic_craterlake/femc_pi0
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 --delete-all-output results/epic_craterlake/femc_pi0
     - mv results{_save,}/

benchmarks/insert_muon/config.yml

@@ -5,7 +5,7 @@ sim:insert_muon:
     matrix:
       - P: 50
   script:
-    - snakemake --cores 5 sim_output/insert_muon/epic_craterlake_sim_mu-_${P}GeV_{0,1,2,3,4}.edm4hep.root
+    - snakemake $SNAKEMAKE_FLAGS --cores 5 sim_output/insert_muon/epic_craterlake_sim_mu-_${P}GeV_{0,1,2,3,4}.edm4hep.root
   retry:
     max: 2
     when:
@@ -16,7 +16,7 @@ bench:insert_muon:
   stage: benchmarks
   needs: ["sim:insert_muon"]
   script:
-    - snakemake --cores 1 results/epic_craterlake/insert_muon
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 results/epic_craterlake/insert_muon
 
 collect_results:insert_muon:
   extends: .det_benchmark
@@ -25,5 +25,5 @@ collect_results:insert_muon:
   script:
     - ls -al
     - mv results{,_save}/ # move results directory out of the way to preserve it
-    - snakemake --cores 1 --delete-all-output results/epic_craterlake/insert_muon
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 --delete-all-output results/epic_craterlake/insert_muon
     - mv results{_save,}/

benchmarks/insert_neutron/config.yml

@@ -11,7 +11,7 @@ sim:insert_neutron:
       - P: 70
       - P: 80
   script:
-    - snakemake --cores 5 sim_output/insert_neutron/epic_craterlake_rec_neutron_${P}GeV_{0,1,2,3,4}.edm4eic.root
+    - snakemake $SNAKEMAKE_FLAGS --cores 5 sim_output/insert_neutron/epic_craterlake_rec_neutron_${P}GeV_{0,1,2,3,4}.edm4eic.root
   retry:
     max: 2
     when:
@@ -22,7 +22,7 @@ bench:insert_neutron:
   stage: benchmarks
   needs: ["sim:insert_neutron"]
   script:
-    - snakemake --cores 1 results/epic_craterlake/insert_neutron
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 results/epic_craterlake/insert_neutron
 
 collect_results:insert_neutron:
   extends: .det_benchmark
@@ -31,5 +31,5 @@ collect_results:insert_neutron:
   script:
     - ls -al
     - mv results{,_save}/ # move results directory out of the way to preserve it
-    - snakemake --cores 1 --delete-all-output results/epic_craterlake/insert_neutron
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 --delete-all-output results/epic_craterlake/insert_neutron
     - mv results{_save,}/

benchmarks/lfhcal/config.yml

@@ -16,7 +16,7 @@ bench:lfhcal:
   needs:
     - ["sim:lfhcal"]
   script:
-    - snakemake --cores 3 lfhcal_local
+    - snakemake $SNAKEMAKE_FLAGS --cores 3 lfhcal_local
 
 collect_results:lfhcal:
   extends: .det_benchmark
@@ -26,15 +26,15 @@ collect_results:lfhcal:
   script:
     - ls -lrht
     - mv results{,_save}/ # move results directory out of the way to preserve it
-    - snakemake --cores 1 --delete-all-output lfhcal_local
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 --delete-all-output lfhcal_local
     - mv results{_save,}/
 
 bench:lfhcal_campaigns:
   extends: .det_benchmark
   stage: benchmarks
   #when: manual
   script:
-    - snakemake --cores 1 lfhcal_campaigns
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 lfhcal_campaigns
 
 collect_results:lfhcal_campaigns:
   extends: .det_benchmark
@@ -44,5 +44,5 @@ collect_results:lfhcal_campaigns:
   script:
     - ls -lrht
     - mv results{,_save}/ # move results directory out of the way to preserve it
-    - snakemake --cores 1 --delete-all-output lfhcal_campaigns
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 --delete-all-output lfhcal_campaigns
     - mv results{_save,}/

benchmarks/material_scan/config.yml

@@ -2,7 +2,7 @@ bench:material_scan:
   extends: .det_benchmark
   stage: benchmarks
   script:
-    - snakemake --cores 1 epic_craterlake/results/material_scan_details.pdf
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 epic_craterlake/results/material_scan_details.pdf
 
 collect_results:material_scan:
   extends: .det_benchmark

benchmarks/tracking_performances/config.yml

@@ -18,7 +18,7 @@ bench:tracking_performance:
   needs:
     - ["sim:tracking_performance"]
   script:
-    - snakemake --cores 3 tracking_performance_local
+    - snakemake $SNAKEMAKE_FLAGS --cores 3 tracking_performance_local
 
 collect_results:tracking_performance:
   extends: .det_benchmark
@@ -27,14 +27,14 @@ collect_results:tracking_performance:
     - "bench:tracking_performance"
   script:
     - ls -lrht
-    - snakemake --cores 1 --delete-all-output tracking_performance_local
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 --delete-all-output tracking_performance_local
 
 bench:tracking_performance_campaigns:
   extends: .det_benchmark
   stage: benchmarks
   #when: manual
   script:
-    - snakemake --cores 1 tracking_performance_campaigns
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 tracking_performance_campaigns
 
 collect_results:tracking_performance_campaigns:
   extends: .det_benchmark
@@ -44,5 +44,5 @@ collect_results:tracking_performance_campaigns:
   script:
     - ls -lrht
     - mv results{,_save}/ # move results directory out of the way to preserve it
-    - snakemake --cores 1 --delete-all-output tracking_performance_campaigns
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 --delete-all-output tracking_performance_campaigns
     - mv results{_save,}/

benchmarks/tracking_performances_dis/config.yml

@@ -2,7 +2,7 @@ sim:tracking_performances_dis:
   extends: .det_benchmark
   stage: simulate
   script:
-    - snakemake --cache --cores 5 results/tracking_performances_dis/epic_craterlake_tracking_only/pythia8NCDIS_18x275_minQ2=1_combined_5/hists.root 
+    - snakemake $SNAKEMAKE_FLAGS --cores 5 results/tracking_performances_dis/epic_craterlake_tracking_only/pythia8NCDIS_18x275_minQ2=1_combined_5/hists.root
   retry:
     max: 2
     when:
@@ -14,7 +14,7 @@ bench:tracking_performances_dis:
   needs:
     - ["sim:tracking_performances_dis"]
   script:
-    - snakemake --cores 1 trk_dis_run_locally_trk_only
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 trk_dis_run_locally_trk_only
     # avoid uploading intermediate results
     - find results/tracking_performances_dis/ -mindepth 2 -maxdepth 2 -type d ! -name "*combined*" | xargs rm -rfv
 
@@ -26,7 +26,7 @@ collect_results:tracking_performances_dis:
   script:
     - ls -lrht
     - mv results{,_save}/ # move results directory out of the way to preserve it
-    - snakemake --cores 1 --delete-all-output trk_dis_run_locally_trk_only
+    - snakemake $SNAKEMAKE_FLAGS --cores 1 --delete-all-output trk_dis_run_locally_trk_only
     - mv results{_save,}/
     # convert to png
     - |