Add support for snakemake 8.x

.gitlab-ci.yml

@@ -88,6 +88,7 @@ common:setup:
       echo "BENCHMARKS_CONTAINER: ${BENCHMARKS_CONTAINER}"
       echo "BENCHMARKS_REGISTRY: ${BENCHMARKS_REGISTRY}"
     - source setup/bin/env.sh && ./setup/bin/install_common.sh
+    - if [[ "$(snakemake --version)" != 7.* ]]; then mv Snakemake.8 Snakemake; fi
 
 
 common:detector:

Snakefile

@@ -17,42 +17,6 @@ elif config["remote"] == "XRootD":
 else:
     raise ValueError(f"Unexpected config[\"remote\"] = {config['remote']}")
 
-include: "benchmarks/backgrounds/Snakefile"
-include: "benchmarks/barrel_ecal/Snakefile"
-include: "benchmarks/ecal_gaps/Snakefile"
+storage = provider.remote # provide Snakemake 8.x interface
 
-
-rule warmup_run:
-    output:
-        "warmup/{DETECTOR_CONFIG}.edm4hep.root",
-    message: "Ensuring that calibrations/fieldmaps are available for {wildcards.DETECTOR_CONFIG}"
-    shell: """
-ddsim \
-  --runType batch \
-  --numberOfEvents 1 \
-  --compactFile "$DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml" \
-  --outputFile "{output}" \
-  --enableGun
-"""
-
-
-rule matplotlibrc:
-    output:
-        ".matplotlibrc",
-    run:
-        with open(output[0], "wt") as fp:
-            fp.write("backend: Agg\n")
-            # interactive mode prevents plt.show() from blocking
-            fp.write("interactive : True\n")
-
-
-rule org2py:
-    input:
-        notebook=workflow.basedir + "/{NOTEBOOK}.org",
-        converter=workflow.source_path("benchmarks/common/org2py.awk"),
-    output:
-        "{NOTEBOOK}.py"
-    shell:
-        """
-awk -f {input.converter} {input.notebook} > {output}
-"""
+include: "Snakefile.common"

Snakefile.8

@@ -0,0 +1,15 @@
+configfile: "config.yaml"
+storage:
+    provider="s3",
+    endpoint_url="https://eics3.sdcc.bnl.gov:9000",
+    access_key=os.environ["S3_ACCESS_KEY"],
+    secret_key=os.environ["S3_SECRET_KEY"],
+
+if config["remote"] == "S3":
+    remote_path = lambda path: f"s3://eictest/{path}"
+elif config["remote"] == "XRootD":
+    remote_path = lambda path: f"root://dtn-eic.jlab.org//work/eic2/{path}"
+else:
+    raise ValueError(f"Unexpected config[\"remote\"] = {config['remote']}")
+
+include: "Snakefile.common"

Snakefile.common

@@ -0,0 +1,39 @@
+include: "benchmarks/backgrounds/Snakefile"
+include: "benchmarks/barrel_ecal/Snakefile"
+include: "benchmarks/ecal_gaps/Snakefile"
+
+
+rule warmup_run:
+    output:
+        "warmup/{DETECTOR_CONFIG}.edm4hep.root",
+    message: "Ensuring that calibrations/fieldmaps are available for {wildcards.DETECTOR_CONFIG}"
+    shell: """
+ddsim \
+  --runType batch \
+  --numberOfEvents 1 \
+  --compactFile "$DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml" \
+  --outputFile "{output}" \
+  --enableGun
+"""
+
+
+rule matplotlibrc:
+    output:
+        ".matplotlibrc",
+    run:
+        with open(output[0], "wt") as fp:
+            fp.write("backend: Agg\n")
+            # interactive mode prevents plt.show() from blocking
+            fp.write("interactive : True\n")
+
+
+rule org2py:
+    input:
+        notebook=workflow.basedir + "/{NOTEBOOK}.org",
+        converter=workflow.source_path("benchmarks/common/org2py.awk"),
+    output:
+        "{NOTEBOOK}.py"
+    shell:
+        """
+awk -f {input.converter} {input.notebook} > {output}
+"""

benchmarks/backgrounds/Snakefile

@@ -4,7 +4,7 @@ import shutil
 
 rule backgrounds_get_beam_gas_electron:
     input:
-        provider.remote(remote_path("EPIC/EVGEN/BACKGROUNDS/BEAMGAS/electron/beam_gas_ep_10GeV_foam_emin10keV_10Mevt_vtx.hepmc")),
+        storage(remote_path("EPIC/EVGEN/BACKGROUNDS/BEAMGAS/electron/beam_gas_ep_10GeV_foam_emin10keV_10Mevt_vtx.hepmc")),
     output:
         "input/backgrounds/beam_gas_electron.hepmc",
     run:
@@ -13,7 +13,7 @@ rule backgrounds_get_beam_gas_electron:
 
 rule backgrounds_get_beam_gas_proton:
     input:
-        provider.remote(remote_path("EPIC/EVGEN/BACKGROUNDS/BEAMGAS/proton/ProtonBeamGasEvents/100GeV/100GeV_1.hepmc")),
+        storage(remote_path("EPIC/EVGEN/BACKGROUNDS/BEAMGAS/proton/ProtonBeamGasEvents/100GeV/100GeV_1.hepmc")),
     output:
         "input/backgrounds/beam_gas_proton.hepmc",
     run:
@@ -22,7 +22,7 @@ rule backgrounds_get_beam_gas_proton:
 
 rule backgrounds_get_DIS:
     input:
-        provider.remote(remote_path("EPIC/EVGEN/DIS/NC/{BEAM}/minQ2={MINQ2}/pythia8NCDIS_{BEAM}_minQ2={MINQ2}_{SUFFIX}.hepmc")),
+        storage(remote_path("EPIC/EVGEN/DIS/NC/{BEAM}/minQ2={MINQ2}/pythia8NCDIS_{BEAM}_minQ2={MINQ2}_{SUFFIX}.hepmc")),
     wildcard_constraints:
         BEAM="\d+x\d+",
         MINQ2="\d+",

benchmarks/ecal_gaps/Snakefile

@@ -3,7 +3,7 @@ import os
 
 rule ecal_gaps_sim:
     input:
-        steering_file=provider.remote(remote_path("EPIC/EVGEN/SINGLE/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.steer")),
+        steering_file=storage(remote_path("EPIC/EVGEN/SINGLE/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.steer")),
         warmup="warmup/{DETECTOR_CONFIG}.edm4hep.root",
     output:
         "sim_output/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX}.edm4hep.root",