snakemake.yml: allow using S3 (XRootD is down sometimes) (#36)

.gitlab-ci.yml

@@ -113,6 +113,7 @@ get_data:
   needs:
     - ["get_data","common:detector"]
   before_script:
+    - mc config host add S3 https://eics3.sdcc.bnl.gov:9000 ${S3_ACCESS_KEY} ${S3_SECRET_KEY}
     - source .local/bin/env.sh
     - ls -lrtha 
     - ln -s "${LOCAL_DATA_PATH}/sim_output" sim_output

Snakefile

@@ -1,15 +1,35 @@
+configfile: "snakemake.yml"
+
 include: "benchmarks/backgrounds/Snakefile"
 include: "benchmarks/barrel_ecal/Snakefile"
 include: "benchmarks/ecal_gaps/Snakefile"
 include: "benchmarks/material_scan/Snakefile"
 include: "benchmarks/tracking_performances/Snakefile"
 include: "benchmarks/zdc_lyso/Snakefile"
 
+use_s3 = config["remote_provider"].lower() == "s3"
+use_xrootd = config["remote_provider"].lower() == "xrootd"
+
+
+def get_remote_path(path):
+    if use_s3:
+        return f"s3https://eics3.sdcc.bnl.gov:9000/eictest/{path}"
+    elif use_xrootd:
+        return f"root://dtn-eic.jlab.org//work/eic2/{path}"
+    else:
+        raise runtime_exception('Unexpected value for config["remote_provider"]: {config["remote_provider"]}')
+
+
 rule fetch_epic:
     output:
         filepath="EPIC/{PATH}"
     shell: """
 xrdcp root://dtn-eic.jlab.org//work/eic2/{output.filepath} {output.filepath}
+""" if use_xrootd else """
+mc cp S3/eictest/{output.filepath} {output.filepath}
+""" if use_s3 else f"""
+echo 'Unexpected value for config["remote_provider"]: {config["remote_provider"]}'
+exit 1
 """
 
 

benchmarks/backgrounds/Snakefile

@@ -10,7 +10,8 @@ rule backgrounds_sim:
     log:
         "sim_output/{DETECTOR_CONFIG}/backgrounds/{PATH}.edm4hep.root.log",
     params:
-        N_EVENTS=100
+        N_EVENTS=100,
+        hepmc=lambda wildcards: get_remote_path(f"{wildcards.PATH}.hepmc3.tree.root"),
     shell:
         """
 set -m # monitor mode to prevent lingering processes
@@ -21,7 +22,7 @@ exec ddsim \
   -v WARNING \
   --numberOfEvents {params.N_EVENTS} \
   --compactFile $DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml \
-  --inputFiles root://dtn-eic.jlab.org//work/eic2/{wildcards.PATH}.hepmc3.tree.root \
+  --inputFiles {params.hepmc} \
   --outputFile {output}
 """
 

snakemake.yml

@@ -0,0 +1 @@
+remote_provider: XRootD