diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5319f8eb..c110892f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -113,6 +113,7 @@ get_data: needs: - ["get_data","common:detector"] before_script: + - mc config host add S3 https://eics3.sdcc.bnl.gov:9000 ${S3_ACCESS_KEY} ${S3_SECRET_KEY} - source .local/bin/env.sh - ls -lrtha - ln -s "${LOCAL_DATA_PATH}/sim_output" sim_output diff --git a/Snakefile b/Snakefile index 151d30d9..c452ba60 100644 --- a/Snakefile +++ b/Snakefile @@ -1,3 +1,5 @@ +configfile: "snakemake.yml" + include: "benchmarks/backgrounds/Snakefile" include: "benchmarks/barrel_ecal/Snakefile" include: "benchmarks/ecal_gaps/Snakefile" @@ -5,11 +7,29 @@ include: "benchmarks/material_scan/Snakefile" include: "benchmarks/tracking_performances/Snakefile" include: "benchmarks/zdc_lyso/Snakefile" +use_s3 = config["remote_provider"].lower() == "s3" +use_xrootd = config["remote_provider"].lower() == "xrootd" + + +def get_remote_path(path): + if use_s3: + return f"s3https://eics3.sdcc.bnl.gov:9000/eictest/{path}" + elif use_xrootd: + return f"root://dtn-eic.jlab.org//work/eic2/{path}" + else: + raise runtime_exception('Unexpected value for config["remote_provider"]: {config["remote_provider"]}') + + rule fetch_epic: output: filepath="EPIC/{PATH}" shell: """ xrdcp root://dtn-eic.jlab.org//work/eic2/{output.filepath} {output.filepath} +""" if use_xrootd else """ +mc cp S3/eictest/{output.filepath} {output.filepath} +""" if use_s3 else f""" +echo 'Unexpected value for config["remote_provider"]: {config["remote_provider"]}' +exit 1 """ diff --git a/benchmarks/backgrounds/Snakefile b/benchmarks/backgrounds/Snakefile index ee6d54c4..7aac8808 100644 --- a/benchmarks/backgrounds/Snakefile +++ b/benchmarks/backgrounds/Snakefile @@ -10,7 +10,8 @@ rule backgrounds_sim: log: "sim_output/{DETECTOR_CONFIG}/backgrounds/{PATH}.edm4hep.root.log", params: - N_EVENTS=100 + N_EVENTS=100, + hepmc=lambda wildcards: get_remote_path(f"{wildcards.PATH}.hepmc3.tree.root"), shell: """ set -m # monitor mode to prevent lingering processes @@ -21,7 +22,7 @@ exec ddsim \ -v WARNING \ --numberOfEvents {params.N_EVENTS} \ --compactFile $DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml \ - --inputFiles root://dtn-eic.jlab.org//work/eic2/{wildcards.PATH}.hepmc3.tree.root \ + --inputFiles {params.hepmc} \ --outputFile {output} """ diff --git a/snakemake.yml b/snakemake.yml new file mode 100644 index 00000000..567e0924 --- /dev/null +++ b/snakemake.yml @@ -0,0 +1 @@ +remote_provider: XRootD