Skip to content

Commit

Permalink
Works locally and has settings to test on github
Browse files Browse the repository at this point in the history
  • Loading branch information
simonge committed May 7, 2024
1 parent c319d83 commit 2d58b4f
Show file tree
Hide file tree
Showing 5 changed files with 167 additions and 78 deletions.
9 changes: 8 additions & 1 deletion benchmarks/LOWQ2/config.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
# Run Snakemake for the training
training:
train:LOWQ2:
extends: .det_benchmark
stage: benchmarks
script:
- snakemake --snakefile training/Snakefile --configfile config.yml


SIM_DIRECTORY: "LowQ2_G4out/"
RECO_DIRECTORY: "LowQ2_ReconOut/"
MODEL_DIRECTORY: "LowQ2_Model/"

94 changes: 21 additions & 73 deletions benchmarks/LOWQ2/training/Snakefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,58 +3,32 @@ from itertools import product

import os
import shutil
from snakemake.remote.S3 import RemoteProvider as S3RemoteProvider

S3 = S3RemoteProvider(
endpoint_url="https://eics3.sdcc.bnl.gov:9000",
access_key_id=os.environ["S3_ACCESS_KEY"],
secret_access_key=os.environ["S3_SECRET_KEY"],
)
configfile: "local_config.yml"

EVENTS_DIRECTORY = "/scratch/EIC/Events/S3in/"
SIM_DIRECTORY = "/scratch/EIC/G4out/S3processed/"
RECO_DIRECTORY = "/scratch/EIC/ReconOut/S3in/"
RECO_DIRECTORY_P = "/scratch/EIC/ReconOut/S3processed/"
MODEL_DIRECTORY = "/scratch/EIC/LowQ2Model/"

REMOTE_EVENTS_DIRECTORY = "EPIC/EVGEN/SIDIS/pythia6-eic/1.0.0/10x100/q2_0to1/"
REMOTE_RECO_DIRECTORY = "EPIC/RECO/24.02.1/epic_craterlake/SIDIS/pythia6-eic/1.0.0/10x100/q2_0to1/"
S3_DIR = "eictest/"
XROOTD_SERVER = "root://dtn-eic.jlab.org/"
XROOTD_DIR = "/work/eic2/"

EVENT_EXTENSION = ".ab.hepmc3.tree.root"
SIM_EXTENSION = ".edm4hep.root"
RECO_EXTENSION = ".eicrecon.tree.edm4eic.root"
EVENT_EXTENSION = ".ab.hepmc3.tree.root"
SIM_EXTENSION = ".edm4hep.root"
RECO_EXTENSION = ".eicrecon.tree.edm4eic.root"

FILE_BASE = "pythia_ep_noradcor_10x100_q2_0.000000001_1.0_run"
XML_FILE = "/home/simong/EIC/epic/install/share/epic/epic_craterlake_10x100.xml"
XML_FILE_TAGGER = "/home/simong/EIC/epic/install/share/epic/epic_ip6.xml"
XML_FILE_EDIT = "/home/simong/EIC/epic/epic_edit.xml"
BEAM_ENERGY = "10"

rule download_event_input:
input:
S3.remote(REMOTE_EVENTS_DIRECTORY+FILE_BASE+"{index}"+EVENT_EXTENSION),
output:
EVENTS_DIRECTORY+FILE_BASE+"{index}"+EVENT_EXTENSION,
run:
shutil.move(input[0], output[0])
REMOTE_EVENTS_DIRECTORY = "root://dtn-eic.jlab.org//work/eic2/EPIC/EVGEN/SIDIS/pythia6-eic/1.0.0/10x100/q2_0to1/"
FILE_BASE = "pythia_ep_noradcor_10x100_q2_0.000000001_1.0_run"

XML_FILE = "epic_edit.xml"
BEAM_ENERGY = "10"

def remote_file_exists(server,url):
try:
subprocess.check_output(['xrdfs', server, 'stat', url])
return True
return url
except subprocess.CalledProcessError:
return False

return None
'''
rule run_simulation_tagger:
params:
XML=XML_FILE_EDIT,
input=lambda wildcards: XROOTD_SERVER+XROOTD_DIR+REMOTE_EVENTS_DIRECTORY+FILE_BASE+wildcards.index+EVENT_EXTENSION if remote_file_exists(XROOTD_SERVER,XROOTD_DIR+REMOTE_EVENTS_DIRECTORY+FILE_BASE+wildcards.index+EVENT_EXTENSION) else None,
XML=XML_FILE,
input=remote_file_exists(REMOTE_EVENTS_DIRECTORY+FILE_BASE+wildcards.index+EVENT_EXTENSION),
output:
SIM_DIRECTORY+FILE_BASE+"{index}_tagger"+SIM_EXTENSION,
config["SIM_DIRECTORY"]+FILE_BASE+"{index}_tagger"+SIM_EXTENSION,
shell: """
npsim \
--inputFiles {params.input} \
Expand All @@ -67,47 +41,21 @@ npsim \
--field.eps_max 1e-04 \
--physics.rangecut 50 \
"""

rule download_recon:
input:
S3.remote(S3_DIR+REMOTE_RECO_DIRECTORY+FILE_BASE+"{runindex}"+".ab."+"{subindex}"+RECO_EXTENSION),
output:
RECO_DIRECTORY+FILE_BASE+"{runindex}"+".ab."+"{subindex}"+RECO_EXTENSION,
run:
shutil.move(input[0], output[0])

'''
rule run_reconstruction:
params:
XML=XML_FILE,
beam_energy=BEAM_ENERGY,
collections="TaggerTrackerProjectedTracks,MCScatteredElectrons,MCParticles,EventHeader",
input:
expand(
SIM_DIRECTORY+FILE_BASE+"{index}_tagger"+SIM_EXTENSION,
index=range(1,10),
),
output:
RECO_DIRECTORY_P+FILE_BASE+".fab"+RECO_EXTENSION,
shell: """
/home/simong/EIC/EICrecon/bin/eicrecon {input} -Pdd4hep:xml_files={params.XML} -Ppodio:output_include_collections={params.collections} -Ppodio:output_file={output[0]} -PLOWQ2:LowQ2Trajectories:electron_beamE={params.beam_energy}
"""

rule run_reconstruction_remote:
params:
XML=XML_FILE,
beam_energy=BEAM_ENERGY,
collections="TaggerTrackerProjectedTracks,MCScatteredElectrons,MCParticles,EventHeader",
stringValue=RECO_DIRECTORY+FILE_BASE+"*.ab.*"+RECO_EXTENSION,
input:
expand(
RECO_DIRECTORY+FILE_BASE+"{runindex}"+".ab."+"{subindex:04d}"+RECO_EXTENSION,
subindex=range(1,4),
runindex=1
config["SIM_DIRECTORY"]+FILE_BASE+"{index}_tagger"+SIM_EXTENSION,
index=range(1,4),
),
output:
RECO_DIRECTORY_P+FILE_BASE+".ab"+RECO_EXTENSION,
config["RECO_DIRECTORY"]+FILE_BASE+".tagger_recon"+RECO_EXTENSION,
shell: """
/home/simong/EIC/EICrecon/bin/eicrecon {input} -Pdd4hep:xml_files={params.XML} -Ppodio:output_include_collections={params.collections} -Ppodio:output_file={output[0]} -PLOWQ2:LowQ2Trajectories:electron_beamE={params.beam_energy}
/home/simong/EIC/EICrecon/bin/eicrecon {input} -Pdd4hep:xml_files={params.XML} -Ppodio:output_include_collections={params.collections} -Ppodio:output_file={output} -PLOWQ2:LowQ2Trajectories:electron_beamE={params.beam_energy}
"""

rule low_q2_train_network:
Expand All @@ -117,9 +65,9 @@ rule low_q2_train_network:
method_name="DNN",
model_dir="LowQ2Model"
input:
train_data=RECO_DIRECTORY_P+FILE_BASE+".fab"+RECO_EXTENSION,
train_data=config["RECO_DIRECTORY"]+FILE_BASE+".tagger_recon"+RECO_EXTENSION,
output:
root_output=MODEL_DIRECTORY+"trainedData.root",
root_output=config["MODEL_DIRECTORY"]+"trainedData.root",
shell:
"""
root -l -b -q 'TaggerRegressionEICrecon.C++("{input.train_data}", "{output.root_output}", "{params.model_dir}", "{params.beam_energy}", "{params.type_name}", "{params.method_name}")'
Expand Down
6 changes: 2 additions & 4 deletions benchmarks/LOWQ2/training/TaggerRegressionEICrecon.C
Original file line number Diff line number Diff line change
Expand Up @@ -97,14 +97,12 @@ void TaggerRegressionEICrecon(
TString layoutString("Layout=TANH|1024,TANH|128,TANH|64,TANH|32,LINEAR");

TString trainingStrategyString("TrainingStrategy=");
//trainingStrategyString +="LearningRate=1e-4,Momentum=0,MaxEpochs=2000,ConvergenceSteps=200,BatchSize=64,TestRepetitions=1,Regularization=None,Optimizer=ADAM";
trainingStrategyString +="LearningRate=1e-4,Momentum=0,MaxEpochs=2000,ConvergenceSteps=200,BatchSize=10,TestRepetitions=1,Regularization=None,Optimizer=ADAM";
trainingStrategyString +="LearningRate=1e-4,Momentum=0,MaxEpochs=2000,ConvergenceSteps=200,BatchSize=64,TestRepetitions=1,Regularization=None,Optimizer=ADAM";

TString nnOptions("!H:V:ErrorStrategy=SUMOFSQUARES:WeightInitialization=XAVIERUNIFORM:RandomSeed=1234");

// Use GPU if possible on the machine
//TString architectureString("Architecture=GPU");
TString architectureString("Architecture=CPU");
TString architectureString("Architecture=GPU");
// Transformation of data prior to training layers - decorrelate and normalise whole dataset
TString transformString("VarTransform=D,N");
Expand Down
133 changes: 133 additions & 0 deletions benchmarks/LOWQ2/training/epic_edit.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
<lccdd xmlns:compact="http://www.lcsim.org/schemas/compact/1.0"
xmlns:xs="http://www.w3.org/2001/XMLSchema"
xs:noNamespaceSchemaLocation="http://www.lcsim.org/schemas/compact/1.0/compact.xsd">

<debug>
<type name="surface" value="0"/>
<type name="material" value="0"/>
<type name="readout" value="0"/>
<type name="segmentation" value="0"/>
<type name="limits" value="0"/>
<type name="region" value="0"/>
<type name="includes" value="0"/>
</debug>

<documentation level="-1">
# EPIC Detector
- https://github.com/eic/epic
- https://github.com/eic/ip6
</documentation>

<!-- Some information about detector -->
<info name="EPIC Detector" title="EPIC Detector"
author="EPIC Collaboration"
url="https://github.com/eic/epic"
status="development"
version="v1 2021-03-16">
<comment> EPIC </comment>
</info>

<!-- Standard environmental conditions: 20 °C, 1 atm -->
<std_conditions type="NTP"/>

<define>
<documentation level="2">
## Main Constant Definitions

The ip6 (or other ip) defines should be included first.
These files have only a define tags.
</documentation>
<include ref="${DETECTOR_PATH}/compact/fields/beamline_10x100.xml" />
<include ref="${DETECTOR_PATH}/compact/far_forward/definitions.xml" />
<include ref="${DETECTOR_PATH}/compact/far_backward/definitions.xml" />
<include ref="${DETECTOR_PATH}/compact/definitions.xml" />
<include ref="${DD4hepINSTALL}/DDDetectors/compact/detector_types.xml"/>
</define>


<includes>
<gdmlFile ref="${DD4hepINSTALL}/DDDetectors/compact/elements.xml"/>
<gdmlFile ref="${DETECTOR_PATH}/compact/materials.xml"/>
</includes>
<include ref="${DETECTOR_PATH}/compact/optical_materials.xml"/>

<limits>
<limitset name="world_limits">
</limitset>
<limitset name="beamline_limits">
<limit name="step_length_max" particles="*" value="1.0" unit="mm" />
<limit name="track_length_max" particles="*" value="1.0" unit="mm" />
<limit name="time_max" particles="*" value="0.1" unit="ns" />
<limit name="ekin_min" particles="*" value="0.001" unit="MeV" />
<limit name="range_min" particles="*" value="0.1" unit="mm" />
</limitset>
<limitset name="cal_limits">
<limit name="step_length_max" particles="*" value="5.0" unit="mm"/>
</limitset>
</limits>

<regions>
<region name="world_region" eunit="MeV" lunit="mm" cut="0.001" threshold="0.001">
<limitsetref name="world_limits"/>
</region>
</regions>

<display>
<include ref="${DETECTOR_PATH}/compact/colors.xml"/>
<include ref="${DETECTOR_PATH}/compact/display.xml"/>
</display>

<documentation level="0">
## World Volume

The world is a simple box, but could be a union of multiple regions.
</documentation>
<world material="Air">
<shape type="Box" dx="world_dx" dy="world_dy" dz="world_dz"/>
<!--regionref name="world_region"/-->
<!--limitsetref name="world_limits"/-->
</world>

<documentation level="0">
## Detector Subsystems

### IP Subsystems

The interaction point subsystems are included before the central detector subsystems.
This is becuase the IP subsystems, for example the beampipe, will define paramters
which are subsquently used in the central detector construction -- e.g. the vertex tracker
uses the beampipe OD to help define its placement.

The IP subsystems include the Far forward and backward regions. The list of subsystem includes:
- Interaction region beampipe
- B0 tracker
- Off-momentum tracker
- Far forward roman pots
- Zero Degree Calorimeter
- Beam line magnets.
- and more...
</documentation>


<documentation level="5">
## Main magnet and its field
</documentation>
<include ref="${DETECTOR_PATH}/compact/solenoid.xml"/>
<include ref="${DETECTOR_PATH}/compact/fields/marco.xml"/>

<documentation level="10">
## Central tracking detectors
</documentation>

<include ref="${DETECTOR_PATH}/compact/tracking/definitions_craterlake.xml"/>

<documentation level="11">
## Central beam pipe
</documentation>
<include ref="${DETECTOR_PATH}/compact/central_beampipe.xml"/>

<documentation level="11">
## Far backward detectors
</documentation>
<include ref="${DETECTOR_PATH}/compact/far_backward/default.xml"/>
</lccdd>
3 changes: 3 additions & 0 deletions benchmarks/LOWQ2/training/local_config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
SIM_DIRECTORY: "/scratch/EIC/G4out/S3processed/"
RECO_DIRECTORY: "/scratch/EIC/ReconOut/S3processed/"
MODEL_DIRECTORY: "/scratch/EIC/LowQ2Model/"

0 comments on commit 2d58b4f

Please sign in to comment.