Exclude submodules in linting

.githooks/pre-commit

@@ -83,10 +83,10 @@ done
 for FILE in $STAGED_FILES_PY; do
 
   # Remove unused imports
-  autoflake --in-place --expand-star-imports --remove-all-unused-imports "$FILE"
+  autoflake --in-place --expand-star-imports --remove-all-unused-imports "$FILE" --exclude=narf,wremnants-data
 
   # Run isort to organize imports
-  isort "$FILE"
+  isort "$FILE" --skip narf --skip wremnants-data --profile black --line-length 88
 
   # PEP 8 formatting
   black "$FILE"

.github/workflows/main.yml

@@ -4,10 +4,14 @@ name: CI
 
 # Controls when the workflow will run
 on:
-  workflow_run:
-    workflows: ["Base"]
-    types:
-      - completed
+  # Triggers the workflow on push or pull request events but only for the master branch
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 1 * * 2,4,6' # Run on Tuesday, Thursday, and Saturday morning at 1h00 UTC
+    - cron: '30 5 * * 1-5' # Run on weekdays
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
 
 env: 
   MAX_FILES: 10
@@ -55,9 +59,77 @@ jobs:
   #              body: ${{env.MESSAGE}}
   #            })
 
+  linting:
+    # The type of runner that the job will run on
+    runs-on: [self-hosted, linux, x64]
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+          lfs: 'true'
+
+      - name: run isort
+        run: |
+          scripts/ci/run_with_singularity.sh isort . --check-only --diff --skip narf --skip wremnants-data --profile black --line-length 88
+
+      - name: run Flake8
+        run: |
+          scripts/ci/run_with_singularity.sh flake8 . --select=F401 --exclude=narf,wremnants-data --max-line-length 88
+
+      - name: run Black
+        run: |
+          scripts/ci/run_with_singularity.sh black --exclude '(^\.git|\.github|narf|wremnants-data)' --check .
+
+      - name: check Python Files
+        run: |
+          # Find all Python files and check their syntax in parallel
+          find . -name '*.py' -not -path '*/narf/*' -not -path '*/wremnants-data/*' | \
+          xargs -P 16 -I {} bash -c '
+            echo "Checking python file: {}"
+            scripts/ci/run_with_singularity.sh python -m py_compile "{}" || \
+            { echo "Invalid python syntax in {}"; exit 1; }
+          '
+
+      - name: check JSON Files
+        run: |
+          # Find all JSON files and check their syntax
+          for FILE in $(find . -name '*.json' -not -path '*/narf/*' -not -path '*/wremnants-data/*'); do
+            echo "Checking JSON file: $FILE"
+            scripts/ci/run_with_singularity.sh python -m json.tool "$FILE" > /dev/null
+            if [ $? -ne 0 ]; then
+              echo "Invalid JSON syntax in $FILE"
+              exit 1
+            fi
+          done
+
+      - name: check YAML Files
+        run: |
+          # Find all YAML files and check their syntax
+          for FILE in $(find . -name '*.yaml' -not -path '*/narf/*' -not -path '*/wremnants-data/*'); do
+            echo "Checking YAML file: $FILE"
+            scripts/ci/run_with_singularity.sh python -c "import yaml, sys; yaml.safe_load(open('$FILE'))" > /dev/null
+            if [ $? -ne 0 ]; then
+              echo "Invalid YAML syntax in $FILE"
+              exit 1
+            fi
+          done
+
+      - name: check C++ Files
+        run: |
+          # Find all C++ files and check their syntax in parallel
+          find . \( -name '*.c' -o -name '*.cpp' -o -name '*.hpp' -o -name '*.h' \) -not -path '*/narf/*' -not -path '*/wremnants-data/*' | \
+          xargs -P 16 -I {} bash -c '
+            echo "Checking {}"
+            scripts/ci/run_with_singularity.sh clang++ -I./narf/narf/include/ -I./wremnants/include/ -std=c++20 -fsyntax-only "{}" || \
+            { echo "Syntax error in {}"; exit 1; }
+          '
+
   setenv:
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}  # Only run if linting succeeded
     runs-on: [self-hosted, linux, x64]
+    needs: linting
 
     outputs:
       WREMNANTS_OUTDIR: ${{steps.export.outputs.WREMNANTS_OUTDIR}}
@@ -79,23 +151,39 @@ jobs:
         run: echo PLOT_DIR=PR$(echo $GITHUB_REF | awk 'BEGIN { FS = "/" } ; { print $3 }')/$(date +%Y_%m_%d) >> $GITHUB_ENV
 
       - name: setup scheduled build
-        if: github.event.workflow_run.outputs.run_on == 'scheduled'
+        if: github.event.schedule == '0 1 * * 2,4,6'
         run: |
           echo PLOT_DIR=ScheduledBuilds/$(date +%Y_%m_%d)_$(git rev-parse --short "$GITHUB_SHA") >> $GITHUB_ENV
 
       - name: setup reference run
-        if: github.event.workflow_run.outputs.run_on == 'reference'
+        if: github.event.schedule == '30 5 * * 1-5'
         run: |
           echo PLOT_DIR=ReferenceRuns/$(date +%Y_%m_%d)_$(git rev-parse --short "$GITHUB_SHA") >> $GITHUB_ENV
 
       - name: setup 1:1 data:mc events
-        if: github.event.workflow_run.outputs.run_on == 'dispatch' || github.event.workflow_run.outputs.run_on == 'scheduled'
+        if: github.event_name != 'pull_request' && github.event.schedule != '30 5 * * 1-5'
         run: |
           echo "NTHREADS=64" >> $GITHUB_ENV
           echo "MAX_FILES=-2" >> $GITHUB_ENV
           echo "LUMI_SCALE=1" >> $GITHUB_ENV
           echo "NOMINAL_FAKE_SMOOTHING=full" >> $GITHUB_ENV
 
+      - name: setup kerberos
+        run: |
+          kinit -kt ~/private/.keytab [email protected]
+          klist -k -t -e ~/private/.keytab
+          klist
+          echo "xrdfs root://eosuser.cern.ch// ls $EOS_DIR"
+          xrdfs root://eosuser.cern.ch// ls $EOS_DIR
+
+      - name: setup kerberos within singularity image
+        run: |
+          scripts/ci/run_with_singularity.sh kinit -kt ~/private/.keytab [email protected]
+          scripts/ci/run_with_singularity.sh klist -k -t -e ~/private/.keytab
+          scripts/ci/run_with_singularity.sh klist
+          echo "xrdfs root://eoscms.cern.ch// ls $EOS_DATA_DIR"
+          scripts/ci/run_with_singularity.sh xrdfs root://eoscms.cern.ch// ls $EOS_DATA_DIR
+
       - name: setup outdir
         run: echo "WREMNANTS_OUTDIR=/tmp/${USER}/$(uuidgen)" >> $GITHUB_ENV
 
@@ -170,7 +258,7 @@ jobs:
           lfs: 'true'
 
       - name: wmass setup lumi scale
-        if: github.event.workflow_run.outputs.run_on == 'pull' || github.event.workflow_run.outputs.run_on == 'scheduled'
+        if: github.event_name == 'pull_request' || github.event.schedule == '30 5 * * 1-5'
         run: | 
           echo "LUMI_SCALE=120" >> $GITHUB_ENV
 
@@ -747,7 +835,7 @@ jobs:
 
       # As of now the gen output isn't used for anything, so there's no point in running more files
       - name: setup nonpr
-        if: github.event.workflow_run.outputs.run_on != 'pull'
+        if: github.event_name != 'pull_request'
         run: |
           echo "NTHREADS=8" >> $GITHUB_ENV
           echo "MAX_FILES=10" >> $GITHUB_ENV
@@ -802,7 +890,7 @@ jobs:
           rm -r $WREMNANTS_OUTDIR/$LOCAL_WEB_DIR
 
       - name: save analysis files
-        if: github.event.workflow_run.outputs.run_on == 'scheduled'
+        if: github.event.schedule == '0 1 * * 2,4,6'
         run: |
           echo "xrdcp --parallel 4 -R -f $WREMNANTS_OUTDIR root://eosuser.cern.ch//$EOS_DIR/ScheduledBuilds/"
           xrdcp --parallel 4 -R -f $WREMNANTS_OUTDIR root://eosuser.cern.ch//$EOS_DIR/ScheduledBuilds/