v1.3.0 - Into the multi-GPU-niverse

.github/workflows/cleanup-caches.yml

@@ -0,0 +1,37 @@
+name: Cleanup caches
+on:
+  pull_request_target:
+    types:
+      - closed
+  push:
+    # Trigger on pushes to master or develop and for git tag pushes
+    branches:
+      - master
+      - develop
+    tags:
+      - v*
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cleanup caches
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          echo "Fetching list of cache key"
+          cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH -L 100 | cut -f 1 )
+
+          ## Setting this to not fail the workflow while deleting cache keys.
+          set +e
+          echo "Deleting caches..."
+          for cacheKey in $cacheKeysForPR
+          do
+              echo $cacheKey
+              gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm
+          done
+          echo "Done"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          BRANCH: refs/pull/${{ github.event.pull_request.number }}/merge

.github/workflows/cleanup.yml

@@ -0,0 +1,15 @@
+name: Delete Untagged Container Versions
+
+on:
+  workflow_dispatch:
+
+jobs:
+  delete-untagged-containers:
+    runs-on: ubuntu-latest
+    steps:
+      - name: mala_conda_cpu
+        uses: actions/delete-package-versions@v5
+        with:
+          package-name: 'mala_conda_cpu'
+          package-type: 'container'
+          delete-only-untagged-versions: 'true'

.github/workflows/cpu-tests.yml

@@ -1,8 +1,15 @@
 name: CPU tests
 
 on:
+  workflow_dispatch:
   pull_request:
-    # Trigger on pull requests to master or develop
+    # Trigger on pull requests to master or develop that are
+    # marked as "ready for review" (non-draft PRs)
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - ready_for_review
     branches:
       - master
       - develop
@@ -21,11 +28,15 @@ env:
 
 jobs:
   build-docker-image-cpu:
+    # do not trigger on draft PRs
+    if: ${{ ! github.event.pull_request.draft }}
     # Build and push temporary Docker image to GitHub's container registry
     runs-on: ubuntu-22.04
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: '1'
 
       - name: Set environment variables
         run: |
@@ -37,7 +48,7 @@ jobs:
           echo "IMAGE_REPO=$IMAGE_REPO"
 
       - name: Restore cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         id: cache-docker
         with:
           path: ${{ env.DOCKER_CACHE_PATH }}
@@ -53,7 +64,7 @@ jobs:
           fi
 
       - name: Pull latest image from container registry
-        run: docker pull $IMAGE_REPO/$IMAGE_NAME || true
+        run: docker pull $IMAGE_REPO/$IMAGE_NAME --quiet || true
 
       - name: Build temporary Docker image
         run: |
@@ -65,7 +76,7 @@ jobs:
             CACHE=$IMAGE_REPO/$IMAGE_NAME:latest
           fi
 
-          docker build . --file Dockerfile --tag $IMAGE_NAME:local --cache-from=$CACHE --build-arg DEVICE=cpu
+          DOCKER_BUILDKIT=0 docker build . --file Dockerfile --tag $IMAGE_NAME:local --cache-from=$CACHE --build-arg DEVICE=cpu
 
           # Show images
           docker images --filter=reference=$IMAGE_NAME --filter=reference=$IMAGE_REPO/$IMAGE_NAME
@@ -114,20 +125,20 @@ jobs:
     steps:
       - name: "Prepare environment: Restore cache"
         if: env.DOCKER_TAG != 'latest'
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         id: cache-docker
         with:
           path: ${{ env.DOCKER_CACHE_PATH }}
           key: ${{ github.run_id }}
 
       - name: "Prepare environment: Load Docker image from cache"
         if: env.DOCKER_TAG != 'latest'
-        run: docker load -i $DOCKER_CACHE_PATH/docker-image.tar.gz
+        run: docker load -i $DOCKER_CACHE_PATH/docker-image.tar.gz --quiet
 
       - name: "Prepare environment: Pull latest image from container registry"
         if: env.DOCKER_TAG == 'latest'
         run: |
-          docker pull $IMAGE_REPO/$IMAGE_NAME:latest
+          docker pull $IMAGE_REPO/$IMAGE_NAME:latest --quiet
           docker image tag $IMAGE_REPO/$IMAGE_NAME:latest $IMAGE_NAME:latest
 
       - name: "Prepare environment: Run Docker container"
@@ -145,39 +156,81 @@ jobs:
           [[ $(docker inspect --format '{{json .State.Running}}' mala-cpu) == 'true' ]]
 
       - name: Check out repository (mala)
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: '1'
 
       - name: Install mala package
         # Exec all commands inside the mala-cpu container
         shell: 'bash -c "docker exec -i mala-cpu bash < {0}"'
         run: |
-          # epxort Docker image Conda environment for a later comparison
-          conda env export -n mala-cpu > env_1.yml
+          # export Docker image Conda environment for a later comparison
+          conda env export -n mala-cpu > env_before.yml
 
           # install mala package
-          pip --no-cache-dir install -e .[opt,test]
+          pip --no-cache-dir install -e .[opt,test] --no-build-isolation
+
 
       - name: Check if Conda environment meets the specified requirements
         shell: 'bash -c "docker exec -i mala-cpu bash < {0}"'
         run: |
           # export Conda environment _with_ mala package installed in it (and extra dependencies)
-          conda env export -n mala-cpu > env_2.yml
+          conda env export -n mala-cpu > env_after.yml
+
+          # This command is necessary because conda includes even editable
+          # packages in an export, at least in the versions we recently used.
+          # That of course leads to the diff failing, since MALA can never
+          # be there before it has been installed.
+          sed -i '/materials-learning-algorithms/d' ./env_after.yml
 
           # if comparison fails, `install/mala_cpu_[base]_environment.yml` needs to be aligned with
           # `requirements.txt` and/or extra dependencies are missing in the Docker Conda environment
-          diff env_1.yml env_2.yml
 
-      - name: Check out repository (data)
-        uses: actions/checkout@v3
-        with:
-          repository: mala-project/test-data
-          path: mala_data
-          ref: v1.7.0
-          lfs: true
+          if diff --brief env_before.yml env_after.yml
+          then
+            echo "Files env_before.yml and env_after.yml do not differ."
+          else
+            diff --side-by-side env_before.yml env_after.yml
+          fi
+
+      - name: Download test data repository from RODARE
+        shell: 'bash -c "docker exec -i mala-cpu python < {0}"'
+        run: |
+          import requests, shutil, zipfile
+
+          # This DOI represents all versions, and will always resolve to the latest one
+          DOI = "https://doi.org/10.14278/rodare.2900"
+
+          # Resolve DOI and get record ID and the associated API URL
+          response = requests.get(DOI)
+          *_, record_id = response.url.split("/")
+          api_url = f"https://rodare.hzdr.de/api/records/{record_id}"
+
+          # Download record from API and get the first file
+          response = requests.get(api_url)
+          record = response.json()
+          size = record["files"][0]["size"]
+          download_link = record["files"][0]["links"]["self"]
+
+          print(size, "bytes", "--", download_link)
+
+          # TODO: implement some sort of auto retry for failed HTTP requests
+          response = requests.get(download_link)
+
+          # Saving downloaded content to a file
+          with open("test-data.zip", mode="wb") as file:
+            file.write(response.content)
+
+          # Get top level directory name
+          dir_name = zipfile.ZipFile("test-data.zip").namelist()[0]
+          shutil.unpack_archive("test-data.zip", ".")
+
+          print(f"Rename {dir_name} to mala_data")
+          shutil.move(dir_name, "mala_data")
 
       - name: Test mala
         shell: 'bash -c "docker exec -i mala-cpu bash < {0}"'
-        run: MALA_DATA_REPO=$(pwd)/mala_data pytest -m "not examples" --disable-warnings
+        run: MALA_DATA_REPO=$(pwd)/mala_data pytest --cov=mala --cov-fail-under=60 -m "not examples" --disable-warnings
 
   retag-docker-image-cpu:
     needs: [cpu-tests, build-docker-image-cpu]
@@ -193,34 +246,29 @@ jobs:
       ((contains(github.ref_name, 'develop') || contains(github.ref_name, 'master')) && needs.build-docker-image-cpu.outputs.docker-tag != 'latest')
       || startsWith(github.ref, 'refs/tags/')
     steps:
-      - name: Check out repository
-        uses: actions/checkout@v3
-
       - name: "Prepare environment: Restore cache"
         if: env.DOCKER_TAG != 'latest'
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         id: cache-docker
         with:
           path: ${{ env.DOCKER_CACHE_PATH }}
           key: ${{ github.run_id }}
 
       - name: "Prepare environment: Load Docker image from cache"
         if: env.DOCKER_TAG != 'latest'
-        run: docker load -i $DOCKER_CACHE_PATH/docker-image.tar.gz
+        run: docker load -i $DOCKER_CACHE_PATH/docker-image.tar.gz --quiet
 
       - name: "Prepare environment: Pull latest image from container registry"
         if: env.DOCKER_TAG == 'latest'
-        run: docker pull $IMAGE_REPO/$IMAGE_NAME:latest
+        run: docker pull $IMAGE_REPO/$IMAGE_NAME:latest --quiet
 
       - name: Tag Docker image
         run: |
           # Execute on change of Docker image
           if [[ "$DOCKER_TAG" != 'latest' ]]; then
-            GIT_SHA=${GITHUB_REF_NAME}-$(git rev-parse --short "$GITHUB_SHA")
-            echo "GIT_SHA=$GIT_SHA"
 
             docker tag $IMAGE_NAME:$GITHUB_RUN_ID $IMAGE_REPO/$IMAGE_NAME:latest
-            docker tag $IMAGE_NAME:$GITHUB_RUN_ID $IMAGE_REPO/$IMAGE_NAME:$GIT_SHA
+            docker tag $IMAGE_NAME:$GITHUB_RUN_ID $IMAGE_REPO/$IMAGE_NAME:${GITHUB_REF_NAME}-${GITHUB_SHA:0:7}
           fi
 
           # Execute on push of git tag
@@ -236,5 +284,4 @@ jobs:
         run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
 
       - name: Push Docker image
-        run: docker push $IMAGE_REPO/$IMAGE_NAME --all-tags
-
+        run: docker push $IMAGE_REPO/$IMAGE_NAME --all-tags | grep -v -E 'Waiting|Layer already|Preparing|Pushed'

.github/workflows/gh-pages.yml

@@ -1,7 +1,12 @@
-name: docs
+name: Documenation
 
 on:
   pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - ready_for_review
     branches:
       - master
       - develop
@@ -11,15 +16,17 @@ on:
 
 jobs:
   test-docstrings:
-    runs-on: ubuntu-22.04
+    # do not trigger on draft PRs
+    if: ${{ ! github.event.pull_request.draft }}
+    runs-on: ubuntu-24.04
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.10.4'
 
       - name: Upgrade pip
         run: python3 -m pip install --upgrade pip
@@ -29,21 +36,21 @@ jobs:
 
       - name: Check docstrings
          # Ignoring the cached_properties because pydocstyle (sometimes?) treats them as functions.
-        run: pydocstyle --convention=numpy --ignore-decorators=[cached_property,property] mala
+        run: pydocstyle --convention=numpy mala
 
   build-and-deploy-pages:
     needs: test-docstrings
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0  # 0 fetches complete history and tags
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.10.4'
 
       - name: Upgrade pip
         run: python3 -m pip install --upgrade pip

.github/workflows/mirror-to-casus.yml

@@ -1,18 +1,19 @@
-name: mirror
+name: Mirror to CASUS
 
 on: [push, delete]
 
 jobs:
   mirror-to-CASUS:
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-    - name: mirror-repository
-      uses: spyoungtech/[email protected]
-      with:
-        REMOTE: 'ssh://[email protected]/casus/mala.git'
-        GIT_SSH_PRIVATE_KEY: ${{ secrets.GIT_SSH_KEY }}
-        GIT_SSH_NO_VERIFY_HOST: "true"
-        DEBUG: "true"
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: mirror-repository
+        uses: spyoungtech/[email protected]
+        with:
+          REMOTE: 'ssh://[email protected]/casus/mala.git'
+          GIT_SSH_PRIVATE_KEY: ${{ secrets.GIT_SSH_KEY }}
+          GIT_SSH_NO_VERIFY_HOST: "true"
+          DEBUG: "true"

.gitignore

@@ -158,6 +158,13 @@ cython_debug/
 # JupyterNotebooks
 .ipynb_checkpoints
 */.ipynb_checkpoints/*
+*.ipynb
+
+# Lightning
+lightning_logs/
+
+# wandb
+wandb/
 
 # SQLite
 *.db

.pre-commit-config.yaml

@@ -0,0 +1,8 @@
+# https://black.readthedocs.io/en/stable/integrations/source_version_control.html
+
+repos:
+  # Using this mirror lets us use mypyc-compiled black, which is about 2x faster
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 24.4.0
+    hooks:
+      - id: black