From 88d718dfc7eb55233d90533e51d74e52e215da94 Mon Sep 17 00:00:00 2001
From: Petr Cagas <p.cagas@gmail.com>
Date: Thu, 30 May 2024 14:49:11 +0200
Subject: [PATCH 01/19] Updating the examples_test so it ccan be run from any
 directory. Test execution now uses an absolute path and the test results are
 also placed to pytest temp directory to avoid artefact generation.

---
 test/examples_test.py | 110 +++++++++++++++++++++++++++++++-----------
 1 file changed, 81 insertions(+), 29 deletions(-)

diff --git a/test/examples_test.py b/test/examples_test.py
index 5d74ec164..1586b17b8 100644
--- a/test/examples_test.py
+++ b/test/examples_test.py
@@ -1,61 +1,113 @@
 """Test whether the examples are still working."""
 
+import os
 import importlib
 import runpy
 
 import pytest
 
-
 @pytest.mark.examples
 class TestExamples:
-    def test_basic_ex01(self):
-        runpy.run_path("../examples/basic/ex01_train_network.py")
+    dir_path = os.path.dirname(__file__)
+
+    def test_basic_ex01(self, tmp_path):
+        os.chdir(tmp_path / "..")
+        runpy.run_path(
+            self.dir_path +
+            "/../examples/basic/ex01_train_network.py"
+        )
 
-    def test_basic_ex02(self):
-        runpy.run_path("../examples/basic/ex02_test_network.py")
+    def test_basic_ex02(self, tmp_path):
+        os.chdir(tmp_path / "..")
+        runpy.run_path(
+            self.dir_path +
+            "/../examples/basic/ex02_test_network.py"
+        )
 
-    def test_basic_ex03(self):
-        runpy.run_path("../examples/basic/ex03_preprocess_data.py")
+    def test_basic_ex03(self, tmp_path):
+        os.chdir(tmp_path / "..")
+        runpy.run_path(
+            self.dir_path +
+            "/../examples/basic/ex03_preprocess_data.py"
+        )
 
-    def test_basic_ex04(self):
-        runpy.run_path("../examples/basic/ex04_hyperparameter_optimization.py")
+    def test_basic_ex04(self, tmp_path):
+        os.chdir(tmp_path / "..")
+        runpy.run_path(
+            self.dir_path +
+            "/../examples/basic/ex04_hyperparameter_optimization.py"
+        )
 
-    def test_basic_ex05(self):
-        runpy.run_path("../examples/basic/ex05_run_predictions.py")
+    def test_basic_ex05(self, tmp_path):
+        os.chdir(tmp_path / "..")
+        runpy.run_path(
+            self.dir_path +
+            "/../examples/basic/ex05_run_predictions.py"
+        )
 
-    def test_basic_ex06(self):
-        runpy.run_path("../examples/basic/ex06_ase_calculator.py")
+    def test_basic_ex06(self, tmp_path):
+        os.chdir(tmp_path / "..")
+        runpy.run_path(
+            self.dir_path +
+            "/../examples/basic/ex06_ase_calculator.py"
+        )
 
-    def test_advanced_ex01(self):
-        runpy.run_path("../examples/advanced/ex01_checkpoint_training.py")
+    def test_advanced_ex01(self, tmp_path):
+        os.chdir(tmp_path / "..")
+        runpy.run_path(
+            self.dir_path +
+            "/../examples/advanced/ex01_checkpoint_training.py"
+        )
 
-    def test_advanced_ex02(self):
-        runpy.run_path("../examples/advanced/ex02_shuffle_data.py")
+    def test_advanced_ex02(self, tmp_path):
+        os.chdir(tmp_path / "..")
+        runpy.run_path(
+            self.dir_path +
+            "/../examples/advanced/ex02_shuffle_data.py"
+        )
 
-    def test_advanced_ex03(self):
-        runpy.run_path("../examples/advanced/ex03_tensor_board.py")
+    def test_advanced_ex03(self, tmp_path):
+        os.chdir(tmp_path / "..")
+        runpy.run_path(
+            self.dir_path +
+            "/../examples/advanced/ex03_tensor_board.py"
+        )
 
-    def test_advanced_ex04(self):
-        runpy.run_path("../examples/advanced/ex04_acsd.py")
+    def test_advanced_ex04(self, tmp_path):
+        os.chdir(tmp_path / "..")
+        runpy.run_path(
+            self.dir_path +
+            "/../examples/advanced/ex04_acsd.py"
+        )
 
-    def test_advanced_ex05(self):
+    def test_advanced_ex05(self, tmp_path):
+        os.chdir(tmp_path / "..")
         runpy.run_path(
-            "../examples/advanced/ex05_checkpoint_hyperparameter_optimization.py"
+            self.dir_path +
+            "/../examples/advanced/ex05_checkpoint_hyperparameter_optimization.py"
         )
 
-    def test_advanced_ex06(self):
+    def test_advanced_ex06(self, tmp_path):
+        os.chdir(tmp_path / "..")
         runpy.run_path(
-            "../examples/advanced/ex06_distributed_hyperparameter_optimization.py"
+            self.dir_path +
+            "/../examples/advanced/ex06_distributed_hyperparameter_optimization.py"
         )
 
     @pytest.mark.skipif(
         importlib.util.find_spec("oapackage") is None,
         reason="No OAT found on this machine, skipping this " "test.",
     )
-    def test_advanced_ex07(self):
+    def test_advanced_ex07(self, tmp_path):
+        os.chdir(tmp_path / "..")
         runpy.run_path(
-            "../examples/advanced/ex07_advanced_hyperparameter_optimization.py"
+            self.dir_path +
+            "/../examples/advanced/ex07_advanced_hyperparameter_optimization.py"
         )
 
-    def test_advanced_ex08(self):
-        runpy.run_path("../examples/advanced/ex08_visualize_observables.py")
+    def test_advanced_ex08(self, tmp_path):
+        os.chdir(tmp_path / "..")
+        runpy.run_path(
+            self.dir_path +
+            "/../examples/advanced/ex08_visualize_observables.py"
+        )

From b4947ba360baf79a43845781e9718e9328205b93 Mon Sep 17 00:00:00 2001
From: Petr Cagas <p.cagas@gmail.com>
Date: Fri, 31 May 2024 15:06:05 +0200
Subject: [PATCH 02/19] Removed the dependency of examples on example 01. If
 the Be_model.zip is not found, it is loaded from hte test-data repository. In
 addition, `data_path` variable was added to the `datahandling` submodule
 which points directly to the `Be2` subdirectory

---
 examples/advanced/ex01_checkpoint_training.py | 12 +++---
 examples/advanced/ex02_shuffle_data.py        |  8 ++--
 examples/advanced/ex03_tensor_board.py        |  7 +--
 examples/advanced/ex04_acsd.py                |  6 +--
 ..._checkpoint_hyperparameter_optimization.py | 10 ++---
 ...distributed_hyperparameter_optimization.py |  8 ++--
 ...07_advanced_hyperparameter_optimization.py |  6 +--
 .../advanced/ex08_visualize_observables.py    | 13 +++---
 examples/basic/ex01_train_network.py          |  7 +--
 examples/basic/ex02_test_network.py           | 11 ++---
 examples/basic/ex03_preprocess_data.py        |  6 +--
 .../basic/ex04_hyperparameter_optimization.py |  6 +--
 examples/basic/ex05_run_predictions.py        | 16 +++----
 examples/basic/ex06_ase_calculator.py         | 18 ++++----
 mala/datahandling/data_repo.py                |  2 +
 test/examples_test.py                         | 43 ++++++++++++-------
 16 files changed, 85 insertions(+), 94 deletions(-)

diff --git a/examples/advanced/ex01_checkpoint_training.py b/examples/advanced/ex01_checkpoint_training.py
index 341ff5c6f..01bb9b486 100644
--- a/examples/advanced/ex01_checkpoint_training.py
+++ b/examples/advanced/ex01_checkpoint_training.py
@@ -3,18 +3,16 @@
 import mala
 from mala import printout
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 """
-Shows how a training run can be paused and 
+Shows how a training run can be paused and
 resumed. Delete the ex07.zip file prior to execution to see the effect
 of checkpointing.
-Afterwards, execute this script twice to see how MALA progresses from a 
+Afterwards, execute this script twice to see how MALA progresses from a
 checkpoint. As the number of total epochs cannot be divided by the number
-of epochs after which a checkpoint is created without residual, this will 
-lead to MALA performing the missing epochs again. 
+of epochs after which a checkpoint is created without residual, this will
+lead to MALA performing the missing epochs again.
 """
 
 
diff --git a/examples/advanced/ex02_shuffle_data.py b/examples/advanced/ex02_shuffle_data.py
index 467da7922..db75d5154 100644
--- a/examples/advanced/ex02_shuffle_data.py
+++ b/examples/advanced/ex02_shuffle_data.py
@@ -2,14 +2,12 @@
 
 import mala
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 """
 Shows how data can be shuffled amongst multiple
-snapshots, which is very useful in the lazy loading case, where this cannot be 
-easily done in memory. 
+snapshots, which is very useful in the lazy loading case, where this cannot be
+easily done in memory.
 """
 
 
diff --git a/examples/advanced/ex03_tensor_board.py b/examples/advanced/ex03_tensor_board.py
index 00728a560..b15239495 100644
--- a/examples/advanced/ex03_tensor_board.py
+++ b/examples/advanced/ex03_tensor_board.py
@@ -3,13 +3,10 @@
 import mala
 from mala import printout
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
-
+from mala.datahandling.data_repo import data_path
 
 """
-Shows how a NN training by MALA can be visualized using 
+Shows how a NN training by MALA can be visualized using
 tensorboard. The training is a basic MALA network training.
 """
 
diff --git a/examples/advanced/ex04_acsd.py b/examples/advanced/ex04_acsd.py
index 5390ae210..53b4b82bd 100644
--- a/examples/advanced/ex04_acsd.py
+++ b/examples/advanced/ex04_acsd.py
@@ -1,13 +1,11 @@
 import os
 
 import mala
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 """
 Shows how MALA can be used to optimize descriptor
-parameters based on the ACSD analysis (see hyperparameter paper in the 
+parameters based on the ACSD analysis (see hyperparameter paper in the
 documentation for mathematical details).
 """
 
diff --git a/examples/advanced/ex05_checkpoint_hyperparameter_optimization.py b/examples/advanced/ex05_checkpoint_hyperparameter_optimization.py
index c7f741d70..cef7c8f4f 100644
--- a/examples/advanced/ex05_checkpoint_hyperparameter_optimization.py
+++ b/examples/advanced/ex05_checkpoint_hyperparameter_optimization.py
@@ -2,16 +2,14 @@
 
 import mala
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 """
-Shows how a hyperparameter optimization run can 
+Shows how a hyperparameter optimization run can
 be paused and resumed. Delete all ex04_*.pkl and ex04_*.pth prior to execution.
-Afterwards, execute this script twice to see how MALA progresses from a 
+Afterwards, execute this script twice to see how MALA progresses from a
 checkpoint. As the number of trials cannot be divided by the number
-of epochs after which a checkpoint is created without residual, this will 
+of epochs after which a checkpoint is created without residual, this will
 lead to MALA performing the missing trials again.
 """
 
diff --git a/examples/advanced/ex06_distributed_hyperparameter_optimization.py b/examples/advanced/ex06_distributed_hyperparameter_optimization.py
index 2a67acb3c..b34f9bb8b 100644
--- a/examples/advanced/ex06_distributed_hyperparameter_optimization.py
+++ b/examples/advanced/ex06_distributed_hyperparameter_optimization.py
@@ -2,14 +2,12 @@
 
 import mala
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 """
-ex09_distributed_hyperopt.py: Shows how a hyperparameter 
+ex09_distributed_hyperopt.py: Shows how a hyperparameter
 optimization can be sped up using a RDB storage. Ideally this should be done
-using a database server system, such as PostgreSQL or MySQL. 
+using a database server system, such as PostgreSQL or MySQL.
 For this easy example, sqlite will be used. It is highly advisory not to
 to use this for actual, at-scale calculations!
 
diff --git a/examples/advanced/ex07_advanced_hyperparameter_optimization.py b/examples/advanced/ex07_advanced_hyperparameter_optimization.py
index 629d47962..8165ef01e 100644
--- a/examples/advanced/ex07_advanced_hyperparameter_optimization.py
+++ b/examples/advanced/ex07_advanced_hyperparameter_optimization.py
@@ -3,12 +3,10 @@
 import mala
 from mala import printout
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 """
-Shows how recent developments in hyperparameter optimization techniques can be 
+Shows how recent developments in hyperparameter optimization techniques can be
 used (OAT / training-free NAS).
 
 REQUIRES OAPACKAGE.
diff --git a/examples/advanced/ex08_visualize_observables.py b/examples/advanced/ex08_visualize_observables.py
index 3b8bbed3d..be344b878 100644
--- a/examples/advanced/ex08_visualize_observables.py
+++ b/examples/advanced/ex08_visualize_observables.py
@@ -2,18 +2,15 @@
 
 import mala
 
-from mala.datahandling.data_repo import data_repo_path
+from mala.datahandling.data_repo import data_path
 
-atoms_path = os.path.join(
-    os.path.join(data_repo_path, "Be2"), "Be_snapshot1.out"
-)
-ldos_path = os.path.join(
-    os.path.join(data_repo_path, "Be2"), "Be_snapshot1.out.npy"
-)
 """
-Shows how MALA can be used to visualize observables of interest. 
+Shows how MALA can be used to visualize observables of interest.
 """
 
+atoms_path = os.path.join(data_path, "Be_snapshot1.out")
+ldos_path = os.path.join(data_path, "Be_snapshot1.out.npy")
+
 ####################
 # 1. READ ELECTRONIC STRUCTURE DATA
 # This data may be read as part of an ML-DFT model inference.
diff --git a/examples/basic/ex01_train_network.py b/examples/basic/ex01_train_network.py
index a5d14d890..95eb2d51b 100644
--- a/examples/basic/ex01_train_network.py
+++ b/examples/basic/ex01_train_network.py
@@ -2,9 +2,7 @@
 
 import mala
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 """
 This example shows how a neural network can be trained on material
@@ -12,7 +10,6 @@
 from *.npy files.
 """
 
-
 ####################
 # 1. PARAMETERS
 # The first step of each MALA workflow is to define a parameters object and
@@ -93,5 +90,5 @@
 test_trainer.train_network()
 additional_calculation_data = os.path.join(data_path, "Be_snapshot0.out")
 test_trainer.save_run(
-    "be_model", additional_calculation_data=additional_calculation_data
+    "Be_model", additional_calculation_data=additional_calculation_data
 )
diff --git a/examples/basic/ex02_test_network.py b/examples/basic/ex02_test_network.py
index 6ef81f880..2e4b8953c 100644
--- a/examples/basic/ex02_test_network.py
+++ b/examples/basic/ex02_test_network.py
@@ -3,17 +3,16 @@
 import mala
 from mala import printout
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 """
 This example shows how a trained network can be tested
 with additional test snapshots. Either execute ex01 before executing this one
 or download the appropriate model from the provided test data repo.
 """
-assert os.path.exists("be_model.zip"), "Be model missing, run ex01 first."
 
+model_name = "Be_model"
+model_path = "./" if os.path.exists("Be_model.zip") else data_path
 
 ####################
 # 1. LOADING A NETWORK
@@ -27,7 +26,9 @@
 # (output_format="list") or as an averaged value (output_format="mae")
 ####################
 
-parameters, network, data_handler, tester = mala.Tester.load_run("be_model")
+parameters, network, data_handler, tester = mala.Tester.load_run(
+    run_name=model_name, path=model_path
+)
 tester.observables_to_test = ["band_energy", "number_of_electrons"]
 tester.output_format = "list"
 parameters.data.use_lazy_loading = True
diff --git a/examples/basic/ex03_preprocess_data.py b/examples/basic/ex03_preprocess_data.py
index 72ec9490a..b0a104885 100644
--- a/examples/basic/ex03_preprocess_data.py
+++ b/examples/basic/ex03_preprocess_data.py
@@ -2,13 +2,11 @@
 
 import mala
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 """
 Shows how this framework can be used to preprocess
-data. Preprocessing here means converting raw DFT calculation output into 
+data. Preprocessing here means converting raw DFT calculation output into
 numpy arrays of the correct size. For the input data, this means descriptor
 calculation.
 
diff --git a/examples/basic/ex04_hyperparameter_optimization.py b/examples/basic/ex04_hyperparameter_optimization.py
index 77985f033..4c68179c2 100644
--- a/examples/basic/ex04_hyperparameter_optimization.py
+++ b/examples/basic/ex04_hyperparameter_optimization.py
@@ -2,14 +2,12 @@
 
 import mala
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 """
 Shows how a hyperparameter optimization can be done using this framework. There
 are multiple hyperparameter optimizers available in this framework. This example
-focusses on the most universal one - optuna.  
+focusses on the most universal one - optuna.
 """
 
 
diff --git a/examples/basic/ex05_run_predictions.py b/examples/basic/ex05_run_predictions.py
index 4e0d72e3b..05deb857e 100644
--- a/examples/basic/ex05_run_predictions.py
+++ b/examples/basic/ex05_run_predictions.py
@@ -4,19 +4,19 @@
 import mala
 from mala import printout
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
-
-assert os.path.exists("be_model.zip"), "Be model missing, run ex01 first."
+from mala.datahandling.data_repo import data_path
 
 """
-Show how a prediction can be made using MALA, based on only a
-trained network and atomic configurations.
+Show how a prediction can be made using MALA, based on only a trained network and atomic
+configurations. Either execute ex01 before executing this one or download the
+appropriate model from the provided test data repo.
 
 REQUIRES LAMMPS (and potentiall the total energy module).
 """
 
+model_name = "Be_model"
+model_path = "./" if os.path.exists("Be_model.zip") else data_path
+
 
 ####################
 # 1. LOADING A NETWORK
@@ -24,7 +24,7 @@
 # Tester class interface. Afterwards, set the necessary parameters.
 ####################
 parameters, network, data_handler, predictor = mala.Predictor.load_run(
-    "be_model"
+    run_name=model_name, path=model_path
 )
 
 
diff --git a/examples/basic/ex06_ase_calculator.py b/examples/basic/ex06_ase_calculator.py
index f4ab2d337..7ba0eee1b 100644
--- a/examples/basic/ex06_ase_calculator.py
+++ b/examples/basic/ex06_ase_calculator.py
@@ -1,21 +1,21 @@
 import os
 
-import mala
 from ase.io import read
+import mala
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
-
-assert os.path.exists("be_model.zip"), "Be model missing, run ex01 first."
+from mala.datahandling.data_repo import data_path
 
 """
-Shows how MALA can be used as an ASE calculator. 
-Currently, calculation of forces is not supported. 
+Shows how MALA can be used as an ASE calculator.
+Currently, calculation of forces is not supported. Either execute ex01 before executing
+this one or download the appropriate model from the provided test data repo.
 
 REQUIRES LAMMPS AND QUANTUM ESPRESSO (TOTAL ENERGY MODULE).
 """
 
+model_name = "Be_model"
+model_path = "./" if os.path.exists("Be_model.zip") else data_path
+
 
 ####################
 # 1. LOADING A NETWORK
@@ -23,7 +23,7 @@
 # Further make sure to set the path to the pseudopotential used during
 # data generation-
 ####################
-calculator = mala.MALA.load_model("be_model")
+calculator = mala.MALA.load_model(run_name=model_name, path=model_path)
 calculator.mala_parameters.targets.pseudopotential_path = data_path
 
 ####################
diff --git a/mala/datahandling/data_repo.py b/mala/datahandling/data_repo.py
index 178872b60..203885c12 100644
--- a/mala/datahandling/data_repo.py
+++ b/mala/datahandling/data_repo.py
@@ -14,9 +14,11 @@
 name = "MALA_DATA_REPO"
 if name in os.environ:
     data_repo_path = os.environ[name]
+    data_path = os.path.join(data_repo_path, "Be2")
 else:
     parallel_warn(
         f"Environment variable {name} not set. You won't be able "
         "to run all examples and tests."
     )
     data_repo_path = None
+    data_path = None
diff --git a/test/examples_test.py b/test/examples_test.py
index 1586b17b8..b5aa9143a 100644
--- a/test/examples_test.py
+++ b/test/examples_test.py
@@ -1,7 +1,7 @@
 """Test whether the examples are still working."""
 
-import os
 import importlib
+import os
 import runpy
 
 import pytest
@@ -11,84 +11,95 @@ class TestExamples:
     dir_path = os.path.dirname(__file__)
 
     def test_basic_ex01(self, tmp_path):
-        os.chdir(tmp_path / "..")
+        os.chdir(tmp_path)
         runpy.run_path(
             self.dir_path +
             "/../examples/basic/ex01_train_network.py"
         )
 
+    @pytest.mark.order(after="test_basic_ex01")
     def test_basic_ex02(self, tmp_path):
-        os.chdir(tmp_path / "..")
+        os.chdir(tmp_path)
         runpy.run_path(
             self.dir_path +
             "/../examples/basic/ex02_test_network.py"
         )
 
+    @pytest.mark.order(after="test_basic_ex01")
     def test_basic_ex03(self, tmp_path):
-        os.chdir(tmp_path / "..")
+        os.chdir(tmp_path)
         runpy.run_path(
             self.dir_path +
             "/../examples/basic/ex03_preprocess_data.py"
         )
 
+    @pytest.mark.order(after="test_basic_ex01")
     def test_basic_ex04(self, tmp_path):
-        os.chdir(tmp_path / "..")
+        os.chdir(tmp_path)
         runpy.run_path(
             self.dir_path +
             "/../examples/basic/ex04_hyperparameter_optimization.py"
         )
 
+    @pytest.mark.order(after="test_basic_ex01")
     def test_basic_ex05(self, tmp_path):
-        os.chdir(tmp_path / "..")
+        os.chdir(tmp_path)
         runpy.run_path(
             self.dir_path +
             "/../examples/basic/ex05_run_predictions.py"
         )
 
+    @pytest.mark.order(after="test_basic_ex01")
     def test_basic_ex06(self, tmp_path):
-        os.chdir(tmp_path / "..")
+        os.chdir(tmp_path)
         runpy.run_path(
             self.dir_path +
             "/../examples/basic/ex06_ase_calculator.py"
         )
 
+    @pytest.mark.order(after="test_basic_ex01")
     def test_advanced_ex01(self, tmp_path):
-        os.chdir(tmp_path / "..")
+        os.chdir(tmp_path)
         runpy.run_path(
             self.dir_path +
             "/../examples/advanced/ex01_checkpoint_training.py"
         )
 
+    @pytest.mark.order(after="test_basic_ex01")
     def test_advanced_ex02(self, tmp_path):
-        os.chdir(tmp_path / "..")
+        os.chdir(tmp_path)
         runpy.run_path(
             self.dir_path +
             "/../examples/advanced/ex02_shuffle_data.py"
         )
 
+    @pytest.mark.order(after="test_basic_ex01")
     def test_advanced_ex03(self, tmp_path):
-        os.chdir(tmp_path / "..")
+        os.chdir(tmp_path)
         runpy.run_path(
             self.dir_path +
             "/../examples/advanced/ex03_tensor_board.py"
         )
 
+    @pytest.mark.order(after="test_basic_ex01")
     def test_advanced_ex04(self, tmp_path):
-        os.chdir(tmp_path / "..")
+        os.chdir(tmp_path)
         runpy.run_path(
             self.dir_path +
             "/../examples/advanced/ex04_acsd.py"
         )
 
+    @pytest.mark.order(after="test_basic_ex01")
     def test_advanced_ex05(self, tmp_path):
-        os.chdir(tmp_path / "..")
+        os.chdir(tmp_path)
         runpy.run_path(
             self.dir_path +
             "/../examples/advanced/ex05_checkpoint_hyperparameter_optimization.py"
         )
 
+    @pytest.mark.order(after="test_basic_ex01")
     def test_advanced_ex06(self, tmp_path):
-        os.chdir(tmp_path / "..")
+        os.chdir(tmp_path)
         runpy.run_path(
             self.dir_path +
             "/../examples/advanced/ex06_distributed_hyperparameter_optimization.py"
@@ -98,15 +109,17 @@ def test_advanced_ex06(self, tmp_path):
         importlib.util.find_spec("oapackage") is None,
         reason="No OAT found on this machine, skipping this " "test.",
     )
+    @pytest.mark.order(after="test_basic_ex01")
     def test_advanced_ex07(self, tmp_path):
-        os.chdir(tmp_path / "..")
+        os.chdir(tmp_path)
         runpy.run_path(
             self.dir_path +
             "/../examples/advanced/ex07_advanced_hyperparameter_optimization.py"
         )
 
+    @pytest.mark.order(after="test_basic_ex01")
     def test_advanced_ex08(self, tmp_path):
-        os.chdir(tmp_path / "..")
+        os.chdir(tmp_path)
         runpy.run_path(
             self.dir_path +
             "/../examples/advanced/ex08_visualize_observables.py"

From b41232348bc93103a700562e6d9c065f7fd2c6fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Franz=20P=C3=B6schel?= <franz.poeschel@gmail.com>
Date: Thu, 6 Jun 2024 17:55:57 +0200
Subject: [PATCH 03/19] Workaround for force-flushing in parallel

---
 mala/common/physical_data.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/mala/common/physical_data.py b/mala/common/physical_data.py
index e756e96d1..7ec85623d 100644
--- a/mala/common/physical_data.py
+++ b/mala/common/physical_data.py
@@ -642,6 +642,11 @@ def write_to_openpmd_iteration(
 
         # Third loop: Extra flushes to harmonize ranks
         for _ in range(extra_flushes):
+            # This following line is a workaround for issue
+            # https://github.com/openPMD/openPMD-api/issues/1616
+            # Fixed in openPMD-api 0.16 by
+            # https://github.com/openPMD/openPMD-api/pull/1619
+            iteration.dt = iteration.dt
             iteration.series_flush()
 
         iteration.close(flush=True)

From a8a2a0ade003552fdb8fb301dc59c92747a71f80 Mon Sep 17 00:00:00 2001
From: Petr Cagas <p.cagas@gmail.com>
Date: Fri, 7 Jun 2024 10:46:09 +0200
Subject: [PATCH 04/19] Updating the RODARE path

---
 .github/workflows/cpu-tests.yml | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/cpu-tests.yml b/.github/workflows/cpu-tests.yml
index b18305a23..063363405 100644
--- a/.github/workflows/cpu-tests.yml
+++ b/.github/workflows/cpu-tests.yml
@@ -1,6 +1,7 @@
 name: CPU tests
 
 on:
+  workflow_dispatch:
   pull_request:
     # Trigger on pull requests to master or develop that are
     # marked as "ready for review" (non-draft PRs)
@@ -174,16 +175,16 @@ jobs:
         shell: 'bash -c "docker exec -i mala-cpu bash < {0}"'
         run: |
           # Download test data repository from RODARE. If the version changes
-          # this URL has to be adapted (the number after /record/ and the 
+          # this URL has to be adapted (the number after /record/ and the
           # version have to be incremented)
-          wget "https://rodare.hzdr.de/record/2999/files/mala-project/test-data-1.8.0.zip"
-          
+          wget "https://rodare.hzdr.de/record/3004/files/mala-project/test-data-1.8.1.zip"
+
           # Once downloaded, we have to unzip the file. The name of the root
           # folder in the zip file has to be updated for data repository
-          # updates as well - the string at the end is the hash of the data 
-          # repository commit. 
-          unzip -q test-data-1.8.0.zip
-          mv mala-project-test-data-d5694c7  mala_data
+          # updates as well - the string at the end is the hash of the data
+          # repository commit.
+          unzip -q test-data-1.8.1.zip
+          mv mala-project-test-data-741eda6 mala_data
 
       - name: Test mala
         shell: 'bash -c "docker exec -i mala-cpu bash < {0}"'

From 67e7c3f42b97f6e92ed0ce5aab6335a5f38925e3 Mon Sep 17 00:00:00 2001
From: Petr Cagas <p.cagas@gmail.com>
Date: Fri, 7 Jun 2024 13:53:27 +0200
Subject: [PATCH 05/19] Updating the path to test-data in the test suite and
 redirecting workflow_test to Be_model

---
 test/all_lazy_loading_test.py    | 4 +---
 test/basic_gpu_test.py           | 8 +++-----
 test/checkpoint_hyperopt_test.py | 4 +---
 test/checkpoint_training_test.py | 3 +--
 test/complete_interfaces_test.py | 4 +---
 test/descriptor_test.py          | 4 +---
 test/hyperopt_test.py            | 4 +---
 test/inference_test.py           | 8 +++-----
 test/integration_test.py         | 3 +--
 test/parallel_run_test.py        | 4 +---
 test/scaling_test.py             | 4 +---
 test/shuffling_test.py           | 4 +---
 test/tensor_memory_test.py       | 4 +---
 test/workflow_test.py            | 9 ++++-----
 14 files changed, 21 insertions(+), 46 deletions(-)

diff --git a/test/all_lazy_loading_test.py b/test/all_lazy_loading_test.py
index f5cc74006..065cbb86e 100644
--- a/test/all_lazy_loading_test.py
+++ b/test/all_lazy_loading_test.py
@@ -7,9 +7,7 @@
 import torch
 import pytest
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 # This test compares the data scaling using the regular scaling procedure and
 # the lazy-loading one (incremental fitting).
diff --git a/test/basic_gpu_test.py b/test/basic_gpu_test.py
index 943862b3d..dcd588ad1 100644
--- a/test/basic_gpu_test.py
+++ b/test/basic_gpu_test.py
@@ -6,9 +6,9 @@
 which MALA relies on). Two things are tested:
 
 1. Whether or not your system has GPU support.
-2. Whether or not the GPU does what it is supposed to. For this, 
+2. Whether or not the GPU does what it is supposed to. For this,
 a training is performed. It is measured whether or not the utilization
-of the GPU results in a speed up. 
+of the GPU results in a speed up.
 """
 import os
 import time
@@ -19,9 +19,7 @@
 import pytest
 import torch
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 test_checkpoint_name = "test"
 
diff --git a/test/checkpoint_hyperopt_test.py b/test/checkpoint_hyperopt_test.py
index f3435e7ab..28889c2df 100644
--- a/test/checkpoint_hyperopt_test.py
+++ b/test/checkpoint_hyperopt_test.py
@@ -4,9 +4,7 @@
 from mala import printout
 import numpy as np
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 checkpoint_name = "test_ho"
 
diff --git a/test/checkpoint_training_test.py b/test/checkpoint_training_test.py
index bf7f62090..4c56ed8eb 100644
--- a/test/checkpoint_training_test.py
+++ b/test/checkpoint_training_test.py
@@ -4,9 +4,8 @@
 from mala import printout
 import numpy as np
 
-from mala.datahandling.data_repo import data_repo_path
+from mala.datahandling.data_repo import data_path
 
-data_path = os.path.join(data_repo_path, "Be2")
 test_checkpoint_name = "test"
 
 # Define the accuracy used in the tests.
diff --git a/test/complete_interfaces_test.py b/test/complete_interfaces_test.py
index 127ba8f82..d793da77f 100644
--- a/test/complete_interfaces_test.py
+++ b/test/complete_interfaces_test.py
@@ -8,9 +8,7 @@
 import pytest
 
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 
 # This test checks whether MALA interfaces to other codes, mainly the ASE
diff --git a/test/descriptor_test.py b/test/descriptor_test.py
index 4a208f832..74cae40f5 100644
--- a/test/descriptor_test.py
+++ b/test/descriptor_test.py
@@ -6,9 +6,7 @@
 import numpy as np
 import pytest
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 # Accuracy of test.
 accuracy_descriptors = 5e-8
diff --git a/test/hyperopt_test.py b/test/hyperopt_test.py
index b2d93f872..bb003082a 100644
--- a/test/hyperopt_test.py
+++ b/test/hyperopt_test.py
@@ -7,9 +7,7 @@
 import mala
 import numpy as np
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 # Control how much the loss should be better after hyperopt compared to
 # before. This value is fairly high, but we're training on absolutely
diff --git a/test/inference_test.py b/test/inference_test.py
index 4e874570b..84e0e9cca 100644
--- a/test/inference_test.py
+++ b/test/inference_test.py
@@ -3,10 +3,8 @@
 import numpy as np
 from mala import Tester, Runner
 
-from mala.datahandling.data_repo import data_repo_path
+from mala.datahandling.data_repo import data_path
 
-data_path = os.path.join(data_repo_path, "Be2")
-param_path = os.path.join(data_repo_path, "workflow_test/")
 accuracy_strict = 1e-16
 accuracy_coarse = 5e-7
 accuracy_very_coarse = 3
@@ -18,7 +16,7 @@ class TestInference:
     def test_unit_conversion(self):
         """Test that RAM inexpensive unit conversion works."""
         parameters, network, data_handler = Runner.load_run(
-            "workflow_test", load_runner=False, path=param_path
+            "Be_model", load_runner=False, path=data_path
         )
         parameters.data.use_lazy_loading = False
         parameters.running.mini_batch_size = 50
@@ -99,7 +97,7 @@ def test_inference_lazy_loading(self):
     def __run(use_lazy_loading=False, batchsize=46):
         # First we load Parameters and network.
         parameters, network, data_handler, tester = Tester.load_run(
-            "workflow_test", path=param_path
+            "Be_model", path=data_path
         )
         parameters.data.use_lazy_loading = use_lazy_loading
         parameters.running.mini_batch_size = batchsize
diff --git a/test/integration_test.py b/test/integration_test.py
index b27abb872..e4e22ea95 100644
--- a/test/integration_test.py
+++ b/test/integration_test.py
@@ -6,7 +6,7 @@
 import scipy as sp
 import pytest
 
-from mala.datahandling.data_repo import data_repo_path
+from mala.datahandling.data_repo import data_path
 
 # In order to test the integration capabilities of MALA we need a
 # QuantumEspresso
@@ -18,7 +18,6 @@
 # Scripts to reproduce the data files used in this test script can be found
 # in the data repo.
 
-data_path = os.path.join(data_repo_path, "Be2")
 path_to_out = os.path.join(data_path, "Be_snapshot0.out")
 path_to_ldos_npy = os.path.join(data_path, "Be_snapshot0.out.npy")
 path_to_dos_npy = os.path.join(data_path, "Be_snapshot0.dos.npy")
diff --git a/test/parallel_run_test.py b/test/parallel_run_test.py
index 89b0cbad8..6ca5c8c8d 100644
--- a/test/parallel_run_test.py
+++ b/test/parallel_run_test.py
@@ -6,9 +6,7 @@
 from ase.io import read
 import pytest
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 # Control the various accuracies..
 accuracy_snaps = 1e-4
diff --git a/test/scaling_test.py b/test/scaling_test.py
index d43648430..b7925cd9f 100644
--- a/test/scaling_test.py
+++ b/test/scaling_test.py
@@ -4,9 +4,7 @@
 import numpy as np
 import torch
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 # This test checks that all scaling options are working and are not messing
 # up the data.
diff --git a/test/shuffling_test.py b/test/shuffling_test.py
index 202e40c9d..e637c7d2b 100644
--- a/test/shuffling_test.py
+++ b/test/shuffling_test.py
@@ -3,9 +3,7 @@
 import mala
 import numpy as np
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 # Accuracy for the shuffling test.
 accuracy = np.finfo(float).eps
diff --git a/test/tensor_memory_test.py b/test/tensor_memory_test.py
index 4a70d9719..b3cb25672 100644
--- a/test/tensor_memory_test.py
+++ b/test/tensor_memory_test.py
@@ -5,9 +5,7 @@
 from torch.utils.data import TensorDataset
 from torch.utils.data import DataLoader
 
-from mala.datahandling.data_repo import data_repo_path
-
-data_path = os.path.join(data_repo_path, "Be2")
+from mala.datahandling.data_repo import data_path
 
 # Define the accuracy used in the tests.
 accuracy = 1e-5
diff --git a/test/workflow_test.py b/test/workflow_test.py
index a652546fd..fa7dee018 100644
--- a/test/workflow_test.py
+++ b/test/workflow_test.py
@@ -5,9 +5,8 @@
 import numpy as np
 import pytest
 
-from mala.datahandling.data_repo import data_repo_path
+from mala.datahandling.data_repo import data_path
 
-data_path = os.path.join(data_repo_path, "Be2")
 # Control how much the loss should be better after training compared to
 # before. This value is fairly high, but we're training on absolutely
 # minimal amounts of data.
@@ -382,7 +381,7 @@ def test_training_with_postprocessing_data_repo(self):
         """
         # Load parameters, network and data scalers.
         parameters, network, data_handler, tester = mala.Tester.load_run(
-            "workflow_test", path=os.path.join(data_repo_path, "workflow_test")
+            "Be_model", path=data_path
         )
 
         parameters.targets.target_type = "LDOS"
@@ -431,7 +430,7 @@ def test_predictions(self):
         ####################
 
         parameters, network, data_handler, tester = mala.Tester.load_run(
-            "workflow_test", path=os.path.join(data_repo_path, "workflow_test")
+            "Be_model", path=data_path
         )
         parameters.targets.target_type = "LDOS"
         parameters.targets.ldos_gridsize = 11
@@ -518,7 +517,7 @@ def test_total_energy_predictions(self):
         ####################
 
         parameters, network, data_handler, predictor = mala.Predictor.load_run(
-            "workflow_test", path=os.path.join(data_repo_path, "workflow_test")
+            "Be_model", path=data_path
         )
         parameters.targets.target_type = "LDOS"
         parameters.targets.ldos_gridsize = 11

From 6c2d43858ec0e225601fc05a6feb2a99e536e339 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Franz=20P=C3=B6schel?= <franz.poeschel@gmail.com>
Date: Thu, 6 Jun 2024 18:09:09 +0200
Subject: [PATCH 06/19] Fix CI installation of openPMD-api

---
 Dockerfile                            | 1 -
 install/mala_cpu_base_environment.yml | 1 +
 install/mala_cpu_environment.yml      | 1 +
 3 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 724ed44e5..3167d4ed7 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -21,7 +21,6 @@ RUN conda env create -f mala_${DEVICE}_environment.yml && rm -rf /opt/conda/pkgs
 RUN /opt/conda/envs/mala-${DEVICE}/bin/pip install --no-input --no-cache-dir \
     pytest \
     oapackage==2.6.8 \
-    openpmd-api==0.15.1 \
     pqkmeans
 
 RUN echo "source activate mala-${DEVICE}" > ~/.bashrc
diff --git a/install/mala_cpu_base_environment.yml b/install/mala_cpu_base_environment.yml
index 626008b16..459fa3231 100644
--- a/install/mala_cpu_base_environment.yml
+++ b/install/mala_cpu_base_environment.yml
@@ -14,3 +14,4 @@ dependencies:
   - mpmath
   - tensorboard
   - scikit-spatial
+  - openpmd_api>=0.15.1
diff --git a/install/mala_cpu_environment.yml b/install/mala_cpu_environment.yml
index 97fb82bd8..3c386b932 100644
--- a/install/mala_cpu_environment.yml
+++ b/install/mala_cpu_environment.yml
@@ -95,6 +95,7 @@ dependencies:
   - numpy=1.24.0
   - oauthlib=3.2.2
   - openjpeg=2.5.0
+  - openpmd-api=0.15.2
   - openssl=3.0.7
   - optuna=3.0.5
   - packaging=22.0

From 35a4a8f4bc77f7335cc2d7b47a17d092762d96a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Franz=20P=C3=B6schel?= <franz.poeschel@gmail.com>
Date: Fri, 7 Jun 2024 14:04:40 +0200
Subject: [PATCH 07/19] Adapt further dependencies

---
 install/mala_cpu_environment.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/install/mala_cpu_environment.yml b/install/mala_cpu_environment.yml
index 3c386b932..43dad4444 100644
--- a/install/mala_cpu_environment.yml
+++ b/install/mala_cpu_environment.yml
@@ -63,8 +63,8 @@ dependencies:
   - libdeflate=1.14
   - libffi=3.4.2
   - libgcc-ng=12.2.0
-  - libgfortran-ng=12.2.0
-  - libgfortran5=12.2.0
+  - libgfortran-ng=12.3.0
+  - libgfortran5=12.3.0
   - libgrpc=1.51.1
   - libhwloc=2.8.0
   - libiconv=1.17
@@ -95,8 +95,8 @@ dependencies:
   - numpy=1.24.0
   - oauthlib=3.2.2
   - openjpeg=2.5.0
-  - openpmd-api=0.15.2
-  - openssl=3.0.7
+  - openpmd-api=0.15.2=nompi_py38h766a7de_102
+  - openssl=3.3.1
   - optuna=3.0.5
   - packaging=22.0
   - pandas=1.5.2
@@ -114,7 +114,7 @@ dependencies:
   - pyparsing=3.0.9
   - pyperclip=1.8.2
   - pysocks=1.7.1
-  - python=3.8.15
+  - python=3.8.16
   - python-dateutil=2.8.2
   - python_abi=3.8
   - pytorch=1.13.0
@@ -154,4 +154,4 @@ dependencies:
   - yarl=1.8.1
   - zipp=3.11.0
   - zlib=1.2.13
-  - zstd=1.5.2
+  - zstd=1.5.5

From b60fdd133f91b2513b7076be47b2784ef45bd1d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Franz=20P=C3=B6schel?= <franz.poeschel@gmail.com>
Date: Fri, 7 Jun 2024 16:07:41 +0200
Subject: [PATCH 08/19] Fix typo

---
 install/mala_cpu_base_environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/install/mala_cpu_base_environment.yml b/install/mala_cpu_base_environment.yml
index 459fa3231..a1b125831 100644
--- a/install/mala_cpu_base_environment.yml
+++ b/install/mala_cpu_base_environment.yml
@@ -14,4 +14,4 @@ dependencies:
   - mpmath
   - tensorboard
   - scikit-spatial
-  - openpmd_api>=0.15.1
+  - openpmd-api>=0.15.1

From 9923992c5aa7a33635d640eb4cf6c0819007ab99 Mon Sep 17 00:00:00 2001
From: Daniel Kotik <d.kotik@hzdr.de>
Date: Mon, 10 Jun 2024 10:57:24 +0200
Subject: [PATCH 09/19] Use RODARE api instead of hard coded URL:

Its better to use the DOI which always points to the latest version of
the test data repo. This avoids updating the CI at several places each
time there is a new version of the test data repo.

Co-authored-by: David Pape <d.pape@hzdr.de>
---
 .github/workflows/cpu-tests.yml | 36 +++++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/cpu-tests.yml b/.github/workflows/cpu-tests.yml
index a7cbdc339..f28293e14 100644
--- a/.github/workflows/cpu-tests.yml
+++ b/.github/workflows/cpu-tests.yml
@@ -176,20 +176,40 @@ jobs:
           # `requirements.txt` and/or extra dependencies are missing in the Docker Conda environment
           diff env_1.yml env_2.yml
 
-      - name: Download test data repository
-        shell: 'bash -c "docker exec -i mala-cpu bash < {0}"'
+      - name: Download test data repository from RODARE
+        shell: 'bash -c "docker exec -i mala-cpu python < {0}"'
         run: |
-          # Download test data repository from RODARE. If the version changes
-          # this URL has to be adapted (the number after /record/ and the
-          # version have to be incremented)
-          wget "https://rodare.hzdr.de/record/3004/files/mala-project/test-data-1.8.1.zip"
+          import requests
+
+          # This DOI represents all versions, and will always resolve to the latest one
+          DOI = "https://doi.org/10.14278/rodare.2900"
+
+          # Resolve DOI and get record ID and the associated API URL
+          response = requests.get(DOI)
+          *_, record_id = response.url.split("/")
+          api_url = f"https://rodare.hzdr.de/api/records/{record_id}"
+
+          # Download record from API and get the first file
+          response = requests.get(api_url)
+          record = response.json()
+          size = record["files"][0]["size"]
+          download_link = record["files"][0]["links"]["self"]
+
+          print(size, "bytes", "--", download_link)
+
+          # TODO: implement some sort of auto retry for failed HTTP requests
+          response = requests.get(download_link)
+
+          # Saving Downloaded Content to a File
+          with open("test-data.zip", mode="wb") as file:
+            file.write(response.content)
 
           # Once downloaded, we have to unzip the file. The name of the root
           # folder in the zip file has to be updated for data repository
           # updates as well - the string at the end is the hash of the data
           # repository commit.
-          unzip -q test-data-1.8.1.zip
-          mv mala-project-test-data-741eda6 mala_data
+          #unzip -q test-data-1.8.1.zip
+          #mv mala-project-test-data-741eda6 mala_data
 
       - name: Test mala
         shell: 'bash -c "docker exec -i mala-cpu bash < {0}"'

From 849ff7d977e3670a87ebe3f8e47a24f85366e3b5 Mon Sep 17 00:00:00 2001
From: Daniel Kotik <d.kotik@hzdr.de>
Date: Mon, 10 Jun 2024 13:33:12 +0200
Subject: [PATCH 10/19] Avoid necessity to rename top level directory:

The top level directory in the zip file is suffixed with a commit hash
that relates to the downloaded test data repository. Subsequent steps in
the pipeline expect this directory have the name `test_data`. This snippet
avoids manual renaming of the extracted folder with each newer version
of the test data repository.
---
 .github/workflows/cpu-tests.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/cpu-tests.yml b/.github/workflows/cpu-tests.yml
index f28293e14..1fdf7c451 100644
--- a/.github/workflows/cpu-tests.yml
+++ b/.github/workflows/cpu-tests.yml
@@ -179,7 +179,7 @@ jobs:
       - name: Download test data repository from RODARE
         shell: 'bash -c "docker exec -i mala-cpu python < {0}"'
         run: |
-          import requests
+          import requests, shutil, zipfile
 
           # This DOI represents all versions, and will always resolve to the latest one
           DOI = "https://doi.org/10.14278/rodare.2900"
@@ -200,16 +200,16 @@ jobs:
           # TODO: implement some sort of auto retry for failed HTTP requests
           response = requests.get(download_link)
 
-          # Saving Downloaded Content to a File
+          # Saving downloaded content to a file
           with open("test-data.zip", mode="wb") as file:
             file.write(response.content)
 
-          # Once downloaded, we have to unzip the file. The name of the root
-          # folder in the zip file has to be updated for data repository
-          # updates as well - the string at the end is the hash of the data
-          # repository commit.
-          #unzip -q test-data-1.8.1.zip
-          #mv mala-project-test-data-741eda6 mala_data
+          # Get top level directory name
+          dir_name = zipfile.ZipFile("test-data.zip").namelist()[0]
+          shutil.unpack_archive("test-data.zip", ".")
+
+          print(f"Rename {dir_name} to mala_data")
+          shutil.move(dir_name, "mala_data")
 
       - name: Test mala
         shell: 'bash -c "docker exec -i mala-cpu bash < {0}"'

From e4a410d66fc8a690d51ffac391cee80f7b24da68 Mon Sep 17 00:00:00 2001
From: Daniel Kotik <d.kotik@hzdr.de>
Date: Mon, 10 Jun 2024 18:23:39 +0200
Subject: [PATCH 11/19] This fixes Node.js 16 deprecation warning:

Node.js 16 actions are deprecated. Please update the following actions
to use Node.js 20: actions/checkout@v3, actions/cache@v3.
---
 .github/workflows/cpu-tests.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/cpu-tests.yml b/.github/workflows/cpu-tests.yml
index a7cbdc339..ed1253613 100644
--- a/.github/workflows/cpu-tests.yml
+++ b/.github/workflows/cpu-tests.yml
@@ -34,7 +34,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set environment variables
         run: |
@@ -46,7 +46,7 @@ jobs:
           echo "IMAGE_REPO=$IMAGE_REPO"
 
       - name: Restore cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         id: cache-docker
         with:
           path: ${{ env.DOCKER_CACHE_PATH }}
@@ -123,7 +123,7 @@ jobs:
     steps:
       - name: "Prepare environment: Restore cache"
         if: env.DOCKER_TAG != 'latest'
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         id: cache-docker
         with:
           path: ${{ env.DOCKER_CACHE_PATH }}
@@ -154,7 +154,7 @@ jobs:
           [[ $(docker inspect --format '{{json .State.Running}}' mala-cpu) == 'true' ]]
 
       - name: Check out repository (mala)
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install mala package
         # Exec all commands inside the mala-cpu container
@@ -210,11 +210,11 @@ jobs:
       || startsWith(github.ref, 'refs/tags/')
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: "Prepare environment: Restore cache"
         if: env.DOCKER_TAG != 'latest'
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         id: cache-docker
         with:
           path: ${{ env.DOCKER_CACHE_PATH }}

From 68fead8d267930fe636373522f24d628cbc2e6db Mon Sep 17 00:00:00 2001
From: Daniel Kotik <d.kotik@hzdr.de>
Date: Tue, 11 Jun 2024 10:57:18 +0200
Subject: [PATCH 12/19] Remove caches after pushes to develop/master (+tags)

---
 .github/workflows/cleanup-caches.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/cleanup-caches.yml b/.github/workflows/cleanup-caches.yml
index 2b65aa371..6cda2d438 100644
--- a/.github/workflows/cleanup-caches.yml
+++ b/.github/workflows/cleanup-caches.yml
@@ -3,6 +3,13 @@ on:
   pull_request_target:
     types:
       - closed
+  push:
+    # Trigger on pushes to master or develop and for git tag pushes
+    branches:
+      - master
+      - develop
+    tags:
+      - v*
 
 jobs:
   cleanup:

From 940fd2184f5d56e4db683102c868b6717a5d1d80 Mon Sep 17 00:00:00 2001
From: Daniel Kotik <d.kotik@hzdr.de>
Date: Tue, 11 Jun 2024 11:22:13 +0200
Subject: [PATCH 13/19] Update cleanup-caches.yml

---
 .github/workflows/cleanup-caches.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cleanup-caches.yml b/.github/workflows/cleanup-caches.yml
index 6cda2d438..86dd7569d 100644
--- a/.github/workflows/cleanup-caches.yml
+++ b/.github/workflows/cleanup-caches.yml
@@ -15,7 +15,7 @@ jobs:
   cleanup:
     runs-on: ubuntu-latest
     steps:
-      - name: Cleanup
+      - name: Cleanup caches
         run: |
           gh extension install actions/gh-actions-cache
 

From 979bab5a4db76b8c1585011a1bc6d30e2f850198 Mon Sep 17 00:00:00 2001
From: Daniel Kotik <d.kotik@hzdr.de>
Date: Wed, 12 Jun 2024 13:26:32 +0200
Subject: [PATCH 14/19] Enhance diff output of Conda environments:

The diffs of the two Conda environments are now displayed next to each other
to make it easier to spot a discrepancy between the two.
---
 .github/workflows/cpu-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cpu-tests.yml b/.github/workflows/cpu-tests.yml
index 14a7dd4b5..ceb7f59a7 100644
--- a/.github/workflows/cpu-tests.yml
+++ b/.github/workflows/cpu-tests.yml
@@ -174,7 +174,7 @@ jobs:
 
           # if comparison fails, `install/mala_cpu_[base]_environment.yml` needs to be aligned with
           # `requirements.txt` and/or extra dependencies are missing in the Docker Conda environment
-          diff env_1.yml env_2.yml
+          diff --side-by-side --color=always env_1.yml env_2.yml
 
       - name: Download test data repository from RODARE
         shell: 'bash -c "docker exec -i mala-cpu python < {0}"'

From 62248242601d91738bfbf0a5f463018423457d92 Mon Sep 17 00:00:00 2001
From: Daniel Kotik <d.kotik@hzdr.de>
Date: Wed, 12 Jun 2024 13:56:55 +0200
Subject: [PATCH 15/19] Fix a typo and rename files for a better understanding

---
 .github/workflows/cpu-tests.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/cpu-tests.yml b/.github/workflows/cpu-tests.yml
index ceb7f59a7..84241a4fa 100644
--- a/.github/workflows/cpu-tests.yml
+++ b/.github/workflows/cpu-tests.yml
@@ -160,8 +160,8 @@ jobs:
         # Exec all commands inside the mala-cpu container
         shell: 'bash -c "docker exec -i mala-cpu bash < {0}"'
         run: |
-          # epxort Docker image Conda environment for a later comparison
-          conda env export -n mala-cpu > env_1.yml
+          # export Docker image Conda environment for a later comparison
+          conda env export -n mala-cpu > env_before.yml
 
           # install mala package
           pip --no-cache-dir install -e .[opt,test] --no-build-isolation
@@ -170,11 +170,11 @@ jobs:
         shell: 'bash -c "docker exec -i mala-cpu bash < {0}"'
         run: |
           # export Conda environment _with_ mala package installed in it (and extra dependencies)
-          conda env export -n mala-cpu > env_2.yml
+          conda env export -n mala-cpu > env_after.yml
 
           # if comparison fails, `install/mala_cpu_[base]_environment.yml` needs to be aligned with
           # `requirements.txt` and/or extra dependencies are missing in the Docker Conda environment
-          diff --side-by-side --color=always env_1.yml env_2.yml
+          diff --side-by-side --color=always env_before.yml env_after.yml
 
       - name: Download test data repository from RODARE
         shell: 'bash -c "docker exec -i mala-cpu python < {0}"'

From 29348ee45ba0f3fafaa30349ca148e01a62e0b91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Franz=20P=C3=B6schel?= <franz.poeschel@gmail.com>
Date: Fri, 7 Jun 2024 16:47:22 +0200
Subject: [PATCH 16/19] be less specific about openpmd-api version

---
 install/mala_cpu_base_environment.yml | 2 +-
 install/mala_cpu_environment.yml      | 2 +-
 requirements.txt                      | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/install/mala_cpu_base_environment.yml b/install/mala_cpu_base_environment.yml
index a1b125831..1f5f61308 100644
--- a/install/mala_cpu_base_environment.yml
+++ b/install/mala_cpu_base_environment.yml
@@ -14,4 +14,4 @@ dependencies:
   - mpmath
   - tensorboard
   - scikit-spatial
-  - openpmd-api>=0.15.1
+  - openpmd-api
diff --git a/install/mala_cpu_environment.yml b/install/mala_cpu_environment.yml
index 43dad4444..8fcd3ba02 100644
--- a/install/mala_cpu_environment.yml
+++ b/install/mala_cpu_environment.yml
@@ -95,7 +95,7 @@ dependencies:
   - numpy=1.24.0
   - oauthlib=3.2.2
   - openjpeg=2.5.0
-  - openpmd-api=0.15.2=nompi_py38h766a7de_102
+  - openpmd-api=0.15.2
   - openssl=3.3.1
   - optuna=3.0.5
   - packaging=22.0
diff --git a/requirements.txt b/requirements.txt
index b8c1d7b64..b784a6c69 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,5 +8,5 @@ optuna
 scipy
 pandas
 tensorboard
-openpmd-api>=0.15
+openpmd-api
 scikit-spatial

From 2bf4933cd32d32b26906e5443d09dbea0ca01b90 Mon Sep 17 00:00:00 2001
From: "Kotik, Daniel" <d.kotik@hzdr.de>
Date: Wed, 12 Jun 2024 15:44:20 +0200
Subject: [PATCH 17/19] Enforce installation of openPMD via pip

---
 install/mala_cpu_base_environment.yml | 3 ++-
 install/mala_cpu_environment.yml      | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/install/mala_cpu_base_environment.yml b/install/mala_cpu_base_environment.yml
index 1f5f61308..f8309f5b9 100644
--- a/install/mala_cpu_base_environment.yml
+++ b/install/mala_cpu_base_environment.yml
@@ -14,4 +14,5 @@ dependencies:
   - mpmath
   - tensorboard
   - scikit-spatial
-  - openpmd-api
+  - pip:
+      - openpmd-api
diff --git a/install/mala_cpu_environment.yml b/install/mala_cpu_environment.yml
index 8fcd3ba02..eaf4b88bc 100644
--- a/install/mala_cpu_environment.yml
+++ b/install/mala_cpu_environment.yml
@@ -95,7 +95,6 @@ dependencies:
   - numpy=1.24.0
   - oauthlib=3.2.2
   - openjpeg=2.5.0
-  - openpmd-api=0.15.2
   - openssl=3.3.1
   - optuna=3.0.5
   - packaging=22.0
@@ -155,3 +154,5 @@ dependencies:
   - zipp=3.11.0
   - zlib=1.2.13
   - zstd=1.5.5
+  - pip:
+      - openpmd-api==0.15.2

From 031fab34c1d4bfe4021f99721d38b4bd247c994c Mon Sep 17 00:00:00 2001
From: Daniel Kotik <d.kotik@hzdr.de>
Date: Thu, 13 Jun 2024 11:04:48 +0200
Subject: [PATCH 18/19] Use legacy builder to build Docker image:

This is a temporary fix to make the caching mechanism in the CI work
again. Its currently broken due to a switch to BuildKit as the default
builder for Docker Engine as of version 23.0 (2023-02-01).
---
 .github/workflows/cpu-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cpu-tests.yml b/.github/workflows/cpu-tests.yml
index 84241a4fa..48f0a456c 100644
--- a/.github/workflows/cpu-tests.yml
+++ b/.github/workflows/cpu-tests.yml
@@ -74,7 +74,7 @@ jobs:
             CACHE=$IMAGE_REPO/$IMAGE_NAME:latest
           fi
 
-          docker build . --file Dockerfile --tag $IMAGE_NAME:local --cache-from=$CACHE --build-arg DEVICE=cpu
+          DOCKER_BUILDKIT=0 docker build . --file Dockerfile --tag $IMAGE_NAME:local --cache-from=$CACHE --build-arg DEVICE=cpu
 
           # Show images
           docker images --filter=reference=$IMAGE_NAME --filter=reference=$IMAGE_REPO/$IMAGE_NAME

From 4ea9adc2671047f9111d8fd11729bfb55858511b Mon Sep 17 00:00:00 2001
From: Daniel Kotik <d.kotik@hzdr.de>
Date: Thu, 13 Jun 2024 13:42:14 +0200
Subject: [PATCH 19/19] Update mirror-to-casus.yml:

- Update workflow name to match style of the other workflows
- Fix: Node.js 16 actions are deprecated. Please update the following actions
       to use Node.js 20: actions/checkout@v3.
- Fix indentation isues
---
 .github/workflows/mirror-to-casus.yml | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/mirror-to-casus.yml b/.github/workflows/mirror-to-casus.yml
index a231093bc..a862d6cac 100644
--- a/.github/workflows/mirror-to-casus.yml
+++ b/.github/workflows/mirror-to-casus.yml
@@ -1,4 +1,4 @@
-name: mirror
+name: Mirror to CASUS
 
 on: [push, delete]
 
@@ -6,13 +6,14 @@ jobs:
   mirror-to-CASUS:
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-    - name: mirror-repository
-      uses: spyoungtech/mirror-action@v0.6.0
-      with:
-        REMOTE: 'ssh://git@github.com/casus/mala.git'
-        GIT_SSH_PRIVATE_KEY: ${{ secrets.GIT_SSH_KEY }}
-        GIT_SSH_NO_VERIFY_HOST: "true"
-        DEBUG: "true"
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: mirror-repository
+        uses: spyoungtech/mirror-action@v0.6.0
+        with:
+          REMOTE: 'ssh://git@github.com/casus/mala.git'
+          GIT_SSH_PRIVATE_KEY: ${{ secrets.GIT_SSH_KEY }}
+          GIT_SSH_NO_VERIFY_HOST: "true"
+          DEBUG: "true"