Wikitext - [WIP]

.github/workflows/run_singularity_versions.yml

@@ -1,6 +1,16 @@
 name: Test Support for different Singularity Versions
 
-on: [push]
+on:
+ pull_request:
+ types: [ready_for_review]
+
+ pull_request_review:
+ types: [submitted]
+
+ push:
+ branches:
+ - 'main'
+ - 'development'
 
 jobs:
  Tests:
@@ -10,25 +20,25 @@ jobs:
  matrix:
  include:
  - python-version: 3.7
- DISPLAY_NAME: "Singularity Container Examples with S3.5"
+ DISPLAY_NAME: "Singularity Container Examples with S3.7"
  RUN_CONTAINER_EXAMPLES: true
  USE_SINGULARITY: false
- SINGULARITY_VERSION: "3.5"
+ SINGULARITY_VERSION: "3.7"
  - python-version: 3.7
- DISPLAY_NAME: "Singularity Container Examples with S3.6"
+ DISPLAY_NAME: "Singularity Container Examples with S3.8"
  RUN_CONTAINER_EXAMPLES: true
  USE_SINGULARITY: false
- SINGULARITY_VERSION: "3.6"
+ SINGULARITY_VERSION: "3.8"
  - python-version: 3.7
- DISPLAY_NAME: "Singularity Container Examples with S3.7"
+ DISPLAY_NAME: "Singularity Container Examples with S3.9"
  RUN_CONTAINER_EXAMPLES: true
  USE_SINGULARITY: false
- SINGULARITY_VERSION: "3.7"
+ SINGULARITY_VERSION: "3.9"
  - python-version: 3.7
- DISPLAY_NAME: "Singularity Container Examples with S3.8"
+ DISPLAY_NAME: "Singularity Container Examples with S3.10"
  RUN_CONTAINER_EXAMPLES: true
  USE_SINGULARITY: false
- SINGULARITY_VERSION: "3.8"
+ SINGULARITY_VERSION: "3.10"
 
  fail-fast: false
 

.github/workflows/run_tests.yml

@@ -15,33 +15,44 @@ jobs:
  DISPLAY_NAME: "Singularity Tests + CODECOV"
  RUN_TESTS: true
  USE_SINGULARITY: true
+ SINGULARITY_VERSION: "3.8"
  RUN_CODECOV: true
+
  - python-version: 3.7
  DISPLAY_NAME: "Codestyle"
  RUN_CODESTYLE: true
+ USE_SINGULARITY: false
+
  - python-version: 3.7
  DISPLAY_NAME: "Singularity Container Examples"
  RUN_CONTAINER_EXAMPLES: true
  USE_SINGULARITY: true
+ SINGULARITY_VERSION: "3.8"
+
  - python-version: 3.7
  DISPLAY_NAME: "Local Examples"
  RUN_LOCAL_EXAMPLES: true
  USE_SINGULARITY: false
+
  - python-version: 3.8
  DISPLAY_NAME: "Singularity Tests"
  RUN_TESTS: true
  USE_SINGULARITY: true
+ SINGULARITY_VERSION: "3.8"
+
  - python-version: 3.9
  DISPLAY_NAME: "Singularity Tests"
  RUN_TESTS: true
  USE_SINGULARITY: true
+ SINGULARITY_VERSION: "3.8"
  fail-fast: false
 
  name: Tests ${{ matrix.python-version }} ${{ matrix.DISPLAY_NAME }}
 
  env:
  RUN_TESTS: ${{ matrix.RUN_TESTS }}
  USE_SINGULARITY: ${{ matrix.USE_SINGULARITY }}
+ SINGULARITY_VERSION: ${{ matrix.SINGULARITY_VERSION }}
  RUN_CODECOV: ${{ matrix.RUN_CODECOV }}
  RUN_CODESTYLE: ${{ matrix.RUN_CODESTYLE }}
  RUN_CONTAINER_EXAMPLES: ${{ matrix.RUN_CONTAINER_EXAMPLES }}
@@ -58,6 +69,10 @@ jobs:
  uses: actions/setup-go@v2
  with:
  go-version: '1.14.15' # The Go version to download (if necessary) and use.
+ - name: Set up Singularity
+ if: matrix.USE_SINGULARITY == true
+ run: |
+ chmod +x ci_scripts/install_singularity.sh && source ./ci_scripts/install_singularity.sh
  - name: Install dependencies
  run: |
  python -m pip install --upgrade pip

README.md

@@ -54,14 +54,14 @@ cd HPOBench
 pip install .
 ```
 
-**Note:** This does not install *singularity (version 3.6)*. Please follow the steps described here: [user-guide](https://sylabs.io/guides/3.6/user-guide/quick_start.html#quick-installation-steps). 
+**Note:** This does not install *singularity (version 3.8)*. Please follow the steps described here: [user-guide](https://sylabs.io/guides/3.8/user-guide/quick_start.html#quick-installation-steps). 
 If you run into problems, using the most recent singularity version might help: [here](https://singularity.hpcng.org/admin-docs/master/installation.html)
 
 ## Containerized Benchmarks
 
-We provide all benchmarks as containerized versions to (i) isolate their dependencies and (ii) keep them reproducible. Our containerized benchmarks do not rely on external dependencies and thus do not change over time. For this, we rely on [Singularity (version 3.6)](https://sylabs.io/guides/3.6/user-guide/) and for now upload all containers to a [gitlab registry](https://gitlab.tf.uni-freiburg.de/muelleph/hpobench-registry/container_registry)
+We provide all benchmarks as containerized versions to (i) isolate their dependencies and (ii) keep them reproducible. Our containerized benchmarks do not rely on external dependencies and thus do not change over time. For this, we rely on [Singularity (version 3.8)](https://sylabs.io/guides/3.8/user-guide/) and for now upload all containers to a [gitlab registry](https://gitlab.tf.uni-freiburg.de/muelleph/hpobench-registry/container_registry)
 
-The only other requirements are: [ConfigSpace](https://github.com/automl/ConfigSpace), *scipy* and *numpy* 
+The only other requirements are: [ConfigSpace](https://github.com/automl/ConfigSpace), *numpy*, *oslo* and *Pyro4* 
 
 ### Run a Benchmark Locally
 
@@ -139,10 +139,9 @@ If you use a benchmark in your experiments, please specify the version number of
 the used container to ensure reproducibility. When starting an experiment, HPOBench writes automatically these two version numbers to the log. 
 
 ### Troubleshooting and Further Notes
-
  - **Singularity throws an 'Invalid Image format' exception**
- Use a singularity version > 3. For users of the Meta-Cluster in Freiburg, you have to set the following path:
- ```export PATH=/usr/local/kislurm/singularity-3.5/bin/:$PATH```
+ Use a singularity version >= 3.8. If you have multiple singularity installations, you have to add the correct singularity version to your $PATH, e.g.
+ ```export PATH=/usr/local/kislurm/singularity-3.8/bin/:$PATH```
 
  - **A Benchmark fails with `SystemError: Could not start an instance of the benchmark. Retried 5 times` but the container 
 can be started locally with `singularity instance start <pathtocontainer> test`**

ci_scripts/install.sh

@@ -40,35 +40,9 @@ else
  echo "Skip installing packages for local examples"
 fi
 
-if [[ "$USE_SINGULARITY" == "true" ]]; then
- echo "Install Singularity"
-
- sudo apt-get update && sudo apt-get install -y \
- build-essential \
- libssl-dev \
- uuid-dev \
- libgpgme11-dev \
- squashfs-tools \
- libseccomp-dev \
- wget \
- pkg-config \
- git \
- cryptsetup
-
- export VERSION=3.5.3 && # adjust this as necessary \
- wget https://github.com/sylabs/singularity/archive/refs/tags/v${VERSION}.tar.gz && \
- tar -xzf v${VERSION}.tar.gz && \
- cd singularity-${VERSION}
-
- ./mconfig && \
- make -C builddir && \
- sudo make -C builddir install
-
- cd ..
- install_packages="${install_packages}placeholder,"
-else
- echo "Skip installing Singularity"
-fi
+# We add a placeholder / No-OP operator. When running the container examples, we don't install any
+# additional packages. That causes an error, since `pip install .[]` does not work.
+install_packages="${install_packages}NOP,"
 
 # remove the trailing comma
 install_packages="$(echo ${install_packages} | sed 's/,*\r*$//')"

ci_scripts/install_singularity.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env sh
 
-echo "Install Singularity"
+echo "Inside Singularity Installation Script"
 
 sudo apt-get update && sudo apt-get install -y \
  build-essential \
@@ -14,21 +14,33 @@ sudo apt-get update && sudo apt-get install -y \
  git \
  cryptsetup
 
-if [[ "$SINGULARITY_VERSION" == "3.5" ]]; then
- export VERSION=3.5.3
-elif [[ "$SINGULARITY_VERSION" == "3.6" ]]; then
- export VERSION=3.6.4
-elif [[ "$SINGULARITY_VERSION" == "3.7" ]]; then
+if [[ "$SINGULARITY_VERSION" == "3.7" ]]; then
  export VERSION=3.7.3
+ export FILENAME=singularity-"${VERSION}"
+ export EXTRACTED_FILENAME=singularity
+
 elif [[ "$SINGULARITY_VERSION" == "3.8" ]]; then
- export VERSION=3.8.0
+ export VERSION=3.8.4
+ export FILENAME=singularity-ce-"${VERSION}"
+ export EXTRACTED_FILENAME=singularity-ce-"${VERSION}"
+
+elif [[ "$SINGULARITY_VERSION" == "3.9" ]]; then
+ export VERSION=3.9.3
+ export FILENAME=singularity-ce-"${VERSION}"
+ export EXTRACTED_FILENAME=singularity-ce-"${VERSION}"
+
+elif [[ "$SINGULARITY_VERSION" == "3.10" ]]; then
+ export VERSION=3.10.0
+ export FILENAME=singularity-ce-"${VERSION}"
+ export EXTRACTED_FILENAME=singularity-ce-"${VERSION}"
+
 else
  echo "Skip installing Singularity"
 fi
 
-wget https://github.com/sylabs/singularity/archive/refs/tags/v${VERSION}.tar.gz && \
-tar -xzf v${VERSION}.tar.gz && \
-cd singularity-${VERSION} && \
+wget https://github.com/sylabs/singularity/releases/download/v"${VERSION}"/"${FILENAME}".tar.gz && \
+tar -xzf "${FILENAME}".tar.gz && \
+cd "${EXTRACTED_FILENAME}" && \
 ./mconfig && \
 make -C builddir && \
 sudo make -C builddir install

extra_requirements/lm_benchmark.json

@@ -0,0 +1,6 @@
+{
+ "lm": [
+ "torch==1.3.0",
+ "tqdm>=3.0.0"
+ ]
+}

extra_requirements/mo_cnn.json

@@ -0,0 +1,7 @@
+{
+ "mo_cnn": [
+ "tqdm>=3.0.0",
+ "torch==1.9.0",
+ "pandas==1.2.4"
+ ]
+}

extra_requirements/multi_objective.json

@@ -0,0 +1,3 @@
+{
+ "mo_adult": ["pandas==1.2.4","scikit-learn==0.24.2","tqdm>=3.1.4"]
+}

extra_requirements/yahpo_gym.json

@@ -0,0 +1,3 @@
+{
+ "yahpo_gym": ["yahpo_gym@git+https://github.com/pfistfl/yahpo_gym#egg=yahpo_gym&subdirectory=yahpo_gym"]
+}