Fix Mag setup in case of non public UPath. (#1222) #4132
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| tags: | |
| - "**" | |
| pull_request: {} | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} | |
| jobs: | |
| changes: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| webknossos: ${{ steps.filter.outputs.webknossos || github.ref == 'refs/heads/master' }} | |
| cluster_tools: ${{ steps.filter.outputs.cluster_tools || github.ref == 'refs/heads/master' }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| webknossos: | |
| - 'webknossos/**' | |
| cluster_tools: | |
| - 'cluster_tools/**' | |
| cluster_tools: | |
| needs: changes | |
| if: ${{ needs.changes.outputs.cluster_tools == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| max-parallel: 4 | |
| matrix: | |
| executors: [multiprocessing, slurm, kubernetes, dask] | |
| python-version: ["3.12", "3.11", "3.10", "3.9"] | |
| defaults: | |
| run: | |
| working-directory: cluster_tools | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| version: "0.4.22" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Build/pull dockered-slurm image | |
| if: ${{ matrix.executors == 'slurm' }} | |
| run: | | |
| cd ./dockered-slurm | |
| echo docker compose up | |
| docker compose up -d | |
| # Register cluster (with retry) | |
| for i in {1..5}; do | |
| echo register_cluster | |
| ./register_cluster.sh && s=0 && break || s=$? | |
| sleep 10 | |
| done | |
| # Show log output for debugging | |
| docker logs slurmctld | |
| docker logs c1 | |
| docker logs c2 | |
| # Run setup.py on all three nodes | |
| docker exec -w /cluster_tools slurmctld bash -c "uv sync --frozen" & | |
| docker exec -w /cluster_tools c1 bash -c "uv sync --frozen" & | |
| docker exec -w /cluster_tools c2 bash -c "uv sync --frozen" & | |
| wait | |
| - name: Setup Kubernetes-in-Docker | |
| if: ${{ matrix.executors == 'kubernetes' }} | |
| run: | | |
| curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 | |
| chmod +x ./kind | |
| sed -i "s#__PATH__#$(pwd)#g" tests/cluster-config.yaml | |
| ./kind create cluster --config=tests/cluster-config.yaml | |
| ./kind export kubeconfig | |
| docker build -f tests/Dockerfile -t scalableminds/cluster-tools:latest . | |
| ./kind load docker-image scalableminds/cluster-tools:latest | |
| - name: Install dependencies (without docker) | |
| if: ${{ matrix.executors == 'multiprocessing' }} | |
| run: | | |
| uv sync --frozen | |
| - name: Install dependencies (without docker) | |
| if: ${{ matrix.executors == 'kubernetes' || matrix.executors == 'dask' }} | |
| run: | | |
| uv sync --all-extras --frozen | |
| - name: Check typing | |
| if: ${{ matrix.executors == 'multiprocessing' && matrix.python-version == '3.11' }} | |
| run: ./typecheck.sh | |
| - name: Check formatting | |
| if: ${{ matrix.executors == 'multiprocessing' && matrix.python-version == '3.11' }} | |
| run: ./format.sh check | |
| - name: Lint code | |
| if: ${{ matrix.executors == 'multiprocessing' && matrix.python-version == '3.11' }} | |
| run: ./lint.sh | |
| - name: Run multiprocessing tests | |
| if: ${{ matrix.executors == 'multiprocessing' }} | |
| run: | | |
| cd tests | |
| PYTEST_EXECUTORS=multiprocessing,sequential,multiprocessing_with_pickling,sequential_with_pickling \ | |
| uv run --frozen python -m pytest -sv test_all.py test_multiprocessing.py | |
| - name: Run slurm tests | |
| if: ${{ matrix.executors == 'slurm' }} | |
| run: | | |
| cd ./dockered-slurm | |
| docker exec \ | |
| -w /cluster_tools/tests \ | |
| -e PYTEST_EXECUTORS=slurm \ | |
| slurmctld bash -c "uv run --frozen python -m pytest -sv test_all.py test_slurm.py" | |
| docker exec \ | |
| -w /cluster_tools/tests \ | |
| slurmctld bash -c "uv run --frozen python test_deref_main.py" | |
| - name: Run kubernetes tests | |
| if: ${{ matrix.executors == 'kubernetes' }} | |
| run: | | |
| cd tests | |
| PYTEST_EXECUTORS=kubernetes uv run --frozen python -m pytest -sv test_all.py test_kubernetes.py | |
| - name: Run dask tests | |
| if: ${{ matrix.executors == 'dask' }} | |
| run: | | |
| cd tests | |
| PYTEST_EXECUTORS=dask uv run --frozen python -m pytest -sv test_all.py test_dask.py | |
| webknossos_linux: | |
| needs: changes | |
| if: | | |
| ${{ needs.changes.outputs.cluster_tools == 'true' }} || | |
| ${{ needs.changes.outputs.webknossos == 'true' }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| max-parallel: 4 | |
| matrix: | |
| python-version: ["3.12", "3.11", "3.10", "3.9"] | |
| group: [1, 2, 3] | |
| fail-fast: false | |
| defaults: | |
| run: | |
| working-directory: webknossos | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| # Install a specific version of uv. | |
| version: "0.4.22" | |
| - name: Install proxay | |
| run: npm install -g proxay | |
| - name: Set up Python ${{ matrix.python-version }} | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Check formatting | |
| if: ${{ matrix.group == 1 && matrix.python-version == '3.11' }} | |
| run: ./format.sh check | |
| - name: Lint code | |
| if: ${{ matrix.group == 1 && matrix.python-version == '3.11' }} | |
| run: ./lint.sh | |
| - name: Check typing | |
| if: ${{ matrix.group == 1 && matrix.python-version == '3.11' }} | |
| run: ./typecheck.sh | |
| - name: Python tests | |
| timeout-minutes: 30 | |
| env: | |
| WK_TOKEN: ${{ secrets.WK_TOKEN }} | |
| run: ./test.sh -vv -p no:faulthandler --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration | |
| - name: Check if git is dirty | |
| run: | | |
| git diff --no-ext-diff --exit-code | |
| [[ -z $(git status -s) ]] | |
| webknossos_cli_docker: | |
| needs: [cluster_tools, webknossos_linux] | |
| if: | | |
| always() && | |
| !contains(needs.*.result, 'failure') && | |
| !contains(needs.*.result, 'cancelled') && | |
| !github.event.pull_request.head.repo.fork | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| # Install a specific version of uv. | |
| version: "0.4.22" | |
| - name: Write version file | |
| run: | | |
| pushd webknossos | |
| PKG_VERSION="$(uvx dunamai from git)" | |
| echo "__version__ = '$PKG_VERSION'" > ./webknossos/version.py | |
| sed -i 's/version = "0.0.0"/version = "'"${PKG_VERSION}"'"/g' pyproject.toml | |
| popd | |
| - name: Build docker image | |
| run: docker build -t scalableminds/webknossos-cli:$GITHUB_SHA -f webknossos/Dockerfile . | |
| - name: Smoke test docker | |
| run: | | |
| docker run --rm \ | |
| -v$(pwd)/webknossos/testdata:/webknossos/testdata \ | |
| scalableminds/webknossos-cli:$GITHUB_SHA \ | |
| webknossos convert \ | |
| --jobs 2 \ | |
| --voxel-size 1,1,1 \ | |
| testdata/tiff testoutput/tiff | |
| - name: Login to docker | |
| env: | |
| DOCKER_USER: ${{ secrets.DOCKER_USER }} | |
| DOCKER_PASS: ${{ secrets.DOCKER_PASS }} | |
| run: | | |
| echo $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin | |
| - name: Push docker images | |
| run: | | |
| docker push scalableminds/webknossos-cli:$GITHUB_SHA | |
| - name: Push docker images (for tag) | |
| if: github.ref_type == 'tag' | |
| run: | | |
| CI_TAG=$(git describe --tags) | |
| docker tag \ | |
| scalableminds/webknossos-cli:$GITHUB_SHA \ | |
| scalableminds/webknossos-cli:$CI_TAG | |
| docker push scalableminds/webknossos-cli:$CI_TAG | |
| docker tag \ | |
| scalableminds/webknossos-cli:$GITHUB_SHA \ | |
| scalableminds/webknossos-cli:latest | |
| docker push scalableminds/webknossos-cli:latest | |
| - name: Push docker images (for branch) | |
| if: github.ref_type == 'branch' | |
| run: | | |
| CI_BRANCH=${GITHUB_HEAD_REF:-$GITHUB_REF_NAME} | |
| NORMALIZED_CI_BRANCH=${CI_BRANCH//[\/-]/_} | |
| docker tag \ | |
| scalableminds/webknossos-cli:$GITHUB_SHA \ | |
| scalableminds/webknossos-cli:$NORMALIZED_CI_BRANCH | |
| docker push scalableminds/webknossos-cli:$NORMALIZED_CI_BRANCH | |
| docs: | |
| needs: [cluster_tools, webknossos_linux] | |
| runs-on: ubuntu-latest | |
| if: | | |
| always() && | |
| !contains(needs.*.result, 'failure') && | |
| !contains(needs.*.result, 'cancelled') && | |
| !github.event.pull_request.head.repo.fork | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: scalableminds/webknossos | |
| path: docs/wk-repo | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| # Install a specific version of uv. | |
| version: "0.4.22" | |
| - name: Build Docs | |
| run: | | |
| cd docs | |
| ./generate.sh --persist | |
| - name: Push docs (for branch) | |
| if: github.ref_type == 'branch' | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: "eu-west-1" | |
| run: | | |
| CI_BRANCH=${GITHUB_HEAD_REF:-$GITHUB_REF_NAME} | |
| NORMALIZED_CI_BRANCH=${CI_BRANCH//[\/-]/_} | |
| aws s3 sync --acl public-read docs/out s3://static.webknossos.org/docs/${NORMALIZED_CI_BRANCH} | |
| - name: Push docs (for tag) | |
| if: github.ref_type == 'tag' | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: "eu-west-1" | |
| run: | | |
| CI_TAG=$(git describe --tags) | |
| aws s3 sync --acl public-read docs/out s3://static.webknossos.org/docs/${CI_TAG} | |
| - name: Check links (on master) | |
| if: github.ref == 'refs/heads/master' | |
| env: # Or as an environment variable | |
| SLACK_HOOK: ${{ secrets.LINK_CHECKER_SLACK_HOOK }} | |
| run: | | |
| cd docs | |
| uv run --frozen linkchecker --config linkcheckerrc https://docs.webknossos.org > link_status || \ | |
| curl -X POST --data-urlencode "payload={\"text\": \":warning: Broken Links on doc.webknossos.org :warning:\n"'```'"\n$(cat link_status)\n"'```"}' \ | |
| "$SLACK_HOOK" | |
| pypi_and_gh_release: | |
| needs: [cluster_tools, webknossos_linux] | |
| if: | | |
| always() && | |
| !contains(needs.*.result, 'failure') && | |
| !contains(needs.*.result, 'cancelled') && | |
| github.ref_type == 'tag' && | |
| !github.event.pull_request.head.repo.fork | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| # Install a specific version of uv. | |
| version: "0.4.22" | |
| - name: Publish python packages | |
| env: | |
| UV_PUBLISH_TOKEN: ${{ secrets.PYPI_PASSWORD }} | |
| run: _tooling/publish.sh | |
| - name: Prepare github release | |
| run: | | |
| VERSION="$(uvx dunamai from git)" | |
| _tooling/changelog_for_version.sh $VERSION > Changelog.md | |
| - name: Publish github release | |
| id: create_release | |
| uses: actions/create-release@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| tag_name: ${{ github.ref }} | |
| release_name: ${{ github.ref }} | |
| body_path: Changelog.md | |
| draft: false | |
| prerelease: false | |
| complete: | |
| needs: | |
| [ | |
| cluster_tools, | |
| webknossos_linux, | |
| webknossos_cli_docker, | |
| docs, | |
| pypi_and_gh_release, | |
| ] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check failure | |
| if: | | |
| contains(needs.*.result, 'failure') || | |
| contains(needs.*.result, 'cancelled') | |
| run: exit 1 | |
| - name: Success | |
| run: echo Success! |