Refactor Docker image and package builds #1399
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Development | |
| on: | |
| push: | |
| branches: | |
| - "**" # every branch | |
| - "!gh-pages" # exclude gh-pages branch | |
| - "!stage*" # exclude branches beginning with stage | |
| tags: | |
| - '\d+\.\d+\.\d+' # only semver tags | |
| pull_request: | |
| branches: | |
| - "**" # every branch | |
| - "!gh-pages" # exclude gh-pages branch | |
| - "!stage*" # exclude branches beginning with stage | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - py_ver: "3.9" | |
| distro: debian | |
| image: djbase | |
| env: | |
| PY_VER: ${{matrix.py_ver}} | |
| DISTRO: ${{matrix.distro}} | |
| IMAGE: ${{matrix.image}} | |
| DOCKER_CLIENT_TIMEOUT: "120" | |
| COMPOSE_HTTP_TIMEOUT: "120" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{matrix.py_ver}} | |
| - name: Validate version and release notes | |
| run: | | |
| DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py) | |
| RELEASE_BODY=$(python3 -c \ | |
| 'print(open("./CHANGELOG.md").read().split("\n\n")[1].split("\n", 1)[1])' \ | |
| ) | |
| echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV | |
| echo "RELEASE_BODY<<EOF" >> $GITHUB_ENV | |
| echo "$RELEASE_BODY" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - name: Build pip artifacts | |
| run: | | |
| python3 -m pip install build | |
| python3 -m build . | |
| echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV | |
| - if: matrix.py_ver == '3.9' && matrix.distro == 'debian' | |
| name: Add pip artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: pip-datajoint-${{env.DJ_VERSION}} | |
| path: dist | |
| retention-days: 1 | |
| test: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| py_ver: ["3.9"] | |
| mysql_ver: ["8.0", "5.7"] | |
| include: | |
| - py_ver: "3.12" | |
| mysql_ver: "8.0" | |
| - py_ver: "3.11" | |
| mysql_ver: "8.0" | |
| - py_ver: "3.10" | |
| mysql_ver: "8.0" | |
| - py_ver: "3.8" | |
| mysql_ver: "5.7" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{matrix.py_ver}} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{matrix.py_ver}} | |
| - name: Run primary tests | |
| env: | |
| PY_VER: ${{matrix.py_ver}} | |
| DJ_PASS: password | |
| MYSQL_VER: ${{matrix.mysql_ver}} | |
| DISTRO: alpine | |
| MINIO_VER: RELEASE.2021-09-03T03-56-13Z | |
| DOCKER_CLIENT_TIMEOUT: "120" | |
| COMPOSE_HTTP_TIMEOUT: "120" | |
| run: | | |
| export HOST_UID=$(id -u) | |
| docker compose --profile test up --quiet-pull --build --exit-code-from djtest djtest | |
| lint: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| py_ver: ["3.11"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{matrix.py_ver}} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{matrix.py_ver}} | |
| - name: Install dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install ".[test]" | |
| - name: Run syntax tests | |
| run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics | |
| - name: Run style tests | |
| run: | | |
| flake8 --ignore=E203,E722,W503 datajoint \ | |
| --count --max-complexity=62 --max-line-length=127 --statistics \ | |
| --per-file-ignores='datajoint/diagram.py:C901' | |
| black --required-version '24.2.0' --check -v datajoint tests | |
| codespell: | |
| name: Check for spelling errors | |
| permissions: | |
| contents: read | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Codespell | |
| uses: codespell-project/actions-codespell@v2 | |
| publish-docs: | |
| if: | | |
| github.event_name == 'push' && | |
| startsWith(github.ref, 'refs/tags') | |
| needs: test | |
| runs-on: ubuntu-latest | |
| env: | |
| DOCKER_CLIENT_TIMEOUT: "120" | |
| COMPOSE_HTTP_TIMEOUT: "120" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Deploy docs | |
| run: | | |
| export MODE=BUILD | |
| export PACKAGE=datajoint | |
| export UPSTREAM_REPO=https://github.com/${GITHUB_REPOSITORY}.git | |
| export HOST_UID=$(id -u) | |
| docker compose -f docs/docker-compose.yaml up --quiet-pull --exit-code-from docs --build | |
| git push origin gh-pages | |
| publish-release: | |
| if: | | |
| github.event_name == 'push' && | |
| startsWith(github.ref, 'refs/tags') | |
| needs: test | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - py_ver: "3.9" | |
| distro: debian | |
| image: djbase | |
| env: | |
| PY_VER: ${{matrix.py_ver}} | |
| DISTRO: ${{matrix.distro}} | |
| IMAGE: ${{matrix.image}} | |
| TWINE_USERNAME: ${{secrets.twine_username}} | |
| TWINE_PASSWORD: ${{secrets.twine_password}} | |
| DOCKER_CLIENT_TIMEOUT: "120" | |
| COMPOSE_HTTP_TIMEOUT: "120" | |
| outputs: | |
| release_upload_url: ${{steps.create_gh_release.outputs.upload_url}} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{matrix.py_ver}} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{matrix.py_ver}} | |
| - name: Determine package version | |
| run: | | |
| DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py) | |
| RELEASE_BODY=$(python -c \ | |
| 'print(open("./CHANGELOG.md").read().split("\n\n")[1].split("\n", 1)[1])' \ | |
| ) | |
| echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV | |
| echo "RELEASE_BODY<<EOF" >> $GITHUB_ENV | |
| echo "$RELEASE_BODY" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - name: Create GH release | |
| id: create_gh_release | |
| uses: actions/create-release@v1 | |
| env: | |
| GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} | |
| with: | |
| tag_name: ${{env.DJ_VERSION}} | |
| release_name: Release ${{env.DJ_VERSION}} | |
| body: ${{env.RELEASE_BODY}} | |
| prerelease: false | |
| draft: false | |
| - name: Fetch pip artifacts | |
| uses: actions/[email protected] | |
| with: | |
| name: pip-datajoint-${{env.DJ_VERSION}} | |
| path: dist | |
| - name: Determine pip artifact paths | |
| run: | | |
| echo "DJ_WHEEL_PATH=$(ls dist/datajoint-*.whl)" >> $GITHUB_ENV | |
| echo "DJ_SDIST_PATH=$(ls dist/datajoint-*.tar.gz)" >> $GITHUB_ENV | |
| - name: Upload pip wheel asset to release | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} | |
| with: | |
| upload_url: ${{steps.create_gh_release.outputs.upload_url}} | |
| asset_path: ${{env.DJ_WHEEL_PATH}} | |
| asset_name: pip-datajoint-${{env.DJ_VERSION}}.whl | |
| asset_content_type: application/zip | |
| - name: Upload pip sdist asset to release | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} | |
| with: | |
| upload_url: ${{steps.create_gh_release.outputs.upload_url}} | |
| asset_path: ${{env.DJ_SDIST_PATH}} | |
| asset_name: pip-datajoint-${{env.DJ_VERSION}}.tar.gz | |
| asset_content_type: application/gzip | |
| - name: Publish pip release | |
| run: | | |
| export HOST_UID=$(id -u) | |
| docker compose run --build --quiet-pull \ | |
| -e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD} app \ | |
| sh -c "pip install twine && python -m twine upload dist/*" | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v1 | |
| with: | |
| username: ${{secrets.docker_username}} | |
| password: ${{secrets.docker_password}} | |
| - name: Publish image | |
| run: | | |
| IMAGE=$(docker images --filter "reference=datajoint/datajoint*" --format "{{.Repository}}") | |
| TAG=$(docker images --filter "reference=datajoint/datajoint*" --format "{{.Tag}}") | |
| docker push "${IMAGE}:${TAG}" | |
| docker tag "${IMAGE}:${TAG}" "${IMAGE}:${TAG}-${GITHUB_SHA:0:7}" | |
| docker push "${IMAGE}:${TAG}-${GITHUB_SHA:0:7}" | |
| [ "$PY_VER" == "3.9" ] && [ "$DISTRO" == "debian" ] \ | |
| && docker tag "${IMAGE}:${TAG}" "${IMAGE}:latest" \ | |
| && docker push "${IMAGE}:latest" \ | |
| || echo "skipping 'latest' tag..." |