Update dependencies #779
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build dependencies | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| branches: | |
| - master | |
| - 7.*.* | |
| paths: | |
| - .github/workflows/build-deps.yml | |
| - .builders/** | |
| - agent_requirements.in | |
| push: | |
| branches: | |
| - master | |
| - 7.*.* | |
| paths: | |
| - .builders/** | |
| - agent_requirements.in | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' && true || false }} | |
| defaults: | |
| run: | |
| shell: bash | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| PYTHON_VERSION: "3.11" | |
| DIRECT_DEPENDENCY_FILE: agent_requirements.in | |
| # https://reproducible-builds.org/specs/source-date-epoch/ | |
| SOURCE_DATE_EPOCH: "1580601600" | |
| jobs: | |
| test: | |
| name: Run tests | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ env.PYTHON_VERSION }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: | | |
| pip install -r .builders/deps/host_dependencies.txt | |
| pip install -r .builders/test_dependencies.txt | |
| - name: Run tests | |
| run: | | |
| cd .builders | |
| pytest -vvv | |
| build: | |
| name: Target ${{ matrix.job.image }} on ${{ matrix.job.os }} | |
| runs-on: ${{ matrix.job.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| job: | |
| - os: arm-4core-linux | |
| image: linux-aarch64 | |
| - os: ubuntu-22.04 | |
| image: linux-x86_64 | |
| - os: windows-2022 | |
| image: windows-x86_64 | |
| permissions: | |
| packages: write | |
| env: | |
| OUT_DIR: output/${{ matrix.job.image }} | |
| BUILDER_IMAGE: ghcr.io/datadog/agent-int-builder:${{ matrix.job.image }} | |
| DOCKER: docker | |
| steps: | |
| - name: Checkout code | |
| if: github.event_name != 'pull_request' | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| # On pull requests, ensure that changed files are determined before checking out the code so | |
| # that we use the GitHub API, otherwise we would have to fetch the entire history (depth: 0) | |
| - name: Get changed files | |
| id: changed-files | |
| uses: tj-actions/changed-files@v42 | |
| with: | |
| files_yaml: |- | |
| builders: | |
| - .builders/** | |
| dependencies: | |
| - ${{ env.DIRECT_DEPENDENCY_FILE }} | |
| - name: Checkout code | |
| if: github.event_name == 'pull_request' | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ env.PYTHON_VERSION }} | |
| if: matrix.job.image != 'linux-aarch64' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Set up Python (with miniconda) and other aarch64 requirements | |
| if: matrix.job.image == 'linux-aarch64' | |
| run: | | |
| mkdir -p ~/miniconda3 | |
| wget https://repo.anaconda.com/miniconda/Miniconda3-py311_24.1.2-0-Linux-aarch64.sh -O ~/miniconda3/miniconda.sh | |
| bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3 | |
| rm -rf ~/miniconda3/miniconda.sh | |
| ~/miniconda3/bin/conda init bash | |
| # jq | |
| wget https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-arm64 -O ~/miniconda3/bin/jq | |
| chmod +x ~/miniconda3/bin/jq | |
| echo "PATH=~/miniconda3/bin/:${PATH}" >> "$GITHUB_ENV" | |
| echo DOCKER="sudo docker" >> "$GITHUB_ENV" | |
| - name: Install management dependencies | |
| run: | | |
| pip install -r .builders/deps/host_dependencies.txt | |
| - name: Install docker and log in (arm64) | |
| if: matrix.job.image == 'linux-aarch64' | |
| run: | | |
| curl -fsSL https://get.docker.com -o get-docker.sh | |
| sudo sh get-docker.sh | |
| # Logging in with sudo is necessary to get authorized with the registry when running docker under sudo | |
| echo ${{ secrets.GITHUB_TOKEN }} | sudo docker login --username ${{ github.actor }} --password-stdin ghcr.io | |
| - name: Log in to GitHub Packages | |
| if: matrix.job.image != 'linux-aarch64' | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build image and wheels (arm64) | |
| if: steps.changed-files.outputs.builders_any_changed == 'true' && matrix.job.image == 'linux-aarch64' | |
| run: |- | |
| sudo /home/runner/miniconda3/bin/python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 | |
| sudo /home/runner/miniconda3/bin/python .builders/build.py ${{ matrix.job.image }} --python 2 ${{ env.OUT_DIR }}/py2 | |
| # Give ownership of the output back to the user | |
| sudo chown ${USER} ${{ env.OUT_DIR }} | |
| - name: Pull image and build wheels (arm64) | |
| if: steps.changed-files.outputs.builders_any_changed != 'true' && matrix.job.image == 'linux-aarch64' | |
| run: |- | |
| digest=$(jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json) | |
| sudo /home/runner/miniconda3/bin/python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --digest $digest | |
| sudo /home/runner/miniconda3/bin/python .builders/build.py ${{ matrix.job.image }} --python 2 ${{ env.OUT_DIR }}/py2 --digest $digest | |
| - name: Build image and wheels | |
| if: steps.changed-files.outputs.builders_any_changed == 'true' && matrix.job.image != 'linux-aarch64' | |
| run: |- | |
| python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 | |
| python .builders/build.py ${{ matrix.job.image }} --python 2 ${{ env.OUT_DIR }}/py2 | |
| - name: Pull image and build wheels | |
| if: steps.changed-files.outputs.builders_any_changed != 'true' && matrix.job.image != 'linux-aarch64' | |
| run: |- | |
| digest=$(jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json) | |
| python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --digest $digest | |
| python .builders/build.py ${{ matrix.job.image }} --python 2 ${{ env.OUT_DIR }}/py2 --digest $digest | |
| - name: Change permissions | |
| if: matrix.job.image == 'linux-aarch64' | |
| run: | | |
| sudo chmod 777 ${{ env.OUT_DIR }} | |
| - name: Publish image | |
| if: github.event_name == 'push' && steps.changed-files.outputs.builders_any_changed == 'true' | |
| run: ${DOCKER} push ${{ env.BUILDER_IMAGE }} | |
| - name: Save new image digest | |
| if: github.event_name == 'push' && steps.changed-files.outputs.builders_any_changed == 'true' | |
| run: >- | |
| ${DOCKER} inspect --format "{{index .RepoDigests 0}}" ${{ env.BUILDER_IMAGE }} | |
| | cut -d '@' -f 2 | |
| > ${{ env.OUT_DIR }}/image_digest | |
| - name: Persist current image digest | |
| if: github.event_name == 'push' && steps.changed-files.outputs.builders_any_changed != 'true' | |
| run: >- | |
| jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json | |
| > ${{ env.OUT_DIR }}/image_digest | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: target-${{ matrix.job.image }} | |
| path: output | |
| build-macos: | |
| name: Target macOS | |
| runs-on: macos-12 | |
| env: | |
| TARGET_NAME: macos-x86_64 | |
| OUT_DIR: output/macos-x86_64 | |
| DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.11/bin/python" | |
| DD_PYTHON2: "/Library/Frameworks/Python.framework/Versions/2.7/bin/python" | |
| steps: | |
| - name: Set up environment | |
| run: |- | |
| # We remove everything that comes pre-installed via Homebrew to avoid depending on or shipping stuff that | |
| # comes in the runner through Homebrew to better control what might get shipped in the wheels via `delocate` | |
| brew remove --force --ignore-dependencies $(brew list --formula) | |
| brew install coreutils | |
| - name: Set up Python | |
| env: | |
| # Despite the name, this is built for the macOS 11 SDK on arm64 and 10.9+ on intel | |
| PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.11.5/python-3.11.5-macos11.pkg" | |
| PYTHON2_DOWNLOAD_URL: "https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg" | |
| run: |- | |
| curl "$PYTHON3_DOWNLOAD_URL" -o python3.pkg | |
| sudo installer -pkg python3.pkg -target / | |
| curl "$PYTHON2_DOWNLOAD_URL" -o python2.pkg | |
| sudo installer -pkg python2.pkg -target / | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install management dependencies | |
| run: | | |
| ${DD_PYTHON3} -m pip install -r .builders/deps/host_dependencies.txt | |
| ${DD_PYTHON3} -m pip install --no-warn-script-location -r ".builders/images/runner_dependencies.txt" | |
| - name: Cache builder root | |
| id: cache-builder-root | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/builder_root | |
| key: macos-deps-builder-root-cache-${{ hashFiles('./.builders/images/macos/*', './.builders/images/*', './.builders/deps/*', './.builders/build.py') }} | |
| - name: Run the build | |
| env: | |
| # This sets the minimum macOS version compatible for all built artifacts | |
| MACOSX_DEPLOYMENT_TARGET: "10.12" | |
| CACHE_HIT: ${{ steps.cache-builder-root.outputs.cache-hit }} | |
| run: |- | |
| # If we hit the cache, we can skip the builder setup | |
| if [[ ${CACHE_HIT} == "true" ]]; then | |
| ${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 3 ${{ env.OUT_DIR }}/py3 --skip-setup | |
| else | |
| mkdir ~/builder_root | |
| ${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 3 ${{ env.OUT_DIR }}/py3 | |
| fi | |
| # At this point we can always skip builder setup since it's equivalent for python versions | |
| ${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 2 ${{ env.OUT_DIR }}/py2 --skip-setup | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: target-macos | |
| path: output | |
| publish: | |
| name: Publish artifacts | |
| if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref == github.event.repository.default_branch || startsWith(github.ref, '7.'))) | |
| needs: | |
| - build | |
| - build-macos | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: "${{ github.head_ref }}" | |
| - name: Set up Python ${{ env.PYTHON_VERSION }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install management dependencies | |
| run: pip install -r .builders/deps/host_dependencies.txt | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: targets | |
| pattern: target-* | |
| merge-multiple: true | |
| - name: Get credentials | |
| id: auth | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| project_id: datadog-agent-int-build | |
| workload_identity_provider: projects/574011472402/locations/global/workloadIdentityPools/github/providers/integrations-core | |
| - name: Upload wheels | |
| run: python .builders/upload.py targets | |
| - name: Lock files | |
| run: python .builders/lock.py targets | |
| - name: Clean up repository | |
| run: |- | |
| rm ${{ steps.auth.outputs.credentials_file_path }} | |
| rm -rf targets | |
| - name: Create token | |
| uses: actions/create-github-app-token@v1 | |
| id: token-generator | |
| with: | |
| app-id: ${{ vars.DD_GITHUB_TOKEN_GENERATOR_APP_ID }} | |
| private-key: ${{ secrets.DD_GITHUB_TOKEN_GENERATOR_PRIVATE_KEY }} | |
| repositories: integrations-core | |
| - name: Create pull request | |
| uses: peter-evans/create-pull-request@v5 | |
| with: | |
| token: ${{ steps.token-generator.outputs.token }} | |
| title: Update dependency resolution | |
| commit-message: Update dependency resolution | |
| branch: bot/update-dependency-resolution | |
| branch-suffix: timestamp | |
| delete-branch: true | |
| labels: bot,qa/skip-qa | |
| body: |- | |
| ### Motivation | |
| Direct dependencies were updated in ${{ github.sha }}. | |
| ### Additional Notes | |
| This PR was automatically generated by the following workflow: | |
| ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} |