Skip to content

datadog-agent/7.57.1 package update (#28672) #17300

datadog-agent/7.57.1 package update (#28672)

datadog-agent/7.57.1 package update (#28672) #17300

Workflow file for this run

name: Build Wolfi OS
on:
push:
branches: ['main']
paths-ignore:
- '**.md'
- '**.txt'
workflow_dispatch:
# Only run one build at a time to prevent out of sync signatures.
concurrency: build
jobs:
build:
name: Build packages
if: github.repository == 'wolfi-dev/os'
strategy:
matrix:
arch: [ "x86_64", "aarch64" ]
fail-fast: false
runs-on:
group: wolfi-os-builder-${{ matrix.arch }}
permissions:
contents: read
container:
image: ghcr.io/wolfi-dev/sdk:latest@sha256:e8c9680e3262d27b28c38e84f51f8a8587c84dc192b0f198b96b11de27aafc34
# TODO: Deprivilege
options: |
--cap-add NET_ADMIN --cap-add SYS_ADMIN --device /dev/fuse --security-opt seccomp=unconfined --security-opt apparmor:unconfined
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- name: 'Trust the github workspace'
run: |
# This is to avoid fatal errors about "dubious ownership" because we are
# running inside of a container action with the workspace mounted in.
git config --global --add safe.directory "$(pwd)"
# Build with a local key, we'll resign this with the real key later
- name: 'Generate local signing key'
run: |
make local-melange.rsa
# Touch it with the epoch date to convince `make` that we don't need to
# rebuild the targets that depend on this (all)
touch -d @0 local-melange.rsa
- name: 'Prepare package repository'
run: |
# yay wolfi!
apk add gcsfuse google-cloud-sdk
# Set up a gcsfuse RO mount to the public bucket. This is a cheap and
# cheerful way to recreate the make targets (class A HEADs) locally
# without syncing the whole bucket (class A+B).
mkdir -p /gcsfuse/wolfi-registry
gcsfuse -o ro --implicit-dirs --only-dir os wolfi-production-registry-destination /gcsfuse/wolfi-registry
mkdir -p ./packages/${{ matrix.arch }}
# Symlink the gcsfuse mount to ./packages/ to workaround the Makefile CWD assumptions
for f in /gcsfuse/wolfi-registry/${{ matrix.arch }}/*.apk; do
ln -s "$f" ./packages/${{ matrix.arch }}/
done
# Make a copy of the APKINDEX.* since we'll need to write to it on package builds
cp /gcsfuse/wolfi-registry/${{ matrix.arch }}/APKINDEX.* ./packages/${{ matrix.arch }}/
# TODO: Remove the previous and next steps by folding the logic into wolfictl.
- name: 'Build Wolfi'
run: |
wolfictl build \
--runner bubblewrap \
--keyring-append /gcsfuse/wolfi-registry/wolfi-signing.rsa.pub \
--repository-append ./packages \
--keyring-append local-melange.rsa.pub \
--repository-append https://packages.wolfi.dev/os \
--keyring-append https://packages.wolfi.dev/os/wolfi-signing.rsa.pub \
--signing-key local-melange.rsa \
--arch ${{ matrix.arch }} \
--namespace wolfi \
--pipeline-dir ./pipelines/ \
--destination-repository https://packages.wolfi.dev/os \
--trace ./packages/${{ matrix.arch }}/trace.json
# Always run this step for https://github.com/wolfi-dev/os/issues/8698
- if: ${{ always() }}
name: 'Create artifacts tarball'
run: |
set -x
set -e
set -o pipefail
# Overwrite the APKINDEX.tar.gz we just generated with the original one.
# Since we use a temporary key during the build, we need to re-sign the APKs later.
# We don't want to keep the newly indexed stuff because the size field is probably wrong.
cp /gcsfuse/wolfi-registry/${{ matrix.arch }}/APKINDEX.* ./packages/${{ matrix.arch }}/
# Pick up any stragglers that didn't get uploaded in previous builds.
cat ./packages/${{ matrix.arch }}/APKINDEX.tar.gz | tar -Oxz APKINDEX | awk -F':' '$1 == "P" {printf "%s-", $2} $1 == "V" {printf "%s.apk\n", $2}' | sort > indexed.txt
# TODO: Figure out why ls through gcsfuse is so slow.
gcloud storage ls gs://wolfi-production-registry-destination/os/${{ matrix.arch }} | grep ".apk$" | xargs -n1 basename | sort > uploaded.txt
# Lines that are only in uploaded.txt and not indexed.txt.
comm -13 indexed.txt uploaded.txt > missing.txt
# Clean up the symlinks to keep only packages we built.
find ./packages/${{ matrix.arch }} -type l -exec rm -f {} \;
# Merge any missing APKs into our new index.
for missed in $(cat missing.txt); do
# We could do this in one command instead of a loop, but it takes things on argv, which is a bit annoying.
melange index --merge \
--source ./packages/${{ matrix.arch }}/APKINDEX.tar.gz \
--output new.tar.gz \
/gcsfuse/wolfi-registry/${{ matrix.arch }}/${missed}
# Overwrite what we're going to upload (and for the next loop).
mv new.tar.gz ./packages/${{ matrix.arch }}/APKINDEX.tar.gz
done
diff \
<(cat /gcsfuse/wolfi-registry/${{ matrix.arch }}/APKINDEX.tar.gz | tar -Oxz APKINDEX) \
<(cat ./packages/${{ matrix.arch }}/APKINDEX.tar.gz | tar -Oxz APKINDEX) || true
# Move logs so we can upload them separately.
mv ./packages/${{ matrix.arch }}/buildlogs /tmp/buildlogs
# Move trace so we can upload it separately.
mv ./packages/${{ matrix.arch }}/trace.json /tmp/trace.json
# Create an archive for uploading
tar -cvzf /tmp/packages-${{ matrix.arch }}.tar.gz ./packages/${{ matrix.arch }}
# Always run these steps for https://github.com/wolfi-dev/os/issues/8698
- if: ${{ always() }}
name: 'Upload logs archive to GitHub Artifacts'
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:
name: logs-${{ matrix.arch }}
path: /tmp/buildlogs/
if-no-files-found: warn
- if: ${{ always() }}
name: 'Upload trace to GitHub Artifacts'
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:
name: trace-${{ matrix.arch }}
path: /tmp/trace.json
if-no-files-found: warn
- if: ${{ always() }}
name: 'Upload built packages archive to GitHub Artifacts'
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:
name: packages-${{ matrix.arch }}
path: /tmp/packages-${{ matrix.arch }}.tar.gz
retention-days: 1 # Low ttl since this is just an intermediary used once
if-no-files-found: warn
upload-packages:
runs-on: ubuntu-latest-16-cores
needs: build
# Always run this job for https://github.com/wolfi-dev/os/issues/8698
if: ${{ always() }}
permissions:
id-token: write
contents: read
container:
# NOTE: This step only signs and uploads, so it doesn't need any privileges
image: ghcr.io/wolfi-dev/sdk:latest@sha256:e8c9680e3262d27b28c38e84f51f8a8587c84dc192b0f198b96b11de27aafc34
steps:
- name: Harden Runner
uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
with:
egress-policy: audit
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- name: Free up runner disk space
run: |
set -x
rm -rf /usr/share/dotnet
rm -rf "$AGENT_TOOLSDIRECTORY"
- name: 'Trust the github workspace'
run: |
# This is to avoid fatal errors about "dubious ownership" because we are
# running inside of a container action with the workspace mounted in.
git config --global --add safe.directory "$(pwd)"
- name: 'Download x86_64 package archives'
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
path: /tmp/artifacts/
name: packages-x86_64
- name: 'Download aarch64 package archives'
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
path: /tmp/artifacts/
name: packages-aarch64
# This is managed here: https://github.com/chainguard-dev/secrets/blob/main/wolfi-dev.tf
- uses: google-github-actions/auth@62cf5bd3e4211a0a0b51f2c6d6a37129d828611d # v2.1.5
id: auth
with:
workload_identity_provider: "projects/12758742386/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
service_account: "[email protected]"
- uses: google-github-actions/setup-gcloud@f0990588f1e5b5af6827153b93673613abdc6ec7 # v2.1.1
with:
project_id: "chainguard-github-secrets"
- uses: 'google-github-actions/get-secretmanager-secrets@95a0b09b8348ef3d02c68c6ba5662a037e78d713' # v2.1.4
id: secrets
with:
secrets: |-
token:chainguard-github-secrets/wolfi-dev-signing-key
- run: echo "${{ steps.secrets.outputs.token }}" > ./wolfi-signing.rsa
- run: |
mkdir -p /etc/apk/keys
cp ./wolfi-signing.rsa.pub /etc/apk/keys/wolfi-signing.rsa.pub
- name: 'Update the APKINDEX'
run: |
for arch in "aarch64" "x86_64"; do
mkdir -p ./packages/${arch}
# Consolidate with the built artifacts
tar xvf /tmp/artifacts/packages-${arch}.tar.gz
# Only attempt to sign when *.apk's exist.
apks=$(ls ./packages/${arch}/*.apk 2>/dev/null || true)
if [ -n "$apks" ]; then
melange sign --signing-key ./wolfi-signing.rsa ./packages/${arch}/*.apk
melange index --merge \
--source ./packages/${arch}/APKINDEX.tar.gz \
--output ./packages/${arch}/APKINDEX.tar.gz \
./packages/${arch}/*.apk
fi
# Sign the APK index
melange sign-index -f --signing-key ./wolfi-signing.rsa packages/${arch}/APKINDEX.tar.gz
done
# Clean up the signing key before uploading to storage out
# of an abundance of caution.
- run: rm ./wolfi-signing.rsa
# We use a different GSA for our interaction with GCS.
- uses: google-github-actions/auth@62cf5bd3e4211a0a0b51f2c6d6a37129d828611d # v2.1.5
with:
workload_identity_provider: "projects/618116202522/locations/global/workloadIdentityPools/prod-shared-e350/providers/prod-shared-gha"
service_account: "[email protected]"
- uses: google-github-actions/setup-gcloud@f0990588f1e5b5af6827153b93673613abdc6ec7 # v2.1.1
with:
project_id: "prod-images-c6e5"
- name: 'Upload packages to GCS'
run: |
for arch in "aarch64" "x86_64"; do
# Only attempt to upload when *.apk's exist
apks=$(ls ./packages/${arch}/*.apk 2>/dev/null || true)
if [ -n "$apks" ]; then
# apks will be cached in CDN for an hour by default.
# Don't upload the object if it already exists.
gcloud --quiet storage cp \
--no-clobber \
"./packages/${arch}/*.apk" "gs://wolfi-production-registry-destination/os/${arch}/"
fi
done
- name: 'Create APKINDEX tarball'
run: |
# Tar up any 'APKINDEX.*' files {aarch64,x86_64} x {tar.gz,json}
find ./packages/ -name 'APKINDEX.*' > to-include
tar -cvzf /tmp/indexes.tar.gz --files-from to-include
- name: 'Upload APKINDEX archive to GitHub Artifacts'
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:
name: indexes
path: /tmp/indexes.tar.gz
retention-days: 1 # Low ttl since this is just an intermediary used once
if-no-files-found: warn
upload-index:
runs-on: ubuntu-latest-16-cores
needs: upload-packages
permissions:
id-token: write
contents: read
container:
# NOTE: This step only signs and uploads, so it doesn't need any privileges
image: ghcr.io/wolfi-dev/sdk:latest@sha256:e8c9680e3262d27b28c38e84f51f8a8587c84dc192b0f198b96b11de27aafc34
steps:
- name: Harden Runner
uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
with:
egress-policy: audit
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- name: 'Trust the github workspace'
run: |
# This is to avoid fatal errors about "dubious ownership" because we are
# running inside of a container action with the workspace mounted in.
git config --global --add safe.directory "$(pwd)"
- id: auth
name: 'Authenticate to Google Cloud'
uses: google-github-actions/auth@62cf5bd3e4211a0a0b51f2c6d6a37129d828611d # v2.1.5
with:
workload_identity_provider: "projects/618116202522/locations/global/workloadIdentityPools/prod-shared-e350/providers/prod-shared-gha"
service_account: "[email protected]"
- uses: google-github-actions/setup-gcloud@f0990588f1e5b5af6827153b93673613abdc6ec7 # v2.1.1
with:
project_id: prod-images-c6e5
- name: 'Download index archive'
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
path: /tmp/artifacts/
name: indexes
- name: 'Upload indexes to GCS'
run: |
tar xvf /tmp/artifacts/indexes.tar.gz
for arch in "aarch64" "x86_64"; do
# Don't cache the APKINDEX.
gcloud --quiet storage cp \
--cache-control=no-store \
"./packages/${arch}/APKINDEX.tar.gz" "gs://wolfi-production-registry-destination/os/${arch}/"
gcloud --quiet storage cp \
--cache-control=no-store \
"./packages/${arch}/APKINDEX.json" "gs://wolfi-production-registry-destination/os/${arch}/"
done
upload-packages-to-cgr:
runs-on: ubuntu-latest
needs: build
# Always run this job for https://github.com/wolfi-dev/os/issues/8698
if: ${{ always() }}
permissions:
id-token: write
contents: read
steps:
- uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
with:
egress-policy: audit
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- name: 'Trust the github workspace'
run: |
# This is to avoid fatal errors about "dubious ownership" because we are
# running inside of a container action with the workspace mounted in.
git config --global --add safe.directory "$(pwd)"
- name: 'Download x86_64 package archives'
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
path: /tmp/artifacts/
name: packages-x86_64
- name: 'Download aarch64 package archives'
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
path: /tmp/artifacts/
name: packages-aarch64
- name: 'Unpack the package archives'
run: |
for arch in "aarch64" "x86_64"; do
mkdir -p ./packages/${arch}
# Consolidate with the built artifacts
tar xvf /tmp/artifacts/packages-${arch}.tar.gz
done
# use public chainguard provider.
- uses: chainguard-dev/setup-chainctl@f52718d822dc73d21a04ef2082822c4a203163b3 # v0.2.2
with:
# Managed here:
# https://github.com/chainguard-dev/mono/blob/main/env/chainguard-images/iac/wolfi-os-pusher.tf
identity: "720909c9f5279097d847ad02a2f24ba8f59de36a/6a26f2970f880c31"
- name: 'Upload packages to apk.cgr.dev'
run: |
set -ex
# Populate the token here, since chainctl auth token
# doesn't support all of the options we need.
chainctl auth login --audience apk.cgr.dev \
--identity "720909c9f5279097d847ad02a2f24ba8f59de36a/6a26f2970f880c31"
tok=$(chainctl auth token --audience apk.cgr.dev)
echo "::add-mask::${tok}"
for arch in "aarch64" "x86_64"; do
# Only attempt to upload when *.apk's exist
apks=$(ls ./packages/${arch}/*.apk 2>/dev/null || true)
if [ -n "$apks" ]; then
for apk in ${apks}; do
package="$(basename ${apk})"
# Check if package already exists in apk.cgr.dev
code=$(curl -s -o /dev/null --head -w "%{http_code}" --user "user:${tok}" "https://apk.cgr.dev/chainguard/${arch}/${package}")
if [ $code == "303" ]; then
echo "Package already exists: ${package}"
continue
elif [ $code != "404" ]; then
echo "Unexpected response code: $code"
exit 1
fi
curl --fail -X POST \
--user "user:${tok}" \
--data-binary "@${apk}" \
"https://apk.cgr.dev/chainguard/${arch}/${package}"
done
fi
done
postrun:
name: Notify Slack
runs-on: ubuntu-latest
if: failure()
needs: [build, upload-packages, upload-packages-to-cgr, upload-index]
steps:
- name: Harden Runner
uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
with:
egress-policy: audit
- uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 # v2.2.1
env:
SLACK_ICON: http://github.com/chainguard-dev.png?size=48
SLACK_USERNAME: guardian
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_CHANNEL: C047DK5BUNP
SLACK_MSG_AUTHOR: wolfi-bot
SLACK_COLOR: '#8E1600'
MSG_MINIMAL: 'true'
SLACK_TITLE: '[build-wolfi-os] failure: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}'
SLACK_MESSAGE: |
https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}