From 71994bf681c7bb2864a8a398cbe9b8021b29fd4f Mon Sep 17 00:00:00 2001 From: "Austin S. Hemmelgarn" Date: Wed, 12 Jun 2024 10:45:37 -0400 Subject: [PATCH] Update CI infrastructure to publish to secondary packaging host. (#17863) * Update CI infrastructure to publish to secondary packaging host. * Fix typo. --- .github/scripts/package-upload.sh | 10 +++++----- .github/scripts/upload-new-version-tags.sh | 3 ++- .github/workflows/kickstart-upload.yml | 11 ++++++++--- .github/workflows/monitor-releases.yml | 11 +++++++++-- .github/workflows/packaging.yml | 13 +++++++++++++ .github/workflows/repoconfig-packages.yml | 16 ++++++++++++++++ 6 files changed, 53 insertions(+), 11 deletions(-) diff --git a/.github/scripts/package-upload.sh b/.github/scripts/package-upload.sh index 13d63b4a74c3ea..b4e432852a9ef6 100755 --- a/.github/scripts/package-upload.sh +++ b/.github/scripts/package-upload.sh @@ -2,13 +2,13 @@ set -e -host="packages.netdata.cloud" user="netdatabot" -distro="${1}" -arch="${2}" -format="${3}" -repo="${4}" +host="${1}" +distro="${2}" +arch="${3}" +format="${4}" +repo="${5}" staging="${TMPDIR:-/tmp}/package-staging" prefix="/home/netdatabot/incoming/${repo}/" diff --git a/.github/scripts/upload-new-version-tags.sh b/.github/scripts/upload-new-version-tags.sh index a9b0cd303099dc..ffdfadfa0ec482 100755 --- a/.github/scripts/upload-new-version-tags.sh +++ b/.github/scripts/upload-new-version-tags.sh @@ -2,9 +2,10 @@ set -e -host="packages.netdata.cloud" user="netdatabot" +host="${1}" + prefix="/var/www/html/releases" staging="${TMPDIR:-/tmp}/staging-new-releases" diff --git a/.github/workflows/kickstart-upload.yml b/.github/workflows/kickstart-upload.yml index 77c26d7bcafa7f..46505a4a37930c 100644 --- a/.github/workflows/kickstart-upload.yml +++ b/.github/workflows/kickstart-upload.yml @@ -30,9 +30,13 @@ jobs: key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }} name: id_ecdsa known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }} - - name: Upload - id: upload + - name: Upload to packages.netdata.cloud + id: upload-primary + continue-on-error: true run: rsync -vp packaging/installer/kickstart.sh netdatabot@packages.netdata.cloud:/home/netdatabot/incoming/kickstart.sh + - name: Upload to packages2.netdata.cloud + id: upload-packages2 + run: rsync -vp packaging/installer/kickstart.sh netdatabot@packages2.netdata.cloud:/home/netdatabot/incoming/kickstart.sh - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: @@ -44,7 +48,8 @@ jobs: ${{ github.repository }}: Failed to upload updated kickstart script to repo server. Checkout: ${{ steps.checkout.outcome }} Import SSH Key: ${{ steps.ssh-setup.outcome }} - Upload: ${{ steps.upload.outcome }} + Upload to packages.netdata.cloud: ${{ steps.upload-packages.outcome }} + Upload to packages2.netdata.cloud: ${{ steps.upload-packages2.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ diff --git a/.github/workflows/monitor-releases.yml b/.github/workflows/monitor-releases.yml index 77ec06bcf53a5e..e4269f3c19ca62 100644 --- a/.github/workflows/monitor-releases.yml +++ b/.github/workflows/monitor-releases.yml @@ -55,11 +55,17 @@ jobs: key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }} name: id_ecdsa known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }} - - name: Sync newer releases + - name: Sync newer releases to packages.netdata.cloud id: sync-releases + continue-on-error: true if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-newer-releases.outputs.versions_needs_update == 'true' run: | - .github/scripts/upload-new-version-tags.sh + .github/scripts/upload-new-version-tags.sh packages.netdata.cloud + - name: Sync newer releases to packages2.netdata.cloud + id: sync-releases2 + if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-newer-releases.outputs.versions_needs_update == 'true' + run: | + .github/scripts/upload-new-version-tags.sh packages2.netdata.cloud - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: @@ -77,5 +83,6 @@ jobs: Check for newer stable releaes: ${{ steps.check-newer-releases.outcome }} Setup ssh: ${{ steps.ssh-setup.outcome }} Syncing newer release to packages.netdata.cloud : ${{ steps.sync-releases.outcome }} + Syncing newer release to packages2.netdata.cloud : ${{ steps.sync-releases2.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: failure() diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index ad5b8faaa73e1e..dcf6ee8c44040d 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -300,9 +300,21 @@ jobs: known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }} - name: Upload to packages.netdata.cloud id: package-upload + continue-on-error: true + if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true' + run: | + .github/scripts/package-upload.sh \ + packages.netdata.cloud \ + ${{ matrix.repo_distro }} \ + ${{ matrix.arch }} \ + ${{ matrix.format }} \ + ${{ needs.version-check.outputs.repo }} + - name: Upload to packages2.netdata.cloud + id: package2-upload if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true' run: | .github/scripts/package-upload.sh \ + packages2.netdata.cloud \ ${{ matrix.repo_distro }} \ ${{ matrix.arch }} \ ${{ matrix.format }} \ @@ -325,6 +337,7 @@ jobs: Publish to PackageCloud: ${{ steps.upload.outcome }} Import SSH Key: ${{ steps.ssh-setup.outcome }} Publish to packages.netdata.cloud: ${{ steps.package-upload.outcome }} + Publish to packages2.netdata.cloud: ${{ steps.package2-upload.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ diff --git a/.github/workflows/repoconfig-packages.yml b/.github/workflows/repoconfig-packages.yml index 57995847398e15..f151671ed776a5 100644 --- a/.github/workflows/repoconfig-packages.yml +++ b/.github/workflows/repoconfig-packages.yml @@ -118,12 +118,27 @@ jobs: name: id_ecdsa known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }} - name: Upload to packages.netdata.cloud + id: package-upload + continue-on-error: true + if: github.event_name != 'pull_request' && github.repository == 'netdata/netdata' + run: | + # shellcheck disable=SC2043 + for arch in ${{ matrix.arches }}; do + .github/scripts/package-upload.sh \ + packages.netdata.cloud \ + "${{ matrix.pkgclouddistro }}" \ + "${arch}" \ + "${{ matrix.format }}" \ + netdata/netdata-repoconfig + done + - name: Upload to packages2.netdata.cloud id: package-upload if: github.event_name != 'pull_request' && github.repository == 'netdata/netdata' run: | # shellcheck disable=SC2043 for arch in ${{ matrix.arches }}; do .github/scripts/package-upload.sh \ + packages2.netdata.cloud \ "${{ matrix.pkgclouddistro }}" \ "${arch}" \ "${{ matrix.format }}" \ @@ -146,4 +161,5 @@ jobs: Publish to PackageCloud: ${{ steps.publish.outcome }} Import SSH Key: ${{ steps.ssh-setup.outcome }} Publish to packages.netdata.cloud: ${{ steps.package-upload.outcome }} + Publish to packages2.netdata.cloud: ${{ steps.package2-upload.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}