From b86c5e5df78cbb2741190ae21637646c904a0efe Mon Sep 17 00:00:00 2001 From: Gleb Sizov Date: Fri, 16 Aug 2024 09:52:30 +0200 Subject: [PATCH] chore: migrate workflow to mirror Copr RPMs into Cloudsmith archive to Github Actions (#32143) --- .../publish-unpublished-rpms-to-archive.sh | 88 +++++++++++++++++++ .github/scripts/upload-rpm-to-cloudsmith.sh | 36 ++++++++ .../workflows/mirror-copr-rpms-to-archive.yml | 49 +++++++++++ screwdriver.yaml | 19 ---- .../publish-unpublished-rpms-to-archive.sh | 2 +- 5 files changed, 174 insertions(+), 20 deletions(-) create mode 100755 .github/scripts/publish-unpublished-rpms-to-archive.sh create mode 100755 .github/scripts/upload-rpm-to-cloudsmith.sh create mode 100644 .github/workflows/mirror-copr-rpms-to-archive.yml diff --git a/.github/scripts/publish-unpublished-rpms-to-archive.sh b/.github/scripts/publish-unpublished-rpms-to-archive.sh new file mode 100755 index 000000000000..7f34c6954d53 --- /dev/null +++ b/.github/scripts/publish-unpublished-rpms-to-archive.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +set -euo pipefail +set -x + +if (( $# < 1 )); then + echo "Usage: $0 " + exit 1 +fi + +RPMARCH=$1 +ALLOWED_ARCHS=("x86_64" "aarch64") + +if [[ ! ${ALLOWED_ARCHS[@]} =~ $RPMARCH ]]; then + echo "Architecture $RPMARCH not in allowed archs: ${ALLOWED_ARCHS[@]}" + exit 1 +fi + +readonly MYDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# Copr repo +dnf config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-8/group_vespa-vespa-epel-8.repo +sed -i "s,\$basearch,$RPMARCH,g" /etc/yum.repos.d/group_vespa-vespa-epel-8.repo + +# Cloudsmith repo +rpm --import 'https://dl.cloudsmith.io/public/vespa/open-source-rpms/gpg.0F3DA3C70D35DA7B.key' +curl -1sLf 'https://dl.cloudsmith.io/public/vespa/open-source-rpms/config.rpm.txt?distro=el&codename=8' > /tmp/vespa-open-source-rpms.repo +dnf config-manager --add-repo '/tmp/vespa-open-source-rpms.repo' +rm -f /tmp/vespa-open-source-rpms.repo + +readonly COPR_PACKAGES=$(mktemp) +trap "rm -f $COPR_PACKAGES" EXIT +readonly DLDIR=$(mktemp -d) +trap "rm -rf $DLDIR" EXIT + +cd $DLDIR + +readonly DNF="dnf -y -q --forcearch $RPMARCH" + +$DNF list --disablerepo='*' --enablerepo=copr:copr.fedorainfracloud.org:group_vespa:vespa --showduplicates 'vespa*' | grep "Available Packages" -A 100000 | tail -n +2 | sed '/\.src\ */d' | sed -E "s/\.($RPMARCH|noarch)\ */-/" | awk '{print $1}' | grep -v '.src$' > $COPR_PACKAGES + +echo "Packages on Copr:" +cat $COPR_PACKAGES +echo + +for pv in $(cat $COPR_PACKAGES); do + if ! $DNF list --disablerepo='*' --enablerepo=vespa-open-source-rpms $pv &> /dev/null; then + # Need one extra check here for noarch packages + if ! dnf -y -q --forcearch noarch list --disablerepo='*' --enablerepo=vespa-open-source-rpms $pv &> /dev/null; then + echo "$pv not found on in archive. Downloading..." + $DNF download --disablerepo='*' --enablerepo=copr:copr.fedorainfracloud.org:group_vespa:vespa $pv + echo "$pv downloaded." + fi + fi +done +echo + +if ! ls *.rpm &> /dev/null; then + echo "All packages already in archive." + exit 0 +fi + +echo "RPMs missing in archive:" +ls -lh *.rpm +echo + + +UPLOAD_FAILED=false +echo "GitHub event: $GITHUB_EVENT_NAME" + +if [ "$GITHUB_EVENT_NAME" == "schedule" ]; then + for rpm in $(ls *.rpm); do + echo "Uploading $rpm ..." + if ! $MYDIR/upload-rpm-to-cloudsmith.sh $rpm ; then + echo "Could not upload $rpm" + UPLOAD_FAILED=true + else + echo "$rpm uploaded" + fi + done + echo +fi + +if $UPLOAD_FAILED; then + echo "Some RPMs failed to upload" + exit 1 +fi diff --git a/.github/scripts/upload-rpm-to-cloudsmith.sh b/.github/scripts/upload-rpm-to-cloudsmith.sh new file mode 100755 index 000000000000..b20bf033373c --- /dev/null +++ b/.github/scripts/upload-rpm-to-cloudsmith.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +set -euo pipefail + +if (( $# < 1 )); then + echo "Usage: $0 " + exit 1 +fi + +if [[ -z $CLOUDSMITH_API_TOKEN ]]; then + echo "Environment CLOUDSMITH_API_TOKEN not set. Exiting." + exit 1 +fi + +RPM=$1 +OS_DISTRO=el +RELEASEVER=8 + +main() { + + FID=$(curl -sLf \ + --upload-file $RPM \ + -H "X-Api-Key: $CLOUDSMITH_API_TOKEN" \ + -H "Content-Sha256: $(sha256sum $RPM | cut -f1 -d' ')" \ + https://upload.cloudsmith.io/vespa/open-source-rpms/$RPM | jq -re '.identifier') + + if [[ -n $FID ]]; then + curl -sLf -X POST -H "Content-Type: application/json" \ + -H "X-Api-Key: $CLOUDSMITH_API_TOKEN" \ + -d "{\"package_file\": \"$FID\", \"distribution\": \"$OS_DISTRO/$RELEASEVER\"}" \ + https://api-prd.cloudsmith.io/v1/packages/vespa/open-source-rpms/upload/rpm/ + fi +} + +main "$@" diff --git a/.github/workflows/mirror-copr-rpms-to-archive.yml b/.github/workflows/mirror-copr-rpms-to-archive.yml new file mode 100644 index 000000000000..4841d20e24d3 --- /dev/null +++ b/.github/workflows/mirror-copr-rpms-to-archive.yml @@ -0,0 +1,49 @@ +name: Mirror Copr RPMs to archive + +on: + workflow_dispatch: + + pull_request: + paths: + - .github/workflows/mirror-copr-rpms-to-archive.yml + - .github/scripts/publish-unpublished-rpms-to-archive.sh + - .github/scripts/upload-rpm-to-cloudsmith.sh + branches: + - master + + schedule: + - cron: '0 6 * * *' + +jobs: + mirror-copr-rpms-to-archive: + runs-on: ubuntu-latest + + container: + image: almalinux:8 + env: + GITHUB_EVENT_NAME: ${{ github.event_name }} + CLOUDSMITH_API_TOKEN: ${{ secrets.CLOUDSMITH_API_TOKEN }} + volumes: + - ${{ github.workspace }}:/workspace + + defaults: + run: + working-directory: /workspace + + strategy: + matrix: + arch: + - x86_64 + - aarch64 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install tools + run: | + dnf install -y dnf-plugins-core jq + + - name: Publish to ${{ matrix.arch }} mirror + run: | + .github/scripts/publish-unpublished-rpms-to-archive.sh ${{ matrix.arch }} diff --git a/screwdriver.yaml b/screwdriver.yaml index 9439c88b2abb..195eb79c8236 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -302,25 +302,6 @@ jobs: exit 1 fi - mirror-copr-rpms-to-archive: - requires: [publish-release] - image: docker.io/almalinux:8 - annotations: - screwdriver.cd/cpu: LOW - screwdriver.cd/ram: LOW - screwdriver.cd/disk: HIGH - screwdriver.cd/timeout: 60 - screwdriver.cd/buildPeriodically: H 6 * * * - secrets: - - CLOUDSMITH_API_TOKEN - steps: - - install: | - dnf install -y dnf-plugins-core jq - - mirror-x86-64: | - screwdriver/publish-unpublished-rpms-to-archive.sh x86_64 - - mirror-aarch64: | - screwdriver/publish-unpublished-rpms-to-archive.sh aarch64 - delete-old-versions-in-archive: annotations: screwdriver.cd/cpu: LOW diff --git a/screwdriver/publish-unpublished-rpms-to-archive.sh b/screwdriver/publish-unpublished-rpms-to-archive.sh index 32e1b07d3dbf..e3e2b7b1c021 100755 --- a/screwdriver/publish-unpublished-rpms-to-archive.sh +++ b/screwdriver/publish-unpublished-rpms-to-archive.sh @@ -42,7 +42,7 @@ $DNF list --disablerepo='*' --enablerepo=copr:copr.fedorainfracloud.org:group_ve echo "Packages on Copr:" cat $COPR_PACKAGES -echo +echo for pv in $(cat $COPR_PACKAGES); do if ! $DNF list --disablerepo='*' --enablerepo=vespa-open-source-rpms $pv &> /dev/null; then