chore: migrate workflow to mirror Copr RPMs into Cloudsmith archive t…

…o Github Actions (#32143)

.github/scripts/publish-unpublished-rpms-to-archive.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+set -euo pipefail
+set -x
+
+if (( $# < 1 )); then
+    echo "Usage: $0 <RPM architecture>"
+    exit 1
+fi
+
+RPMARCH=$1
+ALLOWED_ARCHS=("x86_64" "aarch64")
+
+if [[ ! ${ALLOWED_ARCHS[@]} =~ $RPMARCH ]]; then
+  echo "Architecture $RPMARCH not in allowed archs: ${ALLOWED_ARCHS[@]}"
+  exit 1
+fi
+
+readonly MYDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# Copr repo
+dnf config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-8/group_vespa-vespa-epel-8.repo
+sed -i "s,\$basearch,$RPMARCH,g" /etc/yum.repos.d/group_vespa-vespa-epel-8.repo
+
+# Cloudsmith repo
+rpm --import 'https://dl.cloudsmith.io/public/vespa/open-source-rpms/gpg.0F3DA3C70D35DA7B.key'
+curl -1sLf 'https://dl.cloudsmith.io/public/vespa/open-source-rpms/config.rpm.txt?distro=el&codename=8' > /tmp/vespa-open-source-rpms.repo
+dnf config-manager --add-repo '/tmp/vespa-open-source-rpms.repo'
+rm -f /tmp/vespa-open-source-rpms.repo
+
+readonly COPR_PACKAGES=$(mktemp)
+trap "rm -f $COPR_PACKAGES" EXIT
+readonly DLDIR=$(mktemp -d)
+trap "rm -rf $DLDIR" EXIT
+
+cd $DLDIR
+
+readonly DNF="dnf -y -q --forcearch $RPMARCH"
+
+$DNF list --disablerepo='*' --enablerepo=copr:copr.fedorainfracloud.org:group_vespa:vespa --showduplicates 'vespa*' | grep "Available Packages" -A 100000 | tail -n +2 | sed '/\.src\ */d' | sed -E "s/\.($RPMARCH|noarch)\ */-/" | awk '{print $1}' | grep -v '.src$' > $COPR_PACKAGES
+
+echo "Packages on Copr:"
+cat $COPR_PACKAGES
+echo
+
+for pv in $(cat $COPR_PACKAGES); do
+  if ! $DNF list --disablerepo='*' --enablerepo=vespa-open-source-rpms $pv &> /dev/null; then
+    # Need one extra check here for noarch packages
+    if ! dnf -y -q --forcearch noarch list --disablerepo='*' --enablerepo=vespa-open-source-rpms $pv &> /dev/null; then
+      echo "$pv not found on in archive. Downloading..."
+      $DNF download --disablerepo='*' --enablerepo=copr:copr.fedorainfracloud.org:group_vespa:vespa $pv
+      echo "$pv downloaded."
+    fi
+  fi
+done
+echo
+
+if ! ls *.rpm &> /dev/null; then
+  echo "All packages already in archive."
+  exit 0
+fi
+
+echo "RPMs missing in archive:"
+ls -lh  *.rpm
+echo
+
+
+UPLOAD_FAILED=false
+echo "GitHub event: $GITHUB_EVENT_NAME"
+
+if [ "$GITHUB_EVENT_NAME" == "schedule" ]; then
+  for rpm in $(ls *.rpm); do
+    echo "Uploading $rpm ..."
+    if ! $MYDIR/upload-rpm-to-cloudsmith.sh $rpm ; then
+      echo "Could not upload $rpm"
+      UPLOAD_FAILED=true
+    else
+      echo "$rpm uploaded"
+    fi
+  done
+  echo
+fi
+
+if $UPLOAD_FAILED; then
+  echo "Some RPMs failed to upload"
+  exit 1
+fi

.github/scripts/upload-rpm-to-cloudsmith.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+set -euo pipefail
+
+if (( $# < 1 )); then
+  echo "Usage: $0 <RPM file>"
+  exit 1
+fi
+
+if [[ -z $CLOUDSMITH_API_TOKEN ]]; then
+  echo "Environment CLOUDSMITH_API_TOKEN not set. Exiting."
+  exit 1
+fi
+
+RPM=$1
+OS_DISTRO=el
+RELEASEVER=8
+
+main() {
+
+  FID=$(curl -sLf \
+    --upload-file $RPM \
+    -H "X-Api-Key: $CLOUDSMITH_API_TOKEN" \
+    -H "Content-Sha256: $(sha256sum $RPM | cut -f1 -d' ')" \
+    https://upload.cloudsmith.io/vespa/open-source-rpms/$RPM | jq -re '.identifier')
+
+  if [[ -n $FID ]]; then
+    curl -sLf -X POST -H "Content-Type: application/json" \
+      -H "X-Api-Key: $CLOUDSMITH_API_TOKEN" \
+      -d "{\"package_file\": \"$FID\", \"distribution\": \"$OS_DISTRO/$RELEASEVER\"}" \
+      https://api-prd.cloudsmith.io/v1/packages/vespa/open-source-rpms/upload/rpm/
+  fi
+}
+
+main "$@"

.github/workflows/mirror-copr-rpms-to-archive.yml

@@ -0,0 +1,49 @@
+name: Mirror Copr RPMs to archive
+
+on:
+  workflow_dispatch:
+
+  pull_request:
+    paths:
+      - .github/workflows/mirror-copr-rpms-to-archive.yml
+      - .github/scripts/publish-unpublished-rpms-to-archive.sh
+      - .github/scripts/upload-rpm-to-cloudsmith.sh
+    branches:
+      - master
+
+  schedule:
+   - cron: '0 6 * * *'
+
+jobs:
+  mirror-copr-rpms-to-archive:
+    runs-on: ubuntu-latest
+
+    container:
+        image: almalinux:8
+        env:
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+          CLOUDSMITH_API_TOKEN: ${{ secrets.CLOUDSMITH_API_TOKEN }}
+        volumes:
+          - ${{ github.workspace }}:/workspace
+
+    defaults:
+      run:
+        working-directory: /workspace
+
+    strategy:
+      matrix:
+        arch:
+          - x86_64
+          - aarch64
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install tools
+        run: |
+          dnf install -y dnf-plugins-core jq
+
+      - name: Publish to ${{ matrix.arch }} mirror
+        run: |
+          .github/scripts/publish-unpublished-rpms-to-archive.sh ${{ matrix.arch }}

screwdriver.yaml

@@ -302,25 +302,6 @@ jobs:
             exit 1
           fi
 
-  mirror-copr-rpms-to-archive:
-    requires: [publish-release]
-    image: docker.io/almalinux:8
-    annotations:
-      screwdriver.cd/cpu: LOW
-      screwdriver.cd/ram: LOW
-      screwdriver.cd/disk: HIGH
-      screwdriver.cd/timeout: 60
-      screwdriver.cd/buildPeriodically: H 6 * * *
-    secrets:
-      - CLOUDSMITH_API_TOKEN
-    steps:
-      - install: |
-          dnf install -y dnf-plugins-core jq
-      - mirror-x86-64: |
-          screwdriver/publish-unpublished-rpms-to-archive.sh x86_64
-      - mirror-aarch64: |
-          screwdriver/publish-unpublished-rpms-to-archive.sh aarch64
-
   delete-old-versions-in-archive:
     annotations:
       screwdriver.cd/cpu: LOW

screwdriver/publish-unpublished-rpms-to-archive.sh

@@ -42,7 +42,7 @@ $DNF list --disablerepo='*' --enablerepo=copr:copr.fedorainfracloud.org:group_ve
 
 echo "Packages on Copr:"
 cat $COPR_PACKAGES
-echo 
+echo
 
 for pv in $(cat $COPR_PACKAGES); do
   if ! $DNF list --disablerepo='*' --enablerepo=vespa-open-source-rpms $pv &> /dev/null; then