import.sh: Handle multiple artifacts

ubi-agni · Nov 13, 2024 · 40995b1 · 40995b1
1 parent d0e846e
commit 40995b1
Show file tree

Hide file tree

Showing 3 changed files with 71 additions and 53 deletions.
diff --git a/.github/workflows/scheduled.yaml b/.github/workflows/scheduled.yaml
@@ -41,4 +41,4 @@ jobs:
       - name: Import build artifacts to reprepro server
         uses: ubi-agni/ros-builder-action/reprepro@main
         with:
-          url: "${{ vars.DEPLOY_URL }}?distro=${{ matrix.DEB_DISTRO }}&run_id=${{ github.run_id }}&arch=x64"
+          url: "${{ vars.DEPLOY_URL }}?run_id=${{ github.run_id }}&arch=x64"
diff --git a/reprepro/import.sh b/reprepro/import.sh
@@ -7,73 +7,87 @@ fi
 
 # Sanity checks
 [ ! -d "$INCOMING_DIR" ] && echo "Invalid incoming directory" && exit 1
-[ -z "$DISTRO" ] && echo "Distribution undefined" && exit 1
 [ -z "$ARCH" ] && echo "ARCH undefined" && exit 1
 [ -z "$REPO" ] && echo "github repo undefined" && exit 1
 
 # Translate ARCH x64 -> amd64
 [ "$ARCH" == "x64" ] && ARCH="amd64"
 
-# Operate on the -testing distro
-DISTRO="${DISTRO}-testing"
-
-if [ -n "$RUN_ID" ] ; then
-	echo "Fetching artifact"
-	gh --repo "$REPO" run download --name debs --dir "$INCOMING_DIR" "$RUN_ID"
-elif [ "$(ls -A "$INCOMING_DIR")" ]; then
-	echo "Importing existing files from incoming directory"
-elif [ -n "$GH_TOKEN" ]; then
-   echo "Fetching last debs artifact from https://github.com/$REPO"
-	gh --repo "$REPO" run download --name debs --dir "$INCOMING_DIR"
-fi
-
 function filter {
 	grep -vE "Exporting indices...|Deleting files no longer referenced..."
 }
 
-# Import sources
-if [ "$ARCH" == "amd64" ]; then
-	printf "\nImporting source packages\n"
-	for f in "$INCOMING_DIR"/*.dsc; do
+function import {
+	local distro="$1-testing" # operate on -testing distro
+
+	# Import sources
+	if [ "$ARCH" == "amd64" ]; then
+		printf "\nImporting source packages\n"
+		for f in "$INCOMING_DIR"/*.dsc; do
+			[ -f "$f" ] || break  # Handle case of no files found
+			echo "${f#"$INCOMING_DIR/"}"
+			reprepro includedsc "$distro" "$f" | filter
+		done
+	fi
+
+	# Import packages
+	printf "\nImporting binary packages\n"
+	for f in "$INCOMING_DIR"/*.deb; do
 		[ -f "$f" ] || break  # Handle case of no files found
 		echo "${f#"$INCOMING_DIR/"}"
-		reprepro includedsc "$DISTRO" "$f" | filter
+		reprepro -A "$ARCH" includedeb "$distro" "$f" | filter
 	done
-fi
 
-# Import packages
-printf "\nImporting binary packages\n"
-for f in "$INCOMING_DIR"/*.deb; do
-	[ -f "$f" ] || break  # Handle case of no files found
-	echo "${f#"$INCOMING_DIR/"}"
-	reprepro -A "$ARCH" includedeb "$DISTRO" "$f" | filter
-done
+	# Save log files
+	mkdir -p "log/${distro%-testing}.$ARCH"
+	mv "$INCOMING_DIR"/*.log "log/${distro%-testing}.$ARCH"
 
-# Save log files
-mkdir -p "log/${DISTRO%-testing}.$ARCH"
-mv "$INCOMING_DIR"/*.log "log/${DISTRO%-testing}.$ARCH"
+	# Cleanup files
+	(cd "$INCOMING_DIR" || exit 1; rm -f ./*.log ./*.deb ./*.dsc ./*.tar.gz ./*.tar.xz ./*.changes ./*.buildinfo)
 
-# Cleanup files
-(cd "$INCOMING_DIR" || exit 1; rm -f ./*.log ./*.deb ./*.dsc ./*.tar.gz ./*.tar.xz ./*.changes ./*.buildinfo)
+	# Rename, Import, and Cleanup ddeb files (if existing)
+	printf "\nImporting debug packages\n"
+	for f in "$INCOMING_DIR"/*.ddeb; do
+		[ -f "$f" ] || break  # Handle case of no files found
+		echo "${f#"$INCOMING_DIR/"}"
+		# remove .ddeb suffix
+		f=${f%.ddeb}
+		mv "${f}.ddeb" "${f}.deb"
+		reprepro -A "$ARCH" -C main-dbg includedeb "$distro" "${f}.deb" | filter
+	done
+	(cd "$INCOMING_DIR" || exit 1; rm ./*.deb)
 
-# Rename, Import, and Cleanup ddeb files (if existing)
-printf "\nImporting debug packages\n"
-for f in "$INCOMING_DIR"/*.ddeb; do
-	[ -f "$f" ] || break  # Handle case of no files found
-	echo "${f#"$INCOMING_DIR/"}"
-	# remove .ddeb suffix
-	f=${f%.ddeb}
-	mv "${f}.ddeb" "${f}.deb"
-	reprepro -A "$ARCH" -C main-dbg includedeb "$DISTRO" "${f}.deb" | filter
-done
-(cd "$INCOMING_DIR" || exit 1; rm ./*.deb)
+	printf "\nExporting\n"
+	reprepro export "$distro"
 
-printf "\nExporting\n"
-reprepro export "$DISTRO"
+	# Merge local.yaml into ros-one.yaml
+	cat "$INCOMING_DIR/local.yaml" >> "ros-one.yaml"
+	"$(dirname "${BASH_SOURCE[0]}")/../src/scripts/yaml_remove_duplicates.py" ros-one.yaml
 
-# Merge local.yaml into ros-one.yaml
-cat "$INCOMING_DIR/local.yaml" >> "ros-one.yaml"
-"$(dirname "${BASH_SOURCE[0]}")/../src/scripts/yaml_remove_duplicates.py" ros-one.yaml
+	# Remove remaining files
+	rm -rf "${INCOMING_DIR:?}"/*
+}
 
-# Remove remaining files
-(cd "$INCOMING_DIR" || exit 1; rm -f ./Packages ./Release ./README.md.in ./local.yaml)
+# Download debs artifact(s)
+if [ "$(ls -A "$INCOMING_DIR")" ]; then
+	[ -z "$DISTRO" ] && echo "Distribution undefined" && exit 1
+	echo "Importing existing files from incoming directory"
+	import "$DISTRO"
+else
+	if [ -n "$RUN_ID" ] ; then
+		# Retrieve RUN_ID of latest workflow run
+		RUN_ID=$(gh api -X GET "/repos/$REPO/actions/runs" | jq ".workflow_runs[0] | .id")
+	fi
+	# Retrieve names of artifacts in that workflow run
+	artifacts=$(gh api -X GET "/repos/$REPO/actions/artifacts" | jq --raw-output ".artifacts[] | select(.workflow_run.id == $RUN_ID) | .name")
+	for a in $artifacts; do
+		echo "Fetching artifact \"$a\" from https://github.com/$REPO"
+		gh --repo "$REPO" run download --name "$a" --dir "$INCOMING_DIR" "$RUN_ID" || continue
+		if [ "$distro" == "debs" ]; then
+			distro=$DISTRO
+		else
+			distro=${a%-debs}  # Remove -debs suffix from <distro>-debs artifact name
+		fi
+		import "$distro"
+	done
+fi
diff --git a/reprepro/server.py b/reprepro/server.py
@@ -79,7 +79,9 @@ def process(q: queue.Queue, distro: str, repo: str, arch: str, run_id: str):
 
 
 @app.get("/import")
-def reprepro_import(request: Request, distro: str, run_id: str, arch: str = "x64"):
+def reprepro_import(
+    request: Request, run_id: str, arch: str = "x64", distro: str = "jammy"
+):
     kwargs = dict(
         repo="ubi-agni/ros-builder-action", distro=distro, run_id=run_id, arch=arch
     )
@@ -96,8 +98,9 @@ async def processor():
             while True:
                 try:
                     response = q.get_nowait()
-                    if status == Status.STARTED and response.startswith("Fetching "):
+                    if response.startswith("Fetching "):
                         status = Status.DOWNLOADING
+                        size = -1
                     elif status == Status.DOWNLOADING or status == Status.IMPORTING:
                         status = Status.IMPORTING
                         size = 0
@@ -116,6 +119,7 @@ async def processor():
                     elif status == Status.IMPORTING:
                         size += 1
                         if size >= 10:
+                            size = 0
                             yield f"{colorama.Fore.RED}Import stalled. Killing unzstd.{colorama.Style.RESET_ALL}"
                             subprocess.run(["pkill", "unzstd"], check=False)