diff --git a/.github/workflows/scheduled.yaml b/.github/workflows/scheduled.yaml index e28c282..11b851e 100644 --- a/.github/workflows/scheduled.yaml +++ b/.github/workflows/scheduled.yaml @@ -41,4 +41,4 @@ jobs: - name: Import build artifacts to reprepro server uses: ubi-agni/ros-builder-action/reprepro@main with: - url: "${{ vars.DEPLOY_URL }}?distro=${{ matrix.DEB_DISTRO }}&run_id=${{ github.run_id }}&arch=x64" + url: "${{ vars.DEPLOY_URL }}?run_id=${{ github.run_id }}&arch=x64" diff --git a/reprepro/import.sh b/reprepro/import.sh index 5a6ff34..1b1952e 100755 --- a/reprepro/import.sh +++ b/reprepro/import.sh @@ -7,73 +7,87 @@ fi # Sanity checks [ ! -d "$INCOMING_DIR" ] && echo "Invalid incoming directory" && exit 1 -[ -z "$DISTRO" ] && echo "Distribution undefined" && exit 1 [ -z "$ARCH" ] && echo "ARCH undefined" && exit 1 [ -z "$REPO" ] && echo "github repo undefined" && exit 1 # Translate ARCH x64 -> amd64 [ "$ARCH" == "x64" ] && ARCH="amd64" -# Operate on the -testing distro -DISTRO="${DISTRO}-testing" - -if [ -n "$RUN_ID" ] ; then - echo "Fetching artifact" - gh --repo "$REPO" run download --name debs --dir "$INCOMING_DIR" "$RUN_ID" -elif [ "$(ls -A "$INCOMING_DIR")" ]; then - echo "Importing existing files from incoming directory" -elif [ -n "$GH_TOKEN" ]; then - echo "Fetching last debs artifact from https://github.com/$REPO" - gh --repo "$REPO" run download --name debs --dir "$INCOMING_DIR" -fi - function filter { grep -vE "Exporting indices...|Deleting files no longer referenced..." } -# Import sources -if [ "$ARCH" == "amd64" ]; then - printf "\nImporting source packages\n" - for f in "$INCOMING_DIR"/*.dsc; do +function import { + local distro="$1-testing" # operate on -testing distro + + # Import sources + if [ "$ARCH" == "amd64" ]; then + printf "\nImporting source packages\n" + for f in "$INCOMING_DIR"/*.dsc; do + [ -f "$f" ] || break # Handle case of no files found + echo "${f#"$INCOMING_DIR/"}" + reprepro includedsc "$distro" "$f" | filter + done + fi + + # Import packages + printf "\nImporting binary packages\n" + for f in "$INCOMING_DIR"/*.deb; do [ -f "$f" ] || break # Handle case of no files found echo "${f#"$INCOMING_DIR/"}" - reprepro includedsc "$DISTRO" "$f" | filter + reprepro -A "$ARCH" includedeb "$distro" "$f" | filter done -fi -# Import packages -printf "\nImporting binary packages\n" -for f in "$INCOMING_DIR"/*.deb; do - [ -f "$f" ] || break # Handle case of no files found - echo "${f#"$INCOMING_DIR/"}" - reprepro -A "$ARCH" includedeb "$DISTRO" "$f" | filter -done + # Save log files + mkdir -p "log/${distro%-testing}.$ARCH" + mv "$INCOMING_DIR"/*.log "log/${distro%-testing}.$ARCH" -# Save log files -mkdir -p "log/${DISTRO%-testing}.$ARCH" -mv "$INCOMING_DIR"/*.log "log/${DISTRO%-testing}.$ARCH" + # Cleanup files + (cd "$INCOMING_DIR" || exit 1; rm -f ./*.log ./*.deb ./*.dsc ./*.tar.gz ./*.tar.xz ./*.changes ./*.buildinfo) -# Cleanup files -(cd "$INCOMING_DIR" || exit 1; rm -f ./*.log ./*.deb ./*.dsc ./*.tar.gz ./*.tar.xz ./*.changes ./*.buildinfo) + # Rename, Import, and Cleanup ddeb files (if existing) + printf "\nImporting debug packages\n" + for f in "$INCOMING_DIR"/*.ddeb; do + [ -f "$f" ] || break # Handle case of no files found + echo "${f#"$INCOMING_DIR/"}" + # remove .ddeb suffix + f=${f%.ddeb} + mv "${f}.ddeb" "${f}.deb" + reprepro -A "$ARCH" -C main-dbg includedeb "$distro" "${f}.deb" | filter + done + (cd "$INCOMING_DIR" || exit 1; rm ./*.deb) -# Rename, Import, and Cleanup ddeb files (if existing) -printf "\nImporting debug packages\n" -for f in "$INCOMING_DIR"/*.ddeb; do - [ -f "$f" ] || break # Handle case of no files found - echo "${f#"$INCOMING_DIR/"}" - # remove .ddeb suffix - f=${f%.ddeb} - mv "${f}.ddeb" "${f}.deb" - reprepro -A "$ARCH" -C main-dbg includedeb "$DISTRO" "${f}.deb" | filter -done -(cd "$INCOMING_DIR" || exit 1; rm ./*.deb) + printf "\nExporting\n" + reprepro export "$distro" -printf "\nExporting\n" -reprepro export "$DISTRO" + # Merge local.yaml into ros-one.yaml + cat "$INCOMING_DIR/local.yaml" >> "ros-one.yaml" + "$(dirname "${BASH_SOURCE[0]}")/../src/scripts/yaml_remove_duplicates.py" ros-one.yaml -# Merge local.yaml into ros-one.yaml -cat "$INCOMING_DIR/local.yaml" >> "ros-one.yaml" -"$(dirname "${BASH_SOURCE[0]}")/../src/scripts/yaml_remove_duplicates.py" ros-one.yaml + # Remove remaining files + rm -rf "${INCOMING_DIR:?}"/* +} -# Remove remaining files -(cd "$INCOMING_DIR" || exit 1; rm -f ./Packages ./Release ./README.md.in ./local.yaml) +# Download debs artifact(s) +if [ "$(ls -A "$INCOMING_DIR")" ]; then + [ -z "$DISTRO" ] && echo "Distribution undefined" && exit 1 + echo "Importing existing files from incoming directory" + import "$DISTRO" +else + if [ -n "$RUN_ID" ] ; then + # Retrieve RUN_ID of latest workflow run + RUN_ID=$(gh api -X GET "/repos/$REPO/actions/runs" | jq ".workflow_runs[0] | .id") + fi + # Retrieve names of artifacts in that workflow run + artifacts=$(gh api -X GET "/repos/$REPO/actions/artifacts" | jq --raw-output ".artifacts[] | select(.workflow_run.id == $RUN_ID) | .name") + for a in $artifacts; do + echo "Fetching artifact \"$a\" from https://github.com/$REPO" + gh --repo "$REPO" run download --name "$a" --dir "$INCOMING_DIR" "$RUN_ID" || continue + if [ "$distro" == "debs" ]; then + distro=$DISTRO + else + distro=${a%-debs} # Remove -debs suffix from -debs artifact name + fi + import "$distro" + done +fi diff --git a/reprepro/server.py b/reprepro/server.py index 33b12f2..2f924c8 100644 --- a/reprepro/server.py +++ b/reprepro/server.py @@ -79,7 +79,9 @@ def process(q: queue.Queue, distro: str, repo: str, arch: str, run_id: str): @app.get("/import") -def reprepro_import(request: Request, distro: str, run_id: str, arch: str = "x64"): +def reprepro_import( + request: Request, run_id: str, arch: str = "x64", distro: str = "jammy" +): kwargs = dict( repo="ubi-agni/ros-builder-action", distro=distro, run_id=run_id, arch=arch ) @@ -96,8 +98,9 @@ async def processor(): while True: try: response = q.get_nowait() - if status == Status.STARTED and response.startswith("Fetching "): + if response.startswith("Fetching "): status = Status.DOWNLOADING + size = -1 elif status == Status.DOWNLOADING or status == Status.IMPORTING: status = Status.IMPORTING size = 0 @@ -116,6 +119,7 @@ async def processor(): elif status == Status.IMPORTING: size += 1 if size >= 10: + size = 0 yield f"{colorama.Fore.RED}Import stalled. Killing unzstd.{colorama.Style.RESET_ALL}" subprocess.run(["pkill", "unzstd"], check=False)