diff --git a/.github/actions/docker-build-artifacts/action.yml b/.github/actions/docker-build-artifacts/action.yml new file mode 100644 index 00000000..39fc3b53 --- /dev/null +++ b/.github/actions/docker-build-artifacts/action.yml @@ -0,0 +1,110 @@ +name: Build artifacts using Dockerfile + +inputs: + REPO_DOMAIN: + required: false + description: Domain name of repository + PLATFORM: + required: true + description: Default Linux Arch (amd64/arm32v7/...) + DOCKERFILE: + required: true + description: Path to Dockerfile + MAINTAINER: + required: true + description: Package maintainer + WORKING_DIRECTORY: + required: true + default: '.' + description: Working directory + ARTIFACTS_PATTERN: + required: false + default: '.*\.(deb|rpm)$' + description: Regexp that matches artifacts + ARTIFACTS_DIR: + required: false + default: 'BUILD' + description: Output directory for artifacts + BUILD_LOG_FILENAME: + required: false + default: 'build.log' + description: Build log filename + +runs: + using: "composite" + steps: + + - name: Set up QEMU for Docker + uses: docker/setup-qemu-action@v3 + + - name: Build Docker image + shell: bash + working-directory: ${{ inputs.WORKING_DIRECTORY }} + env: + REPO_PASSWORD: ${{ env.REPO_PASSWORD }} + run: | + docker build \ + --build-arg BUILD_NUMBER="${GITHUB_RUN_ID}" \ + --build-arg GIT_SHA="$(echo ${GITHUB_SHA} | cut -c1-10)" \ + --build-arg MAINTAINER="${{ inputs.MAINTAINER }}" \ + --build-arg REPO_DOMAIN="${{ inputs.REPO_DOMAIN }}" \ + --build-arg REPO_USERNAME="${{ env.REPO_USERNAME }}" \ + --file "${{ inputs.DOCKERFILE }}" \ + --no-cache \ + --platform linux/${{ inputs.PLATFORM }} \ + --progress=plain \ + --secret id=REPO_PASSWORD,env=REPO_PASSWORD \ + --tag artifacts-${GITHUB_RUN_ID}:${GITHUB_SHA} \ + --ulimit nofile=1024000:1024000 \ + . 2>&1 | tee -a ${{ inputs.BUILD_LOG_FILENAME }} + + - name: Extract artifacts from image + shell: bash + working-directory: ${{ inputs.WORKING_DIRECTORY }} + run: | + set -euo pipefail + + export TEMP_DIR=$(mktemp -d) + + # dump Docker image blobs + docker save artifacts-${GITHUB_RUN_ID}:${GITHUB_SHA} --output "${TEMP_DIR}/artifacts-${GITHUB_RUN_ID}-${GITHUB_SHA}.tar" && \ + tar -xf "${TEMP_DIR}/artifacts-${GITHUB_RUN_ID}-${GITHUB_SHA}.tar" -C "${TEMP_DIR}" && \ + rm -f "${TEMP_DIR}/artifacts-${GITHUB_RUN_ID}-${GITHUB_SHA}.tar" + + # extract blobs content + mkdir -p "${{ inputs.ARTIFACTS_DIR }}" && find "${TEMP_DIR}/" -type f -exec file {} + \ + | grep -E ":.*tar archive" \ + | cut -d: -f1 \ + | xargs -rI{} tar --keep-newer-files -xf {} -C "${{ inputs.ARTIFACTS_DIR }}" + + # cleanup + docker image rm artifacts-${GITHUB_RUN_ID}:${GITHUB_SHA} && \ + rm -rf "${TEMP_DIR}" + + if [ "$(find "${{ inputs.ARTIFACTS_DIR }}" -type f | wc -l)" -lt 1 ]; then + echo "No files found in ${{ inputs.ARTIFACTS_DIR }}." + exit 1 + fi + + - name: Filter artifacts by pattern + shell: bash + working-directory: ${{ inputs.WORKING_DIRECTORY }} + run: | + set -euo pipefail + + export TEMP_DIR=$(mktemp -d) + + find "${{ inputs.ARTIFACTS_DIR }}" \ + -type f \ + -regextype posix-extended \ + -regex "${{ inputs.ARTIFACTS_PATTERN }}" \ + -exec sh -c 'mv -vf "$1" "${TEMP_DIR}/$(basename "$1")"' _ {} \; && \ + rm -rvf "${{ inputs.ARTIFACTS_DIR }}" && \ + mv -v "${TEMP_DIR}" "${{ inputs.ARTIFACTS_DIR }}" + + if [ "$(find "${{ inputs.ARTIFACTS_DIR }}" -type f | wc -l)" -lt 1 ]; then + echo "No files found in ${{ inputs.ARTIFACTS_DIR }}." + exit 1 + fi + + printf ${GITHUB_SHA} | tee "${{ inputs.ARTIFACTS_DIR }}/hash.txt" diff --git a/.github/actions/teleport/action.yml b/.github/actions/teleport/action.yml index 95b92ba0..a3902f24 100644 --- a/.github/actions/teleport/action.yml +++ b/.github/actions/teleport/action.yml @@ -13,9 +13,12 @@ inputs: K8S_CLUSTER_NAME: required: false description: 'kubernetes cluster name to connect' + EXEC_COMMANDS_PRE: + required: false + description: 'whenever you would like to execute commands on remote before files copy' EXEC_COMMANDS: required: false - description: 'whenever you would like to execute commands on remote' + description: 'whenever you would like to execute commands on remote after files copy' FILES: required: false description: 'Files to move to remote host' @@ -83,14 +86,20 @@ runs: certificate-ttl: 1h kubernetes-cluster: ${{ inputs.K8S_CLUSTER_NAME }} - + + - name: Execute commands on remote before files copy + if: inputs.EXEC_COMMANDS_PRE != '' + run: > + tsh -i ${{ steps.auth.outputs.identity-file }} --login=${{env.USERNAME}} ssh ${{ env.HOSTNAME }} '${{ inputs.EXEC_COMMANDS_PRE }}' + shell: bash + - name: Copy files to remote if: inputs.FILES != '' run: > tsh scp -i ${{ steps.auth.outputs.identity-file }} --login=${{env.USERNAME}} ${{ inputs.FILES }} ${{ env.HOSTNAME }}:${{ inputs.FILES_FOLDER }} shell: bash - - name: Execute commands on remote + - name: Execute commands on remote after files copy if: inputs.EXEC_COMMANDS != '' run: > tsh -i ${{ steps.auth.outputs.identity-file }} --login=${{env.USERNAME}} ssh ${{ env.HOSTNAME }} '${{ inputs.EXEC_COMMANDS }}' diff --git a/.github/workflows/cicd-docker-build-and-distribute.yml b/.github/workflows/cicd-docker-build-and-distribute.yml index 84d86583..15d58af1 100644 --- a/.github/workflows/cicd-docker-build-and-distribute.yml +++ b/.github/workflows/cicd-docker-build-and-distribute.yml @@ -42,21 +42,6 @@ on: type: boolean default: true description: Enable upload build artifacts related steps - CLEAN_BUILD_ARTIFACTS: - required: false - type: boolean - default: true - description: Remove build artifacts after publish - CREATE_DESTINATION_FOLDERS: - required: false - type: boolean - default: true - description: Create folder on destination - BUILD_ARTIFACT_RETENTION_DAYS: - required: false - type: number - default: 1 - description: Number of days to keep build artifacts LOG_RETENTION_DAYS: required: false type: number @@ -82,6 +67,16 @@ on: type: string default: 'main' description: Target meta repo default branch name + CODE_WORKING_DIRECTORY: + required: false + type: string + default: 'code' + description: Working directory to place code into + ARTIFACTS_DIR: + required: false + type: string + default: 'build' + description: Directory to place produced artifacts into (will be inside CODE_WORKING_DIRECTORY) secrets: HOSTNAME: @@ -100,7 +95,7 @@ on: required: true jobs: - build: + build-and-distribute: runs-on: ${{ inputs.RUNNER }} permissions: contents: read @@ -112,152 +107,63 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - path: code - - - name: Set up QEMU for Docker - uses: docker/setup-qemu-action@v3 - - - name: Build Docker image - shell: bash - working-directory: code - run: | - env REPO_PASSWORD='${{ secrets.REPO_PASSWORD }}' docker build \ - --build-arg BUILD_NUMBER="${GITHUB_RUN_ID}" \ - --build-arg GIT_SHA="$(echo ${GITHUB_SHA} | cut -c1-10)" \ - --build-arg MAINTAINER="${{ inputs.MAINTAINER }}" \ - --build-arg REPO_DOMAIN="${{ inputs.REPO_DOMAIN }}" \ - --build-arg REPO_USERNAME="${{ secrets.REPO_USERNAME }}" \ - --file "${{ inputs.DOCKERFILE }}" \ - --no-cache \ - --platform linux/${{ inputs.PLATFORM }} \ - --progress=plain \ - --secret id=REPO_PASSWORD,env=REPO_PASSWORD \ - --tag artifacts-${GITHUB_RUN_ID}:${GITHUB_SHA} \ - --ulimit nofile=1024000:1024000 \ - . 2>&1 | tee artifacts-${GITHUB_RUN_ID}-${GITHUB_SHA}.log - - - name: Extract artifacts from image - shell: bash - working-directory: code - run: | - set -euo pipefail - - export ARTIFACTS_DIR="./out" - echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" | tee -a "${GITHUB_ENV}" + path: ${{ inputs.CODE_WORKING_DIRECTORY }} - export TEMP_DIR=$(mktemp -d) - - # dump Docker image blobs - docker save artifacts-${GITHUB_RUN_ID}:${GITHUB_SHA} --output "${TEMP_DIR}/artifacts-${GITHUB_RUN_ID}-${GITHUB_SHA}.tar" && \ - tar -xf "${TEMP_DIR}/artifacts-${GITHUB_RUN_ID}-${GITHUB_SHA}.tar" -C "${TEMP_DIR}" && \ - rm -f "${TEMP_DIR}/artifacts-${GITHUB_RUN_ID}-${GITHUB_SHA}.tar" - - # extract blobs content - mkdir -p "${ARTIFACTS_DIR}" && find "${TEMP_DIR}/" -type f -exec file {} + \ - | grep -E ":.*tar archive" \ - | cut -d: -f1 \ - | xargs -rI{} tar --keep-newer-files -xf {} -C "${ARTIFACTS_DIR}" - - # cleanup - docker image rm artifacts-${GITHUB_RUN_ID}:${GITHUB_SHA} && \ - rm -rf "${TEMP_DIR}" - - if [ "$(find "${ARTIFACTS_DIR}" -type f | wc -l)" -lt 1 ]; then - echo "No files found in ${ARTIFACTS_DIR}." - exit 1 - fi - - - name: Filter artifacts by pattern - shell: bash - working-directory: code - run: | - set -euo pipefail - - export TEMP_DIR=$(mktemp -d) - - find "${ARTIFACTS_DIR}" \ - -type f \ - -regextype posix-extended \ - -regex "${{ inputs.ARTIFACTS_PATTERN || env.ARTIFACTS_PATTERN }}" \ - -exec sh -c 'mv -vf "$1" "${TEMP_DIR}/$(basename "$1")"' _ {} \; && \ - rm -rvf "${ARTIFACTS_DIR}" && \ - mv -v "${TEMP_DIR}" "${ARTIFACTS_DIR}" - - if [ "$(find "${ARTIFACTS_DIR}" -type f | wc -l)" -lt 1 ]; then - echo "No files found in ${ARTIFACTS_DIR}." - exit 1 - fi + - name: Checkout reusable actions + uses: actions/checkout@v4 + with: + repository: signalwire/actions-template + ref: main + fetch-depth: 1 + path: actions + sparse-checkout: | + .github/actions/teleport/action.yml + .github/actions/docker-build-artifacts/action.yml + sparse-checkout-cone-mode: false - printf ${GITHUB_SHA} | tee "${ARTIFACTS_DIR}/hash.txt" + - name: Build artifacts via Docker + uses: ./actions/.github/actions/docker-build-artifacts + with: + REPO_DOMAIN: ${{ inputs.REPO_DOMAIN }} + PLATFORM: ${{ inputs.PLATFORM }} + DOCKERFILE: ${{ inputs.DOCKERFILE }} + MAINTAINER: ${{ inputs.MAINTAINER }} + WORKING_DIRECTORY: ${{ inputs.CODE_WORKING_DIRECTORY }} + ARTIFACTS_PATTERN: ${{ inputs.ARTIFACTS_PATTERN }} + ARTIFACTS_DIR: ${{ inputs.ARTIFACTS_DIR }} + BUILD_LOG_FILENAME: 'artifacts-${GITHUB_RUN_ID}-${GITHUB_SHA}.log' + env: + REPO_USERNAME: ${{ secrets.REPO_USERNAME }} + REPO_PASSWORD: ${{ secrets.REPO_PASSWORD }} - name: Upload build logs uses: actions/upload-artifact@v4 with: name: ${{ inputs.TARGET_ARTIFACT_NAME }}.log - path: code/artifacts-*.log + path: ${{ inputs.CODE_WORKING_DIRECTORY }}/artifacts-*.log if-no-files-found: warn retention-days: ${{ inputs.LOG_RETENTION_DAYS }} - name: Compress build artifacts if: ${{ inputs.UPLOAD_BUILD_ARTIFACTS }} shell: bash - working-directory: code + working-directory: ${{ inputs.CODE_WORKING_DIRECTORY }} + env: + ARTIFACTS_DIR: ${{ inputs.ARTIFACTS_DIR }} run: | tar -czvf ${{ inputs.TARGET_ARTIFACT_NAME }}.tar.gz -C "${ARTIFACTS_DIR}" $(ls -1 "${ARTIFACTS_DIR}") sha512sum ${{ inputs.TARGET_ARTIFACT_NAME }}.tar.gz | tee ${{ inputs.TARGET_ARTIFACT_NAME }}.tar.gz.sha512 - - name: Upload build artifacts - if: ${{ inputs.UPLOAD_BUILD_ARTIFACTS }} - uses: actions/upload-artifact@v4 - with: - name: ${{ inputs.TARGET_ARTIFACT_NAME }} - path: | - code/*.tar.gz - code/*.sha512 - if-no-files-found: error - retention-days: ${{ inputs.BUILD_ARTIFACT_RETENTION_DAYS }} - - distribute: - runs-on: ${{ inputs.RUNNER }} - if: (github.ref_type == 'branch' && github.base_ref == '' && inputs.UPLOAD_BUILD_ARTIFACTS) - needs: build - permissions: - id-token: write - contents: read - environment: ${{ inputs.ENVIRONMENT }} - steps: - - - name: Checkout reusable actions - uses: actions/checkout@v4 - with: - repository: signalwire/actions-template - ref: main - fetch-depth: 1 - path: actions - sparse-checkout: | - .github/actions/teleport/action.yml - sparse-checkout-cone-mode: false - - - name: Create destination folder on remote host - if: ${{ inputs.CREATE_DESTINATION_FOLDERS }} - uses: ./actions/.github/actions/teleport - with: - EXEC_COMMANDS: mkdir -p ${{ inputs.META_FILE_PATH_PREFIX }} - env: - HOSTNAME: ${{ secrets.HOSTNAME }} - PROXY_URL: ${{ secrets.PROXY_URL }} - TOKEN: ${{ secrets.TELEPORT_TOKEN }} - USERNAME: ${{ secrets.USERNAME }} + tar -czvf temp_${{ inputs.TARGET_ARTIFACT_NAME }}.tar.gz ${{ inputs.TARGET_ARTIFACT_NAME }}.tar.gz ${{ inputs.TARGET_ARTIFACT_NAME }}.tar.gz.sha512 + rm -v ${{ inputs.TARGET_ARTIFACT_NAME }}.tar.gz ${{ inputs.TARGET_ARTIFACT_NAME }}.tar.gz.sha512 && \ + mv -v temp_${{ inputs.TARGET_ARTIFACT_NAME }}.tar.gz ${{ inputs.TARGET_ARTIFACT_NAME }}.tar.gz - - name: Download build artifacts - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.TARGET_ARTIFACT_NAME }} - - - name: Copy build artifacts to remote host + - name: Copy/Exec artifacts/commands to/on remote host + if: ${{ inputs.UPLOAD_BUILD_ARTIFACTS }} uses: ./actions/.github/actions/teleport with: - FILES: '*.tar.gz' + EXEC_COMMANDS_PRE: 'mkdir -p ${{ inputs.META_FILE_PATH_PREFIX }}' + FILES: './${{ inputs.CODE_WORKING_DIRECTORY }}/*.tar.gz' EXEC_COMMANDS: 'echo "${{ github.sha }}" > ${{ inputs.META_FILE_PATH_PREFIX }}/hash.txt' FILES_FOLDER: ${{ inputs.META_FILE_PATH_PREFIX }} env: @@ -266,19 +172,11 @@ jobs: TOKEN: ${{ secrets.TELEPORT_TOKEN }} USERNAME: ${{ secrets.USERNAME }} - - name: Delete build artifact - if: ${{ inputs.CLEAN_BUILD_ARTIFACTS }} - uses: geekyeggo/delete-artifact@v5 - with: - name: ${{ inputs.TARGET_ARTIFACT_NAME }} - failOnError: false - meta-repo: runs-on: ${{ inputs.RUNNER }} - if: (github.ref_type == 'branch' && github.base_ref == '' && inputs.UPLOAD_BUILD_ARTIFACTS) + if: ${{ inputs.UPLOAD_BUILD_ARTIFACTS }} needs: - - build - - distribute + - build-and-distribute permissions: id-token: write contents: read