Build and Deploy #236
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ## | |
| ## TODO: The following issues still needs to be fixed | |
| ## | |
| ## - [ ] Fix the apt repo itself, so it's a valid apt repo, compatible with Debian etc. | |
| ## | |
| ## - [ ] Build time dependencies, possibly even cached build files, should be cached appropriately, | |
| ## as otherwise builds can take a VERY long time to complete! | |
| ## | |
| ## - [ ] Update the apt repo instead of replacing it, possibly by simply caching the packages? | |
| ## | |
| ## - [ ] Rename or refactor the built kernel version, at the very least append a timestamp | |
| ## and/or a short git commit hash, so we can easily distinguish between different versions | |
| ## | |
| ## - [ ] Decide on the appropriate codename, suite, components etc. and stick with that naming | |
| ## | |
| ## - [ ] Run a quick automated test after deploying the apt repo to GitHub Pages, | |
| ## just to confirm that a) the website is functional and b) it is a valid apt repo (if possible) | |
| ## | |
| name: Build and Deploy | |
| on: | |
| ## TODO: Setup scheduled builds, but only after we have caching working! | |
| ## TODO: Setup proper commit, PR, tags etc. building rules, also after caching works! | |
| #push: | |
| # branches: [ 'build', 'kernel-*' ] | |
| ## FIXME: Add tag based builds, depending on how we want to handle | |
| ## building, deploying and releasing different things?! | |
| # Run on a schedule | |
| # (note that caches are only kept for a week) | |
| schedule: | |
| # Run once a week, every Monday morning | |
| # - cron: '0 3 * * MON' | |
| # Run once a day | |
| - cron: '0 3 * * *' | |
| # Run when manually triggered | |
| workflow_dispatch: | |
| inputs: | |
| force: | |
| type: boolean | |
| description: 'Forced build and deploy (also updates cache)' | |
| required: false | |
| default: false | |
| env: | |
| REPO_PATH: ${{ github.workspace }}/repo | |
| PACKAGES_PATH: ${{ github.workspace }}/packages | |
| KEYS_PATH: ${{ github.workspace }}/keys | |
| BUILD_PATH: ${{ github.workspace }}/build | |
| OUTPUT_PATH: ${{ github.workspace }}/output | |
| TARGET_BRANCH: kernel-5.16 | |
| PACKAGES_ARTIFACT: 'cb1-kernel-packages' | |
| jobs: | |
| # Build job | |
| build: | |
| name: Build | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 300 ## TODO: Increase this if necessary! Benchmark a bit, look into caching etc. | |
| strategy: | |
| fail-fast: true | |
| outputs: | |
| updated: ${{ steps.copy-packages.outputs.update == 'true' || steps.cache-packages.outputs.cache-hit != 'true' }} | |
| steps: | |
| - name: Checkout Repository | |
| timeout-minutes: 15 | |
| uses: actions/checkout@v3 | |
| with: | |
| # Use the target branch instead of our current branch | |
| ref: ${{ env.TARGET_BRANCH }} | |
| # Always limit depth | |
| #fetch-depth: 1 | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Prepare Environment | |
| timeout-minutes: 5 | |
| run: mkdir -p ${{ env.REPO_PATH }} ${{ env.PACKAGES_PATH }} ${{ env.KEYS_PATH }} | |
| - name: Prepare Kernel | |
| id: prepare-kernel | |
| timeout-minutes: 15 | |
| uses: didstopia/cb1-kernel-builder@master | |
| with: | |
| repo: github.com/${{ github.repository }} | |
| branch: ${{ env.TARGET_BRANCH }} | |
| build-path: ${{ env.BUILD_PATH }} | |
| only-detect-kernel-version: yes | |
| - name: Check Kernel Revision | |
| id: check-kernel-revision | |
| timeout-minutes: 5 | |
| run: |- | |
| # Enable debugging | |
| #set -x | |
| ## TODO: Add/change echo statements to utilize GHA logging instead! | |
| ## TODO: Add support for "pre-release" revisions, based on | |
| ## commit hashes, if possible or feasible? | |
| # Prepare the target kernel revision and the revision update state | |
| CURRENT_REVISION="current" | |
| TARGET_REVISION="current" | |
| IS_NEW_REVISION="false" | |
| ## | |
| ## NOTE: We can detect the current revision like so: | |
| ## grep -oP '(?<=REVISION=")[^"]+' scripts/main.sh | |
| ## | |
| # Detect and validate the current kernel revision (eg. "2.3.2") | |
| CURRENT_REVISION_NEW=${{ steps.prepare-kernel.outputs.revision }} | |
| if [ -z "$CURRENT_REVISION_NEW" ]; then | |
| # Output the current values and exit with an error code | |
| echo "target_revision=${TARGET_REVISION}" > $GITHUB_OUTPUT | |
| echo "is_new_revision=${IS_NEW_REVISION}" > $GITHUB_OUTPUT | |
| echo "Failed to get the current kernel revision" | |
| exit 1 | |
| fi | |
| # Store and log the current kernel revision | |
| CURRENT_REVISION="${CURRENT_REVISION_NEW}" | |
| echo "Detected current kernel revision: ${CURRENT_REVISION_NEW}" | |
| # Fetch the latest tags | |
| #git remote add upstream [email protected]:bigtreetech/CB1-Kernel.git | |
| #git fetch --tags upstream | |
| git fetch --tags || true | |
| # Get the latest repository tag, remove the "V" prefix | |
| # (if any), and validate it as the new kernel revision | |
| TARGET_REVISION_NEW=$(git describe --tags `git rev-list --tags --max-count=1 2>/dev/null` 2>/dev/null) || true | |
| if [ -z "$TARGET_REVISION_NEW" ]; then | |
| # Target the current revision instead | |
| TARGET_REVISION_NEW="${CURRENT_REVISION_NEW}" | |
| echo "No git tags found, falling back to revision ${TARGET_REVISION_NEW}" | |
| # Push the new tag to the remote repository | |
| echo "Pushing a new tag for V${TARGET_REVISION_NEW}" | |
| git config user.email "[email protected]" | |
| git config user.name "GitHub Actions" | |
| git tag -a "V${TARGET_REVISION_NEW}" -m "V${TARGET_REVISION_NEW}" | |
| git push origin "V${TARGET_REVISION_NEW}" --force | |
| # Output the current values and exit with an error code | |
| #echo "target_revision=${TARGET_REVISION}" > $GITHUB_OUTPUT | |
| #echo "is_new_revision=${IS_NEW_REVISION}" > $GITHUB_OUTPUT | |
| #echo "Failed to get the new kernel revision" | |
| #exit 1 | |
| fi | |
| # Remove the "V" prefix if it exists | |
| TARGET_REVISION_NEW=${TARGET_REVISION_NEW#V} | |
| # Store and log the new current kernel revision | |
| TARGET_REVISION="${TARGET_REVISION_NEW}" | |
| echo "Detected new kernel revision: ${TARGET_REVISION_NEW}" | |
| # Compare the current and new kernel revisions | |
| if [ "$CURRENT_REVISION_NEW" = "$TARGET_REVISION" ]; then | |
| echo "Detected no changes between kernel revisions" | |
| IS_NEW_REVISION="false" | |
| else | |
| ## TODO: Wouldn't it make more sense to do semver compatible | |
| ## version comparisons, but also validate them as semvers? | |
| echo "Detected changes between kernel revisions" | |
| IS_NEW_REVISION="true" | |
| fi | |
| # Output the target kernel revision and the revision update state | |
| echo "Generating output data:" | |
| echo "-> target_revision=${TARGET_REVISION} | |
| echo "-> is_new_revision=${IS_NEW_REVISION} | |
| echo "target_revision=${TARGET_REVISION}" > $GITHUB_OUTPUT | |
| echo "is_new_revision=${IS_NEW_REVISION}" > $GITHUB_OUTPUT | |
| ## TODO: This is only a test! This might not work AT ALL because there's a ton of files! | |
| - name: Cache Build | |
| id: cache-build | |
| if: ${{ steps.check-kernel-revision.outputs.is_new_revision == 'true' }} | |
| uses: actions/cache@v3 | |
| with: | |
| #path: ${{ env.BUILD_PATH }} | |
| path: build/ | |
| key: ${{ runner.os }}-build-${{ env.TARGET_BRANCH }} | |
| - name: Build Kernel | |
| if: ${{ steps.check-kernel-revision.outputs.is_new_revision == 'true' }} | |
| timeout-minutes: 120 # NOTE: This typically takes less than an hour, but may take longer! | |
| uses: didstopia/cb1-kernel-builder@master | |
| with: | |
| repo: github.com/${{ github.repository }} | |
| branch: ${{ env.TARGET_BRANCH }} | |
| build-path: ${{ env.BUILD_PATH }} | |
| output-path: ${{ env.OUTPUT_PATH }} | |
| target-board: h616 | |
| target-branch: ${{ steps.check-kernel-revision.outputs.target_revision }} | |
| # target-branch: current # FIXME: Should be keep this "current", to match the official builds? | |
| # target-branch: ${{ env.TARGET_BRANCH }} | |
| # target-branch: cb1-${{ env.TARGET_BRANCH }} | |
| target-build-option: kernel # FIXME: We also want to build u-boot AND the image, right?! | |
| target-release: bullseye # FIXME: Do we want to set this to something else? | |
| - name: Build U-Boot | |
| if: ${{ steps.check-kernel-revision.outputs.is_new_revision == 'true' }} | |
| timeout-minutes: 120 | |
| uses: didstopia/cb1-kernel-builder@master | |
| with: | |
| repo: github.com/${{ github.repository }} | |
| branch: ${{ env.TARGET_BRANCH }} | |
| build-path: ${{ env.BUILD_PATH }} | |
| output-path: ${{ env.OUTPUT_PATH }} | |
| target-board: h616 | |
| target-branch: ${{ steps.check-kernel-revision.outputs.target_revision }} | |
| # target-branch: current # FIXME: Should be keep this "current", to match the official builds? | |
| # target-branch: ${{ env.TARGET_BRANCH }} | |
| # target-branch: cb1-${{ env.TARGET_BRANCH }} | |
| target-build-option: u-boot # FIXME: We also want to build u-boot AND the image, right?! | |
| target-release: bullseye # FIXME: Do we want to set this to something else? | |
| ## TODO: Does this even work correctly? Pretty sure it does NOT, | |
| ## and instead it just updates the packages with every single run?! | |
| - name: Copy Packages | |
| id: copy-packages | |
| timeout-minutes: 5 | |
| run: |- | |
| # Generate hashes for the previous and new packages | |
| OLD_PACKAGES_HASH=${{ hashFiles('packages/**/*') }} | |
| NEW_PACKAGES_HASH=${{ hashFiles('output/**/*') }} | |
| # Check if we are forcefully updating | |
| if [ "${{ github.event.inputs.force }}" = "true" ]; then | |
| echo "Forced update detected, updating all packages (where possible)" | |
| # Copy any built packages to the packages folder | |
| #cp -fr ${{ env.OUTPUT_PATH }}/debs/*.deb ${{ env.PACKAGES_PATH }}/ | |
| find ${{ env.OUTPUT_PATH }} -name '*.deb' -exec cp -prv '{}' '${{ env.PACKAGES_PATH }}/' ';' | |
| echo "update=true" >> "$GITHUB_OUTPUT" | |
| # Check if the package hashes haven't changed | |
| elif [ "$OLD_PACKAGES_HASH" = "$NEW_PACKAGES_HASH" ]; then | |
| echo "No changes detected in packages, skipping updates" | |
| # No changes to packages, no update necessary | |
| echo "update=false" >> "$GITHUB_OUTPUT" | |
| # Check if we have a new kernel revision | |
| elif [ "${{ steps.check-kernel-revision.outputs.is_new_revision }}" = "true" ]; then | |
| echo "New kernel revision detected, updating all packages (where possible)" | |
| # Copy any built packages to the packages folder | |
| #cp -fr ${{ env.OUTPUT_PATH }}/debs/*.deb ${{ env.PACKAGES_PATH }}/ | |
| find ${{ env.OUTPUT_PATH }} -name '*.deb' -exec cp -prv '{}' '${{ env.PACKAGES_PATH }}/' ';' | |
| echo "update=true" >> "$GITHUB_OUTPUT" | |
| # Fallback, update all packages where possible | |
| else | |
| echo "Fallback detected, updating all packages (where possible)" | |
| # Copy any built packages to the packages folder | |
| #cp -fr ${{ env.OUTPUT_PATH }}/debs/*.deb ${{ env.PACKAGES_PATH }}/ | |
| find ${{ env.OUTPUT_PATH }} -name '*.deb' -exec cp -prv '{}' '${{ env.PACKAGES_PATH }}/' ';' | |
| echo "update=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| ## TODO: Remove when done debugging! | |
| - name: List Packages | |
| #if: ${{ steps.copy-packages.outputs.update == 'true' }} | |
| run: ls -la ${{ env.PACKAGES_PATH }}/ || true | |
| ## TODO: Does this work? Verify that it does! | |
| - name: Cache Packages | |
| #if: ${{ steps.copy-packages.outputs.update == 'true' }} | |
| id: cache-packages | |
| uses: actions/cache@v3 | |
| with: | |
| #path: ${{ env.PACKAGES_PATH }} | |
| path: packages/ | |
| # key: ${{ runner.os }}-packages-${{ env.TARGET_BRANCH }} | |
| # key: ${{ runner.os }}-packages-${{ hashFiles('packages/**/*') }} | |
| # key: ${{ runner.os }}-packages-${{ hashFiles('packages/*.deb') }} | |
| key: ${{ runner.os }}-packages-${{ hashFiles('**.deb') }} | |
| ## TODO: Verify that this works as intended, as we're | |
| ## likely getting package updates when we shouldn't be.. | |
| - name: Upload Build Artifacts | |
| if: ${{ steps.copy-packages.outputs.update == 'true' }} | |
| timeout-minutes: 15 | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: build-artifacts-${{ env.TARGET_BRANCH }} | |
| #path: ${{ env.OUTPUT_PATH }} | |
| path: output/ | |
| retention-days: 30 | |
| if-no-files-found: error | |
| - name: Build Apt Repository | |
| if: ${{ (steps.copy-packages.outputs.update == 'true' || steps.cache-packages.outputs.cache-hit != 'true') }} | |
| timeout-minutes: 5 | |
| uses: didstopia/simple-apt-repo@master | |
| with: | |
| # timezone: Europe/Helsinki | |
| timezone: UTC | |
| repo-origin: Didstopia CB1 Apt Repo | |
| repo-label: Didstopia BTT CB1 Apt Repository (2023) | |
| repo-version: 0.1.0 | |
| repo-description: Didstopia custom apt repository for the BIGTREETECH CB1 hardware, including both stock and custom kernel packages. | |
| key-email: [email protected] | |
| key-public: ${{ secrets.APT_REPO_PUBLIC_KEY }} | |
| key-private: ${{ secrets.APT_REPO_PRIVATE_KEY }} | |
| - name: Generate Repository File Index | |
| if: ${{ (steps.copy-packages.outputs.update == 'true' || steps.cache-packages.outputs.cache-hit != 'true') }} | |
| uses: jayanta525/[email protected] | |
| with: | |
| FOLDER: repo | |
| - name: List Repository Contents | |
| if: ${{ (steps.copy-packages.outputs.update == 'true' || steps.cache-packages.outputs.cache-hit != 'true') }} | |
| run: ls -la ${{ env.REPO_PATH }}/ | |
| ## TODO: Customize apt repository HTML/JS/CSS here! | |
| - name: Upload Apt Repository | |
| if: ${{ (steps.copy-packages.outputs.update == 'true' || steps.cache-packages.outputs.cache-hit != 'true') }} | |
| timeout-minutes: 15 | |
| uses: actions/upload-pages-artifact@v1 | |
| with: | |
| path: ${{ env.REPO_PATH }}/ | |
| retention-days: 1 | |
| # Deploy job | |
| deploy: | |
| name: Deploy | |
| # Add a dependency to the build job | |
| needs: build | |
| # Only run if we got new builds | |
| if: ${{ needs.build.outputs.updated == 'true' }} | |
| # Grant GITHUB_TOKEN the permissions required to make a Pages deployment | |
| permissions: | |
| pages: write # to deploy to Pages | |
| id-token: write # to verify the deployment originates from an appropriate source | |
| # Deploy to the github-pages environment | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| # Specify runner + deployment step | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Deploy to GitHub Pages | |
| id: deployment | |
| uses: actions/deploy-pages@v1 | |
| - name: Display Deployment URL | |
| run: | | |
| echo "::notice::${{ steps.deployment.outputs.page_url }}" | |
| echo "URL=${{ steps.deployment.outputs.page_url }}" >> $GITHUB_OUTPUT |