diff --git a/.github/actions/configure_git/README.md b/.github/actions/configure_git/README.md new file mode 100644 index 0000000000..19d3e8aea9 --- /dev/null +++ b/.github/actions/configure_git/README.md @@ -0,0 +1,21 @@ +## Prerequisite + +This action assumes that the repository has already been checked out before +calling the action, typically using `actions/checkout@v4`. If you have not +checked out the code in a previous step, make sure to do so to avoid errors. + +This action does **not** perform a checkout action itself because it would be +redundant. This action is part of the repository's codebase, so if the code +hasn't already been checked out, the action itself wouldn't even be available to +call. + +## Example Usage + +```yaml +steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/configure_git + with: + gpg_signing_key: ${{ secrets.GPG_SIGNING_KEY }} +``` diff --git a/.github/actions/configure_git/action.yml b/.github/actions/configure_git/action.yml new file mode 100644 index 0000000000..6f18cb7a71 --- /dev/null +++ b/.github/actions/configure_git/action.yml @@ -0,0 +1,18 @@ +name: Configure Git +description: 'Configures git with user info and GPG signing' +inputs: + gpg_signing_key: + description: 'The GPG signing key to use for signing commits' + required: true +runs: + using: composite + steps: + - name: Configure git + run: | + # TODO we should use some Action-specific bot account + git config --global user.name 'Jordan Last' + git config --global user.email 'jordan.michael.last@gmail.com' + git config --global commit.gpgsign true + echo -n "${{ inputs.gpg_signing_key }}" | base64 --decode | gpg --import + git config --global user.signingkey C8B77BCBE16CD2B94B43F9C8757397B82D4ED7B0 + shell: bash diff --git a/.github/actions/get_dfx_version/action.yml b/.github/actions/get_dfx_version/action.yml deleted file mode 100644 index 3797571230..0000000000 --- a/.github/actions/get_dfx_version/action.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: Get dfx version -description: Determines Azle's dfx version -outputs: - dfx-version: - description: Returns the version of dfx that Azle will test against and use to generate its Wasm binary templates - value: ${{ steps.get-dfx-version.outputs.dfx-version }} -runs: - using: composite - steps: - - id: get-dfx-version - run: | - DFX_VERSION=$(jq -r '.azle.globalDependencies.dfx // error("dfx version not found")' "package.json") - echo "dfx-version=${DFX_VERSION}" >> "$GITHUB_OUTPUT" - shell: bash diff --git a/.github/actions/get_node_version/action.yml b/.github/actions/get_node_version/action.yml deleted file mode 100644 index b97e3d32e0..0000000000 --- a/.github/actions/get_node_version/action.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: Get node version -description: Determines Azle's node version -outputs: - node-version: - description: Returns the version of node that Azle will test against and use to generate its Wasm binary templates - value: ${{ steps.get-node-version.outputs.node-version }} -runs: - using: composite - steps: - - id: get-node-version - run: | - NODE_VERSION=$(jq -r '.azle.globalDependencies.node // error("node version not found")' "package.json") - echo "node-version=${NODE_VERSION}" >> "$GITHUB_OUTPUT" - shell: bash diff --git a/.github/actions/get_test_infos/action.yml b/.github/actions/get_test_infos/action.yml index 5c421bb733..b2fe19158a 100644 --- a/.github/actions/get_test_infos/action.yml +++ b/.github/actions/get_test_infos/action.yml @@ -23,12 +23,7 @@ outputs: runs: using: composite steps: - - id: get-node-version - uses: ./.github/actions/get_node_version - - - uses: actions/setup-node@v4 - with: - node-version: ${{ steps.get-node-version.outputs.node-version }} + - uses: ./.github/actions/setup-node - name: Get test infos id: get-test-infos @@ -40,3 +35,9 @@ runs: TEST_INFOS=$(./.github/actions/get_test_infos/get_test_infos.sh | base64 -d) echo "test-infos=${TEST_INFOS}" >> "$GITHUB_OUTPUT" shell: bash + + - name: Echo test infos for troubleshooting + run: | + echo "Test Infos Output:" + echo "${{ steps.get-test-infos.outputs.test-infos }}" + shell: bash diff --git a/.github/actions/get_dfx_version/README.md b/.github/actions/setup-node/README.md similarity index 84% rename from .github/actions/get_dfx_version/README.md rename to .github/actions/setup-node/README.md index 6a114e88db..9c5219de3f 100644 --- a/.github/actions/get_dfx_version/README.md +++ b/.github/actions/setup-node/README.md @@ -17,8 +17,7 @@ checking out a specific branch. steps: - uses: actions/checkout@v4 - - id: get-dfx-version - uses: ./.github/actions/get_dfx_version + - uses: ./.github/actions/setup-node - - run: echo ${{ steps.get-dfx-version.outputs.dfx-version }} + - run: echo "${{ steps.setup-node.outputs.version }}" ``` diff --git a/.github/actions/setup-node/action.yml b/.github/actions/setup-node/action.yml new file mode 100644 index 0000000000..1196c6da9b --- /dev/null +++ b/.github/actions/setup-node/action.yml @@ -0,0 +1,43 @@ +name: 'Setup Node.js' +description: 'Sets up Node.js environment and gets Node version from package.json' +inputs: + working-directory: + description: 'Directory containing package.json' + required: false + default: '.' + registry-url: + description: 'Optional registry URL' + required: false + node-version-file: + description: 'File containing the version Spec' + required: false + default: 'package.json' + cache: + description: 'Used to specify a package manager for caching' + required: false + cache-dependency-path: + description: 'Path to package-lock.json file' + required: false + +outputs: + node-version: + description: 'The Node.js version from package.json' + value: ${{ steps.get-version.outputs.version }} + +runs: + using: 'composite' + steps: + - id: get-version + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + VERSION=$(jq -r '.azle.globalDependencies.node // error("node version not found")' "package.json") + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - uses: actions/setup-node@v4 + with: + node-version: ${{ steps.get-version.outputs.version }} + registry-url: ${{ inputs.registry-url }} + node-version-file: ${{ inputs.node-version-file }} + cache: ${{ inputs.cache }} + cache-dependency-path: ${{ inputs.cache-dependency-path }} diff --git a/.github/actions/get_node_version/README.md b/.github/actions/setup_dfx/README.md similarity index 83% rename from .github/actions/get_node_version/README.md rename to .github/actions/setup_dfx/README.md index 478190d010..293d76f9e3 100644 --- a/.github/actions/get_node_version/README.md +++ b/.github/actions/setup_dfx/README.md @@ -17,8 +17,8 @@ checking out a specific branch. steps: - uses: actions/checkout@v4 - - id: get-node-version - uses: ./.github/actions/get_node_version + - id: setup-dfx + uses: ./.github/actions/setup_dfx - - run: echo "${{ steps.get-node-version.outputs.node-version }}" + - run: echo ${{ steps.setup-dfx.outputs.dfx-version }} ``` diff --git a/.github/actions/setup_dfx/action.yml b/.github/actions/setup_dfx/action.yml new file mode 100644 index 0000000000..1d2103cd1a --- /dev/null +++ b/.github/actions/setup_dfx/action.yml @@ -0,0 +1,27 @@ +name: Setup dfx +description: 'Sets up dfx by detecting version from package.json and installing it. (Note: DFX must be installed before `npm install` because the azle installation process requires dfx)' +inputs: + dfx-version: + description: 'The version of dfx to install (optional - will read from package.json if not provided)' + required: false +outputs: + dfx-version: + description: 'The version of dfx that was installed' + value: ${{ steps.get-dfx-version.outputs.dfx-version }} +runs: + using: composite + steps: + - id: get-dfx-version + if: inputs.dfx-version == '' + run: | + DFX_VERSION=$(jq -r '.azle.globalDependencies.dfx // error("dfx version not found")' "package.json") + echo "dfx-version=${DFX_VERSION}" >> "$GITHUB_OUTPUT" + shell: bash + + - name: Install dfx + run: | + VERSION="${{ inputs.dfx-version || steps.get-dfx-version.outputs.dfx-version }}" + # Install dfx (Note: dfx must be installed before `npx azle` because the azle installation process requires dfx) + src/build/stable/commands/install_global_dependencies/install_dfx.sh $VERSION + echo "$HOME/.local/share/dfx/bin" >> $GITHUB_PATH + shell: bash diff --git a/.github/scripts/publish_github_action.sh b/.github/scripts/publish_github_action.sh index df959b3edf..7d326a5bc4 100755 --- a/.github/scripts/publish_github_action.sh +++ b/.github/scripts/publish_github_action.sh @@ -26,34 +26,34 @@ else npm publish fi -# TODO loop through checking for the status instead of sleeping -echo -e "sleeping for 30 seconds to ensure azle@$VERSION is fully registered on npm" +# # TODO loop through checking for the status instead of sleeping +# echo -e "sleeping for 30 seconds to ensure azle@$VERSION is fully registered on npm" -sleep 30 +# sleep 30 -for directory in ${directories[@]} -do - cd "$directory" - echo "updating $directory" +# for directory in ${directories[@]} +# do +# cd "$directory" +# echo "updating $directory" - sed -E -i "s/(\"azle\": \")(.*)(\")/\1$VERSION\3/" package.json - npm install +# sed -E -i "s/(\"azle\": \")(.*)(\")/\1$VERSION\3/" package.json +# npm install - rm -rf node_modules +# rm -rf node_modules - cd $root_dir -done +# cd $root_dir +# done -git add --all -git commit -am "azle-bot automated release $VERSION" -git push origin $GITHUB_HEAD_REF +# git add --all +# git commit -am "azle-bot automated release $VERSION" +# git push origin $GITHUB_HEAD_REF -git tag $VERSION -git push origin $VERSION +# git tag $VERSION +# git push origin $VERSION -if [[ "$VERSION" == *"-rc."* ]]; -then - gh release create "$VERSION" -t "$VERSION" --prerelease -else - gh release create "$VERSION" -t "$VERSION" -fi +# if [[ "$VERSION" == *"-rc."* ]]; +# then +# gh release create "$VERSION" -t "$VERSION" --prerelease +# else +# gh release create "$VERSION" -t "$VERSION" +# fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fc7bbe141b..897a33b66c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,8 @@ on: push: branches: - main - pull_request: # Runs on pull requests to any branch + pull_request: + jobs: determine-should-release: if: ${{ startsWith(github.head_ref, 'release--') }} @@ -17,84 +18,17 @@ jobs: - id: determine-should-release uses: ./.github/actions/should_release - get-test-infos: - needs: determine-should-release - if: ${{ startsWith(github.head_ref, 'release--') && needs.determine-should-release.outputs.should-release }} - name: Get test infos - runs-on: ubuntu-latest - outputs: - test-infos: ${{ steps.get-test-infos.outputs.test-infos }} - steps: - - uses: actions/checkout@v4 - - - id: get-node-version - uses: ./.github/actions/get_node_version - - - name: Get all test infos - id: get-test-infos - uses: ./.github/actions/get_test_infos - with: - node-version: ${{ steps.get-node-version.outputs.node-version }} - directories: | - ./examples - ./tests - release: name: Deploy release # Only run this job if it's a release branch. This job will run instead of run-tests and will automatically publish another commit which will be tested if: ${{ needs.determine-should-release.outputs.should-release == 'true' }} + needs: - determine-should-release - - get-test-infos - runs-on: ubuntu-latest - env: - GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} # All commits must be verified - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.ref || github.ref }} - token: ${{ secrets.LASTMJS_GITHUB_TOKEN || github.token }} - - - id: get-node-version - uses: ./.github/actions/get_node_version - - - uses: actions/setup-node@v4 - with: - node-version: ${{ steps.get-node-version.outputs.node-version }} - registry-url: https://registry.npmjs.org - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - - name: Install curl - run: sudo apt-get install curl -y - - - id: get-dfx-version - uses: ./.github/actions/get_dfx_version - - name: Install dfx - run: | - # Install dfx (Note: dfx must be installed before `npx azle` because the azle installation process requires dfx) - src/build/stable/commands/install_global_dependencies/install_dfx.sh ${{ steps.get-dfx-version.outputs.dfx-version }} - echo "$HOME/.local/share/dfx/bin" >> $GITHUB_PATH - - - run: npm install - - - name: Install global dependencies - run: | - AZLE_VERBOSE=true npx azle install-global-dependencies --rust --wasi2ic - - # TODO we should use some Action-specific bot account - - name: Configure git for publishing release - run: | - git config --global user.name 'Jordan Last' - git config --global user.email 'jordan.michael.last@gmail.com' - git config --global commit.gpgsign true - echo -n "$GPG_SIGNING_KEY" | base64 --decode | gpg --import - git config --global user.signingkey C8B77BCBE16CD2B94B43F9C8757397B82D4ED7B0 - - - name: Publish release - run: | - BRANCH_NAME="${{ github.head_ref }}" - RELEASE_VERSION="${BRANCH_NAME:9}" - ./.github/scripts/publish_github_action.sh $RELEASE_VERSION ${{ toJSON(needs.get-test-infos.outputs.test-infos) }} + uses: ./.github/workflows/release_parallel.yml + secrets: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + LASTMJS_GITHUB_TOKEN: ${{ secrets.LASTMJS_GITHUB_TOKEN }} diff --git a/.github/workflows/release_parallel.yml b/.github/workflows/release_parallel.yml new file mode 100644 index 0000000000..dbd0e179a0 --- /dev/null +++ b/.github/workflows/release_parallel.yml @@ -0,0 +1,222 @@ +name: Parallel Release +on: + workflow_call: + secrets: + GPG_SIGNING_KEY: + required: true + GH_TOKEN: + required: true + NPM_TOKEN: + required: true + LASTMJS_GITHUB_TOKEN: + required: true + +jobs: + prepare-release: + name: Prepare Release + runs-on: ubuntu-latest + env: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} # All commits must be verified + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + outputs: + release-version: ${{ steps.get-version.outputs.release-version }} + test-infos: ${{ steps.get-test-infos.outputs.test-infos }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref || github.ref }} + token: ${{ secrets.LASTMJS_GITHUB_TOKEN || github.token }} + + - id: get-version + run: | + BRANCH_NAME="${{ github.event.pull_request.head.ref || github.ref_name }}" + RELEASE_VERSION="${BRANCH_NAME:9}" + echo "release-version=$RELEASE_VERSION" >> $GITHUB_OUTPUT + + - uses: ./.github/actions/setup-node + with: + registry-url: https://registry.npmjs.org + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + - uses: ./.github/actions/setup_dfx + + - run: npm install + + - name: Install global dependencies + run: | + AZLE_VERBOSE=true npx azle install-global-dependencies --rust --wasi2ic + + - uses: ./.github/actions/configure_git + with: + gpg_signing_key: ${{ secrets.GPG_SIGNING_KEY }} + + - name: Update version and build templates + run: | + VERSION=${{ steps.get-version.outputs.release-version }} + sed -E -i "s/(\"version\": \")(.*)(\")/\1$VERSION\3/" package.json + sed -E -i "s/(\"version\": \")(.*)(\")/\1$VERSION\3/" dfx_extension/extension.json + npm install + AZLE_VERBOSE=true npx azle template + AZLE_VERBOSE=true npx azle template --experimental + + - name: Publish to npm + run: | + if [[ "${{ steps.get-version.outputs.release-version }}" == *"-rc."* ]]; then + npm publish --tag next + else + npm publish + fi + + - id: get-test-infos + uses: ./.github/actions/get_test_infos + with: + directories: | + ./examples + ./tests + + update-test-files-for-release-commit: + needs: prepare-release + name: Update ${{ matrix.test.name }} files for release commit + runs-on: ubuntu-latest + env: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + strategy: + fail-fast: false + matrix: + test: ${{ fromJson(needs.prepare-release.outputs.test-infos) }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref || github.ref }} + token: ${{ secrets.LASTMJS_GITHUB_TOKEN || github.token }} + + - uses: ./.github/actions/setup-node + + - uses: ./.github/actions/setup_dfx + + - name: Update azle version + run: | + npm install + cd ${{ matrix.test.path }} + sed -E -i "s/(\"azle\": \")(.*)(\")/\1${{ needs.prepare-release.outputs.release-version }}\3/" package.json + npm install + + - name: Start dfx with artificial delay 0 + working-directory: ${{ matrix.test.path }} + run: dfx start --clean --background --host 127.0.0.1:8000 --artificial-delay 0 + + - name: Run npm test (continue on error) + working-directory: ${{ matrix.test.path }} + continue-on-error: true + run: npm test + + - uses: ./.github/actions/configure_git + with: + gpg_signing_key: ${{ secrets.GPG_SIGNING_KEY }} + + - name: Commit and push changes + run: | + BRANCH_NAME="update-${{ needs.prepare-release.outputs.release-version }}-$(echo '${{ matrix.test.displayPath }}' | sed 's/\//-/g')" + git switch -c "$BRANCH_NAME" + git add --all + if ! git diff --cached --quiet; then + git commit -m "Update dependencies for ${{ matrix.test.displayPath }}" + else + echo "No changes to commit. Skipping commit and push." + fi + git push origin "$BRANCH_NAME" + + finalize-release: + needs: [prepare-release, update-test-files-for-release-commit] + name: Finalize Release + runs-on: ubuntu-latest + env: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref || github.ref }} + token: ${{ secrets.LASTMJS_GITHUB_TOKEN }} + fetch-depth: 0 + + - uses: ./.github/actions/configure_git + with: + gpg_signing_key: ${{ secrets.GPG_SIGNING_KEY }} + + - name: Collect branches + id: collect-branches + run: | + # Create array of branches + readarray -t BRANCH_ARRAY < <(git branch -r | grep "origin/update-${{ needs.prepare-release.outputs.release-version }}-" | sed 's/origin\///' | xargs -n1) + + # Output array elements as multiline value + echo "branches<> $GITHUB_OUTPUT + printf '%s\n' "${BRANCH_ARRAY[@]}" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Display collected branches + run: | + echo "Collected branches:" + while read branch; do + echo " - $branch" + done < <(echo "${{ steps.collect-branches.outputs.branches }}") + echo "End of branch list" + + - name: Fetch branches + run: | + echo "Fetching all branches..." + BRANCHES_TO_FETCH="" + while read branch; do + BRANCHES_TO_FETCH+=" ${branch}:${branch}" + done < <(echo "${{ steps.collect-branches.outputs.branches }}") + + echo "BRANCHES_TO_FETCH: $BRANCHES_TO_FETCH" + git fetch origin ${BRANCHES_TO_FETCH} + + - name: Squash changes + env: + PAT: ${{ secrets.LASTMJS_GITHUB_TOKEN }} + run: | + CURRENT_BRANCH=$(git branch --show-current) + BASE_COMMIT=$(git rev-parse HEAD) + + while read branch; do + echo "Merging changes from branch: $branch" + git merge --squash "$branch" || { + echo "Failed to merge $branch" + git merge --abort + exit 1 + } + git commit -m "Squashed changes from $branch" || true + done < <(echo "${{ steps.collect-branches.outputs.branches }}") + + git reset --soft $BASE_COMMIT + git commit -am "Update all dependencies for release ${{ needs.prepare-release.outputs.release-version }}" + + git push origin HEAD:$CURRENT_BRANCH + + - name: Delete branches + run: | + echo "Starting branch deletion process..." + BRANCHES_TO_DELETE="" + while read branch; do + BRANCHES_TO_DELETE+=" ${branch}" + done < <(echo "${{ steps.collect-branches.outputs.branches }}") + + echo "BRANCHES_TO_DELETE: $BRANCHES_TO_DELETE" + git push origin --delete ${BRANCHES_TO_DELETE} + + - name: Create release + run: | + VERSION=${{ needs.prepare-release.outputs.release-version }} + git tag $VERSION + git push origin $VERSION + + if [[ "$VERSION" == *"-rc."* ]]; then + gh release create "$VERSION" -t "$VERSION" --prerelease + else + gh release create "$VERSION" -t "$VERSION" + fi diff --git a/.github/workflows/run_test.yml b/.github/workflows/run_test.yml index 49f8e74415..4d4d25c3c0 100644 --- a/.github/workflows/run_test.yml +++ b/.github/workflows/run_test.yml @@ -30,7 +30,7 @@ jobs: # os: [macos-latest, ubuntu-latest] os: [ubuntu-latest] azle_source: - - ${{ inputs.include_npm == 'true' && 'npm' || 'repo' }} + - ${{ inputs.include_npm == true && 'npm' || 'repo' }} tests: ${{ fromJSON(inputs.test_infos) }} steps: - uses: actions/checkout@v4 @@ -58,23 +58,12 @@ jobs: # Just in case the path isn't obvious from the name, this will remove ambiguity run: echo ${{matrix.tests.path}} - - id: get-node-version - uses: ./.github/actions/get_node_version + - uses: ./.github/actions/setup-node - - uses: actions/setup-node@v4 - with: - node-version: ${{ steps.get-node-version.outputs.node-version }} - - - id: get-dfx-version - uses: ./.github/actions/get_dfx_version + - uses: ./.github/actions/setup_dfx - name: Run pre-test Azle setup run: | - - # Install dfx (Note: DFX must be installed before `npm install` because the azle installation process requires dfx) - src/build/stable/commands/install_global_dependencies/install_dfx.sh ${{ steps.get-dfx-version.outputs.dfx-version }} - echo "$HOME/.local/share/dfx/bin" >> $GITHUB_PATH - # MacOS-specific DNS configuration if [[ "${{ matrix.os }}" == "macos-latest" ]]; then sudo networksetup -setdnsservers Ethernet 9.9.9.9 @@ -111,7 +100,8 @@ jobs: working-directory: ${{ matrix.tests.path }} run: dfx start --clean --background --host 127.0.0.1:8000 - - name: Run test + - name: Calculate number of test runs + id: calc-runs run: | RUNS=1 @@ -136,7 +126,10 @@ jobs: fi echo "Running tests $RUNS times" + echo "runs=$RUNS" >> $GITHUB_OUTPUT + shell: bash -l {0} - AZLE_PROPTEST_NUM_RUNS=$RUNS AZLE_PROPTEST_VERBOSE=true npm test + - name: Run tests + run: AZLE_PROPTEST_NUM_RUNS=${{ steps.calc-runs.outputs.runs }} AZLE_PROPTEST_VERBOSE=true npm test shell: bash -l {0} working-directory: ${{ matrix.tests.path }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 61bb09a8be..9f8c43d106 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ on: push: branches: - main - pull_request: # Runs on pull requests to any branch + pull_request: jobs: determine-should-run-tests: