run parallel benchmarks

.github/workflows/benchmark.yml

@@ -0,0 +1,146 @@
+# This GitHub Action flow works as follows:
+# Each directory in the examples and tests directory represents an example project and is intended to have tests that ensure the canisters contained in that example function properly.
+# These tests are currently written in TypeScript and are intended to be run in a Node.js environment.
+# This GitHub Action takes care of deploying to npm and GitHub.
+
+name: Test
+
+on:
+    push:
+        branches:
+            - main
+    pull_request:
+
+jobs:
+    determine-should-run-tests:
+        name: Determine if tests should run
+        runs-on: ubuntu-latest
+        outputs:
+            # If the branch should release then it shouldn't run tests.
+            should-run-tests: ${{ steps.determine-should-run-tests.outputs.should-release == 'false' }}
+        steps:
+            - uses: actions/checkout@v4
+
+            - id: determine-should-run-tests
+              uses: ./.github/actions/should_release
+
+    set-exclude-dirs:
+        name: Set exclude directories
+        runs-on: ubuntu-latest
+        outputs:
+            exclude-dirs: ${{ steps.set-exclude-dirs.outputs.exclude-dirs }}
+        steps:
+            - uses: actions/checkout@v4
+
+            - id: set-conditions
+              uses: ./.github/actions/set_run_conditions
+
+            - id: set-exclude-dirs
+              run: |
+                  RELEASE_TESTS="${{ format('
+                  tests/end_to_end/candid_rpc/class_syntax/new
+                  tests/end_to_end/http_server/new
+                  ') }}"
+
+                  UNSTABLE_TESTS="${{ format('
+                  examples/basic_bitcoin
+                  examples/bitcoin_psbt
+                  examples/ckbtc
+                  tests/end_to_end/http_server/ethers_base
+                  tests/end_to_end/http_server/http_outcall_fetch
+                  tests/end_to_end/http_server/ic_evm_rpc
+                  tests/property/candid_rpc/class_api/stable_b_tree_map
+                  tests/property/candid_rpc/functional_api/stable_b_tree_map
+                  tests/property/ic_api/performance_counter
+                  tests/property/ic_api/instruction_counter
+                  ') }}"
+
+                  SLOW_TESTS="${{ format('
+                  tests/end_to_end/candid_rpc/functional_syntax/ckbtc
+                  tests/end_to_end/candid_rpc/class_syntax/bitcoin
+                  tests/end_to_end/http_server/large_files
+                  tests/end_to_end/http_server/open_value_sharing
+                  tests/end_to_end/candid_rpc/class_syntax/stable_structures
+                  tests/end_to_end/candid_rpc/functional_syntax/bitcoin
+                  tests/end_to_end/candid_rpc/functional_syntax/composite_queries
+                  tests/end_to_end/candid_rpc/functional_syntax/cross_canister_calls
+                  tests/end_to_end/candid_rpc/functional_syntax/management_canister
+                  tests/end_to_end/candid_rpc/functional_syntax/stable_structures
+                  tests/end_to_end/http_server/autoreload
+                  ') }}"
+
+                  AZLE_IS_MAIN_BRANCH_PUSH="${{ steps.set-conditions.outputs.is_main_branch_push }}"
+                  AZLE_IS_MAIN_BRANCH_PUSH_FROM_RELEASE_MERGE="${{ steps.set-conditions.outputs.is_main_branch_push_from_release_merge }}"
+                  AZLE_IS_RELEASE_BRANCH_PR="${{ steps.set-conditions.outputs.is_release_branch_pr }}"
+                  AZLE_IS_FEATURE_BRANCH_PR="${{ steps.set-conditions.outputs.is_feature_branch_pr }}"
+                  AZLE_IS_FEATURE_BRANCH_DRAFT_PR="${{ steps.set-conditions.outputs.is_feature_branch_draft_pr }}"
+
+                  EXCLUDE_DIRS=""
+
+                  if [[ "$AZLE_IS_MAIN_BRANCH_PUSH" == "true" ]]; then
+                      EXCLUDE_DIRS=""
+                  fi
+
+                  if [[ "$AZLE_IS_MAIN_BRANCH_PUSH_FROM_RELEASE_MERGE" == "true" ]]; then
+                      EXCLUDE_DIRS=""
+                  fi
+
+                  if [[ "$AZLE_IS_RELEASE_BRANCH_PR" == "true" ]]; then
+                      EXCLUDE_DIRS=""
+                  fi
+
+                  if [[ "$AZLE_IS_FEATURE_BRANCH_PR" == "true" ]]; then
+                      EXCLUDE_DIRS="$RELEASE_TESTS $UNSTABLE_TESTS"
+                  fi
+
+                  if [[ "$AZLE_IS_FEATURE_BRANCH_DRAFT_PR" == "true" ]]; then
+                      EXCLUDE_DIRS="$RELEASE_TESTS $UNSTABLE_TESTS $SLOW_TESTS"
+                  fi
+
+                  # Trim leading or trailing spaces and save the exclude-dirs in the environment
+                  EXCLUDE_DIRS=$(echo $EXCLUDE_DIRS | xargs)
+                  echo "exclude-dirs=$EXCLUDE_DIRS" >> $GITHUB_OUTPUT
+
+    run-tests:
+        name: ${{ matrix.test_group.name }}
+        needs:
+            - determine-should-run-tests
+            - set-exclude-dirs
+        if: ${{ needs.determine-should-run-tests.outputs.should-run-tests == 'true' }}
+        strategy:
+            fail-fast: false
+            matrix:
+                test_group:
+                    - { name: 'Examples', directories: './examples' }
+                    - {
+                          name: 'E2E Class',
+                          directories: './tests/end_to_end/candid_rpc/class_syntax'
+                      }
+                    - {
+                          name: 'E2E Functional',
+                          directories: './tests/end_to_end/candid_rpc/functional_syntax'
+                      }
+                    - {
+                          name: 'E2E HTTP Server',
+                          directories: './tests/end_to_end/http_server'
+                      }
+                    - {
+                          name: 'Property Class',
+                          directories: './tests/property/candid_rpc/class_api'
+                      }
+                    - {
+                          name: 'Property Functional',
+                          directories: './tests/property/candid_rpc/functional_api'
+                      }
+                    - {
+                          name: 'Property IC API',
+                          directories: './tests/property/ic_api'
+                      }
+        uses: ./.github/workflows/benchmark_parallel.yml
+        secrets:
+            GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
+            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+            LASTMJS_GITHUB_TOKEN: ${{ secrets.LASTMJS_GITHUB_TOKEN }}
+        with:
+            directories: ${{ matrix.test_group.directories }}
+            exclude-dirs: ${{ needs.set-exclude-dirs.outputs.exclude-dirs }}

.github/workflows/benchmark_parallel.yml

@@ -0,0 +1,187 @@
+name: Parallel Release
+on:
+    workflow_call:
+        inputs:
+            directories:
+                required: true
+                type: string
+            exclude-dirs:
+                required: false
+                type: string
+                default: ''
+        secrets:
+            GPG_SIGNING_KEY:
+                required: true
+            GH_TOKEN:
+                required: true
+            LASTMJS_GITHUB_TOKEN:
+                required: true
+
+jobs:
+    prepare-release:
+        name: Prepare Release
+        runs-on: ubuntu-latest
+        env:
+            GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} # All commits must be verified
+            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        outputs:
+            azle-version: ${{ steps.get-version.outputs.azle-version }}
+            test-infos: ${{ steps.get-test-infos.outputs.test-infos }}
+        steps:
+            - uses: actions/checkout@v4
+
+            - id: get-version
+              run: |
+                  RELEASE_VERSION=$(jq -r '.version' package.json)
+                  echo "azle-version=$RELEASE_VERSION" >> $GITHUB_OUTPUT
+
+            - id: get-test-infos
+              uses: ./.github/actions/get_test_infos
+              with:
+                  directories: ${{ inputs.directories }}
+                  exclude-dirs: ${{ inputs.exclude-dirs }}
+
+    run-benchmarks:
+        needs: prepare-release
+        name: Run benchmarks for ${{ matrix.test.name }}
+        runs-on: ubuntu-latest
+        env:
+            GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
+            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        strategy:
+            fail-fast: false
+            matrix:
+                test: ${{ fromJson(needs.prepare-release.outputs.test-infos) }}
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  ref: ${{ github.event.pull_request.head.ref || github.ref }}
+                  token: ${{ secrets.LASTMJS_GITHUB_TOKEN || github.token }}
+
+            - uses: ./.github/actions/setup-node
+
+            - uses: ./.github/actions/setup_dfx
+
+            - name: Install dependencies
+              run: |
+                  npm install
+                  cd ${{ matrix.test.path }}
+                  npm install
+
+            - name: Start dfx with artificial delay 0
+              working-directory: ${{ matrix.test.path }}
+              run: dfx start --clean --background --host 127.0.0.1:8000 --artificial-delay 0
+
+            - name: Run npm test (continue on error)
+              working-directory: ${{ matrix.test.path }}
+              continue-on-error: true
+              run: npm test
+
+            - uses: ./.github/actions/configure_git
+              with:
+                  gpg_signing_key: ${{ secrets.GPG_SIGNING_KEY }}
+
+            - name: Commit and push changes
+              run: |
+                  BRANCH_NAME="benchmark-${{ needs.prepare-release.outputs.azle-version }}-$(echo '${{ matrix.test.displayPath }}' | sed 's/\//-/g')"
+                  git switch -c "$BRANCH_NAME"
+                  git add --all
+                  if ! git diff --cached --quiet; then
+                    git commit -m "Run benchmarks for ${{ matrix.test.displayPath }}"
+                  else
+                    echo "No changes to commit. Skipping commit and push."
+                  fi
+                  git push origin "$BRANCH_NAME"
+
+    finalize-release:
+        needs: [prepare-release, run-benchmarks]
+        name: Finalize Release
+        runs-on: ubuntu-latest
+        env:
+            GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
+            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  ref: ${{ github.event.pull_request.head.ref || github.ref }}
+                  token: ${{ secrets.LASTMJS_GITHUB_TOKEN }}
+                  fetch-depth: 0
+
+            - uses: ./.github/actions/configure_git
+              with:
+                  gpg_signing_key: ${{ secrets.GPG_SIGNING_KEY }}
+
+            - name: Collect branches
+              id: collect-branches
+              run: |
+                  # Create array of branches
+                  readarray -t BRANCH_ARRAY < <(git branch -r | grep "origin/benchmark-${{ needs.prepare-release.outputs.azle-version }}-" | sed 's/origin\///' | xargs -n1)
+
+                  # Output array elements as multiline value
+                  echo "branches<<EOF" >> $GITHUB_OUTPUT
+                  printf '%s\n' "${BRANCH_ARRAY[@]}" >> $GITHUB_OUTPUT
+                  echo "EOF" >> $GITHUB_OUTPUT
+
+            - name: Display collected branches
+              run: |
+                  echo "Collected branches:"
+                  while read branch; do
+                    echo "  - $branch"
+                  done < <(echo "${{ steps.collect-branches.outputs.branches }}")
+                  echo "End of branch list"
+
+            - name: Fetch branches
+              run: |
+                  echo "Fetching all branches..."
+                  BRANCHES_TO_FETCH=""
+                  while read branch; do
+                    BRANCHES_TO_FETCH+=" ${branch}:${branch}"
+                  done < <(echo "${{ steps.collect-branches.outputs.branches }}")
+
+                  echo "BRANCHES_TO_FETCH: $BRANCHES_TO_FETCH"
+                  git fetch origin ${BRANCHES_TO_FETCH}
+
+            - name: Squash changes
+              env:
+                  PAT: ${{ secrets.LASTMJS_GITHUB_TOKEN }}
+              run: |
+                  CURRENT_BRANCH=$(git branch --show-current)
+                  BASE_COMMIT=$(git rev-parse HEAD)
+
+                  while read branch; do
+                    echo "Merging changes from branch: $branch"
+                    git merge --squash "$branch" || {
+                      echo "Failed to merge $branch"
+                      git merge --abort
+                      exit 1
+                    }
+                    git commit -m "Squashed changes from $branch" || true
+                  done < <(echo "${{ steps.collect-branches.outputs.branches }}")
+
+                  git reset --soft $BASE_COMMIT
+                  git commit -am "Update all dependencies for release ${{ needs.prepare-release.outputs.azle-version }}"
+
+                  git push origin HEAD:$CURRENT_BRANCH
+
+            - name: Delete branches
+              run: |
+                  echo "Starting branch deletion process..."
+                  BRANCHES_TO_DELETE=""
+                  while read branch; do
+                    BRANCHES_TO_DELETE+=" ${branch}"
+                  done < <(echo "${{ steps.collect-branches.outputs.branches }}")
+
+                  echo "BRANCHES_TO_DELETE: $BRANCHES_TO_DELETE"
+                  git push origin --delete ${BRANCHES_TO_DELETE}
+
+            - name: Create release
+              run: |
+                  VERSION=${{ needs.prepare-release.outputs.azle-version }}
+                  git tag $VERSION
+                  git push origin $VERSION
+
+                  if [[ "$VERSION" == *"-rc."* ]]; then
+                    gh release create "$VERSION" -t "$VERSION" --prerelease
+                  else
+                    gh release create "$VERSION" -t "$VERSION"
+                  fi