run parallel benchmarks

.github/workflows/benchmark.yml

@@ -0,0 +1,141 @@
+name: Benchmark
+
+on:
+    push:
+        branches:
+            - main
+    pull_request:
+
+jobs:
+    determine-should-run-benchmarks:
+        name: Determine if benchmarks should run
+        runs-on: ubuntu-latest
+        outputs:
+            # If the branch should release then it shouldn't run benchmarks.
+            should-run-benchmarks: ${{ steps.determine-should-run-benchmarks.outputs.should-release == 'false' }}
+        steps:
+            - uses: actions/checkout@v4
+
+            - id: determine-should-run-benchmarks
+              uses: ./.github/actions/should_release
+
+    set-exclude-dirs:
+        name: Set exclude directories
+        runs-on: ubuntu-latest
+        outputs:
+            exclude-dirs: ${{ steps.set-exclude-dirs.outputs.exclude-dirs }}
+        steps:
+            - uses: actions/checkout@v4
+
+            - id: set-conditions
+              uses: ./.github/actions/set_run_conditions
+
+            - id: set-exclude-dirs
+              run: |
+                  RELEASE_BENCHMARKS="${{ format('
+                  tests/end_to_end/candid_rpc/class_syntax/new
+                  tests/end_to_end/http_server/new
+                  ') }}"
+
+                  UNSTABLE_BENCHMARKS="${{ format('
+                  examples/basic_bitcoin
+                  examples/bitcoin_psbt
+                  examples/ckbtc
+                  tests/end_to_end/http_server/ethers_base
+                  tests/end_to_end/http_server/http_outcall_fetch
+                  tests/end_to_end/http_server/ic_evm_rpc
+                  tests/property/candid_rpc/class_api/stable_b_tree_map
+                  tests/property/candid_rpc/functional_api/stable_b_tree_map
+                  tests/property/ic_api/performance_counter
+                  tests/property/ic_api/instruction_counter
+                  ') }}"
+
+                  SLOW_BENCHMARKS="${{ format('
+                  tests/end_to_end/candid_rpc/functional_syntax/ckbtc
+                  tests/end_to_end/candid_rpc/class_syntax/bitcoin
+                  tests/end_to_end/http_server/large_files
+                  tests/end_to_end/http_server/open_value_sharing
+                  tests/end_to_end/candid_rpc/class_syntax/stable_structures
+                  tests/end_to_end/candid_rpc/functional_syntax/bitcoin
+                  tests/end_to_end/candid_rpc/functional_syntax/composite_queries
+                  tests/end_to_end/candid_rpc/functional_syntax/cross_canister_calls
+                  tests/end_to_end/candid_rpc/functional_syntax/management_canister
+                  tests/end_to_end/candid_rpc/functional_syntax/stable_structures
+                  tests/end_to_end/http_server/autoreload
+                  ') }}"
+
+                  AZLE_IS_MAIN_BRANCH_PUSH="${{ steps.set-conditions.outputs.is_main_branch_push }}"
+                  AZLE_IS_MAIN_BRANCH_PUSH_FROM_RELEASE_MERGE="${{ steps.set-conditions.outputs.is_main_branch_push_from_release_merge }}"
+                  AZLE_IS_RELEASE_BRANCH_PR="${{ steps.set-conditions.outputs.is_release_branch_pr }}"
+                  AZLE_IS_FEATURE_BRANCH_PR="${{ steps.set-conditions.outputs.is_feature_branch_pr }}"
+                  AZLE_IS_FEATURE_BRANCH_DRAFT_PR="${{ steps.set-conditions.outputs.is_feature_branch_draft_pr }}"
+
+                  EXCLUDE_DIRS=""
+
+                  if [[ "$AZLE_IS_MAIN_BRANCH_PUSH" == "true" ]]; then
+                      EXCLUDE_DIRS=""
+                  fi
+
+                  if [[ "$AZLE_IS_MAIN_BRANCH_PUSH_FROM_RELEASE_MERGE" == "true" ]]; then
+                      EXCLUDE_DIRS=""
+                  fi
+
+                  if [[ "$AZLE_IS_RELEASE_BRANCH_PR" == "true" ]]; then
+                      EXCLUDE_DIRS=""
+                  fi
+
+                  if [[ "$AZLE_IS_FEATURE_BRANCH_PR" == "true" ]]; then
+                      EXCLUDE_DIRS="$RELEASE_BENCHMARKS $UNSTABLE_BENCHMARKS"
+                  fi
+
+                  if [[ "$AZLE_IS_FEATURE_BRANCH_DRAFT_PR" == "true" ]]; then
+                      EXCLUDE_DIRS="$RELEASE_BENCHMARKS $UNSTABLE_BENCHMARKS $SLOW_BENCHMARKS"
+                  fi
+
+                  # Trim leading or trailing spaces and save the exclude-dirs in the environment
+                  EXCLUDE_DIRS=$(echo $EXCLUDE_DIRS | xargs)
+                  echo "exclude-dirs=$EXCLUDE_DIRS" >> $GITHUB_OUTPUT
+
+    run-benchmarks:
+        name: ${{ matrix.benchmark_group.name }}
+        needs:
+            - determine-should-run-benchmarks
+            - set-exclude-dirs
+        if: ${{ needs.determine-should-run-benchmarks.outputs.should-run-benchmarks == 'true' }}
+        strategy:
+            fail-fast: false
+            matrix:
+                benchmark_group:
+                    - { name: 'Examples', directories: './examples' }
+                    - {
+                          name: 'E2E Class',
+                          directories: './tests/end_to_end/candid_rpc/class_syntax'
+                      }
+                    - {
+                          name: 'E2E Functional',
+                          directories: './tests/end_to_end/candid_rpc/functional_syntax'
+                      }
+                    - {
+                          name: 'E2E HTTP Server',
+                          directories: './tests/end_to_end/http_server'
+                      }
+                    - {
+                          name: 'Property Class',
+                          directories: './tests/property/candid_rpc/class_api'
+                      }
+                    - {
+                          name: 'Property Functional',
+                          directories: './tests/property/candid_rpc/functional_api'
+                      }
+                    - {
+                          name: 'Property IC API',
+                          directories: './tests/property/ic_api'
+                      }
+        uses: ./.github/workflows/benchmark_parallel.yml
+        secrets:
+            GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
+            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+            LASTMJS_GITHUB_TOKEN: ${{ secrets.LASTMJS_GITHUB_TOKEN }}
+        with:
+            directories: ${{ matrix.benchmark_group.directories }}
+            exclude-dirs: ${{ needs.set-exclude-dirs.outputs.exclude-dirs }}

.github/workflows/benchmark_parallel.yml

@@ -0,0 +1,156 @@
+name: Parallel Benchmark
+on:
+    workflow_call:
+        inputs:
+            directories:
+                required: true
+                type: string
+            exclude-dirs:
+                required: false
+                type: string
+                default: ''
+        secrets:
+            GPG_SIGNING_KEY:
+                required: true
+            GH_TOKEN:
+                required: true
+            LASTMJS_GITHUB_TOKEN:
+                required: true
+
+jobs:
+    prepare-benchmark:
+        name: Prepare Benchmark
+        runs-on: ubuntu-latest
+        env:
+            GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} # All commits must be verified
+            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        outputs:
+            azle-version: ${{ steps.get-version.outputs.azle-version }}
+            test-infos: ${{ steps.get-test-infos.outputs.test-infos }}
+        steps:
+            - uses: actions/checkout@v4
+
+            - id: get-version
+              run: |
+                  VERSION=$(jq -r '.version' package.json)
+                  echo "azle-version=$VERSION" >> $GITHUB_OUTPUT
+
+            - id: get-test-infos
+              uses: ./.github/actions/get_test_infos
+              with:
+                  directories: ${{ inputs.directories }}
+                  exclude-dirs: ${{ inputs.exclude-dirs }}
+
+    run-benchmarks:
+        needs: prepare-benchmark
+        name: Run benchmarks for ${{ matrix.test.name }}
+        runs-on: ubuntu-latest
+        env:
+            GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
+            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        strategy:
+            fail-fast: false
+            matrix:
+                test: ${{ fromJson(needs.prepare-benchmark.outputs.test-infos) }}
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  ref: ${{ github.event.pull_request.head.ref || github.ref }}
+                  token: ${{ secrets.LASTMJS_GITHUB_TOKEN || github.token }}
+
+            - uses: ./.github/actions/setup_node
+
+            - uses: ./.github/actions/setup_dfx
+
+            - name: Install dependencies
+              run: |
+                  npm install
+                  cd ${{ matrix.test.path }}
+                  npm install
+
+            - name: Start dfx with artificial delay 0
+              working-directory: ${{ matrix.test.path }}
+              run: dfx start --clean --background --host 127.0.0.1:8000 --artificial-delay 0
+
+            - name: Run npm test (continue on error)
+              working-directory: ${{ matrix.test.path }}
+              continue-on-error: true
+              run: npm test
+
+            - uses: ./.github/actions/configure_git
+              with:
+                  gpg_signing_key: ${{ secrets.GPG_SIGNING_KEY }}
+
+            - name: Commit and push changes
+              run: |
+                  BRANCH_NAME="benchmark--${{ needs.prepare-benchmark.outputs.azle-version }}-$(echo '${{ matrix.test.displayPath }}' | sed 's/\//-/g')"
+                  git switch -c "$BRANCH_NAME"
+                  git add --all
+                  if ! git diff --cached --quiet; then
+                    git commit -m "Run benchmarks for ${{ matrix.test.displayPath }}"
+                  else
+                    echo "No changes to commit. Skipping commit and push."
+                  fi
+                  git push origin "$BRANCH_NAME"
+
+    finalize-benchmark:
+        needs: [prepare-benchmark, run-benchmarks]
+        name: Finalize Benchmark
+        runs-on: ubuntu-latest
+        env:
+            GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
+            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  ref: ${{ github.event.pull_request.head.ref || github.ref }}
+                  token: ${{ secrets.LASTMJS_GITHUB_TOKEN }}
+                  fetch-depth: 0
+
+            - uses: ./.github/actions/configure_git
+              with:
+                  gpg_signing_key: ${{ secrets.GPG_SIGNING_KEY }}
+
+            - name: Collect branches
+              id: collect-branches
+              run: |
+                  # Get branches and convert to space-separated list
+                  BRANCHES=$(git branch -r | grep "origin/benchmark--${{ needs.prepare-benchmark.outputs.azle-version }}-" | sed 's/origin\///' | xargs)
+                  echo "branches=$BRANCHES" >> $GITHUB_OUTPUT
+
+            - name: Display collected branches
+              run: |
+                  echo "Collected branches:"
+                  for branch in ${{ steps.collect-branches.outputs.branches }}; do
+                    echo "  - $branch"
+                  done
+                  echo "End of branch list"
+
+            - name: Fetch branches
+              run: |
+                  echo "Fetching all branches..."
+                  BRANCHES_TO_FETCH=""
+                  for branch in ${{ steps.collect-branches.outputs.branches }}; do
+                    BRANCHES_TO_FETCH+=" $branch:$branch"
+                  done
+                  git fetch origin $BRANCHES_TO_FETCH
+
+            - name: Squash changes
+              env:
+                  PAT: ${{ secrets.LASTMJS_GITHUB_TOKEN }}
+              run: |
+                  CURRENT_BRANCH=$(git branch --show-current)
+
+                  for branch in ${{ steps.collect-branches.outputs.branches }}; do
+                    git merge --squash $branch
+                  done
+
+                  # Create a merge commit with a descriptive message
+                  git commit -am "Update all dependencies for benchmark ${{ needs.prepare-benchmark.outputs.azle-version }}"
+
+                  git push origin HEAD:$CURRENT_BRANCH
+
+            - name: Delete branches
+              run: |
+                  echo "Starting branch deletion process..."
+                  git push origin --delete ${{ steps.collect-branches.outputs.branches }}