Parallel benchmarking

.github/actions/commit_and_push/README.md

@@ -0,0 +1,23 @@
+## Prerequisite
+
+This action assumes that the repository has already been checked out before
+calling the action, typically using `actions/checkout@v4`. If you have not
+checked out the code in a previous step, make sure to do so to avoid errors.
+
+This action does **not** perform a checkout action itself because it would be
+redundant. This action is part of the repository's codebase, so if the code
+hasn't already been checked out, the action itself wouldn't even be available to
+call.
+
+## Example Usage
+
+```yaml
+steps:
+    - uses: actions/checkout@v4
+
+    - uses: ./.github/actions/commit_and_push
+      with:
+          gpg_signing_key: ${{ secrets.GPG_SIGNING_KEY }}
+          branch-name: 'branch-name'
+          commit-message: 'commit message'
+```

.github/actions/commit_and_push/action.yml

@@ -0,0 +1,37 @@
+name: Commit and Push Changes
+description: 'Configures git, commits changes and pushes to a new branch'
+inputs:
+    branch-name:
+        description: 'Name of the branch to create'
+        required: true
+    commit-message:
+        description: 'Commit message'
+        required: true
+    gpg_signing_key:
+        description: 'The GPG signing key to use for signing commits'
+        required: true
+runs:
+    using: composite
+    steps:
+        - uses: ./.github/actions/configure_git
+          with:
+              gpg_signing_key: ${{ inputs.gpg_signing_key }}
+
+        - name: Commit and push changes
+          shell: bash
+          run: |
+              # Create and switch to new branch
+              git switch -c "${{ inputs.branch-name }}"
+
+              # Show status of working directory
+              echo "Current git status:"
+              git status
+
+              # Add and commit changes if there are any
+              git add --all
+              if ! git diff --cached --quiet; then
+                git commit -m "${{ inputs.commit-message }}"
+                git push origin "${{ inputs.branch-name }}"
+              else
+                echo "No changes to commit. Skipping commit and push."
+              fi

.github/actions/get_exclude_dirs/README.md

@@ -0,0 +1,15 @@
+## Example Usage
+
+```yaml
+steps:
+    - uses: actions/checkout@v4
+
+    - id: get-exclude-dirs
+      uses: ./.github/actions/get_exclude_dirs
+      with:
+          exclude-slow: true
+          exclude-unstable: true
+          exclude-release: true
+
+    - run: echo "${{ steps.get-exclude-dirs.outputs.exclude-dirs }}"
+```

.github/actions/get_exclude_dirs/action.yml

@@ -0,0 +1,71 @@
+name: Get Exclude Directories
+description: 'Gets a list of directories to exclude based on input parameters'
+inputs:
+    exclude-slow:
+        description: 'Whether to exclude slow benchmarks'
+        required: true
+    exclude-unstable:
+        description: 'Whether to exclude unstable benchmarks'
+        required: true
+    exclude-release-only:
+        description: 'Whether to exclude release benchmarks only'
+        required: true
+outputs:
+    exclude-dirs:
+        description: 'Space-separated list of directories to exclude'
+        value: ${{ steps.set-exclude-dirs.outputs.exclude-dirs }}
+runs:
+    using: composite
+    steps:
+        - id: set-exclude-dirs
+          shell: bash
+          run: |
+              RELEASE_ONLY_EXCLUSIONS="${{ format('
+              tests/end_to_end/candid_rpc/class_syntax/new
+              tests/end_to_end/http_server/new
+              ') }}"
+
+              UNSTABLE_EXCLUSIONS="${{ format('
+              examples/basic_bitcoin
+              examples/bitcoin_psbt
+              examples/ckbtc
+              tests/end_to_end/http_server/ethers_base
+              tests/end_to_end/http_server/http_outcall_fetch
+              tests/end_to_end/http_server/ic_evm_rpc
+              tests/property/candid_rpc/class_api/stable_b_tree_map
+              tests/property/candid_rpc/functional_api/stable_b_tree_map
+              tests/property/ic_api/performance_counter
+              tests/property/ic_api/instruction_counter
+              ') }}"
+
+              SLOW_EXCLUSIONS="${{ format('
+              tests/end_to_end/candid_rpc/functional_syntax/ckbtc
+              tests/end_to_end/candid_rpc/class_syntax/bitcoin
+              tests/end_to_end/http_server/large_files
+              tests/end_to_end/http_server/open_value_sharing
+              tests/end_to_end/candid_rpc/class_syntax/stable_structures
+              tests/end_to_end/candid_rpc/functional_syntax/bitcoin
+              tests/end_to_end/candid_rpc/functional_syntax/composite_queries
+              tests/end_to_end/candid_rpc/functional_syntax/cross_canister_calls
+              tests/end_to_end/candid_rpc/functional_syntax/management_canister
+              tests/end_to_end/candid_rpc/functional_syntax/stable_structures
+              tests/end_to_end/http_server/autoreload
+              ') }}"
+
+              EXCLUDE_DIRS=""
+
+              if [[ "${{ inputs.exclude-release-only }}" == "true" ]]; then
+                  EXCLUDE_DIRS="$EXCLUDE_DIRS $RELEASE_ONLY_EXCLUSIONS"
+              fi
+
+              if [[ "${{ inputs.exclude-unstable }}" == "true" ]]; then
+                  EXCLUDE_DIRS="$EXCLUDE_DIRS $UNSTABLE_EXCLUSIONS"
+              fi
+
+              if [[ "${{ inputs.exclude-slow }}" == "true" ]]; then
+                  EXCLUDE_DIRS="$EXCLUDE_DIRS $SLOW_EXCLUSIONS"
+              fi
+
+              # Trim leading or trailing spaces and save the exclude-dirs in the environment
+              EXCLUDE_DIRS=$(echo $EXCLUDE_DIRS | xargs)
+              echo "exclude-dirs=$EXCLUDE_DIRS" >> $GITHUB_OUTPUT

.github/workflows/benchmark.yml

@@ -0,0 +1,134 @@
+name: Benchmark
+on:
+    workflow_dispatch:
+        inputs:
+            exclude-slow-benchmarks:
+                description: 'Exclude slow benchmarks'
+                required: true
+                type: boolean
+                default: false
+            exclude-unstable-benchmarks:
+                description: 'Exclude unstable benchmarks'
+                required: true
+                type: boolean
+                default: false
+            exclude-release-benchmarks:
+                description: 'Exclude release benchmarks'
+                required: true
+                type: boolean
+                default: false
+
+jobs:
+    get-exclude-dirs:
+        name: Get exclude directories
+        runs-on: ubuntu-latest
+        outputs:
+            exclude-dirs: ${{ steps.get-exclude-dirs.outputs.exclude-dirs }}
+        steps:
+            - uses: actions/checkout@v4
+
+            - id: get-exclude-dirs
+              uses: ./.github/actions/get_exclude_dirs
+              with:
+                  exclude-slow: ${{ github.event.inputs.exclude-slow-benchmarks == true }}
+                  exclude-unstable: ${{ github.event.inputs.exclude-unstable-benchmarks == true }}
+                  exclude-release-only: ${{ github.event.inputs.exclude-release-benchmarks == true }}
+
+    create-branch-prefix:
+        name: Create Branch and Branch Prefix
+        runs-on: ubuntu-latest
+        outputs:
+            branch-prefix: ${{ steps.create-prefix.outputs.branch-prefix }}
+            base-branch: ${{ steps.create-prefix.outputs.base-branch }}
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  ref: ${{ github.ref }}
+
+            - id: create-prefix
+              run: |
+                  VERSION=$(jq -r '.version' package.json)
+                  echo "branch-prefix=benchmark--$VERSION-" >> $GITHUB_OUTPUT
+                  echo "base-branch=benchmark--$VERSION" >> $GITHUB_OUTPUT
+
+            - uses: ./.github/actions/configure_git
+              with:
+                  gpg_signing_key: ${{ secrets.GPG_SIGNING_KEY }}
+
+            - name: Create base branch
+              run: |
+                  git checkout -b ${{ steps.create-prefix.outputs.base-branch }}
+                  git push origin ${{ steps.create-prefix.outputs.base-branch }}
+
+    run-benchmarks:
+        name: ${{ matrix.benchmark_group.name }}
+        needs:
+            - get-exclude-dirs
+            - create-branch-prefix
+        strategy:
+            fail-fast: false
+            matrix:
+                benchmark_group:
+                    - { name: 'Examples', directories: './examples' }
+                    - {
+                          name: 'E2E Class',
+                          directories: './tests/end_to_end/candid_rpc/class_syntax'
+                      }
+                    - {
+                          name: 'E2E Functional',
+                          directories: './tests/end_to_end/candid_rpc/functional_syntax'
+                      }
+                    - {
+                          name: 'E2E HTTP Server',
+                          directories: './tests/end_to_end/http_server'
+                      }
+                    - {
+                          name: 'Property Class',
+                          directories: './tests/property/candid_rpc/class_api'
+                      }
+                    - {
+                          name: 'Property Functional',
+                          directories: './tests/property/candid_rpc/functional_api'
+                      }
+                    - {
+                          name: 'Property IC API',
+                          directories: './tests/property/ic_api'
+                      }
+        uses: ./.github/workflows/benchmark_parallel.yml
+        secrets:
+            GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
+            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+            LASTMJS_GITHUB_TOKEN: ${{ secrets.LASTMJS_GITHUB_TOKEN }}
+        with:
+            directories: ${{ matrix.benchmark_group.directories }}
+            exclude-dirs: ${{ needs.get-exclude-dirs.outputs.exclude-dirs }}
+            branch-prefix: ${{ needs.create-branch-prefix.outputs.branch-prefix }}
+
+    finalize-benchmark:
+        needs: [run-benchmarks, create-branch-prefix]
+        uses: ./.github/workflows/squash_branches.yml
+        with:
+            branch-prefix: ${{ needs.create-branch-prefix.outputs.branch-prefix }}
+            commit-message: 'run benchmarks'
+        secrets:
+            GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
+            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+            LASTMJS_GITHUB_TOKEN: ${{ secrets.LASTMJS_GITHUB_TOKEN }}
+
+    create-pr:
+        needs: [finalize-benchmark, create-branch-prefix]
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  ref: ${{ github.ref }}
+
+            - name: Create Pull Request
+              env:
+                  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              run: |
+                  gh pr create \
+                    --base ${{ github.ref_name }} \
+                    --head ${{ needs.create-branch-prefix.outputs.base-branch }} \
+                    --title "Benchmark Results for ${{ needs.create-branch-prefix.outputs.base-branch }}" \
+                    --body "Automated PR for benchmark results"