diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000000..8038acf3c3 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,141 @@ +name: Benchmark + +on: + push: + branches: + - main + pull_request: + +jobs: + determine-should-run-benchmarks: + name: Determine if benchmarks should run + runs-on: ubuntu-latest + outputs: + # If the branch should release then it shouldn't run benchmarks. + should-run-benchmarks: ${{ steps.determine-should-run-benchmarks.outputs.should-release == 'false' }} + steps: + - uses: actions/checkout@v4 + + - id: determine-should-run-benchmarks + uses: ./.github/actions/should_release + + set-exclude-dirs: + name: Set exclude directories + runs-on: ubuntu-latest + outputs: + exclude-dirs: ${{ steps.set-exclude-dirs.outputs.exclude-dirs }} + steps: + - uses: actions/checkout@v4 + + - id: set-conditions + uses: ./.github/actions/set_run_conditions + + - id: set-exclude-dirs + run: | + RELEASE_BENCHMARKS="${{ format(' + tests/end_to_end/candid_rpc/class_syntax/new + tests/end_to_end/http_server/new + ') }}" + + UNSTABLE_BENCHMARKS="${{ format(' + examples/basic_bitcoin + examples/bitcoin_psbt + examples/ckbtc + tests/end_to_end/http_server/ethers_base + tests/end_to_end/http_server/http_outcall_fetch + tests/end_to_end/http_server/ic_evm_rpc + tests/property/candid_rpc/class_api/stable_b_tree_map + tests/property/candid_rpc/functional_api/stable_b_tree_map + tests/property/ic_api/performance_counter + tests/property/ic_api/instruction_counter + ') }}" + + SLOW_BENCHMARKS="${{ format(' + tests/end_to_end/candid_rpc/functional_syntax/ckbtc + tests/end_to_end/candid_rpc/class_syntax/bitcoin + tests/end_to_end/http_server/large_files + tests/end_to_end/http_server/open_value_sharing + tests/end_to_end/candid_rpc/class_syntax/stable_structures + tests/end_to_end/candid_rpc/functional_syntax/bitcoin + tests/end_to_end/candid_rpc/functional_syntax/composite_queries + tests/end_to_end/candid_rpc/functional_syntax/cross_canister_calls + tests/end_to_end/candid_rpc/functional_syntax/management_canister + tests/end_to_end/candid_rpc/functional_syntax/stable_structures + tests/end_to_end/http_server/autoreload + ') }}" + + AZLE_IS_MAIN_BRANCH_PUSH="${{ steps.set-conditions.outputs.is_main_branch_push }}" + AZLE_IS_MAIN_BRANCH_PUSH_FROM_RELEASE_MERGE="${{ steps.set-conditions.outputs.is_main_branch_push_from_release_merge }}" + AZLE_IS_RELEASE_BRANCH_PR="${{ steps.set-conditions.outputs.is_release_branch_pr }}" + AZLE_IS_FEATURE_BRANCH_PR="${{ steps.set-conditions.outputs.is_feature_branch_pr }}" + AZLE_IS_FEATURE_BRANCH_DRAFT_PR="${{ steps.set-conditions.outputs.is_feature_branch_draft_pr }}" + + EXCLUDE_DIRS="" + + if [[ "$AZLE_IS_MAIN_BRANCH_PUSH" == "true" ]]; then + EXCLUDE_DIRS="" + fi + + if [[ "$AZLE_IS_MAIN_BRANCH_PUSH_FROM_RELEASE_MERGE" == "true" ]]; then + EXCLUDE_DIRS="" + fi + + if [[ "$AZLE_IS_RELEASE_BRANCH_PR" == "true" ]]; then + EXCLUDE_DIRS="" + fi + + if [[ "$AZLE_IS_FEATURE_BRANCH_PR" == "true" ]]; then + EXCLUDE_DIRS="$RELEASE_BENCHMARKS $UNSTABLE_BENCHMARKS" + fi + + if [[ "$AZLE_IS_FEATURE_BRANCH_DRAFT_PR" == "true" ]]; then + EXCLUDE_DIRS="$RELEASE_BENCHMARKS $UNSTABLE_BENCHMARKS $SLOW_BENCHMARKS" + fi + + # Trim leading or trailing spaces and save the exclude-dirs in the environment + EXCLUDE_DIRS=$(echo $EXCLUDE_DIRS | xargs) + echo "exclude-dirs=$EXCLUDE_DIRS" >> $GITHUB_OUTPUT + + run-benchmarks: + name: ${{ matrix.benchmark_group.name }} + needs: + - determine-should-run-benchmarks + - set-exclude-dirs + if: ${{ needs.determine-should-run-benchmarks.outputs.should-run-benchmarks == 'true' }} + strategy: + fail-fast: false + matrix: + benchmark_group: + - { name: 'Examples', directories: './examples' } + - { + name: 'E2E Class', + directories: './tests/end_to_end/candid_rpc/class_syntax' + } + - { + name: 'E2E Functional', + directories: './tests/end_to_end/candid_rpc/functional_syntax' + } + - { + name: 'E2E HTTP Server', + directories: './tests/end_to_end/http_server' + } + - { + name: 'Property Class', + directories: './tests/property/candid_rpc/class_api' + } + - { + name: 'Property Functional', + directories: './tests/property/candid_rpc/functional_api' + } + - { + name: 'Property IC API', + directories: './tests/property/ic_api' + } + uses: ./.github/workflows/benchmark_parallel.yml + secrets: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LASTMJS_GITHUB_TOKEN: ${{ secrets.LASTMJS_GITHUB_TOKEN }} + with: + directories: ${{ matrix.benchmark_group.directories }} + exclude-dirs: ${{ needs.set-exclude-dirs.outputs.exclude-dirs }} diff --git a/.github/workflows/benchmark_parallel.yml b/.github/workflows/benchmark_parallel.yml new file mode 100644 index 0000000000..3a94273c40 --- /dev/null +++ b/.github/workflows/benchmark_parallel.yml @@ -0,0 +1,156 @@ +name: Parallel Benchmark +on: + workflow_call: + inputs: + directories: + required: true + type: string + exclude-dirs: + required: false + type: string + default: '' + secrets: + GPG_SIGNING_KEY: + required: true + GH_TOKEN: + required: true + LASTMJS_GITHUB_TOKEN: + required: true + +jobs: + prepare-benchmark: + name: Prepare Benchmark + runs-on: ubuntu-latest + env: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} # All commits must be verified + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + outputs: + azle-version: ${{ steps.get-version.outputs.azle-version }} + test-infos: ${{ steps.get-test-infos.outputs.test-infos }} + steps: + - uses: actions/checkout@v4 + + - id: get-version + run: | + VERSION=$(jq -r '.version' package.json) + echo "azle-version=$VERSION" >> $GITHUB_OUTPUT + + - id: get-test-infos + uses: ./.github/actions/get_test_infos + with: + directories: ${{ inputs.directories }} + exclude-dirs: ${{ inputs.exclude-dirs }} + + run-benchmarks: + needs: prepare-benchmark + name: Run benchmarks for ${{ matrix.test.name }} + runs-on: ubuntu-latest + env: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + strategy: + fail-fast: false + matrix: + test: ${{ fromJson(needs.prepare-benchmark.outputs.test-infos) }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref || github.ref }} + token: ${{ secrets.LASTMJS_GITHUB_TOKEN || github.token }} + + - uses: ./.github/actions/setup_node + + - uses: ./.github/actions/setup_dfx + + - name: Install dependencies + run: | + npm install + cd ${{ matrix.test.path }} + npm install + + - name: Start dfx with artificial delay 0 + working-directory: ${{ matrix.test.path }} + run: dfx start --clean --background --host 127.0.0.1:8000 --artificial-delay 0 + + - name: Run npm test (continue on error) + working-directory: ${{ matrix.test.path }} + continue-on-error: true + run: npm test + + - uses: ./.github/actions/configure_git + with: + gpg_signing_key: ${{ secrets.GPG_SIGNING_KEY }} + + - name: Commit and push changes + run: | + BRANCH_NAME="benchmark--${{ needs.prepare-benchmark.outputs.azle-version }}-$(echo '${{ matrix.test.displayPath }}' | sed 's/\//-/g')" + git switch -c "$BRANCH_NAME" + git add --all + if ! git diff --cached --quiet; then + git commit -m "Run benchmarks for ${{ matrix.test.displayPath }}" + else + echo "No changes to commit. Skipping commit and push." + fi + git push origin "$BRANCH_NAME" + + finalize-benchmark: + needs: [prepare-benchmark, run-benchmarks] + name: Finalize Benchmark + runs-on: ubuntu-latest + env: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref || github.ref }} + token: ${{ secrets.LASTMJS_GITHUB_TOKEN }} + fetch-depth: 0 + + - uses: ./.github/actions/configure_git + with: + gpg_signing_key: ${{ secrets.GPG_SIGNING_KEY }} + + - name: Collect branches + id: collect-branches + run: | + # Get branches and convert to space-separated list + BRANCHES=$(git branch -r | grep "origin/benchmark--${{ needs.prepare-benchmark.outputs.azle-version }}-" | sed 's/origin\///' | xargs) + echo "branches=$BRANCHES" >> $GITHUB_OUTPUT + + - name: Display collected branches + run: | + echo "Collected branches:" + for branch in ${{ steps.collect-branches.outputs.branches }}; do + echo " - $branch" + done + echo "End of branch list" + + - name: Fetch branches + run: | + echo "Fetching all branches..." + BRANCHES_TO_FETCH="" + for branch in ${{ steps.collect-branches.outputs.branches }}; do + BRANCHES_TO_FETCH+=" $branch:$branch" + done + git fetch origin $BRANCHES_TO_FETCH + + - name: Squash changes + env: + PAT: ${{ secrets.LASTMJS_GITHUB_TOKEN }} + run: | + CURRENT_BRANCH=$(git branch --show-current) + + for branch in ${{ steps.collect-branches.outputs.branches }}; do + git merge --squash $branch + done + + # Create a merge commit with a descriptive message + git commit -am "Update all dependencies for benchmark ${{ needs.prepare-benchmark.outputs.azle-version }}" + + git push origin HEAD:$CURRENT_BRANCH + + - name: Delete branches + run: | + echo "Starting branch deletion process..." + git push origin --delete ${{ steps.collect-branches.outputs.branches }}