Cache worker metadata in worker bridge #17
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: On Demand Benchmark | |
| on: | |
| issue_comment: | |
| types: [created] | |
| jobs: | |
| run-benchmark: | |
| if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/run-benchmark') }} | |
| permissions: write-all | |
| runs-on: ubuntu-latest | |
| steps: | |
| # https://github.com/actions/checkout/issues/331#issuecomment-1567674767 | |
| - id: 'get-branch' | |
| run: echo ::set-output name=branch::$(gh pr view $PR_NO --repo $REPO --json headRefName --jq '.headRefName') | |
| env: | |
| REPO: ${{ github.repository }} | |
| PR_NO: ${{ github.event.issue.number }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ steps.get-branch.outputs.branch }} | |
| submodules: recursive | |
| - name: Setup Rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| override: true | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: benchmark-tests | |
| cache-all-crates: true | |
| - name: Run sccache-cache | |
| uses: mozilla-actions/[email protected] | |
| - uses: davidB/rust-cargo-make@v1 | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v1 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Setup Redis | |
| uses: shogo82148/[email protected] | |
| with: | |
| redis-version: latest | |
| auto-start: false | |
| - name: Build Benchmark Report Job | |
| run: | | |
| echo "Building all targets including benchmark-report job" | |
| cargo build | |
| - name: Run Benchmark | |
| run: | | |
| echo "Running all benchmark jobs and saving to files" | |
| - name: Run benchmark and report | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| ISSUE_NUMBER: ${{ github.event.issue.number }} | |
| WORKFLOW_ID: ${{ github.run_id }} | |
| COMMENT_BODY: ${{ github.event.comment.body }} | |
| run: | | |
| #! /bin/bash | |
| # Create a dummy benchmark result | |
| mkdir -p ./benchmark-data-current | |
| echo "Starting cold start large benchmark" | |
| ./target/debug/benchmark_cold_start_large --quiet --json spawned > benchmark-data-current/benchmark_cold_start_large.json | |
| cat benchmark-data-current/benchmark_cold_start_large.json | |
| echo "Successfully ran cold start large benchmark" | |
| echo "Starting cold start medium benchmark" | |
| ./target/debug/benchmark_cold_start_medium --quiet --json spawned > benchmark-data-current/benchmark_cold_start_medium.json | |
| cat benchmark-data-current/benchmark_cold_start_medium.json | |
| echo "Successfully ran cold start medium benchmark" | |
| echo "Starting cold start small benchmark" | |
| ./target/debug/benchmark_cold_start_small --quiet --json spawned > benchmark-data-current/benchmark_cold_start_small.json | |
| cat benchmark-data-current/benchmark_cold_start_small.json | |
| echo "Successfully ran cold start small benchmark" | |
| echo "Starting cold start large benchmark without component compilation" | |
| ./target/debug/benchmark_cold_start_large --quiet --json spawned --component-compilation-disabled > benchmark-data-current/benchmark_cold_start_large_no_compilation.json | |
| cat benchmark-data-current/benchmark_cold_start_large_no_compilation.json | |
| echo "Successfully ran cold start large benchmark without component compilation" | |
| echo "Starting cold start medium benchmark without component compilation" | |
| ./target/debug/benchmark_cold_start_medium --quiet --json spawned --component-compilation-disabled > benchmark-data-current/benchmark_cold_start_medium_no_compilation.json | |
| cat benchmark-data-current/benchmark_cold_start_medium_no_compilation.json | |
| echo "Successfully ran cold start medium benchmark without component compilation" | |
| echo "Starting cold start small benchmark without component compilation" | |
| ./target/debug/benchmark_cold_start_small --quiet --json spawned --component-compilation-disabled> benchmark-data-current/benchmark_cold_start_small_no_compilation.json | |
| cat benchmark-data-current/benchmark_cold_start_small_no_compilation.json | |
| echo "Successfully ran cold start small benchmark without component compilation" | |
| echo "Starting durability overhead benchmark" | |
| ./target/debug/benchmark_durability_overhead --quiet --json spawned > benchmark-data-current/benchmark_durability_overhead.json | |
| cat benchmark-data-current/benchmark_durability_overhead.json | |
| echo "Successfully durability overhead benchmark" | |
| echo "Starting latency large benchmark" | |
| ./target/debug/benchmark_latency_large --quiet --json spawned > benchmark-data-current/benchmark_latency_large.json | |
| cat benchmark-data-current/benchmark_latency_large.json | |
| echo "Successfully ran latency large benchmark" | |
| echo "Starting latency medium benchmark" | |
| ./target/debug/benchmark_latency_medium --quiet --json spawned > benchmark-data-current/benchmark_latency_medium.json | |
| cat benchmark-data-current/benchmark_latency_medium.json | |
| echo "Successfully ran latency medium benchmark" | |
| echo "Starting latency small benchmark" | |
| ./target/debug/benchmark_latency_small --quiet --json spawned > benchmark-data-current/benchmark_latency_small.json | |
| cat benchmark-data-current/benchmark_latency_small.json | |
| echo "Successfully ran latency small benchmark" | |
| echo "Starting suspend benchmark" | |
| ./target/debug/benchmark_suspend_worker --quiet --json spawned > benchmark-data-current/benchmark_suspend_worker.json | |
| cat benchmark-data-current/benchmark_suspend_worker.json | |
| echo "Successfully ran suspend benchmark" | |
| # echo "Starting throughput benchmark" | |
| # ./target/debug/benchmark_throughput --quiet --json spawned > benchmark-data-current/benchmark_throughput.json | |
| # cat benchmark-data-current/benchmark_throughput.json | |
| # echo "Successfully ran throughput benchmark" | |
| echo "Finished Running Benchmark Jobs" | |
| files=$(ls benchmark-data-current) | |
| echo "Checking the following files if it already exists in a previous benchmark run" | |
| for str in ${files[@]}; do | |
| echo $str | |
| done | |
| # Initialize arrays to store present and absent files | |
| present_files=() | |
| absent_files=() | |
| compare_command="./target/debug/benchmark_report compare-benchmarks" | |
| get_report_command="./target/debug/benchmark_report get-report" | |
| echo "Starting to pull the existing benchmark results if exists" | |
| if git fetch origin main:main && [[ ! "$COMMENT_BODY" =~ refresh ]]; then | |
| # Check each file | |
| for file in ${files[@]}; do | |
| if git checkout main -- "benchmark-data/$file"; then | |
| present_files+=("$file") | |
| compare_command="$compare_command --files $file=$(pwd)/benchmark-data/$file,$(pwd)/benchmark-data-current/$file" | |
| else | |
| absent_files+=("$file") | |
| get_report_command="$get_report_command --files $file=$(pwd)/benchmark-data-current/$file" | |
| fi | |
| done | |
| else | |
| for file in ${files[@]}; do | |
| absent_files+=("$file") | |
| get_report_command="$get_report_command --files $file=$(pwd)/benchmark-data-current/$file" | |
| done | |
| fi | |
| echo $compare_command | |
| echo $get_report_command | |
| # If there are present files, run the comparison | |
| if [ ${#present_files[@]} -ne 0 ]; then | |
| echo "Running comparison with command: $compare_command" | |
| $compare_command > benchmark_comparison.md | |
| COMPARISON_MD=$(<benchmark_comparison.md) | |
| curl -L \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer $GITHUB_TOKEN" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/golemcloud/golem/issues/$ISSUE_NUMBER/comments \ | |
| -d "{\"body\":\"$COMPARISON_MD \n RunID: $WORKFLOW_ID \"}" | |
| fi | |
| # Process absent files | |
| if [ ${#absent_files[@]} -ne 0 ]; then | |
| echo "Running report" | |
| $get_report_command > report.md | |
| echo "Successfully ran the report command" | |
| REPORT_MD=$(<report.md) | |
| echo "$REPORT_MD" | |
| curl -L \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer $GITHUB_TOKEN" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/golemcloud/golem/issues/$ISSUE_NUMBER/comments \ | |
| -d "{\"body\":\"$REPORT_MD \n RunID: $WORKFLOW_ID \"}" | |
| fi | |
| echo "Done processing files." | |
| - name: "Move benchmark-data-current to benchmark-data" | |
| run: | | |
| # Removing the old benchmark data | |
| rm -rf benchmark-data | |
| rm -rf benchmark_comparison.md | |
| rm -rf report.md | |
| # Moving the current benchmark data to benchmark-data | |
| mv benchmark-data-current benchmark-data | |
| - name: Commit and push benchmark results (not report) back to PR branch | |
| run: | | |
| git branch -a | |
| git config --local user.email "[email protected]" | |
| git config --local user.name "GitHub Action" | |
| git pull | |
| git add benchmark-data | |
| git commit -m "Adding benchmarking data" | |
| if git push; then | |
| echo "Successfully pushed the benchmark data" | |
| else | |
| echo "Benchmark data hasn't been pushed probably due to zero changes" | |
| fi | |