From 2763518d5807457d9b160811c0fb4fc6675ea27d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Gustavo=20A=2E=20Amorim?= Date: Sun, 4 Feb 2024 19:11:55 -0300 Subject: [PATCH] move to benchmark-action/github-action-benchmar --- .github/workflows/pr_benchmarks.yml | 41 +++++++++++++++++++---------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pr_benchmarks.yml b/.github/workflows/pr_benchmarks.yml index a156a110db..190dc22bd5 100644 --- a/.github/workflows/pr_benchmarks.yml +++ b/.github/workflows/pr_benchmarks.yml @@ -15,8 +15,12 @@ concurrency: jobs: benchmark_cpu: + # NOTE: from https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#stability-of-virtual-environment + #As far as watching the benchmark results of examples in this repository, the amplitude of the benchmarks + #is about +- 10~20%. If your benchmarks use some resources such as networks or file I/O, the amplitude + #might be bigger. name: CPU Pytest benchmark - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: kornia/workflows/.github/actions/env@v1.5.3 @@ -25,8 +29,6 @@ jobs: - name: Setup benchmarks run: | - echo "BASE_SHA=$(echo ${{ github.event.pull_request.base.sha }} | cut -c1-8)" >> $GITHUB_ENV - echo "HEAD_SHA=$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-8)" >> $GITHUB_ENV echo "HEAD_JSON=$(mktemp)" >> $GITHUB_ENV echo "BASE_JSON=$(mktemp)" >> $GITHUB_ENV echo "PR_COMMENT=$(mktemp)" >> $GITHUB_ENV @@ -34,22 +36,33 @@ jobs: - name: Install benchmark requirements run: pip install -r requirements/requirements-benchmarks.txt - - name: Run benchmarks + - name: Run benchmarks BASE + # TODO: Save it using actions/cache then we don't need to regenerate it. + # By caching the result, it will also use the same information across PR's run: | cd benchmarks/ git checkout ${{ github.event.pull_request.base.sha }} pytest ./ -vvv --benchmark-json ${{ env.BASE_JSON }} + + - name: Run benchmarks HEAD + run: | + cd benchmarks/ git checkout ${{ github.sha }} pytest ./ -vvv --benchmark-json ${{ env.HEAD_JSON }} - - name: Comment results - uses: apbard/pytest-benchmark-commenter@v3 + - name: Comment benchmark result + uses: benchmark-action/github-action-benchmark@v1 with: - token: ${{ secrets.GITHUB_TOKEN }} - benchmark-file: ${{ env.HEAD_JSON }} - comparison-benchmark-file: ${{ env.BASE_JSON }} - benchmark-metrics: 'name,max,mean,ops' - comparison-benchmark-metric: 'ops' - comparison-higher-is-better: true - comparison-threshold: 5 - benchmark-title: 'Result of CPU Benchmark Tests' + tool: "pytest" + ref: ${{ github.sha }} + output-file-path: ${{ env.HEAD_JSON }} + external-data-json-path: ${{ env.BASE_JSON }} + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-always: true + comment-on-alert: true + fail-on-alert: true + summary-always: true + skip-fetch-gh-pages: true + auto-push: false + save-data-file: false + # alert-comment-cc-users: '@johnnv1 @edgarriba'