From 05ee7207bd364926145bef934e7f7904b1c52b02 Mon Sep 17 00:00:00 2001
From: Alessandro Pietro Bardelli <apbard@users.noreply.github.com>
Date: Thu, 6 Apr 2023 14:19:14 +0200
Subject: [PATCH] [CI] run benchmarks and comment on PR

---
 .github/workflows/benchmarks_pr.yml | 137 ++++++++++++++++++++++++++++
 1 file changed, 137 insertions(+)
 create mode 100644 .github/workflows/benchmarks_pr.yml

diff --git a/.github/workflows/benchmarks_pr.yml b/.github/workflows/benchmarks_pr.yml
new file mode 100644
index 000000000..fd1167197
--- /dev/null
+++ b/.github/workflows/benchmarks_pr.yml
@@ -0,0 +1,137 @@
+name: Continuous Benchmark
+
+on:
+  pull_request:
+
+permissions: write-all
+
+jobs:
+  benchmark_cpu:
+    name: CPU Pytest benchmark
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Who triggered this?
+        run: |
+          echo "Action triggered by ${{ github.event.pull_request.html_url }}"
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 50 # this is to make sure we obtain the target base commit
+      - name: Python Setup
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Setup Environment
+        run: |
+          pip install -e .
+          pip install pytest pytest-benchmark
+      - name: Setup benchmarks
+        run: |
+          echo "BASE_SHA=$(echo ${{ github.event.pull_request.base.sha }} | cut -c1-8)" >> $GITHUB_ENV
+          echo "HEAD_SHA=$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-8)" >> $GITHUB_ENV
+          echo "BASELINE_JSON=$(mktemp)" >> $GITHUB_ENV
+          echo "CONTENDER_JSON=$(mktemp)" >> $GITHUB_ENV
+          echo "PR_COMMENT=$(mktemp)" >>  $GITHUB_ENV
+      - name: Run benchmarks
+        run: |
+          cd benchmarks/
+          RUN_BENCHMARK="pytest --rank 0 --benchmark-json "
+          git checkout ${{ github.event.pull_request.base.sha }}
+          $RUN_BENCHMARK ${{ env.BASELINE_JSON }}
+          git checkout ${{ github.event.pull_request.head.sha }}
+          $RUN_BENCHMARK ${{ env.CONTENDER_JSON }}          
+      - name: Publish results
+        uses: apbard/pytest-benchmark-commenter@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          benchmark-file: ${{ env.BASELINE_JSON }}
+          comparison-benchmark-file: ${{ env.CONTENDER_JSON }}
+          benchmark-metrics: 'name,max,mean,ops'
+          comparison-benchmark-metric: 'ops'
+          comparison-higher-is-better: true
+          comparison-threshold: 5
+          benchmark-title: 'Result of CPU Benchmark Tests'
+
+  benchmark_gpu:
+    name: GPU Pytest benchmark
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        include:
+         - os: linux.4xlarge.nvidia.gpu
+           python-version: 3.8
+    defaults:
+      run:
+        shell: bash -l {0}
+    container: nvidia/cuda:11.8.0-cudnn8-devel-ubuntu18.04
+    steps:
+      - name: Who triggered this?
+        run: |
+          echo "Action triggered by ${{ github.event.pull_request.html_url }}"
+      - name: Install deps
+        run: |
+          apt-get update -y
+          apt-get install software-properties-common -y
+          add-apt-repository ppa:git-core/candidate -y
+          apt-get update -y
+          apt-get upgrade -y
+          apt-get -y install libglu1-mesa libgl1-mesa-glx libosmesa6 gcc curl g++ unzip wget libglfw3-dev libgles2-mesa-dev libglew-dev sudo git cmake libz-dev
+      - name: Check ldd --version
+        run: ldd --version
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 50 # this is to make sure we obtain the target base commit
+      - name: Python Setup
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Setup conda
+        run: |
+          rm -rf $HOME/miniconda
+          wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
+          bash ~/miniconda.sh -b -p $HOME/miniconda
+      - name: setup Path
+        run: |
+          echo "$HOME/miniconda/bin" >> $GITHUB_PATH
+          echo "CONDA=$HOME/miniconda" >> $GITHUB_PATH
+      - name: create and activate conda env
+        run: |
+          $HOME/miniconda/bin/conda create --name build_binary python=${{ matrix.python-version }}
+          $HOME/miniconda/bin/conda info
+          $HOME/miniconda/bin/activate build_binary
+      - name: Setup git
+        run: git config --global --add safe.directory /__w/tensordict/tensordict
+      - name: setup Path
+        run: |
+          echo /usr/local/bin >> $GITHUB_PATH
+      - name: Setup Environment
+        run: |
+          python -m pip install -e .
+          python -m pip install pytest pytest-benchmark
+      - name: Setup benchmarks
+        run: |
+          echo "BASE_SHA=$(echo ${{ github.event.pull_request.base.sha }} | cut -c1-8)" >> $GITHUB_ENV
+          echo "HEAD_SHA=$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-8)" >> $GITHUB_ENV
+          echo "BASELINE_JSON=$(mktemp)" >> $GITHUB_ENV
+          echo "CONTENDER_JSON=$(mktemp)" >> $GITHUB_ENV
+          echo "PR_COMMENT=$(mktemp)" >>  $GITHUB_ENV
+      - name: Run benchmarks
+        run: |
+          cd benchmarks/
+          RUN_BENCHMARK="pytest --rank 0 --benchmark-json "
+          git checkout ${{ github.event.pull_request.base.sha }}
+          $RUN_BENCHMARK ${{ env.BASELINE_JSON }}
+          git checkout ${{ github.event.pull_request.head.sha }}
+          $RUN_BENCHMARK ${{ env.CONTENDER_JSON }}          
+      - name: Publish results
+        uses: apbard/pytest-benchmark-commenter@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          benchmark-file: ${{ env.BASELINE_JSON }}
+          comparison-benchmark-file: ${{ env.CONTENDER_JSON }}
+          benchmark-metrics: 'name,max,mean,ops'
+          comparison-benchmark-metric: 'ops'
+          comparison-higher-is-better: true
+          comparison-threshold: 5
+          benchmark-title: 'Result of GPU Benchmark Tests'