diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
new file mode 100644
index 00000000000..0b60d55d74a
--- /dev/null
+++ b/.github/workflows/benchmarks.yml
@@ -0,0 +1,54 @@
+name: Benchmark
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  benchmark_cpu:
+    name: CPU Pytest benchmark
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: asottile/workflows/.github/actions/fast-checkout@v1.6.0
+
+      - name: Setup environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Install pytorch
+        shell: bash
+        run: pip install torch==2.2.0 --index-url https://download.pytorch.org/whl/cpu
+
+      - name: Install Kornia dev
+        shell: bash
+        run: pip install .[dev,x]
+
+      - name: Check torch version
+        shell: bash
+        run: pip show torch | grep 2.2.0  || false
+
+      - name: Print dependencies and kornia version
+        shell: bash
+        run: |
+          python -c "import torch;print('Pytorch version: ', torch.__version__)"
+          python -c "import kornia;print('Kornia version: ', kornia.__version__)"
+
+      - name: Install benchmark requirements
+        run: pip install -r requirements/requirements-benchmarks.txt
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v2
+        with:
+          token: ${{ secrets.CODSPEED_TOKEN }}
+          # run only 4 benchmarks because they take a long time to run for the time being
+          run: |
+            pytest -vvv --codspeed -n auto \
+              benchmarks/augmentation/2d_geometric_test.py::test_aug_2d_thin_plate_spline \
+              benchmarks/augmentation/2d_geometric_test.py::test_aug_2d_elastic_transform \
+              benchmarks/augmentation/2d_intensity_test.py::test_aug_2d_linear_corner_illumination \
+              benchmarks/augmentation/2d_intensity_test.py::test_aug_2d_solarize
diff --git a/benchmarks/augmentation/conftest.py b/benchmarks/augmentation/conftest.py
index 7239aa0b3d9..f36ae80612d 100644
--- a/benchmarks/augmentation/conftest.py
+++ b/benchmarks/augmentation/conftest.py
@@ -9,8 +9,11 @@ def shape(B, C, H, W):
 
 
 def pytest_generate_tests(metafunc):
-    B = [1, 5]
-    C = [1, 3]
+    # B = [1, 5]
+    # C = [1, 3]
+    # use only 1x1x128x128 for now, to only have a single test case
+    B = [1]
+    C = [1]
     H = W = [128]  # , 237, 512]
     params = list(product(B, C, H, W))
     metafunc.parametrize("B, C, H, W", params)
diff --git a/requirements/requirements-benchmarks.txt b/requirements/requirements-benchmarks.txt
index ad5b816183e..97fb946a057 100644
--- a/requirements/requirements-benchmarks.txt
+++ b/requirements/requirements-benchmarks.txt
@@ -1 +1,3 @@
 pytest-benchmark
+pytest-codspeed
+pytest-xdist