Benchmark with app harness

.github/workflows/benchmarks.yml

@@ -21,11 +21,10 @@ env:
  PYTHONIOENCODING: 'utf8'
  TELEMETRY_ENABLED: false
  NODE_OPTIONS: '--max_old_space_size=4096'
+ DATABASE_URL: ${{ secrets.DATABASE_URL }}
 
 jobs:
  reflex-web:
- env:
- DATABASE_URL: ${{ secrets.DATABASE_URL }}
  strategy:
  fail-fast: false
  matrix:
@@ -70,8 +69,58 @@ jobs:
  - name: Run Benchmarks
  # Only run if the database creds are available in this context.
  if: ${{ env.DATABASE_URL }}
- working-directory: ./integration/benchmarks
- run: poetry run python benchmarks.py "$GITHUB_SHA" .lighthouseci
+ run: poetry run python scripts/lighthouse_score_upload.py "$GITHUB_SHA" ./integration/benchmarks/.lighthouseci
  env:
  GITHUB_SHA: ${{ github.sha }}
  PR_TITLE: ${{ github.event.pull_request.title }}
+
+ simple-apps-benchmarks:
+ env:
+ OUTPUT_FILE: benchmarks.json
+ timeout-minutes: 50
+ strategy:
+ # Prioritize getting more information out of the workflow (even if something fails)
+ fail-fast: false
+ matrix:
+ # Show OS combos first in GUI
+ os: [ubuntu-latest, windows-latest, macos-latest]
+ python-version: ['3.8.18', '3.9.18', '3.10.13', '3.11.5', '3.12.0']
+ exclude:
+ - os: windows-latest
+ python-version: '3.10.13'
+ - os: windows-latest
+ python-version: '3.9.18'
+ - os: windows-latest
+ python-version: '3.8.18'
+ include:
+ - os: windows-latest
+ python-version: '3.10.11'
+ - os: windows-latest
+ python-version: '3.9.13'
+ - os: windows-latest
+ python-version: '3.8.10'
+
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - uses: ./.github/actions/setup_build_env
+ with:
+ python-version: ${{ matrix.python-version }}
+ run-poetry-install: true
+ create-venv-at-path: .venv
+ - name: Install additional dependencies for DB access
+ run: poetry run pip install psycopg2-binary
+ - name: Run benchmark tests
+ env:
+ APP_HARNESS_HEADLESS: 1
+ PYTHONUNBUFFERED: 1
+ run: |
+ poetry run pytest -v benchmarks/ --benchmark-json=${{ env.OUTPUT_FILE }} -s
+ - name: Upload benchmark results
+ # Only run if the database creds are available in this context.
+ if: ${{ env.DATABASE_URL }}
+ run: poetry run python scripts/simple_app_benchmark_upload.py --os "${{ matrix.os }}"
+ --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}" 
+ --benchmark-json "${{ env.OUTPUT_FILE }}" --pr-title "${{ github.event.pull_request.title }}"
+ --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}"
+ --event-type "${{ github.event_name }}" --actor "${{ github.actor }}"

.pre-commit-config.yaml

@@ -10,6 +10,7 @@ repos:
  hooks:
  - id: ruff
  args: [--fix, --exit-non-zero-on-fix]
+ exclude: '^integration/benchmarks/'
 
  - repo: https://github.com/RobertCraigie/pyright-python
  rev: v1.1.313

benchmarks/__init__.py

@@ -0,0 +1,3 @@
+"""Reflex benchmarks."""
+
+WINDOWS_SKIP_REASON = "Takes too much time as a result of npm"

benchmarks/conftest.py

@@ -0,0 +1,20 @@
+"""Shared conftest for all benchmark tests."""
+
+import pytest
+
+from reflex.testing import AppHarness, AppHarnessProd
+
+
+@pytest.fixture(
+ scope="session", params=[AppHarness, AppHarnessProd], ids=["dev", "prod"]
+)
+def app_harness_env(request):
+ """Parametrize the AppHarness class to use for the test, either dev or prod.
+
+ Args:
+ request: The pytest fixture request object.
+
+ Returns:
+ The AppHarness class to use for the test.
+ """
+ return request.param