Merge branch 'master' into validate-app-fetch

PostHog · Sep 18, 2023 · 02c6e0e · 02c6e0e
2 parents 9962356 + 2edc058
commit 02c6e0e
Show file tree

Hide file tree

Showing 880 changed files with 25,202 additions and 7,938 deletions.
diff --git a/.coveragerc b/.coveragerc
diff --git a/.deepsource.toml b/.deepsource.toml
diff --git a/.environment b/.environment
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -147,6 +147,10 @@ module.exports = {
                         element: 'a',
                         message: 'use <Link> instead',
                     },
+                    {
+                        element: 'ReactMarkdown',
+                        message: 'use <LemonMarkdown> instead',
+                    },
                 ],
             },
         ],
@@ -174,6 +178,10 @@ module.exports = {
                         element: 'Collapse',
                         message: 'use <LemonCollapse> instead',
                     },
+                    {
+                        element: 'MonacoEditor',
+                        message: 'use <CodeEditor> instead',
+                    },
                 ],
             },
         ],

diff --git a/.github/actions/run-backend-tests/action.yml b/.github/actions/run-backend-tests/action.yml
@@ -44,21 +44,27 @@ runs:
           run: echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts
 
         - name: Set up Python
-          id: python
-          uses: buildjet/setup-python@v4
+          uses: actions/setup-python@v4
           with:
               python-version: ${{ inputs.python-version }}
               token: ${{ inputs.token }}
-              cache: pip
 
         - name: Install SAML (python3-saml) dependencies
           shell: bash
           run: |
               sudo apt-get update
               sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
-          if: steps.python.outputs.cache-hit != 'true'
+
+        - uses: syphar/restore-virtualenv@v1
+          id: cache-backend-tests
+          with:
+              custom_cache_key_element: v1
+
+        - uses: syphar/restore-pip-download-cache@v1
+          if: steps.cache-backend-tests.outputs.cache-hit != 'true'
 
         - name: Install python dependencies
+          if: steps.cache-backend-tests.outputs.cache-hit != 'true'
           shell: bash
           run: |
               python -m pip install -r requirements-dev.txt
@@ -97,13 +103,29 @@ runs:
               PERSON_ON_EVENTS_V2_ENABLED: ${{ inputs.person-on-events }}
               GROUPS_ON_EVENTS_ENABLED: ${{ inputs.person-on-events }}
           shell: bash
-          run: | # async_migrations are covered in ci-async-migrations.yml
-              pytest hogvm posthog -m "not async_migrations" \
+          run: | # async_migrations covered in ci-async-migrations.yml
+              pytest ${{
+                  inputs.person-on-events == 'true'
+                  && './posthog/clickhouse/ ./posthog/hogql/ ./posthog/queries/ ./posthog/api/test/test_insight* ./posthog/api/test/dashboards/test_dashboard.py'
+                  || 'hogvm posthog'
+              }} -m "not async_migrations" \
+                  --splits ${{ inputs.concurrency }} --group ${{ inputs.group }} \
+                  --durations=100 --durations-min=1.0 \
+                  $PYTEST_ARGS
+
+        - name: Run EE tests
+          if: ${{ inputs.segment == 'EE' }}
+          env:
+              PERSON_ON_EVENTS_V2_ENABLED: ${{ inputs.person-on-events }}
+              GROUPS_ON_EVENTS_ENABLED: ${{ inputs.person-on-events }}
+          shell: bash
+          run: | # async_migrations covered in ci-async-migrations.yml
+              pytest ${{ inputs.person-on-events == 'true' && 'ee/clickhouse/' || 'ee/' }} -m "not async_migrations" \
                   --splits ${{ inputs.concurrency }} --group ${{ inputs.group }} \
-                  --durations=100 --durations-min=1.0 --store-durations \
+                  --durations=100 --durations-min=1.0 \
                   $PYTEST_ARGS
 
-        - name: Run Decide read replica tests
+        - name: Run /decide read replica tests
           if: ${{ inputs.segment == 'FOSS' && inputs.group == 1 && inputs.person-on-events != 'true' }}
           env:
               POSTHOG_DB_NAME: posthog
@@ -116,24 +138,3 @@ runs:
               pytest posthog/api/test/test_decide.py::TestDecideUsesReadReplica \
                   --durations=100 --durations-min=1.0 \
                   $PYTEST_ARGS
-
-        - name: Run EE tests
-          if: ${{ inputs.segment == 'EE' }}
-          env:
-              PERSON_ON_EVENTS_V2_ENABLED: ${{ inputs.person-on-events }}
-              GROUPS_ON_EVENTS_ENABLED: ${{ inputs.person-on-events }}
-          shell: bash
-          run: | # async_migrations are covered in ci-async-migrations.yml
-              pytest ee -m "not async_migrations" \
-                  --splits ${{ inputs.concurrency }} --group ${{ inputs.group }} \
-                  --durations=100 --durations-min=1.0 --store-durations \
-                  $PYTEST_ARGS
-
-        # Post-tests
-
-        - name: Upload updated timing data as artifacts
-          uses: actions/upload-artifact@v2
-          if: ${{ inputs.segment == 'EE' && inputs.person-on-events != 'true'}}
-          with:
-              name: timing_data-${{ inputs.group }}
-              path: .test_durations
diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml
@@ -15,8 +15,13 @@ on:
                 description: ClickHouse server version. Leave blank for default
                 type: string
 
+concurrency:
+    group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+    # This is so that the workflow run isn't canceled when a snapshot update is pushed within it by posthog-bot
+    # We do however cancel from container-images-ci.yml if a commit is pushed by someone OTHER than posthog-bot
+    cancel-in-progress: false
+
 env:
-    DOCKER_REGISTRY_PREFIX: us-east1-docker.pkg.dev/posthog-301601/mirror/
     SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only
     DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
     REDIS_URL: 'redis://localhost'
@@ -34,7 +39,7 @@ jobs:
     # Job to decide if we should run backend ci
     # See https://github.com/dorny/paths-filter#conditional-execution for more details
     changes:
-        runs-on: buildjet-2vcpu-ubuntu-2204
+        runs-on: ubuntu-latest
         timeout-minutes: 5
         if: github.repository == 'PostHog/posthog'
         name: Determine need to run backend checks
@@ -77,10 +82,10 @@ jobs:
 
     backend-code-quality:
         needs: changes
-        timeout-minutes: 10
+        timeout-minutes: 30
 
         name: Python code quality checks
-        runs-on: buildjet-2vcpu-ubuntu-2204
+        runs-on: ubuntu-latest
 
         steps:
             # If this run wasn't initiated by the bot (meaning: snapshot update) and we've determined
@@ -95,27 +100,27 @@ jobs:
                   fetch-depth: 1
                   path: 'current/'
 
-            - name: Stop/Start stack with Docker Compose
-              run: |
-                  cd current
-                  docker compose -f docker-compose.dev.yml down
-                  docker compose -f docker-compose.dev.yml up -d
-
             - name: Set up Python
-              id: python
-              uses: buildjet/setup-python@v4
+              uses: actions/setup-python@v4
               with:
                   python-version: 3.10.10
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
-                  cache: pip
+
+            - uses: syphar/restore-virtualenv@v1
+              id: cache-backend-tests
+              with:
+                  custom_cache_key_element: v1-
+
+            - uses: syphar/restore-pip-download-cache@v1
+              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
 
             - name: Install SAML (python3-saml) dependencies
               run: |
                   sudo apt-get update
-                  sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
-              if: steps.python.outputs.cache-hit != 'true'
+                  sudo apt-get install libxml2-dev libxmlsec1 libxmlsec1-dev libxmlsec1-openssl
 
             - name: Install python dependencies
+              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
               run: |
                   cd current
                   python -m pip install -r requirements.txt -r requirements-dev.txt
@@ -146,8 +151,7 @@ jobs:
                   # "apt-get install antlr" would install 4.7 which is incompatible with our grammar.
                   export ANTLR_VERSION=4.13.0
                   # java version doesn't matter
-                  sudo apt update -y
-                  sudo apt-get -y install default-jre
+                  sudo apt-get install default-jre
                   mkdir antlr
                   cd antlr
                   curl -o antlr.jar https://www.antlr.org/download/antlr-$ANTLR_VERSION-complete.jar
@@ -168,7 +172,7 @@ jobs:
         timeout-minutes: 5
 
         name: Validate Django migrations
-        runs-on: buildjet-2vcpu-ubuntu-2204
+        runs-on: ubuntu-latest
 
         steps:
             - uses: actions/checkout@v3
@@ -179,20 +183,26 @@ jobs:
                   docker compose -f docker-compose.dev.yml up -d
 
             - name: Set up Python
-              id: python
-              uses: buildjet/setup-python@v4
+              uses: actions/setup-python@v4
               with:
                   python-version: 3.10.10
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
-                  cache: pip
+
+            - uses: syphar/restore-virtualenv@v1
+              id: cache-backend-tests
+              with:
+                  custom_cache_key_element: v1-
+
+            - uses: syphar/restore-pip-download-cache@v1
+              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
 
             - name: Install SAML (python3-saml) dependencies
               run: |
                   sudo apt-get update
                   sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
-              if: steps.python.outputs.cache-hit != 'true'
 
             - name: Install python dependencies
+              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
               run: |
                   python -m pip install -r requirements.txt -r requirements-dev.txt
 
@@ -226,7 +236,7 @@ jobs:
         timeout-minutes: 30
 
         name: Django tests – ${{ matrix.segment }} (persons-on-events ${{ matrix.person-on-events && 'on' || 'off' }}), Py ${{ matrix.python-version }}, ${{ matrix.clickhouse-server-image }} (${{matrix.group}}/${{ matrix.concurrency }})
-        runs-on: buildjet-4vcpu-ubuntu-2204
+        runs-on: ubuntu-latest
 
         strategy:
             fail-fast: false
@@ -303,7 +313,7 @@ jobs:
         name: Async migrations tests
         needs: changes
         if: needs.changes.outputs.backend == 'true'
-        runs-on: buildjet-2vcpu-ubuntu-2204
+        runs-on: ubuntu-latest
         steps:
             - name: 'Checkout repo'
               uses: actions/checkout@v3
@@ -317,20 +327,26 @@ jobs:
                   docker compose -f docker-compose.dev.yml up -d
 
             - name: Set up Python
-              id: python
-              uses: buildjet/setup-python@v4
+              uses: actions/setup-python@v4
               with:
                   python-version: 3.10.10
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
-                  cache: pip
+
+            - uses: syphar/restore-virtualenv@v1
+              id: cache-backend-tests
+              with:
+                  custom_cache_key_element: v1-
+
+            - uses: syphar/restore-pip-download-cache@v1
+              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
 
             - name: Install SAML (python3-saml) dependencies
               run: |
                   sudo apt-get update
                   sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
-              if: steps.python.outputs.cache-hit != 'true'
 
             - name: Install python dependencies
+              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
               shell: bash
               run: |
                   python -m pip install -r requirements.txt -r requirements-dev.txt