Aspicer/trends udf

.dockerignore

@@ -39,3 +39,11 @@
 !test-runner-jest.config.js
 !test-runner-jest-environment.js
 !patches
+!rust
+rust/.env
+rust/.github
+rust/docker
+rust/target
+rust/cyclotron-node/dist
+rust/cyclotron-node/node_modules
+rust/cyclotron-node/index.node

.github/actions/run-backend-tests/action.yml

@@ -202,7 +202,7 @@ runs:
 
         - name: Upload updated timing data as artifacts
           uses: actions/upload-artifact@v4
-          if: ${{ inputs.person-on-events != 'true' && inputs.clickhouse-server-image == 'clickhouse/clickhouse-server:23.12.5.81-alpine' }}
+          if: ${{ inputs.person-on-events != 'true' && inputs.clickhouse-server-image == 'clickhouse/clickhouse-server:23.12.6.19-alpine' }}
           with:
               name: timing_data-${{ inputs.segment }}-${{ inputs.group }}
               path: .test_durations

.github/workflows/ci-backend-update-test-timing.yml

@@ -29,13 +29,13 @@ jobs:
                   group: 1
                   token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
                   python-version: '3.11.9'
-                  clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.5.81-alpine'
+                  clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.6.19-alpine'
                   segment: 'FOSS'
                   person-on-events: false
 
             - name: Upload updated timing data as artifacts
               uses: actions/upload-artifact@v4
-              if: ${{ inputs.person-on-events != 'true' && inputs.clickhouse-server-image == 'clickhouse/clickhouse-server:23.12.5.81-alpine' }}
+              if: ${{ inputs.person-on-events != 'true' && inputs.clickhouse-server-image == 'clickhouse/clickhouse-server:23.12.6.19-alpine' }}
               with:
                   name: timing_data-${{ inputs.segment }}-${{ inputs.group }}
                   path: .test_durations

.github/workflows/ci-backend.yml

@@ -233,7 +233,7 @@ jobs:
             fail-fast: false
             matrix:
                 python-version: ['3.11.9']
-                clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.5.81-alpine']
+                clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.6.19-alpine']
                 segment: ['Core']
                 person-on-events: [false, true]
                 # :NOTE: Keep concurrency and groups in sync
@@ -242,7 +242,7 @@ jobs:
                 include:
                     - segment: 'Temporal'
                       person-on-events: false
-                      clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.5.81-alpine'
+                      clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.6.19-alpine'
                       python-version: '3.11.9'
                       concurrency: 1
                       group: 1
@@ -313,7 +313,7 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.5.81-alpine']
+                clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.6.19-alpine']
         if: needs.changes.outputs.backend == 'true'
         runs-on: ubuntu-latest
         steps:

.github/workflows/ci-hog.yml

@@ -118,9 +118,9 @@ jobs:
                   antlr | grep "Version"
                   npm run grammar:build && git diff --exit-code
               env:
-                  # Installing a version of ANTLR compatible with what's in Homebrew as of October 2023 (version 4.13),
+                  # Installing a version of ANTLR compatible with what's in Homebrew as of August 2024 (version 4.13.2),
                   # as apt-get is quite out of date. The same version must be set in hogql_parser/pyproject.toml
-                  ANTLR_VERSION: '4.13.1'
+                  ANTLR_VERSION: '4.13.2'
 
             - name: Run HogVM Python tests
               if: needs.changes.outputs.hog == 'true'

.github/workflows/container-images-cd.yml

@@ -99,7 +99,8 @@ jobs:
                         "release": "posthog",
                         "commit": ${{ toJson(github.event.head_commit) }},
                         "repository": ${{ toJson(github.repository) }},
-                        "labels": ${{ steps.labels.outputs.labels }}
+                        "labels": ${{ steps.labels.outputs.labels }},
+                        "timestamp": "${{ github.event.head_commit.timestamp }}"
                       }
 
             - name: Check for changes in plugins directory
@@ -124,7 +125,8 @@ jobs:
                         "release": "ingestion",
                         "commit": ${{ toJson(github.event.head_commit) }},
                         "repository": ${{ toJson(github.repository) }},
-                        "labels": ${{ toJson(steps.labels.outputs.labels) }}
+                        "labels": ${{ toJson(steps.labels.outputs.labels) }},
+                        "timestamp": "${{ github.event.head_commit.timestamp }}"
                       }
 
             - name: Check for changes that affect batch exports temporal worker
@@ -149,7 +151,8 @@ jobs:
                         "release": "temporal-worker",
                         "commit": ${{ toJson(github.event.head_commit) }},
                         "repository": ${{ toJson(github.repository) }},
-                        "labels": ${{ steps.labels.outputs.labels }}
+                        "labels": ${{ steps.labels.outputs.labels }},
+                        "timestamp": "${{ github.event.head_commit.timestamp }}"
                       }
 
             - name: Check for changes that affect general purpose temporal worker
@@ -174,7 +177,8 @@ jobs:
                         "release": "temporal-worker-general-purpose",
                         "commit": ${{ toJson(github.event.head_commit) }},
                         "repository": ${{ toJson(github.repository) }},
-                        "labels": ${{ steps.labels.outputs.labels }}
+                        "labels": ${{ steps.labels.outputs.labels }},
+                        "timestamp": "${{ github.event.head_commit.timestamp }}"
                       }
 
             - name: Check for changes that affect data warehouse temporal worker
@@ -199,5 +203,6 @@ jobs:
                         "release": "temporal-worker-data-warehouse",
                         "commit": ${{ toJson(github.event.head_commit) }},
                         "repository": ${{ toJson(github.repository) }},
-                        "labels": ${{ steps.labels.outputs.labels }}
+                        "labels": ${{ steps.labels.outputs.labels }},
+                        "timestamp": "${{ github.event.head_commit.timestamp }}"
                       }

.github/workflows/livestream-docker-image.yml

@@ -84,5 +84,6 @@ jobs:
                         },
                         "release": "livestream",
                         "commit": ${{ toJson(github.event.head_commit) }},
-                        "repository": ${{ toJson(github.repository) }}
+                        "repository": ${{ toJson(github.repository) }},
+                        "timestamp": "${{ github.event.head_commit.timestamp }}"
                       }

.github/workflows/rust-docker-build.yml

@@ -1,4 +1,4 @@
-name: Build rust container images
+name: Build and deploy rust container images
 
 on:
     workflow_dispatch:
@@ -19,12 +19,18 @@ jobs:
                     - hook-api
                     - hook-janitor
                     - hook-worker
+                    - cyclotron-janitor
+                    - cyclotron-fetch
+                    - property-defs-rs
         runs-on: depot-ubuntu-22.04-4
         permissions:
             id-token: write # allow issuing OIDC tokens for this workflow run
             contents: read # allow reading the repo contents
             packages: write # allow push to ghcr.io
 
+        outputs:
+            digest: ${{ steps.docker_build.outputs.digest }}
+
         defaults:
             run:
                 working-directory: rust
@@ -88,3 +94,36 @@ jobs:
 
             - name: Container image digest
               run: echo ${{ steps.docker_build.outputs.digest }}
+
+    deploy:
+        name: Deploy capture-replay
+        runs-on: ubuntu-latest
+        needs: build
+        if: github.ref == 'refs/heads/master'
+        steps:
+            - name: get deployer token
+              id: deployer
+              uses: getsentry/action-github-app-token@v3
+              with:
+                  app_id: ${{ secrets.DEPLOYER_APP_ID }}
+                  private_key: ${{ secrets.DEPLOYER_APP_PRIVATE_KEY }}
+
+            - name: Trigger livestream deployment
+              uses: peter-evans/repository-dispatch@v3
+              with:
+                  token: ${{ steps.deployer.outputs.token }}
+                  repository: PostHog/charts
+                  event-type: commit_state_update
+                  client-payload: |
+                      {
+                        "values": {
+                          "image": {
+                            "sha": "${{ needs.build.outputs.digest }}"
+                          }
+                        },
+                        "release": "capture-replay",
+                        "commit": ${{ toJson(github.event.head_commit) }},
+                        "repository": ${{ toJson(github.repository) }},
+                        "labels": [],
+                        "timestamp": "${{ github.event.head_commit.timestamp }}"
+                      }

.github/workflows/rust-hook-migrator-docker.yml

@@ -69,7 +69,7 @@ jobs:
               uses: depot/build-push-action@v1
               with:
                   context: ./rust/
-                  file: ./rust/Dockerfile.migrate
+                  file: ./rust/Dockerfile.migrate-hooks
                   push: true
                   tags: ${{ steps.meta.outputs.tags }}
                   labels: ${{ steps.meta.outputs.labels }}

.github/workflows/rust.yml

@@ -34,6 +34,7 @@ jobs:
                         - '.github/workflows/rust.yml'
                         - '.github/workflows/rust-docker-build.yml'
                         - '.github/workflows/rust-hook-migrator-docker.yml'
+                        - '.github/workflows/rust-cyclotron-migrator-docker.yml'
                         - 'posthog/management/commands/setup_test_environment.py'
                         - 'posthog/migrations/**'
                         - 'ee/migrations/**'
@@ -164,11 +165,17 @@ jobs:
                   DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
               run: cd ../ && python manage.py setup_test_environment --only-postgres
 
+            - name: Download MaxMind Database
+              if: needs.changes.outputs.rust == 'true'
+              run: |
+                  mkdir -p ../share
+                  curl -L "https://mmdbcdn.posthog.net/" --http1.1 | brotli --decompress --output=../share/GeoLite2-City.mmdb
+
             - name: Run cargo test
               if: needs.changes.outputs.rust == 'true'
               run: |
                   echo "Starting cargo test"
-                  cargo test --all-features ${{ matrix.package == 'feature-flags' && '--package feature-flags' || '--workspace --exclude feature-flags' }}
+                  RUST_BACKTRACE=1 cargo test --all-features ${{ matrix.package == 'feature-flags' && '--package feature-flags' || '--workspace --exclude feature-flags' }}
                   echo "Cargo test completed"
 
     linting:

.github/workflows/vector-docker-build-deploy.yml

@@ -101,5 +101,6 @@ jobs:
                         "release": "replay-capture-vector",
                         "commit": ${{ toJson(github.event.head_commit) }},
                         "repository": ${{ toJson(github.repository) }},
-                        "labels": []
+                        "labels": [],
+                        "timestamp": "${{ github.event.head_commit.timestamp }}"
                       }

.gitignore

@@ -64,3 +64,5 @@ plugin-transpiler/dist
 *-esbuild-bundle-visualization.html
 .dlt
 *.db
+# Ignore any log files that happen to be present
+*.log

bin/start-cyclotron

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -ex
+
+trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
+
+cd rust
+
+cargo build
+
+export RUST_LOG=${DEBUG:-debug}
+SQLX_QUERY_LEVEL=${SQLX_QUERY_LEVEL:-warn}
+export RUST_LOG=$RUST_LOG,sqlx::query=$SQLX_QUERY_LEVEL
+
+export DATABASE_URL=${DATABASE_URL:-postgres://posthog:posthog@localhost:5432/posthog}
+export ALLOW_INTERNAL_IPS=${ALLOW_INTERNAL_IPS:-true}
+
+./target/debug/cyclotron-fetch &
+./target/debug/cyclotron-janitor &
+
+wait

cypress/e2e/alerts.cy.ts

@@ -15,19 +15,19 @@ describe('Alerts', () => {
 
     const createAlert = (
         name: string = 'Alert name',
-        email: string = '[email protected]',
         lowerThreshold: string = '100',
         upperThreshold: string = '200'
     ): void => {
         cy.get('[data-attr=more-button]').click()
-        cy.contains('Alerts').click()
+        cy.contains('Manage alerts').click()
         cy.contains('New alert').click()
 
         cy.get('[data-attr=alert-name]').clear().type(name)
-        cy.get('[data-attr=alert-notification-targets').clear().type(email)
+        cy.get('[data-attr=subscribed-users').click().type('{downarrow}{enter}')
         cy.get('[data-attr=alert-lower-threshold').clear().type(lowerThreshold)
         cy.get('[data-attr=alert-upper-threshold').clear().type(upperThreshold)
         cy.contains('Create alert').click()
+        cy.get('.Toastify__toast-body').should('contain', 'Alert saved')
         cy.url().should('not.include', '/new')
 
         cy.get('[aria-label="close"]').click()
@@ -38,14 +38,15 @@ describe('Alerts', () => {
         cy.get('[data-attr=insight-edit-button]').click()
         cy.get('[data-attr=chart-filter]').click()
         cy.contains(displayType).click()
+        cy.get('.insight-empty-state').should('not.exist')
         cy.get('[data-attr=insight-save-button]').contains('Save').click()
         cy.url().should('not.include', '/edit')
     }
 
     it('Should allow create and delete an alert', () => {
         cy.get('[data-attr=more-button]').click()
         // Alerts should be disabled for trends represented with graphs
-        cy.get('[data-attr=disabled-alerts-button]').should('exist')
+        cy.get('[data-attr=manage-alerts-button]').should('have.attr', 'aria-disabled', 'true')
 
         setInsightDisplayTypeAndSave('Number')
 
@@ -54,10 +55,8 @@ describe('Alerts', () => {
 
         // Check the alert has the same values as when it was created
         cy.get('[data-attr=more-button]').click()
-        cy.contains('Alerts').click()
         cy.contains('Manage alerts').click()
         cy.get('[data-attr=alert-list-item]').contains('Alert name').click()
-        cy.get('[data-attr=alert-notification-targets]').should('have.value', '[email protected]')
         cy.get('[data-attr=alert-name]').should('have.value', 'Alert name')
         cy.get('[data-attr=alert-lower-threshold').should('have.value', '100')
         cy.get('[data-attr=alert-upper-threshold').should('have.value', '200')
@@ -90,7 +89,6 @@ describe('Alerts', () => {
 
         // Assert that saving an insight in an incompatible state removes alerts
         cy.get('[data-attr=more-button]').click()
-        cy.contains('Alerts').click()
         cy.contains('Manage alerts').click()
         cy.contains('Alert to be deleted because of a changed insight').should('not.exist')
     })

cypress/e2e/billing-limits.cy.ts

@@ -61,6 +61,38 @@ describe('Billing Limits', () => {
         )
     })
 
+    it('Show existing limit and allow user to change set to $0', () => {
+        cy.intercept('GET', '/api/billing/', (req) => {
+            req.reply({
+                statusCode: 200,
+                body: {
+                    ...require('../fixtures/api/billing/billing.json'),
+                    custom_limits_usd: { product_analytics: 100 },
+                },
+            })
+        }).as('getBilling')
+        cy.visit('/organization/billing')
+        cy.wait('@getBilling')
+
+        cy.intercept('PATCH', '/api/billing/', (req) => {
+            req.reply({
+                statusCode: 200,
+                body: {
+                    ...require('../fixtures/api/billing/billing.json'),
+                    custom_limits_usd: { product_analytics: 0 },
+                },
+            })
+        }).as('patchBilling')
+
+        cy.get('[data-attr="billing-limit-input-wrapper-product_analytics"]').scrollIntoView()
+        cy.get('[data-attr="billing-limit-set-product_analytics"]').should('be.visible')
+        cy.contains('Edit limit').click()
+        cy.get('[data-attr="billing-limit-input-product_analytics"]').clear().type('0')
+        cy.get('[data-attr="save-billing-limit-product_analytics"]').click()
+        cy.wait('@patchBilling')
+        cy.get('[data-attr="billing-limit-set-product_analytics"]').should('contain', 'You have a $0 billing limit set')
+    })
+
     it('Show existing limit and allow user to remove it', () => {
         cy.intercept('GET', '/api/billing/', (req) => {
             req.reply({

cypress/e2e/dashboard.cy.ts

@@ -95,7 +95,7 @@
         // refresh the dashboard by changing date range
         cy.get('[data-attr="date-filter"]').click()
         cy.contains('span', 'Last 14 days').click()
-        cy.contains('span', 'Apply and save dashboard').click()
+        cy.contains('span', 'Save').click()
 
         cy.contains('span[class="text-primary text-sm font-medium"]', 'Refreshing').should('not.exist')
         cy.get('span').contains('Refreshing').should('not.exist')
@@ -163,7 +163,7 @@
         cy.get('[data-attr=date-filter]').contains('No date range override').click()
         cy.get('div').contains('Yesterday').should('exist').click()
         cy.get('[data-attr=date-filter]').contains('Yesterday')
-        cy.get('button').contains('Apply and save dashboard').click()
+        cy.get('button').contains('Save').click()
         cy.get('.InsightCard h5').should('have.length', 1).contains('Yesterday')
         // Cool, now back to A and make sure the insight is still using the original range there, not the one from B
         cy.clickNavMenu('dashboards')
@@ -347,7 +347,7 @@
      * This throws the error Code: 386. DB::Exception: There is no supertype for types String, Float64 because some of them are String/FixedString and some of them are not. (NO_COMMON_TYPE)
      * All the 'app_ratings' are extracted as strings and 5. is a float
      */
     // it('Opens dashboard item in insights', () => {
     //     cy.get('[data-attr=dashboard-name]').contains('App Analytics').click()
     //     cy.get('.InsightCard [data-attr=insight-card-title]').first().click()
     //     cy.location('pathname').should('include', '/insights')

cypress/e2e/trends.cy.ts

@@ -170,4 +170,8 @@ describe('Trends', () => {
         cy.get('[data-attr=math-monthly_active-0]').trigger('mouseenter') // Activate warning tooltip
         cy.get('.Tooltip').contains('we recommend using "Unique users" here instead').should('exist')
     })
+
+    it('Does not show delete button on single series insight', () => {
+        cy.get('[data-attr=delete-prop-filter-0]').should('not.exist')
+    })
 })