Merge branch 'master' into fix-survey-task

.github/workflows/ci-e2e.yml

@@ -191,7 +191,7 @@ jobs:
                   OBJECT_STORAGE_SECRET_ACCESS_KEY=object_storage_root_password
                   GITHUB_ACTION_RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
                   CELERY_METRICS_PORT=8999
-                  CLOUD_DEPLOYMENT=1
+                  CLOUD_DEPLOYMENT=E2E
                   EOT
 
             - name: Start PostHog

.run/Celery Threads.run.xml

@@ -21,7 +21,7 @@
     <option name="ADD_SOURCE_ROOTS" value="true" />
     <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
     <option name="SCRIPT_NAME" value="$PROJECT_DIR$/manage.py" />
-    <option name="PARAMETERS" value="run_autoreload_celery" />
+    <option name="PARAMETERS" value="run_autoreload_celery --type=worker" />
     <option name="SHOW_COMMAND_LINE" value="false" />
     <option name="EMULATE_TERMINAL" value="false" />
     <option name="MODULE_MODE" value="false" />

.vscode/launch.json

@@ -94,7 +94,7 @@
             },
             "request": "launch",
             "program": "${workspaceFolder}/manage.py",
-            "args": ["run_autoreload_celery"],
+            "args": ["run_autoreload_celery", "--type=worker"],
             "console": "integratedTerminal",
             "python": "${workspaceFolder}/env/bin/python",
             "cwd": "${workspaceFolder}",

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2020-2023 PostHog Inc.
+Copyright (c) 2020-2024 PostHog Inc.
 
 Portions of this software are licensed as follows:
 

README.md

@@ -105,15 +105,15 @@ We <3 contributions big and small. In priority order (although everything is app
 
 Our mission is to increase the number of successful products in the world. To do that, we build product and data tools that help you understand user behavior without losing control of your data.
 
-In our view, third-party analytics tools do not work in a world of cookie deprecation, GDPR, HIPAA, CCPA, and many other four-letter acronyms. PostHog is the alternative to sending all of your customers' personal information and usage data to third-parties.
+In our view, third-party analytics tools do not work in a world of cookie deprecation, GDPR, HIPAA, CCPA, and many other four-letter acronyms. PostHog is the alternative to sending all of your customers' personal information and usage data to third parties.
 
 PostHog gives you every tool you need to understand user behavior, develop and test improvements, and release changes to make your product more successful.
 
 PostHog operates in public as much as possible. We detail how we work and our learning on building and running a fast-growing, product-focused startup in our [handbook](https://posthog.com/handbook/getting-started/start-here).
 
 ## Open-source vs. paid
 
-This repo is available under the [MIT expat license](https://github.com/PostHog/posthog/blob/master/LICENSE), except for the `ee` directory (which has it's [license here](https://github.com/PostHog/posthog/blob/master/ee/LICENSE)) if applicable. 
+This repo is available under the [MIT expat license](https://github.com/PostHog/posthog/blob/master/LICENSE), except for the `ee` directory (which has its [license here](https://github.com/PostHog/posthog/blob/master/ee/LICENSE)) if applicable. 
 
 Need *absolutely 💯% FOSS*? Check out our [posthog-foss](https://github.com/PostHog/posthog-foss) repository, which is purged of all proprietary code and features.
 

bin/e2e-test-runner

@@ -46,15 +46,13 @@ do
     fi
 done
 
-export DEBUG=1
-export NO_RESTART_LOOP=1
 export CYPRESS_BASE_URL=http://localhost:8080
 export SECURE_COOKIES=0
 export SKIP_SERVICE_VERSION_REQUIREMENTS=1
 export KAFKA_HOSTS=kafka:9092
 export CLICKHOUSE_DATABASE=posthog_test
-export TEST=1 # Plugin server and kafka revert to 'default' Clickhouse database if TEST is not set
 export CLICKHOUSE_SECURE=0
+export JS_URL=http://localhost:8234
 export E2E_TESTING=1
 export SECRET_KEY=e2e_test
 export EMAIL_HOST=email.test.posthog.net
@@ -66,7 +64,9 @@ export PGUSER="${PGUSER:=posthog}"
 export PGPASSWORD="${PGPASSWORD:=posthog}"
 export PGPORT="${PGPORT:=5432}"
 export DATABASE_URL="postgres://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT}/${DATABASE}"
-export CLOUD_DEPLOYMENT=1
+export CLOUD_DEPLOYMENT=E2E
+
+source ./bin/celery-queues.env
 
 trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
 
@@ -92,8 +92,7 @@ setupDev() {
   python manage.py setup_dev &
 }
 
-nc -z localhost 9092 || ( echo -e "\033[0;31mKafka isn't running. Please run\n\tdocker compose -f docker-compose.dev.yml up zookeeper kafka clickhouse db redis\nI'll wait while you do that.\033[0m" ; bin/check_kafka_clickhouse_up )
-wget -nv -t1 --spider 'http://localhost:8123/' || ( echo -e "\033[0;31mClickhouse isn't running. Please run\n\tdocker compose -f docker-compose.dev.yml up zookeeper kafka clickhouse db redis.\nI'll wait while you do that.\033[0m" ; bin/check_kafka_clickhouse_up )
+bin/check_kafka_clickhouse_up
 
 $SKIP_RECREATE_DATABASE || recreateDatabases
 $SKIP_MIGRATE || migrateDatabases
@@ -103,4 +102,6 @@ $SKIP_SETUP_DEV || setupDev
 # Only start webpack if not already running
 nc -vz 127.0.0.1 8234 2> /dev/null || ./bin/start-frontend &
 pnpm dlx cypress open --config-file cypress.e2e.config.ts &
+uv pip install -r requirements.txt -r requirements-dev.txt
+python manage.py run_autoreload_celery --type=worker &
 python manage.py runserver 8080

bin/start-worker

@@ -7,8 +7,8 @@ trap 'kill $(jobs -p)' EXIT
 source ./bin/celery-queues.env
 
 # start celery worker with heartbeat (-B)
-SKIP_ASYNC_MIGRATIONS_SETUP=0 CELERY_WORKER_QUEUES=$CELERY_WORKER_QUEUES celery -A posthog worker --without-heartbeat --without-mingle --pool=threads -Ofair -n node@%h &
-celery -A posthog beat -S redbeat.RedBeatScheduler &
+python manage.py run_autoreload_celery --type=worker &
+python manage.py run_autoreload_celery --type=beat &
 
 if [[ "$PLUGIN_SERVER_IDLE" != "1" && "$PLUGIN_SERVER_IDLE" != "true" ]]; then
   ./bin/plugin-server

cypress.e2e.config.ts

@@ -99,15 +99,16 @@ export default defineConfig({
                     const redisClient = await createClient()
                         .on('error', (err) => console.log('Redis client error', err))
                         .connect()
-                    for await (const key of redisClient.scanIterator({
-                        TYPE: 'string',
-                        MATCH: '*cache*',
-                        COUNT: 100,
-                    })) {
+                    // Clear cache
+                    for await (const key of redisClient.scanIterator({ TYPE: 'string', MATCH: '*cache*', COUNT: 500 })) {
+                        await redisClient.del(key)
+                    }
+                    // Also clear the more ephemeral async query statuses
+                    for await (const key of redisClient.scanIterator({ TYPE: 'string', MATCH: 'query_async*', COUNT: 500 })) {
                         await redisClient.del(key)
                     }
                     await redisClient.quit()
-                    return null
+                    return null // Cypress requires _some_ return value
                 },
             })
 

cypress/e2e/billing.cy.ts

@@ -14,7 +14,8 @@ describe('Billing', () => {
 
         cy.get('[data-attr=more-button]').first().click()
         cy.contains('.LemonButton', 'Unsubscribe').click()
-        cy.get('.LemonModal h3').should('contain', 'Why are you unsubscribing from Product analytics?')
+        cy.get('.LemonModal h3').should('contain', 'Unsubscribe from Product analytics')
+        cy.get('[data-attr=unsubscribe-reason-too-expensive]').click()
         cy.get('[data-attr=unsubscribe-reason-survey-textarea]').type('Product analytics')
         cy.contains('.LemonModal .LemonButton', 'Unsubscribe').click()
 
@@ -26,6 +27,8 @@ describe('Billing', () => {
             expect(matchingEvent.properties.$survey_id).to.equal(UNSUBSCRIBE_SURVEY_ID)
             expect(matchingEvent.properties.$survey_response).to.equal('Product analytics')
             expect(matchingEvent.properties.$survey_response_1).to.equal('product_analytics')
+            expect(matchingEvent.properties.$survey_reasons.length).to.equal(1)
+            expect(matchingEvent.properties.$survey_reasons[0]).to.equal('Too expensive')
         })
 
         cy.get('.LemonModal').should('not.exist')
@@ -35,14 +38,14 @@ describe('Billing', () => {
     it('Unsubscribe survey text area maintains unique state between product types', () => {
         cy.get('[data-attr=more-button]').first().click()
         cy.contains('.LemonButton', 'Unsubscribe').click()
-        cy.get('.LemonModal h3').should('contain', 'Why are you unsubscribing from Product analytics?')
+        cy.get('.LemonModal h3').should('contain', 'Unsubscribe from Product analytics')
 
         cy.get('[data-attr=unsubscribe-reason-survey-textarea]').type('Product analytics')
         cy.contains('.LemonModal .LemonButton', 'Cancel').click()
 
         cy.get('[data-attr=more-button]').eq(1).click()
         cy.contains('.LemonButton', 'Unsubscribe').click()
-        cy.get('.LemonModal h3').should('contain', 'Why are you unsubscribing from Session replay?')
+        cy.get('.LemonModal h3').should('contain', 'Unsubscribe from Session replay')
         cy.get('[data-attr=unsubscribe-reason-survey-textarea]').type('Session replay')
         cy.contains('.LemonModal .LemonButton', 'Cancel').click()
 

cypress/e2e/dashboard.cy.ts

@@ -61,6 +61,46 @@ describe('Dashboard', () => {
         }
     })
 
+    it('Refreshing dashboard works', () => {
+        const dashboardName = randomString('Dashboard with insights')
+        const insightName = randomString('insight to add to dashboard')
+
+        // Create and visit a dashboard to get it into turbo mode cache
+        dashboards.createAndGoToEmptyDashboard(dashboardName)
+
+        insight.create(insightName)
+
+        insight.addInsightToDashboard(dashboardName, { visitAfterAdding: true })
+
+        cy.get('.CardMeta h4').should('have.text', insightName)
+        cy.get('h4').contains('Refreshing').should('not.exist')
+        cy.get('main').contains('There are no matching events for this query').should('not.exist')
+
+        cy.intercept('GET', /\/api\/projects\/\d+\/dashboard_templates/, (req) => {
+            req.reply((response) => {
+                response.body.results[0].variables = [
+                    {
+                        id: 'id',
+                        name: 'Unique variable name',
+                        type: 'event',
+                        default: {},
+                        required: true,
+                        description: 'description',
+                    },
+                ]
+                return response
+            })
+        })
+
+        // refresh the dashboard by changing date range
+        cy.get('[data-attr="date-filter"]').click()
+        cy.contains('span', 'Last 14 days').click()
+        cy.contains('span', 'Apply and save dashboard').click()
+
+        cy.contains('span[class="text-primary text-sm font-medium"]', 'Refreshing').should('not.exist')
+        cy.get('span').contains('Refreshing').should('not.exist')
+    })
+
     it('Shows details when moving between dashboard and insight', () => {
         const dashboardName = randomString('Dashboard')
         const insightName = randomString('DashboardInsight')

cypress/e2e/insights-date-picker.ts → cypress/e2e/insights-date-picker.cy.ts

@@ -1,8 +1,11 @@
+import 'cypress-network-idle'
+
 import { urls } from 'scenes/urls'
 
 describe('insights date picker', () => {
     beforeEach(() => {
         cy.visit(urls.insightNew())
+        cy.waitForNetworkIdle(300)
     })
 
     it('Can set the date filter and show the right grouping interval', () => {

cypress/e2e/insights-saved.cy.ts

@@ -0,0 +1,44 @@
+import { urls } from 'scenes/urls'
+
+import { createInsight } from '../productAnalytics'
+
+chai.Assertion.addMethod('neverHaveChild', function (childSelector) {
+    this._obj.on('DOMNodeInserted', () => {
+        const matchCount = cy.$$(childSelector, this._obj).length
+        if (matchCount > 0) {
+            throw new Error(
+                `Expected element to never have child ${childSelector}, but found ${matchCount} match${
+                    matchCount > 1 ? 'es' : ''
+                }`
+            )
+        }
+    })
+})
+
+// For tests related to trends please check trendsElements.js
+// insight tests were split up because Cypress was struggling with this many tests in one file🙈
+describe('Insights - saved', () => {
+    it('Data is available immediately', () => {
+        createInsight('saved insight').then((newInsightId) => {
+            cy.get('[data-attr=trend-line-graph]').should('exist') // Results cached
+            cy.visit(urls.insightView(newInsightId)) // Full refresh
+            cy.get('.InsightViz').should('exist').should('neverHaveChild', '.insight-empty-state') // Only cached data
+            cy.get('[data-attr=trend-line-graph]').should('exist')
+        })
+    })
+
+    it('If cache empty, initiate async refresh', () => {
+        cy.intercept('GET', /\/api\/projects\/\d+\/insights\/?\?[^/]*?refresh=async/).as('getInsightsRefreshAsync')
+        let newInsightId: string
+        createInsight('saved insight').then((insightId) => {
+            newInsightId = insightId
+        })
+        cy.task('resetInsightCache').then(() => {
+            cy.visit(urls.insightView(newInsightId)) // Full refresh
+            cy.get('.insight-empty-state').should('exist') // There should be a loading state for a moment
+            cy.wait('@getInsightsRefreshAsync').then(() => {
+                cy.get('[data-attr=trend-line-graph]').should('exist')
+            })
+        })
+    })
+})

cypress/productAnalytics/index.ts

@@ -192,11 +192,15 @@ export const dashboard = {
     },
 }
 
-export function createInsight(insightName: string): void {
+export function createInsight(insightName: string): Cypress.Chainable<string> {
     savedInsights.createNewInsightOfType('TRENDS')
     insight.applyFilter()
     insight.editName(insightName)
     insight.save()
+    // return insight id from the url
+    return cy.url().then((url) => {
+        return url.split('/').at(-1)
+    })
 }
 
 export function duplicateDashboardFromMenu(duplicateTiles = false): void {

cypress/support/e2e.ts

@@ -23,6 +23,10 @@ Cypress.on('window:before:load', (win) => {
     win._cypress_posthog_captures = []
 })
 
+before(() => {
+    cy.task('resetInsightCache') // Reset insight cache before each suite
+})
+
 beforeEach(() => {
     Cypress.env('POSTHOG_PROPERTY_CURRENT_TEST_TITLE', Cypress.currentTest.title)
     Cypress.env('POSTHOG_PROPERTY_CURRENT_TEST_FULL_TITLE', Cypress.currentTest.titlePath.join(' > '))

docker-compose.hobby.yml

@@ -186,6 +186,7 @@ services:
         environment:
             SENTRY_DSN: $SENTRY_DSN
             SITE_URL: https://$DOMAIN
+            SECRET_KEY: $POSTHOG_SECRET
         depends_on:
             - db
             - redis

ee/billing/billing_types.py

@@ -98,7 +98,7 @@ class CustomerInfo(TypedDict):
     current_total_amount_usd: Optional[str]
     current_total_amount_usd_after_discount: Optional[str]
     products: Optional[list[CustomerProduct]]
-    custom_limits_usd: Optional[dict[str, str]]
+    custom_limits_usd: Optional[dict[str, str | int]]
     usage_summary: Optional[dict[str, dict[str, Optional[int]]]]
     free_trial_until: Optional[str]
     discount_percent: Optional[int]