diff --git a/.storybook/preview.tsx b/.storybook/preview.tsx index 0cda6703cfd37..9b0a76da1d367 100644 --- a/.storybook/preview.tsx +++ b/.storybook/preview.tsx @@ -15,6 +15,17 @@ const setupMsw = () => { // Make sure the msw worker is started worker.start({ quiet: true, + onUnhandledRequest(request, print) { + // MSW warns on all unhandled requests, but we don't necessarily care + const pathAllowList = ['/images/'] + + if (pathAllowList.some((path) => request.url.pathname.startsWith(path))) { + return + } + + // Otherwise, default MSW warning behavior + print.warning() + }, }) ;(window as any).__mockServiceWorker = worker ;(window as any).POSTHOG_APP_CONTEXT = getStorybookAppContext() diff --git a/cypress/e2e/notebooks.cy.ts b/cypress/e2e/notebooks.cy.ts index d44555d42294e..4761c29d56827 100644 --- a/cypress/e2e/notebooks.cy.ts +++ b/cypress/e2e/notebooks.cy.ts @@ -7,16 +7,23 @@ describe('Notebooks', () => { 'loadSessionRecordingsList' ) }) + cy.fixture('api/session-recordings/recording.json').then((recording) => { cy.intercept('GET', /api\/projects\/\d+\/session_recordings\/.*\?.*/, { body: recording }).as( 'loadSessionRecording' ) }) + cy.fixture('api/notebooks/notebooks.json').then((notebook) => { cy.intercept('GET', /api\/projects\/\d+\/notebooks\//, { body: notebook }).as('loadNotebooksList') }) + cy.fixture('api/notebooks/notebook.json').then((notebook) => { cy.intercept('GET', /api\/projects\/\d+\/notebooks\/.*\//, { body: notebook }).as('loadNotebook') + // this means saving doesn't work but so what? + cy.intercept('PATCH', /api\/projects\/\d+\/notebooks\/.*\//, (req, res) => { + res.reply(req.body) + }).as('patchNotebook') }) cy.clickNavMenu('dashboards') @@ -53,4 +60,45 @@ describe('Notebooks', () => { cy.get('.ph-recording.NotebookNode').should('be.visible') cy.get('.NotebookRecordingTimestamp').should('contain.text', '0:00') }) + + it('Can add a number list', () => { + cy.get('li').contains('Notebooks').should('exist').click() + cy.get('[data-attr="new-notebook"]').click() + // we don't actually get a new notebook because the API is mocked + // so, press enter twice to "exit" the timestamp block we start in + cy.get('.NotebookEditor').type('{enter}{enter}') + cy.get('.NotebookEditor').type('{enter}') + cy.get('.NotebookEditor').type('1. the first') + cy.get('.NotebookEditor').type('{enter}') + // no need to type the number now. it should be inserted automatically + cy.get('.NotebookEditor').type('the second') + cy.get('.NotebookEditor').type('{enter}') + cy.get('ol').should('contain.text', 'the first') + cy.get('ol').should('contain.text', 'the second') + // the numbered list auto inserts the next list item + cy.get('.NotebookEditor ol li').should('have.length', 3) + }) + + it('Can add bold', () => { + cy.get('li').contains('Notebooks').should('exist').click() + cy.get('[data-attr="new-notebook"]').click() + // we don't actually get a new notebook because the API is mocked + // so, press enter twice to "exit" the timestamp block we start in + cy.get('.NotebookEditor').type('{enter}{enter}') + cy.get('.NotebookEditor').type('**bold**') + cy.get('.NotebookEditor p').last().should('contain.html', 'bold') + }) + + it('Can add bullet list', () => { + cy.get('li').contains('Notebooks').should('exist').click() + cy.get('[data-attr="new-notebook"]').click() + // we don't actually get a new notebook because the API is mocked + // so, press enter twice to "exit" the timestamp block we start in + cy.get('.NotebookEditor').type('{enter}{enter}') + cy.get('.NotebookEditor').type('* the first{enter}the second{enter}') + cy.get('ul').should('contain.text', 'the first') + cy.get('ul').should('contain.text', 'the second') + // the list auto inserts the next list item + cy.get('.NotebookEditor ul li').should('have.length', 3) + }) }) diff --git a/frontend/__snapshots__/scenes-app-insights--trends-line-edit--webkit.png b/frontend/__snapshots__/scenes-app-insights--trends-line-edit--webkit.png index f9063f35a84fa..23a9edff296e8 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-line-edit--webkit.png and b/frontend/__snapshots__/scenes-app-insights--trends-line-edit--webkit.png differ diff --git a/frontend/__snapshots__/scenes-app-notebooks--bullet-list.png b/frontend/__snapshots__/scenes-app-notebooks--bullet-list.png new file mode 100644 index 0000000000000..00ac16d82c920 Binary files /dev/null and b/frontend/__snapshots__/scenes-app-notebooks--bullet-list.png differ diff --git a/frontend/__snapshots__/scenes-app-notebooks--headings.png b/frontend/__snapshots__/scenes-app-notebooks--headings.png new file mode 100644 index 0000000000000..9e17de9d4803a Binary files /dev/null and b/frontend/__snapshots__/scenes-app-notebooks--headings.png differ diff --git a/frontend/__snapshots__/scenes-app-notebooks--notebooks-template-introduction.png b/frontend/__snapshots__/scenes-app-notebooks--notebooks-template-introduction.png new file mode 100644 index 0000000000000..b6466dd921cf7 Binary files /dev/null and b/frontend/__snapshots__/scenes-app-notebooks--notebooks-template-introduction.png differ diff --git a/frontend/__snapshots__/scenes-app-notebooks--numbered-list.png b/frontend/__snapshots__/scenes-app-notebooks--numbered-list.png new file mode 100644 index 0000000000000..76256d08a1d61 Binary files /dev/null and b/frontend/__snapshots__/scenes-app-notebooks--numbered-list.png differ diff --git a/frontend/__snapshots__/scenes-app-notebooks--recordings-playlist.png b/frontend/__snapshots__/scenes-app-notebooks--recordings-playlist.png new file mode 100644 index 0000000000000..6b1e009909dac Binary files /dev/null and b/frontend/__snapshots__/scenes-app-notebooks--recordings-playlist.png differ diff --git a/frontend/__snapshots__/scenes-app-notebooks--text-formats.png b/frontend/__snapshots__/scenes-app-notebooks--text-formats.png new file mode 100644 index 0000000000000..1c4d6960a0d26 Binary files /dev/null and b/frontend/__snapshots__/scenes-app-notebooks--text-formats.png differ diff --git a/frontend/__snapshots__/scenes-app-recordings--recent-recordings.png b/frontend/__snapshots__/scenes-app-recordings--recent-recordings.png new file mode 100644 index 0000000000000..f4060d584e979 Binary files /dev/null and b/frontend/__snapshots__/scenes-app-recordings--recent-recordings.png differ diff --git a/frontend/__snapshots__/scenes-app-recordings--recordings-play-list-no-pinned-recordings.png b/frontend/__snapshots__/scenes-app-recordings--recordings-play-list-no-pinned-recordings.png index 8e3052db551e1..bba83afd7cd8d 100644 Binary files a/frontend/__snapshots__/scenes-app-recordings--recordings-play-list-no-pinned-recordings.png and b/frontend/__snapshots__/scenes-app-recordings--recordings-play-list-no-pinned-recordings.png differ diff --git a/frontend/__snapshots__/scenes-other-login--cloud-with-google-login-enforcement.png b/frontend/__snapshots__/scenes-other-login--cloud-with-google-login-enforcement.png index b533f8a57619e..e34fe137f3088 100644 Binary files a/frontend/__snapshots__/scenes-other-login--cloud-with-google-login-enforcement.png and b/frontend/__snapshots__/scenes-other-login--cloud-with-google-login-enforcement.png differ diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 0da43300083d7..c56bf0bc8086e 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1321,7 +1321,7 @@ const api = { }, async update( notebookId: NotebookType['short_id'], - data: Pick + data: Pick ): Promise { return await new ApiRequest().notebook(notebookId).update({ data }) }, @@ -1352,7 +1352,7 @@ const api = { } return await apiRequest.withQueryString(q).get() }, - async create(data?: Pick): Promise { + async create(data?: Pick): Promise { return await new ApiRequest().notebooks().create({ data }) }, async delete(notebookId: NotebookType['short_id']): Promise { diff --git a/frontend/src/lib/components/Cards/TextCard/TextCard.scss b/frontend/src/lib/components/Cards/TextCard/TextCard.scss index 0652dd7fa64bb..f88af17286e05 100644 --- a/frontend/src/lib/components/Cards/TextCard/TextCard.scss +++ b/frontend/src/lib/components/Cards/TextCard/TextCard.scss @@ -9,13 +9,13 @@ overflow-y: auto; ul { - list-style: disc; - padding-inline-start: 1.5em; + list-style-type: disc; + list-style-position: inside; } ol { - list-style: numeric; - padding-inline-start: 1.5em; + list-style-type: numeric; + list-style-position: inside; } img { diff --git a/frontend/src/lib/components/Cards/TextCard/TextCard.tsx b/frontend/src/lib/components/Cards/TextCard/TextCard.tsx index 9b2a9d8705ddf..5c54f515dbea8 100644 --- a/frontend/src/lib/components/Cards/TextCard/TextCard.tsx +++ b/frontend/src/lib/components/Cards/TextCard/TextCard.tsx @@ -24,15 +24,15 @@ interface TextCardProps extends React.HTMLAttributes, Resizeable showEditingControls?: boolean } -interface TextCardBodyProps extends Pick, 'style'> { +interface TextCardBodyProps extends Pick, 'style' | 'className'> { text: string closeDetails?: () => void } -export function TextContent({ text, closeDetails, style }: TextCardBodyProps): JSX.Element { +export function TextContent({ text, closeDetails, style, className }: TextCardBodyProps): JSX.Element { return ( // eslint-disable-next-line react/forbid-dom-props -
closeDetails?.()} style={style}> +
closeDetails?.()} style={style}> {text}
) diff --git a/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.tsx b/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.tsx index c8547ae06af50..e9014b314ef91 100644 --- a/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.tsx +++ b/frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.tsx @@ -43,7 +43,6 @@ import { groupDisplayId } from 'scenes/persons/GroupActorDisplay' import { infiniteListLogicType } from 'lib/components/TaxonomicFilter/infiniteListLogicType' import { updatePropertyDefinitions } from '~/models/propertyDefinitionsModel' import { InlineHogQLEditor } from './InlineHogQLEditor' -import { FEATURE_FLAGS } from 'lib/constants' export const eventTaxonomicGroupProps: Pick = { getPopoverHeader: (eventDefinition: EventDefinition): string => { @@ -79,8 +78,6 @@ export const taxonomicFilterLogic = kea({ ['groupTypes', 'aggregationLabel'], groupPropertiesModel, ['allGroupProperties'], - featureFlagsLogic, - ['featureFlags'], ], }, actions: () => ({ @@ -150,15 +147,13 @@ export const taxonomicFilterLogic = kea({ s.groupAnalyticsTaxonomicGroupNames, s.eventNames, s.excludedProperties, - s.featureFlags, ], ( teamId, groupAnalyticsTaxonomicGroups, groupAnalyticsTaxonomicGroupNames, eventNames, - excludedProperties, - featureFlags + excludedProperties ): TaxonomicFilterGroup[] => { const groups = [ { @@ -404,6 +399,16 @@ export const taxonomicFilterLogic = kea({ getValue: (dashboard: DashboardType) => dashboard.id, getPopoverHeader: () => `Dashboards`, }, + { + name: 'Notebooks', + searchPlaceholder: 'notebooks', + type: TaxonomicFilterGroupType.Notebooks, + value: 'notebooks', + endpoint: `api/projects/${teamId}/notebooks/`, + getName: (notebook: NotebookType) => notebook.title || `Notebook ${notebook.short_id}`, + getValue: (notebook: NotebookType) => notebook.short_id, + getPopoverHeader: () => 'Notebooks', + }, { name: 'Sessions', searchPlaceholder: 'sessions', @@ -429,19 +434,6 @@ export const taxonomicFilterLogic = kea({ ...groupAnalyticsTaxonomicGroupNames, ] - if (featureFlags[FEATURE_FLAGS.NOTEBOOKS]) { - groups.push({ - name: 'Notebooks', - searchPlaceholder: 'notebooks', - type: TaxonomicFilterGroupType.Notebooks, - value: 'notebooks', - endpoint: `api/projects/${teamId}/notebooks/`, - getName: (notebook: NotebookType) => notebook.title || `Notebook ${notebook.short_id}`, - getValue: (notebook: NotebookType) => notebook.short_id, - getPopoverHeader: () => 'Notebooks', - }) - } - return groups }, ], diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx index 122f62be237a1..03047c582f7b6 100644 --- a/frontend/src/lib/constants.tsx +++ b/frontend/src/lib/constants.tsx @@ -166,6 +166,7 @@ export const FEATURE_FLAGS = { // owner: #team-monitoring SESSION_RECORDING_ALLOW_V1_SNAPSHOTS: 'session-recording-allow-v1-snapshots', HOGQL_INSIGHTS: 'hogql-insights', // owner: @mariusandra + WEBHOOKS_DENYLIST: 'webhooks-denylist', // owner: #team-pipeline } as const export type FeatureFlagKey = (typeof FEATURE_FLAGS)[keyof typeof FEATURE_FLAGS] diff --git a/frontend/src/lib/lemon-ui/LemonTextArea/LemonTextArea.scss b/frontend/src/lib/lemon-ui/LemonTextArea/LemonTextArea.scss index d15a0b710a0d5..389975e57915a 100644 --- a/frontend/src/lib/lemon-ui/LemonTextArea/LemonTextArea.scss +++ b/frontend/src/lib/lemon-ui/LemonTextArea/LemonTextArea.scss @@ -34,3 +34,15 @@ border: 1px solid var(--danger); } } + +.LemonTextArea--preview { + ul { + list-style-type: disc; + list-style-position: inside; + } + + ol { + list-style-type: decimal; + list-style-position: inside; + } +} diff --git a/frontend/src/lib/lemon-ui/LemonTextArea/LemonTextArea.tsx b/frontend/src/lib/lemon-ui/LemonTextArea/LemonTextArea.tsx index 9a85e20ba4829..4cfbf6bd7648f 100644 --- a/frontend/src/lib/lemon-ui/LemonTextArea/LemonTextArea.tsx +++ b/frontend/src/lib/lemon-ui/LemonTextArea/LemonTextArea.tsx @@ -138,7 +138,11 @@ export function LemonTextMarkdown({ value, onChange, ...editAreaProps }: LemonTe { key: 'preview', label: 'Preview', - content: value ? : Nothing to preview, + content: value ? ( + + ) : ( + Nothing to preview + ), }, ]} /> diff --git a/frontend/src/mocks/handlers.ts b/frontend/src/mocks/handlers.ts index e6a101068a6f9..123fc595765ca 100644 --- a/frontend/src/mocks/handlers.ts +++ b/frontend/src/mocks/handlers.ts @@ -82,6 +82,10 @@ export const defaultMocks: Mocks = { }, // We don't want to show the "new version available" banner in tests 'https://api.github.com/repos/posthog/posthog-js/tags': () => [200, []], + 'https://www.gravatar.com/avatar/:gravatar_id': () => [404, ''], + 'https://app.posthog.com/api/early_access_features': { + earlyAccessFeatures: [], + }, }, post: { 'https://app.posthog.com/e/': (): MockSignature => [200, 'ok'], diff --git a/frontend/src/queries/nodes/DataTable/DataTableExport.tsx b/frontend/src/queries/nodes/DataTable/DataTableExport.tsx index 1de315c2c900e..db3a26d62aba0 100644 --- a/frontend/src/queries/nodes/DataTable/DataTableExport.tsx +++ b/frontend/src/queries/nodes/DataTable/DataTableExport.tsx @@ -1,12 +1,17 @@ +import Papa from 'papaparse' import { LemonButton, LemonButtonWithDropdown } from 'lib/lemon-ui/LemonButton' import { IconExport } from 'lib/lemon-ui/icons' import { triggerExport } from 'lib/components/ExportButton/exporter' import { ExporterFormat } from '~/types' import { DataNode, DataTableNode } from '~/queries/schema' -import { defaultDataTableColumns } from '~/queries/nodes/DataTable/utils' -import { isEventsQuery, isPersonsNode } from '~/queries/utils' +import { defaultDataTableColumns, extractExpressionComment } from '~/queries/nodes/DataTable/utils' +import { isEventsQuery, isHogQLQuery, isPersonsNode } from '~/queries/utils' import { getPersonsEndpoint } from '~/queries/query' import { ExportWithConfirmation } from '~/queries/nodes/DataTable/ExportWithConfirmation' +import { DataTableRow, dataTableLogic } from './dataTableLogic' +import { useValues } from 'kea' +import { LemonDivider, lemonToast } from '@posthog/lemon-ui' +import { asDisplay } from 'scenes/persons/person-utils' const EXPORT_MAX_LIMIT = 10000 @@ -39,18 +44,148 @@ function startDownload(query: DataTableNode, onlySelectedColumns: boolean): void }) } +const columnDisallowList = ['person.$delete', '*'] +const getCsvTableData = (dataTableRows: DataTableRow[], columns: string[], query: DataTableNode): string[][] => { + if (isPersonsNode(query.source)) { + const filteredColumns = columns.filter((n) => !columnDisallowList.includes(n)) + + const csvData = dataTableRows.map((n) => { + const record = n.result as Record | undefined + const recordWithPerson = { ...(record ?? {}), person: record?.name } + + return filteredColumns.map((n) => recordWithPerson[n]) + }) + + return [filteredColumns, ...csvData] + } + + if (isEventsQuery(query.source)) { + const filteredColumns = columns + .filter((n) => !columnDisallowList.includes(n)) + .map((n) => extractExpressionComment(n)) + + const csvData = dataTableRows.map((n) => { + return columns + .map((col, colIndex) => { + if (columnDisallowList.includes(col)) { + return null + } + + if (col === 'person') { + return asDisplay(n.result?.[colIndex]) + } + + return n.result?.[colIndex] + }) + .filter(Boolean) + }) + + return [filteredColumns, ...csvData] + } + + if (isHogQLQuery(query.source)) { + return [columns, ...dataTableRows.map((n) => (n.result as any[]) ?? [])] + } + + return [] +} + +const getJsonTableData = ( + dataTableRows: DataTableRow[], + columns: string[], + query: DataTableNode +): Record[] => { + if (isPersonsNode(query.source)) { + const filteredColumns = columns.filter((n) => !columnDisallowList.includes(n)) + + return dataTableRows.map((n) => { + const record = n.result as Record | undefined + const recordWithPerson = { ...(record ?? {}), person: record?.name } + + return filteredColumns.reduce((acc, cur) => { + acc[cur] = recordWithPerson[cur] + return acc + }, {} as Record) + }) + } + + if (isEventsQuery(query.source)) { + return dataTableRows.map((n) => { + return columns.reduce((acc, col, colIndex) => { + if (columnDisallowList.includes(col)) { + return acc + } + + if (col === 'person') { + acc[col] = asDisplay(n.result?.[colIndex]) + return acc + } + + const colName = extractExpressionComment(col) + + acc[colName] = n.result?.[colIndex] + + return acc + }, {} as Record) + }) + } + + if (isHogQLQuery(query.source)) { + return dataTableRows.map((n) => { + const data = n.result ?? {} + return columns.reduce((acc, cur, index) => { + acc[cur] = data[index] + return acc + }, {} as Record) + }) + } + + return [] +} + +function copyTableToCsv(dataTableRows: DataTableRow[], columns: string[], query: DataTableNode): void { + try { + const tableData = getCsvTableData(dataTableRows, columns, query) + + const csv = Papa.unparse(tableData) + + navigator.clipboard.writeText(csv).then(() => { + lemonToast.success('Table copied to clipboard!') + }) + } catch { + lemonToast.error('Copy failed!') + } +} + +function copyTableToJson(dataTableRows: DataTableRow[], columns: string[], query: DataTableNode): void { + try { + const tableData = getJsonTableData(dataTableRows, columns, query) + + const json = JSON.stringify(tableData, null, 4) + + navigator.clipboard.writeText(json).then(() => { + lemonToast.success('Table copied to clipboard!') + }) + } catch { + lemonToast.error('Copy failed!') + } +} + interface DataTableExportProps { query: DataTableNode setQuery?: (query: DataTableNode) => void } export function DataTableExport({ query }: DataTableExportProps): JSX.Element | null { + const { dataTableRows, columnsInResponse, columnsInQuery, queryWithDefaults } = useValues(dataTableLogic) + const source: DataNode = query.source const filterCount = (isEventsQuery(source) || isPersonsNode(source) ? source.properties?.length || 0 : 0) + (isEventsQuery(source) && source.event ? 1 : 0) + (isPersonsNode(source) && source.search ? 1 : 0) const canExportAllColumns = isEventsQuery(source) || isPersonsNode(source) + const showExportClipboardButtons = isPersonsNode(source) || isEventsQuery(source) || isHogQLQuery(source) return ( , - ].concat( - canExportAllColumns - ? [ - startDownload(query, false)} - actor={isPersonsNode(query.source) ? 'persons' : 'events'} - limit={EXPORT_MAX_LIMIT} - > - - Export all columns - - , - ] - : [] - ), + ] + .concat( + canExportAllColumns + ? [ + startDownload(query, false)} + actor={isPersonsNode(query.source) ? 'persons' : 'events'} + limit={EXPORT_MAX_LIMIT} + > + + Export all columns + + , + ] + : [] + ) + .concat( + showExportClipboardButtons + ? [ + , + { + if (dataTableRows) { + copyTableToCsv( + dataTableRows, + columnsInResponse ?? columnsInQuery, + queryWithDefaults + ) + } + }} + > + Copy CSV to clipboard + , + { + if (dataTableRows) { + copyTableToJson( + dataTableRows, + columnsInResponse ?? columnsInQuery, + queryWithDefaults + ) + } + }} + > + Copy JSON to clipboard + , + ] + : [] + ), }} type="secondary" icon={} diff --git a/frontend/src/scenes/authentication/Login.tsx b/frontend/src/scenes/authentication/Login.tsx index 883c3215db9bb..b3c3cf1f70149 100644 --- a/frontend/src/scenes/authentication/Login.tsx +++ b/frontend/src/scenes/authentication/Login.tsx @@ -168,7 +168,9 @@ export function Login(): JSX.Element {
)} - + {!precheckResponse.saml_available && !precheckResponse.sso_enforcement && ( + + )} ) diff --git a/frontend/src/scenes/batch_exports/BatchExportEditForm.tsx b/frontend/src/scenes/batch_exports/BatchExportEditForm.tsx index 16cf9ccd3f3a0..2ce88feaaddfb 100644 --- a/frontend/src/scenes/batch_exports/BatchExportEditForm.tsx +++ b/frontend/src/scenes/batch_exports/BatchExportEditForm.tsx @@ -180,6 +180,12 @@ export function BatchExportsEditForm(props: BatchExportsEditLogicProps): JSX.Ele ]} /> + + + + + +
+ + + +
- - - +
+ + + {batchExportConfigForm.encryption == 'aws:kms' && ( + + + + )}
+ Test that it works - {`posthog.feature_flags.override({'${flagKey}': '${variant}'})`} + {`posthog.featureFlags.override({'${flagKey}': '${variant}'})`} ) diff --git a/frontend/src/scenes/notebooks/Nodes/NodeWrapper.tsx b/frontend/src/scenes/notebooks/Nodes/NodeWrapper.tsx index 5f58cad124a1b..4a7d18ea42cce 100644 --- a/frontend/src/scenes/notebooks/Nodes/NodeWrapper.tsx +++ b/frontend/src/scenes/notebooks/Nodes/NodeWrapper.tsx @@ -6,6 +6,7 @@ import { ExtendedRegExpMatchArray, Attribute, NodeViewProps, + getExtensionField, } from '@tiptap/react' import { ReactNode, useCallback, useRef } from 'react' import clsx from 'clsx' @@ -212,12 +213,14 @@ export type CreatePostHogWidgetNodeOptions> widgets?: NotebookNodeWidget[] + serializedText?: (attributes: NotebookNodeAttributes) => string } export function createPostHogWidgetNode({ Component, pasteOptions, attributes, + serializedText, ...wrapperProps }: CreatePostHogWidgetNodeOptions): Node { // NOTE: We use NodeViewProps here as we convert them to NotebookNodeViewProps @@ -252,6 +255,19 @@ export function createPostHogWidgetNode( atom: true, draggable: true, + serializedText: serializedText, + + extendNodeSchema(extension) { + const context = { + name: extension.name, + options: extension.options, + storage: extension.storage, + } + return { + serializedText: getExtensionField(extension, 'serializedText', context), + } + }, + addAttributes() { return { height: {}, diff --git a/frontend/src/scenes/notebooks/Nodes/NotebookNodeImage.tsx b/frontend/src/scenes/notebooks/Nodes/NotebookNodeImage.tsx index effdf63d7afcf..8dc4e00839409 100644 --- a/frontend/src/scenes/notebooks/Nodes/NotebookNodeImage.tsx +++ b/frontend/src/scenes/notebooks/Nodes/NotebookNodeImage.tsx @@ -79,6 +79,10 @@ export const NotebookNodeImage = createPostHogWidgetNode { + // TODO file is null when this runs... should it be? + return attrs?.file?.name || '' + }, heightEstimate: 400, minHeight: 100, resizeable: true, diff --git a/frontend/src/scenes/notebooks/Nodes/NotebookNodePerson.tsx b/frontend/src/scenes/notebooks/Nodes/NotebookNodePerson.tsx index a8640e956759a..d582171f9690a 100644 --- a/frontend/src/scenes/notebooks/Nodes/NotebookNodePerson.tsx +++ b/frontend/src/scenes/notebooks/Nodes/NotebookNodePerson.tsx @@ -76,4 +76,9 @@ export const NotebookNodePerson = createPostHogWidgetNode { + const personTitle = attrs?.title || '' + const personId = attrs?.id || '' + return `${personTitle} ${personId}`.trim() + }, }) diff --git a/frontend/src/scenes/notebooks/Nodes/NotebookNodeQuery.tsx b/frontend/src/scenes/notebooks/Nodes/NotebookNodeQuery.tsx index da4c15000bc40..c4bc461c68bcf 100644 --- a/frontend/src/scenes/notebooks/Nodes/NotebookNodeQuery.tsx +++ b/frontend/src/scenes/notebooks/Nodes/NotebookNodeQuery.tsx @@ -12,6 +12,7 @@ import { urls } from 'scenes/urls' import api from 'lib/api' import './NotebookNodeQuery.scss' +import { containsHogQLQuery, isHogQLQuery, isNodeWithSource } from '~/queries/utils' const DEFAULT_QUERY: QuerySchema = { kind: NodeKind.DataTableNode, @@ -161,4 +162,17 @@ export const NotebookNodeQuery = createPostHogWidgetNode { + let text = '' + const q = attrs.query + if (containsHogQLQuery(q)) { + if (isHogQLQuery(q)) { + text = q.query + } + if (isNodeWithSource(q)) { + text = isHogQLQuery(q.source) ? q.source.query : '' + } + } + return text + }, }) diff --git a/frontend/src/scenes/notebooks/Nodes/NotebookNodeRecording.tsx b/frontend/src/scenes/notebooks/Nodes/NotebookNodeRecording.tsx index dafa271b98725..766fd500a2f5b 100644 --- a/frontend/src/scenes/notebooks/Nodes/NotebookNodeRecording.tsx +++ b/frontend/src/scenes/notebooks/Nodes/NotebookNodeRecording.tsx @@ -106,6 +106,9 @@ export const NotebookNodeRecording = createPostHogWidgetNode { + return attrs.id + }, }) export function sessionRecordingPlayerProps(id: SessionRecordingId): SessionRecordingPlayerProps { diff --git a/frontend/src/scenes/notebooks/Nodes/NotebookNodeReplayTimestamp.tsx b/frontend/src/scenes/notebooks/Nodes/NotebookNodeReplayTimestamp.tsx index ec49f4445d005..88db6f4395ffc 100644 --- a/frontend/src/scenes/notebooks/Nodes/NotebookNodeReplayTimestamp.tsx +++ b/frontend/src/scenes/notebooks/Nodes/NotebookNodeReplayTimestamp.tsx @@ -75,6 +75,12 @@ export const NotebookNodeReplayTimestamp = Node.create({ group: 'inline', atom: true, + serializedText: (attrs: NotebookNodeReplayTimestampAttrs): string => { + // timestamp is not a block so `getText` does not add a separator. + // we need to add it manually + return `${attrs.playbackTime ? formatTimestamp(attrs.playbackTime) : '00:00'}:\n` + }, + addAttributes() { return { playbackTime: { default: null, keepOnSplit: false }, diff --git a/frontend/src/scenes/notebooks/Notebook/Editor.tsx b/frontend/src/scenes/notebooks/Notebook/Editor.tsx index 60d1a67d08bcb..2a41bcce88209 100644 --- a/frontend/src/scenes/notebooks/Notebook/Editor.tsx +++ b/frontend/src/scenes/notebooks/Notebook/Editor.tsx @@ -3,7 +3,7 @@ import { useActions } from 'kea' import { useCallback, useRef } from 'react' import { Editor as TTEditor } from '@tiptap/core' -import { useEditor, EditorContent } from '@tiptap/react' +import { EditorContent, useEditor } from '@tiptap/react' import { FloatingMenu } from '@tiptap/extension-floating-menu' import StarterKit from '@tiptap/starter-kit' import ExtensionPlaceholder from '@tiptap/extension-placeholder' @@ -25,7 +25,7 @@ import { lemonToast } from '@posthog/lemon-ui' import { NotebookNodeType } from '~/types' import { NotebookNodeImage } from '../Nodes/NotebookNodeImage' -import { JSONContent, NotebookEditor, EditorFocusPosition, EditorRange, Node } from './utils' +import { EditorFocusPosition, EditorRange, JSONContent, Node, NotebookEditor, textContent } from './utils' import { SlashCommandsExtension } from './SlashCommands' import { BacklinkCommandsExtension } from './BacklinkCommands' import { NotebookNodeEarlyAccessFeature } from '../Nodes/NotebookNodeEarlyAccessFeature' @@ -182,6 +182,7 @@ export function Editor({ onCreate({ getJSON: () => editor.getJSON(), + getText: () => textContent(editor.state.doc), getEndPosition: () => editor.state.doc.content.size, getSelectedNode: () => editor.state.doc.nodeAt(editor.state.selection.$anchor.pos), getAdjacentNodes: (pos: number) => getAdjacentNodes(editor, pos), diff --git a/frontend/src/scenes/notebooks/Notebook/Notebook.scss b/frontend/src/scenes/notebooks/Notebook/Notebook.scss index c6906e0a76f8a..f272cacab53be 100644 --- a/frontend/src/scenes/notebooks/Notebook/Notebook.scss +++ b/frontend/src/scenes/notebooks/Notebook/Notebook.scss @@ -32,7 +32,15 @@ height: 0; } - > ul, + ul { + list-style-type: disc; + } + + ol { + list-style-type: decimal; + } + + ul, ol { padding-left: 1rem; @@ -40,11 +48,11 @@ p { margin-bottom: 0.2rem; } - } - } - > ul { - list-style: initial; + > p { + display: inline-block; + } + } } > pre { diff --git a/frontend/src/scenes/notebooks/Notebook/Notebook.stories.tsx b/frontend/src/scenes/notebooks/Notebook/Notebook.stories.tsx index b19845fe0b917..ecceb26e1ec93 100644 --- a/frontend/src/scenes/notebooks/Notebook/Notebook.stories.tsx +++ b/frontend/src/scenes/notebooks/Notebook/Notebook.stories.tsx @@ -5,6 +5,193 @@ import { router } from 'kea-router' import { urls } from 'scenes/urls' import { App } from 'scenes/App' import notebook12345Json from './__mocks__/notebook-12345.json' +import { notebookTestTemplate } from './__mocks__/notebook-template-for-snapshot' +import { NotebookType } from '~/types' + +// a list of test cases to run, showing different types of content in notebooks +const testCases: Record = { + 'api/projects/:team_id/notebooks/text-formats': notebookTestTemplate('text-formats', [ + { + type: 'paragraph', + content: [ + { + type: 'text', + marks: [ + { + type: 'bold', + }, + ], + text: ' bold ', + }, + ], + }, + { + type: 'paragraph', + content: [ + { + type: 'text', + marks: [ + { + type: 'italic', + }, + ], + text: 'italic', + }, + ], + }, + { + type: 'paragraph', + content: [ + { + type: 'text', + marks: [ + { + type: 'bold', + }, + { + type: 'italic', + }, + ], + text: 'bold _and_ italic', + }, + ], + }, + { + type: 'paragraph', + content: [ + { + type: 'text', + marks: [ + { + type: 'code', + }, + ], + text: 'code', + }, + ], + }, + ]), + 'api/projects/:team_id/notebooks/headings': notebookTestTemplate('headings', [ + { + type: 'heading', + attrs: { + level: 1, + }, + content: [ + { + type: 'text', + text: 'Heading 1', + }, + ], + }, + { + type: 'heading', + attrs: { + level: 2, + }, + content: [ + { + type: 'text', + text: 'Heading 2', + }, + ], + }, + { + type: 'heading', + attrs: { + level: 3, + }, + content: [ + { + type: 'text', + text: 'Heading 3', + }, + ], + }, + ]), + 'api/projects/:team_id/notebooks/numbered-list': notebookTestTemplate('numbered-list', [ + { + type: 'orderedList', + content: [ + { + type: 'listItem', + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'first item', + }, + ], + }, + ], + }, + { + type: 'listItem', + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'second item', + }, + ], + }, + ], + }, + ], + }, + ]), + 'api/projects/:team_id/notebooks/bullet-list': notebookTestTemplate('bullet-list', [ + { + type: 'bulletList', + content: [ + { + type: 'listItem', + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'first item', + }, + ], + }, + ], + }, + { + type: 'listItem', + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'second item', + }, + ], + }, + ], + }, + ], + }, + ]), + 'api/projects/:team_id/notebooks/recordings-playlist': notebookTestTemplate('recordings-playlist', [ + { + type: 'ph-recording-playlist', + attrs: { + height: null, + title: 'Session replays', + nodeId: '41faad12-499f-4a4b-95f7-3a36601317cc', + filters: + '{"session_recording_duration":{"type":"recording","key":"duration","value":3600,"operator":"gt"},"properties":[],"events":[],"actions":[],"date_from":"-7d","date_to":null}', + }, + }, + ]), +} const meta: Meta = { title: 'Scenes-App/Notebooks', @@ -15,6 +202,25 @@ const meta: Meta = { }, decorators: [ mswDecorator({ + post: { + 'api/projects/:team_id/query': { + clickhouse: + "SELECT nullIf(nullIf(events.`$session_id`, ''), 'null') AS session_id, any(events.properties) AS properties FROM events WHERE and(equals(events.team_id, 1), in(events.event, [%(hogql_val_0)s, %(hogql_val_1)s]), ifNull(in(session_id, [%(hogql_val_2)s]), 0), ifNull(greaterOrEquals(toTimeZone(events.timestamp, %(hogql_val_3)s), %(hogql_val_4)s), 0), ifNull(lessOrEquals(toTimeZone(events.timestamp, %(hogql_val_5)s), %(hogql_val_6)s), 0)) GROUP BY session_id LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=True", + columns: ['session_id', 'properties'], + hogql: "SELECT properties.$session_id AS session_id, any(properties) AS properties FROM events WHERE and(in(event, ['$pageview', '$autocapture']), in(session_id, ['018a8a51-a39d-7b18-897f-94054eec5f61']), greaterOrEquals(timestamp, '2023-09-11 16:55:36'), lessOrEquals(timestamp, '2023-09-13 18:07:40')) GROUP BY session_id LIMIT 100", + query: "SELECT properties.$session_id as session_id, any(properties) as properties\n FROM events\n WHERE event IN ['$pageview', '$autocapture']\n AND session_id IN ['018a8a51-a39d-7b18-897f-94054eec5f61']\n -- the timestamp range here is only to avoid querying too much of the events table\n -- we don't really care about the absolute value, \n -- but we do care about whether timezones have an odd impact\n -- so, we extend the range by a day on each side so that timezones don't cause issues\n AND timestamp >= '2023-09-11 16:55:36'\n AND timestamp <= '2023-09-13 18:07:40'\n GROUP BY session_id", + results: [ + [ + '018a8a51-a39d-7b18-897f-94054eec5f61', + '{"$os":"Mac OS X","$os_version":"10.15.7","$browser":"Chrome","$device_type":"Desktop","$current_url":"http://localhost:8000/ingestion/platform","$host":"localhost:8000","$pathname":"/ingestion/platform","$browser_version":116,"$browser_language":"en-GB","$screen_height":982,"$screen_width":1512,"$viewport_height":827,"$viewport_width":1498,"$lib":"web","$lib_version":"1.78.2","$insert_id":"249xj40dkv7x9knp","$time":1694537723.201,"distinct_id":"uLI7S0z6rWQIKAjgXhdUBplxPYymuQqxH5QbJKe2wqr","$device_id":"018a8a51-a39c-78f9-a4e4-1183f059f7cc","$user_id":"uLI7S0z6rWQIKAjgXhdUBplxPYymuQqxH5QbJKe2wqr","is_demo_project":false,"$groups":{"project":"018a8a51-9ee3-0000-0369-ff1924dcba89","organization":"018a8a51-988e-0000-d3e6-477c7cc111f1","instance":"http://localhost:8000"},"$autocapture_disabled_server_side":false,"$active_feature_flags":[],"$feature_flag_payloads":{},"realm":"hosted-clickhouse","email_service_available":false,"slack_service_available":false,"$referrer":"http://localhost:8000/signup","$referring_domain":"localhost:8000","$event_type":"click","$ce_version":1,"token":"phc_awewGgfgakHbaSbprHllKajqoa6iP2nz7OAUou763ie","$session_id":"018a8a51-a39d-7b18-897f-94054eec5f61","$window_id":"018a8a51-a39d-7b18-897f-940673bea28c","$set_once":{"$initial_os":"Mac OS X","$initial_browser":"Chrome","$initial_device_type":"Desktop","$initial_current_url":"http://localhost:8000/ingestion/platform","$initial_pathname":"/ingestion/platform","$initial_browser_version":116,"$initial_referrer":"http://localhost:8000/signup","$initial_referring_domain":"localhost:8000"},"$sent_at":"2023-09-12T16:55:23.743000+00:00","$ip":"127.0.0.1","$group_0":"018a8a51-9ee3-0000-0369-ff1924dcba89","$group_1":"018a8a51-988e-0000-d3e6-477c7cc111f1","$group_2":"http://localhost:8000"}', + ], + ], + types: [ + ['session_id', 'Nullable(String)'], + ['properties', 'String'], + ], + }, + }, get: { 'api/projects/:team_id/notebooks': { count: 1, @@ -66,6 +272,76 @@ const meta: Meta = { ], }, 'api/projects/:team_id/notebooks/12345': notebook12345Json, + 'api/projects/:team_id/session_recordings': { + results: [ + { + id: '018a8a51-a39d-7b18-897f-94054eec5f61', + distinct_id: 'uLI7S0z6rWQIKAjgXhdUBplxPYymuQqxH5QbJKe2wqr', + viewed: true, + recording_duration: 4324, + active_seconds: 21, + inactive_seconds: 4302, + start_time: '2023-09-12T16:55:36.404000Z', + end_time: '2023-09-12T18:07:40.147000Z', + click_count: 3, + keypress_count: 0, + mouse_activity_count: 924, + console_log_count: 37, + console_warn_count: 7, + console_error_count: 9, + start_url: 'http://localhost:8000/replay/recent', + person: { + id: 1, + name: 'paul@posthog.com', + distinct_ids: [ + 'uLI7S0z6rWQIKAjgXhdUBplxPYymuQqxH5QbJKe2wqr', + '018a8a51-a39c-78f9-a4e4-1183f059f7cc', + ], + properties: { + email: 'paul@posthog.com', + $initial_os: 'Mac OS X', + $geoip_latitude: -33.8715, + $geoip_city_name: 'Sydney', + $geoip_longitude: 151.2006, + $geoip_time_zone: 'Australia/Sydney', + $initial_browser: 'Chrome', + $initial_pathname: '/', + $initial_referrer: 'http://localhost:8000/signup', + $geoip_postal_code: '2000', + $creator_event_uuid: '018a8a51-a39d-7b18-897f-9407e795547b', + $geoip_country_code: 'AU', + $geoip_country_name: 'Australia', + $initial_current_url: 'http://localhost:8000/', + $initial_device_type: 'Desktop', + $geoip_continent_code: 'OC', + $geoip_continent_name: 'Oceania', + $initial_geoip_latitude: -33.8715, + $initial_browser_version: 116, + $initial_geoip_city_name: 'Sydney', + $initial_geoip_longitude: 151.2006, + $initial_geoip_time_zone: 'Australia/Sydney', + $geoip_subdivision_1_code: 'NSW', + $geoip_subdivision_1_name: 'New South Wales', + $initial_referring_domain: 'localhost:8000', + $initial_geoip_postal_code: '2000', + $initial_geoip_country_code: 'AU', + $initial_geoip_country_name: 'Australia', + $initial_geoip_continent_code: 'OC', + $initial_geoip_continent_name: 'Oceania', + $initial_geoip_subdivision_1_code: 'NSW', + $initial_geoip_subdivision_1_name: 'New South Wales', + }, + created_at: '2023-09-12T16:55:20.736000Z', + uuid: '018a8a51-a3d3-0000-e8fa-94621f9ddd48', + }, + storage: 'clickhouse', + pinned_count: 0, + }, + ], + has_next: false, + version: 3, + }, + ...testCases, }, }), ], @@ -78,6 +354,41 @@ export function NotebooksList(): JSX.Element { return } +export function Headings(): JSX.Element { + useEffect(() => { + router.actions.push(urls.notebook('headings')) + }, []) + return +} + +export function TextFormats(): JSX.Element { + useEffect(() => { + router.actions.push(urls.notebook('text-formats')) + }, []) + return +} + +export function NumberedList(): JSX.Element { + useEffect(() => { + router.actions.push(urls.notebook('numbered-list')) + }, []) + return +} + +export function BulletList(): JSX.Element { + useEffect(() => { + router.actions.push(urls.notebook('bullet-list')) + }, []) + return +} + +export function RecordingsPlaylist(): JSX.Element { + useEffect(() => { + router.actions.push(urls.notebook('recordings-playlist')) + }, []) + return +} + export function TextOnlyNotebook(): JSX.Element { useEffect(() => { router.actions.push(urls.notebook('12345')) diff --git a/frontend/src/scenes/notebooks/Notebook/__mocks__/notebook-template-for-snapshot.ts b/frontend/src/scenes/notebooks/Notebook/__mocks__/notebook-template-for-snapshot.ts new file mode 100644 index 0000000000000..b87917836a5db --- /dev/null +++ b/frontend/src/scenes/notebooks/Notebook/__mocks__/notebook-template-for-snapshot.ts @@ -0,0 +1,34 @@ +import { NotebookType } from '~/types' +import { MOCK_DEFAULT_BASIC_USER } from 'lib/api.mock' +import { JSONContent } from 'scenes/notebooks/Notebook/utils' + +export const notebookTestTemplate = ( + title: string = 'Notebook for snapshots', + notebookJson: JSONContent[] +): NotebookType => ({ + short_id: 'template-introduction', + title: title, + created_at: '2023-06-02T00:00:00Z', + last_modified_at: '2023-06-02T00:00:00Z', + created_by: MOCK_DEFAULT_BASIC_USER, + last_modified_by: MOCK_DEFAULT_BASIC_USER, + version: 1, + content: { + type: 'doc', + content: [ + { + type: 'heading', + attrs: { + level: 1, + }, + content: [ + { + type: 'text', + text: title, + }, + ], + }, + ...notebookJson, + ], + }, +}) diff --git a/frontend/src/scenes/notebooks/Notebook/notebookLogic.ts b/frontend/src/scenes/notebooks/Notebook/notebookLogic.ts index 84c8efa165372..a19b5f09dd1c5 100644 --- a/frontend/src/scenes/notebooks/Notebook/notebookLogic.ts +++ b/frontend/src/scenes/notebooks/Notebook/notebookLogic.ts @@ -178,6 +178,7 @@ export const notebookLogic = kea([ response = { ...values.scratchpadNotebook, content: {}, + text_content: null, version: 0, } } else if (props.shortId.startsWith('template-')) { @@ -210,6 +211,7 @@ export const notebookLogic = kea([ const response = await api.notebooks.update(values.notebook.short_id, { version: values.notebook.version, content: notebook.content, + text_content: values.editor?.getText() || '', title: notebook.title, }) @@ -242,6 +244,7 @@ export const notebookLogic = kea([ // We use the local content if set otherwise the notebook content. That way it supports templates, scratchpad etc. const response = await api.notebooks.create({ content: values.content || values.notebook.content, + text_content: values.editor?.getText() || '', title: values.title || values.notebook.title, }) @@ -430,6 +433,7 @@ export const notebookLogic = kea([ return } const jsonContent = values.editor.getJSON() + actions.setLocalContent(jsonContent) actions.onUpdateEditor() }, diff --git a/frontend/src/scenes/notebooks/Notebook/utils.ts b/frontend/src/scenes/notebooks/Notebook/utils.ts index 6947a4ef4a186..44feef88726e0 100644 --- a/frontend/src/scenes/notebooks/Notebook/utils.ts +++ b/frontend/src/scenes/notebooks/Notebook/utils.ts @@ -6,6 +6,7 @@ import { getText, JSONContent as TTJSONContent, Range as EditorRange, + TextSerializer, } from '@tiptap/core' import { Node as PMNode } from '@tiptap/pm/model' import { NodeViewProps } from '@tiptap/react' @@ -48,12 +49,13 @@ export type NotebookNodeWidget = { key: string label: string icon: JSX.Element - // using 'any' here shouldn't be necessary but I couldn't figure out how to set a generic on the notebookNodeLogic props + // using 'any' here shouldn't be necessary but, I couldn't figure out how to set a generic on the notebookNodeLogic props Component: ({ attributes, updateAttributes }: NotebookNodeAttributeProperties) => JSX.Element } export interface NotebookEditor { getJSON: () => JSONContent + getText: () => string getEndPosition: () => number getSelectedNode: () => Node | null getAdjacentNodes: (pos: number) => { previous: Node | null; next: Node | null } @@ -88,12 +90,39 @@ export const isCurrentNodeEmpty = (editor: TTEditor): boolean => { return false } -const textContent = (node: any): string => { +export const textContent = (node: any): string => { + // we've extended the node schema to support a custom serializedText function + // each custom node type needs to implement this function, or have an alternative in the map below + const customOrTitleSerializer: TextSerializer = (props): string => { + // TipTap chooses whether to add a separator based on a couple of factors + // but, we always want a separator since this text is for search purposes + const serializedText = props.node.type.spec.serializedText(props.node.attrs) || props.node.attrs?.title || '' + if (serializedText.length > 0 && serializedText[serializedText.length - 1] !== '\n') { + return serializedText + '\n' + } + return serializedText + } + + // we want the type system to complain if we forget to add a custom serializer + const customNodeTextSerializers: Record = { + 'ph-backlink': customOrTitleSerializer, + 'ph-early-access-feature': customOrTitleSerializer, + 'ph-experiment': customOrTitleSerializer, + 'ph-feature-flag': customOrTitleSerializer, + 'ph-feature-flag-code-example': customOrTitleSerializer, + 'ph-image': customOrTitleSerializer, + 'ph-insight': customOrTitleSerializer, + 'ph-person': customOrTitleSerializer, + 'ph-query': customOrTitleSerializer, + 'ph-recording': customOrTitleSerializer, + 'ph-recording-playlist': customOrTitleSerializer, + 'ph-replay-timestamp': customOrTitleSerializer, + 'ph-survey': customOrTitleSerializer, + } + return getText(node, { - blockSeparator: ' ', - textSerializers: { - [NotebookNodeType.ReplayTimestamp]: ({ node }) => `${node.attrs.playbackTime || '00:00'}: `, - }, + blockSeparator: '\n', + textSerializers: customNodeTextSerializers, }) } diff --git a/frontend/src/scenes/project/Settings/WebhookIntegration.tsx b/frontend/src/scenes/project/Settings/WebhookIntegration.tsx index 83ba8c9a13114..23caab8f00533 100644 --- a/frontend/src/scenes/project/Settings/WebhookIntegration.tsx +++ b/frontend/src/scenes/project/Settings/WebhookIntegration.tsx @@ -2,13 +2,18 @@ import { useEffect, useState } from 'react' import { useActions, useValues } from 'kea' import { teamLogic } from 'scenes/teamLogic' import { webhookIntegrationLogic } from './webhookIntegrationLogic' -import { LemonButton, LemonInput } from '@posthog/lemon-ui' +import { LemonButton, LemonInput, Link } from '@posthog/lemon-ui' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' +import { FEATURE_FLAGS } from 'lib/constants' +import { supportLogic } from 'lib/components/Support/supportLogic' export function WebhookIntegration(): JSX.Element { const [webhook, setWebhook] = useState('') const { testWebhook, removeWebhook } = useActions(webhookIntegrationLogic) const { loading } = useValues(webhookIntegrationLogic) const { currentTeam } = useValues(teamLogic) + const { featureFlags } = useValues(featureFlagLogic) + const { openSupportForm } = useActions(supportLogic) useEffect(() => { if (currentTeam?.slack_incoming_webhook) { @@ -16,6 +21,18 @@ export function WebhookIntegration(): JSX.Element { } }, [currentTeam]) + const webhooks_blacklisted = featureFlags[FEATURE_FLAGS.WEBHOOKS_DENYLIST] + if (webhooks_blacklisted) { + return ( +
+

+ Webhooks are currently not available for your organization.{' '} + openSupportForm('support', 'apps')}>Contact support +

+
+ ) + } + return (

diff --git a/frontend/src/scenes/session-recordings/SessionsRecordings.stories.tsx b/frontend/src/scenes/session-recordings/SessionsRecordings-player-success.stories.tsx similarity index 93% rename from frontend/src/scenes/session-recordings/SessionsRecordings.stories.tsx rename to frontend/src/scenes/session-recordings/SessionsRecordings-player-success.stories.tsx index 1ac9395728811..1060246c67d27 100644 --- a/frontend/src/scenes/session-recordings/SessionsRecordings.stories.tsx +++ b/frontend/src/scenes/session-recordings/SessionsRecordings-player-success.stories.tsx @@ -9,7 +9,6 @@ import recordingSnapshotsJson from 'scenes/session-recordings/__mocks__/recordin import recordingMetaJson from 'scenes/session-recordings/__mocks__/recording_meta.json' import recordingEventsJson from 'scenes/session-recordings/__mocks__/recording_events_query' import recording_playlists from './__mocks__/recording_playlists.json' -import { ReplayTabs } from '~/types' const meta: Meta = { title: 'Scenes-App/Recordings', @@ -17,6 +16,7 @@ const meta: Meta = { layout: 'fullscreen', viewMode: 'story', mockDate: '2023-02-01', + waitForSelector: '.PlayerFrame__content .replayer-wrapper iframe', }, decorators: [ mswDecorator({ @@ -81,7 +81,7 @@ const meta: Meta = { }, ] }, - '/api/projects/:team_id/session_recording_playlists/:playlist_id/recordings?limit=100': (req) => { + '/api/projects/:team_id/session_recording_playlists/:playlist_id/recordings': (req) => { const playlistId = req.params.playlist_id const response = playlistId === '1234567' ? recordings : [] return [200, { has_next: false, results: response, version: 1 }] @@ -89,6 +89,12 @@ const meta: Meta = { // without the session-recording-blob-replay feature flag, we only load via ClickHouse '/api/projects/:team/session_recordings/:id/snapshots': recordingSnapshotsJson, '/api/projects/:team/session_recordings/:id': recordingMetaJson, + 'api/projects/:team/notebooks': { + count: 0, + next: null, + previous: null, + results: [], + }, }, post: { '/api/projects/:team/query': recordingEventsJson, @@ -97,16 +103,10 @@ const meta: Meta = { ], } export default meta -export function RecordingsList(): JSX.Element { - useEffect(() => { - router.actions.push(urls.replay()) - }, []) - return -} -export function RecordingsPlayLists(): JSX.Element { +export function RecentRecordings(): JSX.Element { useEffect(() => { - router.actions.push(urls.replay(ReplayTabs.Playlists)) + router.actions.push(urls.replay()) }, []) return } diff --git a/frontend/src/scenes/session-recordings/SessionsRecordings-playlist-listing.stories.tsx b/frontend/src/scenes/session-recordings/SessionsRecordings-playlist-listing.stories.tsx new file mode 100644 index 0000000000000..657fbccf4bc29 --- /dev/null +++ b/frontend/src/scenes/session-recordings/SessionsRecordings-playlist-listing.stories.tsx @@ -0,0 +1,48 @@ +import { Meta } from '@storybook/react' +import { useEffect } from 'react' +import { mswDecorator } from '~/mocks/browser' +import { router } from 'kea-router' +import { urls } from 'scenes/urls' +import { App } from 'scenes/App' +import recording_playlists from './__mocks__/recording_playlists.json' +import { ReplayTabs } from '~/types' +import recordings from 'scenes/session-recordings/__mocks__/recordings.json' +import recordingEventsJson from 'scenes/session-recordings/__mocks__/recording_events_query' + +const meta: Meta = { + title: 'Scenes-App/Recordings', + parameters: { + layout: 'fullscreen', + viewMode: 'story', + mockDate: '2023-02-01', + }, + decorators: [ + mswDecorator({ + get: { + '/api/projects/:team_id/session_recording_playlists': recording_playlists, + '/api/projects/:team_id/session_recordings': (req) => { + const version = req.url.searchParams.get('version') + return [ + 200, + { + has_next: false, + results: recordings, + version, + }, + ] + }, + }, + post: { + '/api/projects/:team/query': recordingEventsJson, + }, + }), + ], +} +export default meta + +export function RecordingsPlayLists(): JSX.Element { + useEffect(() => { + router.actions.push(urls.replay(ReplayTabs.Playlists)) + }, []) + return +} diff --git a/frontend/src/scenes/session-recordings/__mocks__/recording_events.json b/frontend/src/scenes/session-recordings/__mocks__/recording_events.json index f2db148045646..0afa00a98d244 100644 --- a/frontend/src/scenes/session-recordings/__mocks__/recording_events.json +++ b/frontend/src/scenes/session-recordings/__mocks__/recording_events.json @@ -1,6 +1,6 @@ [ { - "id": "$pageview", + "id": "$pageview1", "event": "$pageview", "name": "$event_before_recording_starts", "type": "events", @@ -14,7 +14,7 @@ "elements_hash": "" }, { - "id": "$pageview", + "id": "$pageview2", "name": "$pageview", "event": "$pageview", "type": "events", diff --git a/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx b/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx index f90ee397989cb..00508be3ab649 100644 --- a/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx +++ b/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx @@ -257,11 +257,11 @@ export function RecordingsLists({ data-attr={'expand-replay-listing-from-default-seven-days-to-twenty-one'} onClick={() => { setFilters({ - date_from: '-21d', + date_from: '-30d', }) }} > - Search over the last 21 days + Search over the last 30 days ) : ( diff --git a/frontend/src/scenes/surveys/SurveyView.tsx b/frontend/src/scenes/surveys/SurveyView.tsx index 80c207d3c7644..3b0b46e89ca94 100644 --- a/frontend/src/scenes/surveys/SurveyView.tsx +++ b/frontend/src/scenes/surveys/SurveyView.tsx @@ -319,7 +319,9 @@ function SurveyNPSResults({ survey }: { survey: Survey }): JSX.Element { kind: NodeKind.TrendsQuery, dateRange: { date_from: dayjs(survey.created_at).format('YYYY-MM-DD'), - date_to: dayjs().format('YYYY-MM-DD'), + date_to: survey.end_date + ? dayjs(survey.end_date).format('YYYY-MM-DD') + : dayjs().format('YYYY-MM-DD'), }, series: [ { diff --git a/frontend/src/styles/utilities.scss b/frontend/src/styles/utilities.scss index 126d981427e89..745375f1c3f57 100644 --- a/frontend/src/styles/utilities.scss +++ b/frontend/src/styles/utilities.scss @@ -919,6 +919,13 @@ $decorations: underline, overline, line-through, no-underline; } } +.list-inside { + list-style-position: inside; +} +.list-outside { + list-style-position: outside; +} + .shadow { box-shadow: var(--shadow-elevation); } diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 187bcba34863d..f4413d151bcdc 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -3020,6 +3020,8 @@ export type NotebookListItemType = { export type NotebookType = NotebookListItemType & { content: JSONContent // TODO: Type this better version: number + // used to power text-based search + text_content?: string | null } export enum NotebookNodeType { @@ -3095,6 +3097,8 @@ export type BatchExportDestinationS3 = { aws_secret_access_key: string exclude_events: string[] compression: string | null + encryption: string | null + kms_key_id: string | null } } diff --git a/latest_migrations.manifest b/latest_migrations.manifest index 2db90ee7e8674..233b3d446d5cb 100644 --- a/latest_migrations.manifest +++ b/latest_migrations.manifest @@ -5,7 +5,7 @@ contenttypes: 0002_remove_content_type_name ee: 0015_add_verified_properties otp_static: 0002_throttling otp_totp: 0002_auto_20190420_0723 -posthog: 0349_update_survey_query_name +posthog: 0350_add_notebook_text_content sessions: 0001_initial social_django: 0010_uid_db_index two_factor: 0007_auto_20201201_1019 diff --git a/package.json b/package.json index 684b66f89332d..3f8131541f4c5 100644 --- a/package.json +++ b/package.json @@ -125,7 +125,8 @@ "kea-window-values": "^3.0.0", "md5": "^2.3.0", "monaco-editor": "^0.39.0", - "posthog-js": "1.78.2", + "papaparse": "^5.4.1", + "posthog-js": "1.78.5", "posthog-js-lite": "2.0.0-alpha5", "prettier": "^2.8.8", "prop-types": "^15.7.2", @@ -206,6 +207,7 @@ "@types/jest-image-snapshot": "^6.1.0", "@types/md5": "^2.3.0", "@types/node": "^18.11.9", + "@types/papaparse": "^5.3.8", "@types/pixelmatch": "^5.2.4", "@types/pngjs": "^6.0.1", "@types/query-selector-shadow-dom": "^1.0.0", diff --git a/playwright/e2e-vrt/layout/Navigation.spec.ts-snapshots/Navigation-App-Page-With-Side-Bar-Hidden-Mobile-1-chromium-linux.png b/playwright/e2e-vrt/layout/Navigation.spec.ts-snapshots/Navigation-App-Page-With-Side-Bar-Hidden-Mobile-1-chromium-linux.png index 3b185216c6362..ab79c58ee2eaf 100644 Binary files a/playwright/e2e-vrt/layout/Navigation.spec.ts-snapshots/Navigation-App-Page-With-Side-Bar-Hidden-Mobile-1-chromium-linux.png and b/playwright/e2e-vrt/layout/Navigation.spec.ts-snapshots/Navigation-App-Page-With-Side-Bar-Hidden-Mobile-1-chromium-linux.png differ diff --git a/playwright/e2e-vrt/layout/Navigation.spec.ts-snapshots/Navigation-App-Page-With-Side-Bar-Shown-Mobile-1-chromium-linux.png b/playwright/e2e-vrt/layout/Navigation.spec.ts-snapshots/Navigation-App-Page-With-Side-Bar-Shown-Mobile-1-chromium-linux.png index 2ce1d7971c1e1..55ea6ef92745b 100644 Binary files a/playwright/e2e-vrt/layout/Navigation.spec.ts-snapshots/Navigation-App-Page-With-Side-Bar-Shown-Mobile-1-chromium-linux.png and b/playwright/e2e-vrt/layout/Navigation.spec.ts-snapshots/Navigation-App-Page-With-Side-Bar-Shown-Mobile-1-chromium-linux.png differ diff --git a/plugin-server/functional_tests/webhooks.test.ts b/plugin-server/functional_tests/webhooks.test.ts index 0fb7155790034..82f1bfe9bf186 100644 --- a/plugin-server/functional_tests/webhooks.test.ts +++ b/plugin-server/functional_tests/webhooks.test.ts @@ -199,6 +199,7 @@ test.concurrent(`webhooks: fires zapier REST webhook`, async () => { properties: { $creator_event_uuid: eventUuid, $initial_current_url: 'http://localhost:8000', + $current_url: 'http://localhost:8000', email: 't@t.com', }, uuid: expect.any(String), @@ -208,6 +209,7 @@ test.concurrent(`webhooks: fires zapier REST webhook`, async () => { $sent_at: expect.any(String), $set: { email: 't@t.com', + $current_url: 'http://localhost:8000', }, $set_once: { $initial_current_url: 'http://localhost:8000', diff --git a/plugin-server/src/utils/db/hub.ts b/plugin-server/src/utils/db/hub.ts index 710a163752a6b..4e37d8a5cd715 100644 --- a/plugin-server/src/utils/db/hub.ts +++ b/plugin-server/src/utils/db/hub.ts @@ -91,7 +91,6 @@ export async function createHub( : undefined, rejectUnauthorized: serverConfig.CLICKHOUSE_CA ? false : undefined, }) - await clickhouse.querying('SELECT 1') // test that the connection works status.info('👍', `ClickHouse ready`) status.info('🤔', `Connecting to Kafka...`) diff --git a/plugin-server/src/utils/db/utils.ts b/plugin-server/src/utils/db/utils.ts index 49db8914194f6..9e4eb0a3c11b7 100644 --- a/plugin-server/src/utils/db/utils.ts +++ b/plugin-server/src/utils/db/utils.ts @@ -39,7 +39,22 @@ export function timeoutGuard( }, timeout) } -const campaignParams = new Set([ +const eventToPersonProperties = new Set([ + // mobile params + '$app_build', + '$app_name', + '$app_namespace', + '$app_version', + // web params + '$browser', + '$browser_version', + '$device_type', + '$current_url', + '$pathname', + '$os', + '$referring_domain', + '$referrer', + // campaign params 'utm_source', 'utm_medium', 'utm_campaign', @@ -50,31 +65,29 @@ const campaignParams = new Set([ 'fbclid', 'msclkid', ]) -const initialParams = new Set([ - '$browser', - '$browser_version', - '$device_type', - '$current_url', - '$pathname', - '$os', - '$referring_domain', - '$referrer', -]) -const combinedParams = new Set([...campaignParams, ...initialParams]) /** If we get new UTM params, make sure we set those **/ export function personInitialAndUTMProperties(properties: Properties): Properties { const propertiesCopy = { ...properties } - const maybeSet = Object.entries(properties).filter(([key]) => campaignParams.has(key)) - const maybeSetInitial = Object.entries(properties) - .filter(([key]) => combinedParams.has(key)) - .map(([key, value]) => [`$initial_${key.replace('$', '')}`, value]) - if (Object.keys(maybeSet).length > 0) { + const propertiesForPerson: [string, any][] = Object.entries(properties).filter(([key]) => + eventToPersonProperties.has(key) + ) + + // all potential params are checked for $initial_ values and added to $set_once + const maybeSetOnce: [string, any][] = propertiesForPerson.map(([key, value]) => [ + `$initial_${key.replace('$', '')}`, + value, + ]) + + // all found are also then added to $set + const maybeSet: [string, any][] = propertiesForPerson + + if (maybeSet.length > 0) { propertiesCopy.$set = { ...(properties.$set || {}), ...Object.fromEntries(maybeSet) } } - if (Object.keys(maybeSetInitial).length > 0) { - propertiesCopy.$set_once = { ...(properties.$set_once || {}), ...Object.fromEntries(maybeSetInitial) } + if (maybeSetOnce.length > 0) { + propertiesCopy.$set_once = { ...(properties.$set_once || {}), ...Object.fromEntries(maybeSetOnce) } } return propertiesCopy } diff --git a/plugin-server/src/worker/ingestion/process-event.ts b/plugin-server/src/worker/ingestion/process-event.ts index 3941ef725af0e..44327a6a8bfd5 100644 --- a/plugin-server/src/worker/ingestion/process-event.ts +++ b/plugin-server/src/worker/ingestion/process-event.ts @@ -281,6 +281,8 @@ export interface SummarizedSessionRecordingEvent { console_warn_count: number console_error_count: number size: number + event_count: number + message_count: number } export const createSessionReplayEvent = ( @@ -357,6 +359,8 @@ export const createSessionReplayEvent = ( console_warn_count: Math.trunc(consoleWarnCount), console_error_count: Math.trunc(consoleErrorCount), size: Math.trunc(Buffer.byteLength(JSON.stringify(events), 'utf8')), + event_count: Math.trunc(events.length), + message_count: 1, } return data diff --git a/plugin-server/tests/main/process-event.test.ts b/plugin-server/tests/main/process-event.test.ts index 2e0440d454bf8..94505831b8452 100644 --- a/plugin-server/tests/main/process-event.test.ts +++ b/plugin-server/tests/main/process-event.test.ts @@ -315,7 +315,7 @@ test('capture new person', async () => { let persons = await hub.db.fetchPersons() expect(persons[0].version).toEqual(0) expect(persons[0].created_at).toEqual(now) - let expectedProps = { + let expectedProps: Record = { $creator_event_uuid: uuid, $initial_browser: 'Chrome', $initial_browser_version: '95', @@ -329,6 +329,12 @@ test('capture new person', async () => { msclkid: 'BING ADS ID', $initial_referrer: 'https://google.com/?q=posthog', $initial_referring_domain: 'https://google.com', + $browser: 'Chrome', + $browser_version: '95', + $current_url: 'https://test.com', + $os: 'Mac OS X', + $referrer: 'https://google.com/?q=posthog', + $referring_domain: 'https://google.com', } expect(persons[0].properties).toEqual(expectedProps) @@ -343,7 +349,17 @@ test('capture new person', async () => { expect(events[0].properties).toEqual({ $ip: '127.0.0.1', $os: 'Mac OS X', - $set: { utm_medium: 'twitter', gclid: 'GOOGLE ADS ID', msclkid: 'BING ADS ID' }, + $set: { + utm_medium: 'twitter', + gclid: 'GOOGLE ADS ID', + msclkid: 'BING ADS ID', + $browser: 'Chrome', + $browser_version: '95', + $current_url: 'https://test.com', + $os: 'Mac OS X', + $referrer: 'https://google.com/?q=posthog', + $referring_domain: 'https://google.com', + }, token: 'THIS IS NOT A TOKEN FOR TEAM 2', $browser: 'Chrome', $set_once: { @@ -412,6 +428,12 @@ test('capture new person', async () => { msclkid: 'BING ADS ID', $initial_referrer: 'https://google.com/?q=posthog', $initial_referring_domain: 'https://google.com', + $browser: 'Firefox', + $browser_version: 80, + $current_url: 'https://test.com/pricing', + $os: 'Mac OS X', + $referrer: 'https://google.com/?q=posthog', + $referring_domain: 'https://google.com', } expect(persons[0].properties).toEqual(expectedProps) @@ -425,6 +447,9 @@ test('capture new person', async () => { expect(events[1].properties.$set).toEqual({ utm_medium: 'instagram', + $browser: 'Firefox', + $browser_version: 80, + $current_url: 'https://test.com/pricing', }) expect(events[1].properties.$set_once).toEqual({ $initial_browser: 'Firefox', @@ -481,6 +506,9 @@ test('capture new person', async () => { expect(persons[0].version).toEqual(1) expect(events[2].properties.$set).toEqual({ + $browser: 'Firefox', + $current_url: 'https://test.com/pricing', + utm_medium: 'instagram', }) expect(events[2].properties.$set_once).toEqual({ @@ -1236,6 +1264,8 @@ const sessionReplayEventTestCases: { | 'console_warn_count' | 'console_error_count' | 'size' + | 'event_count' + | 'message_count' > }[] = [ { @@ -1252,6 +1282,8 @@ const sessionReplayEventTestCases: { console_warn_count: 0, console_error_count: 0, size: 73, + event_count: 1, + message_count: 1, }, }, { @@ -1268,6 +1300,8 @@ const sessionReplayEventTestCases: { console_warn_count: 0, console_error_count: 0, size: 73, + event_count: 1, + message_count: 1, }, }, { @@ -1324,6 +1358,8 @@ const sessionReplayEventTestCases: { console_warn_count: 3, console_error_count: 1, size: 762, + event_count: 7, + message_count: 1, }, }, { @@ -1362,6 +1398,8 @@ const sessionReplayEventTestCases: { console_warn_count: 0, console_error_count: 0, size: 213, + event_count: 2, + message_count: 1, }, }, { @@ -1389,6 +1427,8 @@ const sessionReplayEventTestCases: { console_warn_count: 0, console_error_count: 0, size: 433, + event_count: 6, + message_count: 1, }, }, ] diff --git a/plugin-server/tests/utils/db/utils.test.ts b/plugin-server/tests/utils/db/utils.test.ts index 5201b8e60b803..420c645472ff3 100644 --- a/plugin-server/tests/utils/db/utils.test.ts +++ b/plugin-server/tests/utils/db/utils.test.ts @@ -17,41 +17,74 @@ describe('personInitialAndUTMProperties()', () => { { tag_name: 'a', nth_child: 1, nth_of_type: 2, attr__class: 'btn btn-sm' }, { tag_name: 'div', nth_child: 1, nth_of_type: 2, $el_text: '💻' }, ], + $app_build: 2, + $app_name: 'my app', + $app_namespace: 'com.posthog.myapp', + $app_version: '1.2.3', } - expect(personInitialAndUTMProperties(properties)).toEqual({ - distinct_id: 2, - $browser: 'Chrome', - $current_url: 'https://test.com', - $os: 'Mac OS X', - $browser_version: '95', - $referring_domain: 'https://google.com', - $referrer: 'https://google.com/?q=posthog', - utm_medium: 'twitter', - gclid: 'GOOGLE ADS ID', - msclkid: 'BING ADS ID', - $elements: [ - { - tag_name: 'a', - nth_child: 1, - nth_of_type: 2, - attr__class: 'btn btn-sm', + expect(personInitialAndUTMProperties(properties)).toMatchInlineSnapshot(` + Object { + "$app_build": 2, + "$app_name": "my app", + "$app_namespace": "com.posthog.myapp", + "$app_version": "1.2.3", + "$browser": "Chrome", + "$browser_version": "95", + "$current_url": "https://test.com", + "$elements": Array [ + Object { + "attr__class": "btn btn-sm", + "nth_child": 1, + "nth_of_type": 2, + "tag_name": "a", }, - { tag_name: 'div', nth_child: 1, nth_of_type: 2, $el_text: '💻' }, - ], - $set: { utm_medium: 'twitter', gclid: 'GOOGLE ADS ID', msclkid: 'BING ADS ID' }, - $set_once: { - $initial_browser: 'Chrome', - $initial_current_url: 'https://test.com', - $initial_os: 'Mac OS X', - $initial_browser_version: '95', - $initial_utm_medium: 'twitter', - $initial_gclid: 'GOOGLE ADS ID', - $initial_msclkid: 'BING ADS ID', - $initial_referring_domain: 'https://google.com', - $initial_referrer: 'https://google.com/?q=posthog', - }, - }) + Object { + "$el_text": "💻", + "nth_child": 1, + "nth_of_type": 2, + "tag_name": "div", + }, + ], + "$os": "Mac OS X", + "$referrer": "https://google.com/?q=posthog", + "$referring_domain": "https://google.com", + "$set": Object { + "$app_build": 2, + "$app_name": "my app", + "$app_namespace": "com.posthog.myapp", + "$app_version": "1.2.3", + "$browser": "Chrome", + "$browser_version": "95", + "$current_url": "https://test.com", + "$os": "Mac OS X", + "$referrer": "https://google.com/?q=posthog", + "$referring_domain": "https://google.com", + "gclid": "GOOGLE ADS ID", + "msclkid": "BING ADS ID", + "utm_medium": "twitter", + }, + "$set_once": Object { + "$initial_app_build": 2, + "$initial_app_name": "my app", + "$initial_app_namespace": "com.posthog.myapp", + "$initial_app_version": "1.2.3", + "$initial_browser": "Chrome", + "$initial_browser_version": "95", + "$initial_current_url": "https://test.com", + "$initial_gclid": "GOOGLE ADS ID", + "$initial_msclkid": "BING ADS ID", + "$initial_os": "Mac OS X", + "$initial_referrer": "https://google.com/?q=posthog", + "$initial_referring_domain": "https://google.com", + "$initial_utm_medium": "twitter", + }, + "distinct_id": 2, + "gclid": "GOOGLE ADS ID", + "msclkid": "BING ADS ID", + "utm_medium": "twitter", + } + `) }) it('initial current domain regression test', () => { @@ -62,6 +95,7 @@ describe('personInitialAndUTMProperties()', () => { expect(personInitialAndUTMProperties(properties)).toEqual({ $current_url: 'https://test.com', $set_once: { $initial_current_url: 'https://test.com' }, + $set: { $current_url: 'https://test.com' }, }) }) }) diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/event-pipeline-integration.test.ts b/plugin-server/tests/worker/ingestion/event-pipeline/event-pipeline-integration.test.ts index 837079da765eb..343826d81a4f2 100644 --- a/plugin-server/tests/worker/ingestion/event-pipeline/event-pipeline-integration.test.ts +++ b/plugin-server/tests/worker/ingestion/event-pipeline/event-pipeline-integration.test.ts @@ -105,6 +105,7 @@ describe('Event Pipeline integration test', () => { $set: { personProp: 'value', anotherValue: 2, + $browser: 'Chrome', }, $set_once: { $initial_browser: 'Chrome', @@ -118,6 +119,7 @@ describe('Event Pipeline integration test', () => { expect(persons[0].properties).toEqual({ $creator_event_uuid: event.uuid, $initial_browser: 'Chrome', + $browser: 'Chrome', personProp: 'value', anotherValue: 2, }) diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/processPersonsStep.test.ts b/plugin-server/tests/worker/ingestion/event-pipeline/processPersonsStep.test.ts index 71d495bcf9bce..d2ce3aa76e383 100644 --- a/plugin-server/tests/worker/ingestion/event-pipeline/processPersonsStep.test.ts +++ b/plugin-server/tests/worker/ingestion/event-pipeline/processPersonsStep.test.ts @@ -85,6 +85,7 @@ describe.each([[true], [false]])('processPersonsStep()', (poEEmbraceJoin) => { $browser: 'Chrome', $set: { someProp: 'value', + $browser: 'Chrome', }, $set_once: { $initial_browser: 'Chrome', @@ -95,7 +96,12 @@ describe.each([[true], [false]])('processPersonsStep()', (poEEmbraceJoin) => { expect.objectContaining({ id: expect.any(Number), uuid: expect.any(String), - properties: { $initial_browser: 'Chrome', someProp: 'value', $creator_event_uuid: expect.any(String) }, + properties: { + $initial_browser: 'Chrome', + someProp: 'value', + $creator_event_uuid: expect.any(String), + $browser: 'Chrome', + }, version: 0, is_identified: false, }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3ce1f9a71b1d2..301601dff1e49 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,4 +1,4 @@ -lockfileVersion: '6.1' +lockfileVersion: '6.0' settings: autoInstallPeers: true @@ -194,9 +194,12 @@ dependencies: monaco-editor: specifier: ^0.39.0 version: 0.39.0 + papaparse: + specifier: ^5.4.1 + version: 5.4.1 posthog-js: - specifier: 1.78.2 - version: 1.78.2 + specifier: 1.78.5 + version: 1.78.5 posthog-js-lite: specifier: 2.0.0-alpha5 version: 2.0.0-alpha5 @@ -432,6 +435,9 @@ devDependencies: '@types/node': specifier: ^18.11.9 version: 18.11.9 + '@types/papaparse': + specifier: ^5.3.8 + version: 5.3.8 '@types/pixelmatch': specifier: ^5.2.4 version: 5.2.4 @@ -6212,6 +6218,12 @@ packages: resolution: {integrity: sha512-sn7L+qQ6RLPdXRoiaE7bZ/Ek+o4uICma/lBFPyJEKDTPTBP1W8u0c4baj3EiS4DiqLs+Hk+KUGvMVJtAw3ePJg==} dev: false + /@types/papaparse@5.3.8: + resolution: {integrity: sha512-ArKIEOOWULbhi53wkAiRy1ze4wvrTfhpAj7Yfzva+EkmX2sV8PpFB+xqzJfzXNzK4me95FJH9QZt5NXFVGzOoQ==} + dependencies: + '@types/node': 18.11.9 + dev: true + /@types/parse-json@4.0.0: resolution: {integrity: sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==} dev: true @@ -14265,6 +14277,10 @@ packages: resolution: {integrity: sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==} dev: true + /papaparse@5.4.1: + resolution: {integrity: sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw==} + dev: false + /param-case@3.0.4: resolution: {integrity: sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A==} dependencies: @@ -14908,8 +14924,8 @@ packages: resolution: {integrity: sha512-tlkBdypJuvK/s00n4EiQjwYVfuuZv6vt8BF3g1ooIQa2Gz9Vz80p8q3qsPLZ0V5ErGRy6i3Q4fWC9TDzR7GNRQ==} dev: false - /posthog-js@1.78.2: - resolution: {integrity: sha512-jDy0QR+Mt7c4efq4knUsDVx/dT9DKMRLPimR/aSNTPRlAdWDNYD6WFv3oFyUk5tzkOPcKVJItRmmS2ua3tesYA==} + /posthog-js@1.78.5: + resolution: {integrity: sha512-UUipML52LEyks7Pbx/3dpBJc2iPJrW+Ss6Y0BiIygn+QZoBjIe1WjE4Ep+Fnz7+cX1axex/ZiYholBnW7E4Aug==} dependencies: fflate: 0.4.8 dev: false diff --git a/posthog/api/notebook.py b/posthog/api/notebook.py index a7db45a2a9a44..5c25efe42815d 100644 --- a/posthog/api/notebook.py +++ b/posthog/api/notebook.py @@ -1,5 +1,5 @@ from typing import Dict, List, Optional, Any - +from django.db.models import Q import structlog from django.db import transaction from django.db.models import QuerySet @@ -74,6 +74,7 @@ class Meta: "short_id", "title", "content", + "text_content", "version", "deleted", "created_at", @@ -251,7 +252,12 @@ def _filter_request(self, request: request.Request, queryset: QuerySet) -> Query last_modified_at__lt=relative_date_parse(request.GET["date_to"], self.team.timezone_info) ) elif key == "search": - queryset = queryset.filter(title__icontains=request.GET["search"]) + queryset = queryset.filter( + # some notebooks have no text_content until next saved, so we need to check the title too + # TODO this can be removed once all/most notebooks have text_content + Q(title__search=request.GET["search"]) + | Q(text_content__search=request.GET["search"]) + ) elif key == "contains": contains = request.GET["contains"] match_pairs = contains.replace(",", " ").split(" ") diff --git a/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr b/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr index af0efd4023fe7..245b0ceb08720 100644 --- a/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr +++ b/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr @@ -40,6 +40,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -51,6 +52,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -157,6 +159,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -168,6 +171,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -267,6 +271,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -278,6 +283,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -468,6 +474,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -479,6 +486,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -626,6 +634,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -637,6 +646,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -795,6 +805,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -806,6 +817,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -951,6 +963,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -962,6 +975,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -1181,6 +1195,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -1192,6 +1207,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -1230,6 +1246,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -1241,6 +1258,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -1376,6 +1394,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -1387,6 +1406,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -1478,6 +1498,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -1489,6 +1510,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -1527,6 +1549,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -1538,6 +1561,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -1671,6 +1695,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -1682,6 +1707,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -1789,6 +1815,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -1800,6 +1827,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -2040,6 +2068,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -2051,6 +2080,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -2272,6 +2302,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -2283,6 +2314,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -2390,6 +2422,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -2401,6 +2434,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -2501,6 +2535,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -2512,6 +2547,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -2612,6 +2648,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -2623,6 +2660,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -2703,6 +2741,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -2714,6 +2753,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -2845,6 +2885,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -2856,6 +2897,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -2933,6 +2975,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -2944,6 +2987,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -3048,6 +3092,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -3059,6 +3104,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -3163,6 +3209,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -3174,6 +3221,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -3289,6 +3337,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -3300,6 +3349,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -3600,6 +3650,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -3611,6 +3662,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -3750,6 +3802,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -3761,6 +3814,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -3874,6 +3928,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -3885,6 +3940,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -3951,6 +4007,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -3962,6 +4019,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -4104,6 +4162,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -4115,6 +4174,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -4153,6 +4213,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -4164,6 +4225,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -4268,6 +4330,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -4279,6 +4342,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -4409,6 +4473,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -4420,6 +4485,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -4825,6 +4891,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -4836,6 +4903,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -4956,6 +5024,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -4967,6 +5036,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -5033,6 +5103,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -5044,6 +5115,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -5148,6 +5220,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -5159,6 +5232,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -5224,6 +5298,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -5235,6 +5310,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -5273,6 +5349,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -5284,6 +5361,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -5388,6 +5466,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -5399,6 +5478,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -5520,6 +5600,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -5531,6 +5612,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -5674,6 +5756,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -5685,6 +5768,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -6072,6 +6156,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -6083,6 +6168,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -6211,6 +6297,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -6222,6 +6309,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -6383,6 +6471,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -6394,6 +6483,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -6541,6 +6631,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -6552,6 +6643,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -6671,6 +6763,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -6682,6 +6775,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -6752,6 +6846,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -6763,6 +6858,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -6908,6 +7004,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -6919,6 +7016,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -7538,6 +7636,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -7549,6 +7648,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -7780,6 +7880,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -7791,6 +7892,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -7933,6 +8035,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -7944,6 +8047,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -7982,6 +8086,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -7993,6 +8098,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -8097,6 +8203,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -8108,6 +8215,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -8238,6 +8346,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -8249,6 +8358,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -8353,6 +8463,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -8364,6 +8475,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -8480,6 +8592,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -8491,6 +8604,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -8612,6 +8726,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -8623,6 +8738,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -8912,6 +9028,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -8923,6 +9040,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -9058,6 +9176,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -9069,6 +9188,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -9153,6 +9273,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -9164,6 +9285,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -9274,6 +9396,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -9285,6 +9408,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -9392,6 +9516,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -9403,6 +9528,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -9513,6 +9639,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -9524,6 +9651,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -9685,6 +9813,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -9696,6 +9825,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -9834,6 +9964,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -9845,6 +9976,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -9929,6 +10061,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -9940,6 +10073,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -10081,6 +10215,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -10092,6 +10227,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -10248,6 +10384,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -10259,6 +10396,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -10350,6 +10488,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -10361,6 +10500,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -10502,6 +10642,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -10513,6 +10654,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -10631,6 +10773,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -10642,6 +10785,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -10831,6 +10975,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -10842,6 +10987,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", diff --git a/posthog/api/test/notebooks/__snapshots__/test_notebook.ambr b/posthog/api/test/notebooks/__snapshots__/test_notebook.ambr index 396f5103c7ec3..299074ec3d44b 100644 --- a/posthog/api/test/notebooks/__snapshots__/test_notebook.ambr +++ b/posthog/api/test/notebooks/__snapshots__/test_notebook.ambr @@ -40,6 +40,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -51,6 +52,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -76,6 +78,7 @@ "posthog_notebook"."team_id", "posthog_notebook"."title", "posthog_notebook"."content", + "posthog_notebook"."text_content", "posthog_notebook"."deleted", "posthog_notebook"."version", "posthog_notebook"."created_at", @@ -94,6 +97,7 @@ "posthog_notebook"."team_id", "posthog_notebook"."title", "posthog_notebook"."content", + "posthog_notebook"."text_content", "posthog_notebook"."deleted", "posthog_notebook"."version", "posthog_notebook"."created_at", @@ -120,6 +124,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -131,6 +136,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -169,6 +175,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -180,6 +187,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -277,6 +285,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -288,6 +297,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -479,6 +489,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -490,6 +501,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -555,6 +567,7 @@ "posthog_notebook"."team_id", "posthog_notebook"."title", "posthog_notebook"."content", + "posthog_notebook"."text_content", "posthog_notebook"."deleted", "posthog_notebook"."version", "posthog_notebook"."created_at", @@ -572,6 +585,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -583,6 +597,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -671,6 +686,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -682,6 +698,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", diff --git a/posthog/api/test/notebooks/test_notebook.py b/posthog/api/test/notebooks/test_notebook.py index 3f49024d708e9..1b7f36ae54ce3 100644 --- a/posthog/api/test/notebooks/test_notebook.py +++ b/posthog/api/test/notebooks/test_notebook.py @@ -1,4 +1,4 @@ -from typing import List, Dict, Optional +from typing import List, Dict from unittest import mock from freezegun import freeze_time @@ -67,17 +67,20 @@ def test_cannot_list_deleted_notebook(self) -> None: @parameterized.expand( [ - ("without_content", None), - ("with_content", {"some": "kind", "of": "tip", "tap": "content"}), + ("without_content", None, None), + ("with_content", {"some": "kind", "of": "tip", "tap": "content"}, "some kind of tip tap content"), ] ) - def test_create_a_notebook(self, _, content: Optional[Dict]) -> None: - response = self.client.post(f"/api/projects/{self.team.id}/notebooks", data={"content": content}) + def test_create_a_notebook(self, _, content: Dict | None, text_content: str | None) -> None: + response = self.client.post( + f"/api/projects/{self.team.id}/notebooks", data={"content": content, "text_content": text_content} + ) assert response.status_code == status.HTTP_201_CREATED assert response.json() == { "id": response.json()["id"], "short_id": response.json()["short_id"], "content": content, + "text_content": text_content, "title": None, "version": 0, "created_at": mock.ANY, diff --git a/posthog/api/test/notebooks/test_notebook_filtering.py b/posthog/api/test/notebooks/test_notebook_filtering.py index 5881c609ebe81..5f634de548fc7 100644 --- a/posthog/api/test/notebooks/test_notebook_filtering.py +++ b/posthog/api/test/notebooks/test_notebook_filtering.py @@ -42,7 +42,7 @@ }, } -BASIC_TEXT = lambda text: {"type": "paragraph", "content": [{"text": text, "type": "text"}]} +BASIC_TEXT = lambda text: {"type": "paragraph", "content": [{"text": text, "type": "text"}], "text_content": text} class TestNotebooksFiltering(APIBaseTest, QueryMatchingTest): @@ -62,16 +62,18 @@ def _create_notebook_with_content(self, inner_content: List[Dict[str, Any]], tit @parameterized.expand( [ - ["some text", [0]], - ["other text", [1]], - ["text", [0, 1]], + ["i ride", [0]], + ["pony", [0]], + ["ponies", [0]], + ["my hobby", [1]], + ["around", [0, 1]], ["random", []], ] ) def test_filters_based_on_title(self, search_text: str, expected_match_indexes: List[int]) -> None: notebook_ids = [ - self._create_notebook_with_content([BASIC_TEXT("my important notes")], title="some text"), - self._create_notebook_with_content([BASIC_TEXT("my important notes")], title="other text"), + self._create_notebook_with_content([BASIC_TEXT("my important notes")], title="i ride around on a pony"), + self._create_notebook_with_content([BASIC_TEXT("my important notes")], title="my hobby is to fish around"), ] response = self.client.get( @@ -83,6 +85,32 @@ def test_filters_based_on_title(self, search_text: str, expected_match_indexes: assert len(results) == len(expected_match_indexes) assert sorted([r["id"] for r in results]) == sorted([notebook_ids[i] for i in expected_match_indexes]) + @parameterized.expand( + [ + ["pony", [0]], + ["pOnY", [0]], + ["ponies", [0]], + ["goat", [1]], + ["ride", [0, 1]], + ["neither", []], + ] + ) + def test_filters_based_on_text_content(self, search_text: str, expected_match_indexes: List[int]) -> None: + notebook_ids = [ + # will match both pony and ponies + self._create_notebook_with_content([BASIC_TEXT("you may ride a pony")], title="never matches"), + self._create_notebook_with_content([BASIC_TEXT("but may not ride a goat")], title="never matches"), + ] + + response = self.client.get( + f"/api/projects/{self.team.id}/notebooks?search={search_text}", + ) + assert response.status_code == status.HTTP_200_OK + + results = response.json()["results"] + assert len(results) == len(expected_match_indexes) + assert sorted([r["id"] for r in results]) == sorted([notebook_ids[i] for i in expected_match_indexes]) + def test_filters_based_on_params(self) -> None: other_user = User.objects.create_and_join(self.organization, "other@posthog.com", "password") notebook_one = Notebook.objects.create(team=self.team, created_by=self.user) diff --git a/posthog/batch_exports/service.py b/posthog/batch_exports/service.py index 5aa0fa7d18e22..b5eb182e68a70 100644 --- a/posthog/batch_exports/service.py +++ b/posthog/batch_exports/service.py @@ -52,6 +52,8 @@ class S3BatchExportInputs: data_interval_end: str | None = None compression: str | None = None exclude_events: list[str] | None = None + encryption: str | None = None + kms_key_id: str | None = None @dataclass diff --git a/posthog/clickhouse/migrations/0048_session_replay_events_count.py b/posthog/clickhouse/migrations/0048_session_replay_events_count.py new file mode 100644 index 0000000000000..d4676e2794884 --- /dev/null +++ b/posthog/clickhouse/migrations/0048_session_replay_events_count.py @@ -0,0 +1,26 @@ +from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions +from posthog.models.session_replay_event.migrations_sql import ( + DROP_SESSION_REPLAY_EVENTS_TABLE_MV_SQL, + DROP_KAFKA_SESSION_REPLAY_EVENTS_TABLE_SQL, + ADD_EVENT_COUNT_WRITABLE_SESSION_REPLAY_EVENTS_TABLE_SQL, + ADD_EVENT_COUNT_DISTRIBUTED_SESSION_REPLAY_EVENTS_TABLE_SQL, + ADD_EVENT_COUNT_SESSION_REPLAY_EVENTS_TABLE_SQL, +) +from posthog.models.session_replay_event.sql import ( + SESSION_REPLAY_EVENTS_TABLE_MV_SQL, + KAFKA_SESSION_REPLAY_EVENTS_TABLE_SQL, +) + +operations = [ + # we have to drop materialized view first so that we're no longer pulling from kakfa + # then we drop the kafka table + run_sql_with_exceptions(DROP_SESSION_REPLAY_EVENTS_TABLE_MV_SQL()), + run_sql_with_exceptions(DROP_KAFKA_SESSION_REPLAY_EVENTS_TABLE_SQL()), + # now we can alter the target tables + run_sql_with_exceptions(ADD_EVENT_COUNT_WRITABLE_SESSION_REPLAY_EVENTS_TABLE_SQL()), + run_sql_with_exceptions(ADD_EVENT_COUNT_DISTRIBUTED_SESSION_REPLAY_EVENTS_TABLE_SQL()), + run_sql_with_exceptions(ADD_EVENT_COUNT_SESSION_REPLAY_EVENTS_TABLE_SQL()), + # and then recreate the materialized views and kafka tables + run_sql_with_exceptions(KAFKA_SESSION_REPLAY_EVENTS_TABLE_SQL()), + run_sql_with_exceptions(SESSION_REPLAY_EVENTS_TABLE_MV_SQL()), +] diff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambr index 36ab529259c77..ac21b1ac5989f 100644 --- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr +++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr @@ -336,7 +336,9 @@ console_log_count Int64, console_warn_count Int64, console_error_count Int64, - size Int64 + size Int64, + event_count Int64, + message_count Int64 ) ENGINE = Kafka('test.kafka.broker:9092', 'clickhouse_session_replay_events_test', 'group1', 'JSONEachRow') ' @@ -922,7 +924,9 @@ console_log_count Int64, console_warn_count Int64, console_error_count Int64, - size Int64 + size Int64, + event_count Int64, + message_count Int64 ) ENGINE = Kafka('kafka:9092', 'clickhouse_session_replay_events_test', 'group1', 'JSONEachRow') ' @@ -1344,7 +1348,15 @@ console_warn_count SimpleAggregateFunction(sum, Int64), console_error_count SimpleAggregateFunction(sum, Int64), -- this column allows us to estimate the amount of data that is being ingested - size SimpleAggregateFunction(sum, Int64) + size SimpleAggregateFunction(sum, Int64), + -- this allows us to count the number of messages received in a session + -- often very useful in incidents or debugging + message_count SimpleAggregateFunction(sum, Int64), + -- this allows us to count the number of snapshot events received in a session + -- often very useful in incidents or debugging + -- because we batch events we expect message_count to be lower than event_count + event_count SimpleAggregateFunction(sum, Int64), + _timestamp SimpleAggregateFunction(max, DateTime) ) ENGINE = Distributed('posthog', 'posthog_test', 'sharded_session_replay_events', sipHash64(distinct_id)) ' @@ -1377,7 +1389,11 @@ sum(console_log_count) as console_log_count, sum(console_warn_count) as console_warn_count, sum(console_error_count) as console_error_count, - sum(size) as size + sum(size) as size, + -- we can count the number of kafka messages instead of sending it explicitly + sum(message_count) as message_count, + sum(event_count) as event_count, + max(_timestamp) as _timestamp FROM posthog_test.kafka_session_replay_events group by session_id, team_id @@ -1608,7 +1624,15 @@ console_warn_count SimpleAggregateFunction(sum, Int64), console_error_count SimpleAggregateFunction(sum, Int64), -- this column allows us to estimate the amount of data that is being ingested - size SimpleAggregateFunction(sum, Int64) + size SimpleAggregateFunction(sum, Int64), + -- this allows us to count the number of messages received in a session + -- often very useful in incidents or debugging + message_count SimpleAggregateFunction(sum, Int64), + -- this allows us to count the number of snapshot events received in a session + -- often very useful in incidents or debugging + -- because we batch events we expect message_count to be lower than event_count + event_count SimpleAggregateFunction(sum, Int64), + _timestamp SimpleAggregateFunction(max, DateTime) ) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_{shard}/posthog.session_replay_events', '{replica}') PARTITION BY toYYYYMM(min_first_timestamp) @@ -2226,7 +2250,15 @@ console_warn_count SimpleAggregateFunction(sum, Int64), console_error_count SimpleAggregateFunction(sum, Int64), -- this column allows us to estimate the amount of data that is being ingested - size SimpleAggregateFunction(sum, Int64) + size SimpleAggregateFunction(sum, Int64), + -- this allows us to count the number of messages received in a session + -- often very useful in incidents or debugging + message_count SimpleAggregateFunction(sum, Int64), + -- this allows us to count the number of snapshot events received in a session + -- often very useful in incidents or debugging + -- because we batch events we expect message_count to be lower than event_count + event_count SimpleAggregateFunction(sum, Int64), + _timestamp SimpleAggregateFunction(max, DateTime) ) ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/77f1df52-4b43-11e9-910f-b8ca3a9b9f3e_{shard}/posthog.session_replay_events', '{replica}') PARTITION BY toYYYYMM(min_first_timestamp) diff --git a/posthog/errors.py b/posthog/errors.py index 5cd3342f7a3fa..b2d34ed858448 100644 --- a/posthog/errors.py +++ b/posthog/errors.py @@ -206,7 +206,7 @@ def look_up_error_code_meta(error: ServerException) -> ErrorCodeMeta: 131: ErrorCodeMeta("TOO_LARGE_STRING_SIZE"), 133: ErrorCodeMeta("AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS"), 134: ErrorCodeMeta("PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS"), - 135: ErrorCodeMeta("ZERO_ARRAY_OR_TUPLE_INDEX"), + 135: ErrorCodeMeta("ZERO_ARRAY_OR_TUPLE_INDEX", user_safe=True), 137: ErrorCodeMeta("UNKNOWN_ELEMENT_IN_CONFIG"), 138: ErrorCodeMeta("EXCESSIVE_ELEMENT_IN_CONFIG"), 139: ErrorCodeMeta("NO_ELEMENTS_IN_CONFIG"), diff --git a/posthog/hogql/database/schema/session_replay_events.py b/posthog/hogql/database/schema/session_replay_events.py index c4f1980df5491..b8d79e86d9780 100644 --- a/posthog/hogql/database/schema/session_replay_events.py +++ b/posthog/hogql/database/schema/session_replay_events.py @@ -31,6 +31,8 @@ "console_warn_count": IntegerDatabaseField(name="console_warn_count"), "console_error_count": IntegerDatabaseField(name="console_error_count"), "size": IntegerDatabaseField(name="size"), + "event_count": IntegerDatabaseField(name="event_count"), + "message_count": IntegerDatabaseField(name="message_count"), "pdi": LazyJoin( from_field="distinct_id", join_table=PersonDistinctIdsTable(), @@ -77,6 +79,8 @@ def select_from_session_replay_events_table(requested_fields: Dict[str, List[str "console_error_count": ast.Call(name="sum", args=[ast.Field(chain=[table_name, "console_error_count"])]), "distinct_id": ast.Call(name="any", args=[ast.Field(chain=[table_name, "distinct_id"])]), "size": ast.Call(name="sum", args=[ast.Field(chain=[table_name, "size"])]), + "event_count": ast.Call(name="sum", args=[ast.Field(chain=[table_name, "event_count"])]), + "message_count": ast.Call(name="sum", args=[ast.Field(chain=[table_name, "message_count"])]), } select_fields: List[ast.Expr] = [] diff --git a/posthog/hogql/database/test/__snapshots__/test_database.ambr b/posthog/hogql/database/test/__snapshots__/test_database.ambr index 166391d344856..9e1413d84a0bf 100644 --- a/posthog/hogql/database/test/__snapshots__/test_database.ambr +++ b/posthog/hogql/database/test/__snapshots__/test_database.ambr @@ -276,6 +276,14 @@ "key": "size", "type": "integer" }, + { + "key": "event_count", + "type": "integer" + }, + { + "key": "message_count", + "type": "integer" + }, { "key": "pdi", "type": "lazy_table", @@ -405,6 +413,14 @@ "key": "size", "type": "integer" }, + { + "key": "event_count", + "type": "integer" + }, + { + "key": "message_count", + "type": "integer" + }, { "key": "pdi", "type": "lazy_table", @@ -849,6 +865,14 @@ "key": "size", "type": "integer" }, + { + "key": "event_count", + "type": "integer" + }, + { + "key": "message_count", + "type": "integer" + }, { "key": "pdi", "type": "lazy_table", @@ -978,6 +1002,14 @@ "key": "size", "type": "integer" }, + { + "key": "event_count", + "type": "integer" + }, + { + "key": "message_count", + "type": "integer" + }, { "key": "pdi", "type": "lazy_table", diff --git a/posthog/migrations/0350_add_notebook_text_content.py b/posthog/migrations/0350_add_notebook_text_content.py new file mode 100644 index 0000000000000..bfe4b079b9945 --- /dev/null +++ b/posthog/migrations/0350_add_notebook_text_content.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.19 on 2023-09-12 18:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("posthog", "0349_update_survey_query_name"), + ] + + operations = [ + migrations.AddField( + model_name="notebook", + name="text_content", + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/posthog/models/activity_logging/activity_log.py b/posthog/models/activity_logging/activity_log.py index ba47b2c326ff1..f3b36e2c3dbd0 100644 --- a/posthog/models/activity_logging/activity_log.py +++ b/posthog/models/activity_logging/activity_log.py @@ -99,7 +99,7 @@ class Meta: field_exclusions: Dict[ActivityScope, List[str]] = { - "Notebook": ["id", "last_modified_at", "last_modified_by", "created_at", "created_by"], + "Notebook": ["id", "last_modified_at", "last_modified_by", "created_at", "created_by", "text_content"], "FeatureFlag": ["id", "created_at", "created_by", "is_simple_flag", "experiment", "team", "featureflagoverride"], "Person": [ "id", diff --git a/posthog/models/filters/test/__snapshots__/test_filter.ambr b/posthog/models/filters/test/__snapshots__/test_filter.ambr index 922fdf12a27f1..9be8465ff5f0f 100644 --- a/posthog/models/filters/test/__snapshots__/test_filter.ambr +++ b/posthog/models/filters/test/__snapshots__/test_filter.ambr @@ -11,6 +11,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -22,6 +23,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -60,6 +62,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -71,6 +74,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -109,6 +113,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -120,6 +125,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -158,6 +164,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -169,6 +176,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", @@ -207,6 +215,7 @@ "posthog_team"."updated_at", "posthog_team"."anonymize_ips", "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", "posthog_team"."ingested_event", "posthog_team"."autocapture_opt_out", "posthog_team"."autocapture_exceptions_opt_in", @@ -218,6 +227,7 @@ "posthog_team"."signup_token", "posthog_team"."is_demo", "posthog_team"."access_control", + "posthog_team"."week_start_day", "posthog_team"."inject_web_apps", "posthog_team"."test_account_filters", "posthog_team"."test_account_filters_default_checked", diff --git a/posthog/models/notebook/notebook.py b/posthog/models/notebook/notebook.py index dde92fddab944..490645909df26 100644 --- a/posthog/models/notebook/notebook.py +++ b/posthog/models/notebook/notebook.py @@ -12,6 +12,7 @@ class Notebook(UUIDModel): team: models.ForeignKey = models.ForeignKey("Team", on_delete=models.CASCADE) title: models.CharField = models.CharField(max_length=256, blank=True, null=True) content: JSONField = JSONField(default=None, null=True, blank=True) + text_content: models.TextField = models.TextField(blank=True, null=True) deleted: models.BooleanField = models.BooleanField(default=False) version: models.IntegerField = models.IntegerField(default=0) created_at: models.DateTimeField = models.DateTimeField(auto_now_add=True, blank=True) diff --git a/posthog/models/session_replay_event/migrations_sql.py b/posthog/models/session_replay_event/migrations_sql.py index 09f4e300be624..b11f5581c930f 100644 --- a/posthog/models/session_replay_event/migrations_sql.py +++ b/posthog/models/session_replay_event/migrations_sql.py @@ -65,3 +65,29 @@ table_name=SESSION_REPLAY_EVENTS_DATA_TABLE(), cluster=settings.CLICKHOUSE_CLUSTER, ) + +# migration to add size column to the session replay table +ALTER_SESSION_REPLAY_ADD_EVENT_COUNT_COLUMN = """ + ALTER TABLE {table_name} on CLUSTER '{cluster}' + ADD COLUMN IF NOT EXISTS message_count SimpleAggregateFunction(sum, Int64), + ADD COLUMN IF NOT EXISTS event_count SimpleAggregateFunction(sum, Int64), + -- fly by addition so that we can track lag in the data the same way as for other tables + ADD COLUMN IF NOT EXISTS _timestamp SimpleAggregateFunction(max, DateTime) +""" + +ADD_EVENT_COUNT_DISTRIBUTED_SESSION_REPLAY_EVENTS_TABLE_SQL = ( + lambda: ALTER_SESSION_REPLAY_ADD_EVENT_COUNT_COLUMN.format( + table_name="session_replay_events", + cluster=settings.CLICKHOUSE_CLUSTER, + ) +) + +ADD_EVENT_COUNT_WRITABLE_SESSION_REPLAY_EVENTS_TABLE_SQL = lambda: ALTER_SESSION_REPLAY_ADD_EVENT_COUNT_COLUMN.format( + table_name="writable_session_replay_events", + cluster=settings.CLICKHOUSE_CLUSTER, +) + +ADD_EVENT_COUNT_SESSION_REPLAY_EVENTS_TABLE_SQL = lambda: ALTER_SESSION_REPLAY_ADD_EVENT_COUNT_COLUMN.format( + table_name=SESSION_REPLAY_EVENTS_DATA_TABLE(), + cluster=settings.CLICKHOUSE_CLUSTER, +) diff --git a/posthog/models/session_replay_event/sql.py b/posthog/models/session_replay_event/sql.py index 1221fd80bb6de..dfe839843979f 100644 --- a/posthog/models/session_replay_event/sql.py +++ b/posthog/models/session_replay_event/sql.py @@ -27,7 +27,9 @@ console_log_count Int64, console_warn_count Int64, console_error_count Int64, - size Int64 + size Int64, + event_count Int64, + message_count Int64 ) ENGINE = {engine} """ @@ -53,7 +55,15 @@ console_warn_count SimpleAggregateFunction(sum, Int64), console_error_count SimpleAggregateFunction(sum, Int64), -- this column allows us to estimate the amount of data that is being ingested - size SimpleAggregateFunction(sum, Int64) + size SimpleAggregateFunction(sum, Int64), + -- this allows us to count the number of messages received in a session + -- often very useful in incidents or debugging + message_count SimpleAggregateFunction(sum, Int64), + -- this allows us to count the number of snapshot events received in a session + -- often very useful in incidents or debugging + -- because we batch events we expect message_count to be lower than event_count + event_count SimpleAggregateFunction(sum, Int64), + _timestamp SimpleAggregateFunction(max, DateTime) ) ENGINE = {engine} """ @@ -117,7 +127,11 @@ sum(console_log_count) as console_log_count, sum(console_warn_count) as console_warn_count, sum(console_error_count) as console_error_count, -sum(size) as size +sum(size) as size, +-- we can count the number of kafka messages instead of sending it explicitly +sum(message_count) as message_count, +sum(event_count) as event_count, +max(_timestamp) as _timestamp FROM {database}.kafka_session_replay_events group by session_id, team_id """.format( diff --git a/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py b/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py index 392534fc8999c..cb38d818ba9d7 100644 --- a/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py +++ b/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py @@ -3,11 +3,13 @@ import gzip import itertools import json +import os from random import randint from unittest import mock from uuid import uuid4 import boto3 +import botocore.exceptions import brotli import pytest from django.conf import settings @@ -40,6 +42,18 @@ TEST_ROOT_BUCKET = "test-batch-exports" + +def check_valid_credentials() -> bool: + """Check if there are valid AWS credentials in the environment.""" + sts = boto3.client("sts") + try: + sts.get_caller_identity() + except botocore.exceptions.ClientError: + return False + else: + return True + + create_test_client = functools.partial(boto3.client, endpoint_url=settings.OBJECT_STORAGE_ENDPOINT) @@ -422,6 +436,163 @@ async def test_s3_export_workflow_with_minio_bucket( assert_events_in_s3(s3_client, bucket_name, prefix, events, compression, exclude_events) +@pytest.mark.skipif( + "S3_TEST_BUCKET" not in os.environ or not check_valid_credentials(), + reason="AWS credentials not set in environment or missing S3_TEST_BUCKET variable", +) +@pytest.mark.django_db +@pytest.mark.asyncio +@pytest.mark.parametrize( + "interval,compression,encryption,exclude_events", + itertools.product(["hour", "day"], [None, "gzip", "brotli"], [None, "AES256"], [None, ["test-exclude"]]), +) +async def test_s3_export_workflow_with_s3_bucket(interval, compression, encryption, exclude_events): + """Test S3 Export Workflow end-to-end by using an S3 bucket. + + The S3_TEST_BUCKET environment variable is used to set the name of the bucket for this test. + This test will be skipped if no valid AWS credentials exist, or if the S3_TEST_BUCKET environment + variable is not set. + + The workflow should update the batch export run status to completed and produce the expected + records to the S3 bucket. + """ + bucket_name = os.getenv("S3_TEST_BUCKET") + prefix = f"posthog-events-{str(uuid4())}" + destination_data = { + "type": "S3", + "config": { + "bucket_name": bucket_name, + "region": "us-east-1", + "prefix": prefix, + "aws_access_key_id": "object_storage_root_user", + "aws_secret_access_key": "object_storage_root_password", + "compression": compression, + "exclude_events": exclude_events, + "encryption": encryption, + }, + } + + batch_export_data = { + "name": "my-production-s3-bucket-destination", + "destination": destination_data, + "interval": interval, + } + + organization = await acreate_organization("test") + team = await acreate_team(organization=organization) + batch_export = await acreate_batch_export( + team_id=team.pk, + name=batch_export_data["name"], + destination_data=batch_export_data["destination"], + interval=batch_export_data["interval"], + ) + + events: list[EventValues] = [ + { + "uuid": str(uuid4()), + "event": "test", + "timestamp": "2023-04-25 13:30:00.000000", + "created_at": "2023-04-25 13:30:00.000000", + "inserted_at": "2023-04-25 13:30:00.000000", + "_timestamp": "2023-04-25 13:30:00", + "person_id": str(uuid4()), + "person_properties": {"$browser": "Chrome", "$os": "Mac OS X"}, + "team_id": team.pk, + "properties": {"$browser": "Chrome", "$os": "Mac OS X"}, + "distinct_id": str(uuid4()), + "elements_chain": "this is a comman, separated, list, of css selectors(?)", + }, + { + "uuid": str(uuid4()), + "event": "test-exclude", + "timestamp": "2023-04-25 14:29:00.000000", + "created_at": "2023-04-25 14:29:00.000000", + "inserted_at": "2023-04-25 14:29:00.000000", + "_timestamp": "2023-04-25 14:29:00", + "person_id": str(uuid4()), + "person_properties": {"$browser": "Chrome", "$os": "Mac OS X"}, + "team_id": team.pk, + "properties": {"$browser": "Chrome", "$os": "Mac OS X"}, + "distinct_id": str(uuid4()), + "elements_chain": "this is a comman, separated, list, of css selectors(?)", + }, + ] + + if interval == "day": + # Add an event outside the hour range but within the day range to ensure it's exported too. + events_outside_hour: list[EventValues] = [ + { + "uuid": str(uuid4()), + "event": "test", + "timestamp": "2023-04-25 00:30:00.000000", + "created_at": "2023-04-25 00:30:00.000000", + "inserted_at": "2023-04-25 00:30:00.000000", + "_timestamp": "2023-04-25 00:30:00", + "person_id": str(uuid4()), + "person_properties": {"$browser": "Chrome", "$os": "Mac OS X"}, + "team_id": team.pk, + "properties": {"$browser": "Chrome", "$os": "Mac OS X"}, + "distinct_id": str(uuid4()), + "elements_chain": "this is a comman, separated, list, of css selectors(?)", + } + ] + events += events_outside_hour + + ch_client = ClickHouseClient( + url=settings.CLICKHOUSE_HTTP_URL, + user=settings.CLICKHOUSE_USER, + password=settings.CLICKHOUSE_PASSWORD, + database=settings.CLICKHOUSE_DATABASE, + ) + + # Insert some data into the `sharded_events` table. + await insert_events( + client=ch_client, + events=events, + ) + + workflow_id = str(uuid4()) + inputs = S3BatchExportInputs( + team_id=team.pk, + batch_export_id=str(batch_export.id), + data_interval_end="2023-04-25 14:30:00.000000", + interval=interval, + **batch_export.destination.config, + ) + + s3_client = boto3.client("s3") + + def create_s3_client(*args, **kwargs): + """Mock function to return an already initialized S3 client.""" + return s3_client + + async with await WorkflowEnvironment.start_time_skipping() as activity_environment: + async with Worker( + activity_environment.client, + task_queue=settings.TEMPORAL_TASK_QUEUE, + workflows=[S3BatchExportWorkflow], + activities=[create_export_run, insert_into_s3_activity, update_export_run_status], + workflow_runner=UnsandboxedWorkflowRunner(), + ): + with mock.patch("posthog.temporal.workflows.s3_batch_export.boto3.client", side_effect=create_s3_client): + await activity_environment.client.execute_workflow( + S3BatchExportWorkflow.run, + inputs, + id=workflow_id, + task_queue=settings.TEMPORAL_TASK_QUEUE, + retry_policy=RetryPolicy(maximum_attempts=1), + execution_timeout=dt.timedelta(seconds=10), + ) + + runs = await afetch_batch_export_runs(batch_export_id=batch_export.id) + assert len(runs) == 1 + + run = runs[0] + assert run.status == "Completed" + + assert_events_in_s3(s3_client, bucket_name, prefix, events, compression, exclude_events) + + @pytest.mark.django_db @pytest.mark.asyncio @pytest.mark.parametrize("compression", [None, "gzip"]) diff --git a/posthog/temporal/workflows/s3_batch_export.py b/posthog/temporal/workflows/s3_batch_export.py index 028b6f422e26f..13bbf183e5d06 100644 --- a/posthog/temporal/workflows/s3_batch_export.py +++ b/posthog/temporal/workflows/s3_batch_export.py @@ -85,15 +85,20 @@ class S3MultiPartUploadState(typing.NamedTuple): parts: list[dict[str, str | int]] +Part = dict[str, str | int] + + class S3MultiPartUpload: """An S3 multi-part upload.""" - def __init__(self, s3_client, bucket_name, key): + def __init__(self, s3_client, bucket_name: str, key: str, encryption: str | None, kms_key_id: str | None): self.s3_client = s3_client self.bucket_name = bucket_name self.key = key - self.upload_id = None - self.parts = [] + self.encryption = encryption + self.kms_key_id = kms_key_id + self.upload_id: str | None = None + self.parts: list[Part] = [] def to_state(self) -> S3MultiPartUploadState: """Produce state tuple that can be used to resume this S3MultiPartUpload.""" @@ -119,10 +124,21 @@ def start(self) -> str: if self.is_upload_in_progress() is True: raise UploadAlreadyInProgressError(self.upload_id) - multipart_response = self.s3_client.create_multipart_upload(Bucket=self.bucket_name, Key=self.key) - self.upload_id = multipart_response["UploadId"] + optional_kwargs = {} + if self.encryption: + optional_kwargs["ServerSideEncryption"] = self.encryption + if self.kms_key_id: + optional_kwargs["SSEKMSKeyId"] = self.kms_key_id - return self.upload_id + multipart_response = self.s3_client.create_multipart_upload( + Bucket=self.bucket_name, + Key=self.key, + **optional_kwargs, + ) + upload_id: str = multipart_response["UploadId"] + self.upload_id = upload_id + + return upload_id def continue_from_state(self, state: S3MultiPartUploadState): """Continue this S3MultiPartUpload from a previous state.""" @@ -230,6 +246,8 @@ class S3InsertInputs: aws_secret_access_key: str | None = None compression: str | None = None exclude_events: list[str] | None = None + encryption: str | None = None + kms_key_id: str | None = None def initialize_and_resume_multipart_upload(inputs: S3InsertInputs) -> tuple[S3MultiPartUpload, str]: @@ -241,7 +259,7 @@ def initialize_and_resume_multipart_upload(inputs: S3InsertInputs) -> tuple[S3Mu aws_access_key_id=inputs.aws_access_key_id, aws_secret_access_key=inputs.aws_secret_access_key, ) - s3_upload = S3MultiPartUpload(s3_client, inputs.bucket_name, key) + s3_upload = S3MultiPartUpload(s3_client, inputs.bucket_name, key, inputs.encryption, inputs.kms_key_id) details = activity.info().heartbeat_details @@ -442,6 +460,8 @@ async def run(self, inputs: S3BatchExportInputs): data_interval_end=data_interval_end.isoformat(), compression=inputs.compression, exclude_events=inputs.exclude_events, + encryption=inputs.encryption, + kms_key_id=inputs.kms_key_id, ) try: await workflow.execute_activity(