diff --git a/.vscode/launch.json b/.vscode/launch.json index 51c2ed4c02932a..cd681198de8b93 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -121,7 +121,7 @@ "WORKER_CONCURRENCY": "2", "OBJECT_STORAGE_ENABLED": "True", "HOG_HOOK_URL": "http://localhost:3300/hoghook", - "CDP_ASYNC_FUNCTIONS_RUSTY_HOOK_TEAMS": "*" + "CDP_ASYNC_FUNCTIONS_RUSTY_HOOK_TEAMS": "" }, "presentation": { "group": "main" diff --git a/frontend/__snapshots__/exporter-exporter--funnel-top-to-bottom-breakdown-insight--dark.png b/frontend/__snapshots__/exporter-exporter--funnel-top-to-bottom-breakdown-insight--dark.png index badd58a5a74b6a..723c5895b17fc8 100644 Binary files a/frontend/__snapshots__/exporter-exporter--funnel-top-to-bottom-breakdown-insight--dark.png and b/frontend/__snapshots__/exporter-exporter--funnel-top-to-bottom-breakdown-insight--dark.png differ diff --git a/frontend/__snapshots__/exporter-exporter--funnel-top-to-bottom-breakdown-insight--light.png b/frontend/__snapshots__/exporter-exporter--funnel-top-to-bottom-breakdown-insight--light.png index 3d908501e3be54..559ec7fd6866bf 100644 Binary files a/frontend/__snapshots__/exporter-exporter--funnel-top-to-bottom-breakdown-insight--light.png and b/frontend/__snapshots__/exporter-exporter--funnel-top-to-bottom-breakdown-insight--light.png differ diff --git a/frontend/__snapshots__/replay-player-failure--recent-recordings-404--dark.png b/frontend/__snapshots__/replay-player-failure--recent-recordings-404--dark.png index 7c36f82f3c19e7..c6f7ea6c355c15 100644 Binary files a/frontend/__snapshots__/replay-player-failure--recent-recordings-404--dark.png and b/frontend/__snapshots__/replay-player-failure--recent-recordings-404--dark.png differ diff --git a/frontend/__snapshots__/replay-player-failure--recent-recordings-404--light.png b/frontend/__snapshots__/replay-player-failure--recent-recordings-404--light.png index e07d3b87766639..4506520e439101 100644 Binary files a/frontend/__snapshots__/replay-player-failure--recent-recordings-404--light.png and b/frontend/__snapshots__/replay-player-failure--recent-recordings-404--light.png differ diff --git a/frontend/__snapshots__/replay-player-success--recent-recordings--dark.png b/frontend/__snapshots__/replay-player-success--recent-recordings--dark.png index 01a57fdc939a31..af464e4ed5ca94 100644 Binary files a/frontend/__snapshots__/replay-player-success--recent-recordings--dark.png and b/frontend/__snapshots__/replay-player-success--recent-recordings--dark.png differ diff --git a/frontend/__snapshots__/replay-player-success--recent-recordings--light.png b/frontend/__snapshots__/replay-player-success--recent-recordings--light.png index 33a4cd371aba09..50cad866234161 100644 Binary files a/frontend/__snapshots__/replay-player-success--recent-recordings--light.png and b/frontend/__snapshots__/replay-player-success--recent-recordings--light.png differ diff --git a/frontend/__snapshots__/replay-player-success--second-recording-in-list--dark.png b/frontend/__snapshots__/replay-player-success--second-recording-in-list--dark.png index 5f2d993d05efba..2bf8d714033411 100644 Binary files a/frontend/__snapshots__/replay-player-success--second-recording-in-list--dark.png and b/frontend/__snapshots__/replay-player-success--second-recording-in-list--dark.png differ diff --git a/frontend/__snapshots__/replay-player-success--second-recording-in-list--light.png b/frontend/__snapshots__/replay-player-success--second-recording-in-list--light.png index 693947ce7669de..7b64cab21e28de 100644 Binary files a/frontend/__snapshots__/replay-player-success--second-recording-in-list--light.png and b/frontend/__snapshots__/replay-player-success--second-recording-in-list--light.png differ diff --git a/frontend/__snapshots__/scenes-app-dashboards--insight-legend--dark.png b/frontend/__snapshots__/scenes-app-dashboards--insight-legend--dark.png index d1217fba75dfbf..9d811913930caa 100644 Binary files a/frontend/__snapshots__/scenes-app-dashboards--insight-legend--dark.png and b/frontend/__snapshots__/scenes-app-dashboards--insight-legend--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--dark--webkit.png b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--dark--webkit.png index 784f39f6eb8ded..a1b8b4db32e917 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--dark--webkit.png and b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--dark--webkit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--dark.png b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--dark.png index 55e3c8157b614b..804a66a0a880f2 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--dark.png and b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--light--webkit.png b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--light--webkit.png index 87a3743cf6524f..48d9cecc3902ba 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--light--webkit.png and b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--light--webkit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--light.png b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--light.png index e708c340d53665..756c64885336f0 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--light.png and b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--light.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--dark--webkit.png b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--dark--webkit.png index 2678587de54295..3d4ce71a129a46 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--dark--webkit.png and b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--dark--webkit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--dark.png b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--dark.png index 3ff13d640c61a7..4d2ff3308afa8d 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--dark.png and b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--light--webkit.png b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--light--webkit.png index 86d01b2d6c77f5..37ae1a7b3ca1f9 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--light--webkit.png and b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--light--webkit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--light.png b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--light.png index a8b763cfcb7c07..2ea9fc20185371 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--light.png and b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--light.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-edit--light.png b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-edit--light.png index 01df2b71a3a80c..4ca21748c10e4c 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-edit--light.png and b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-edit--light.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--user-paths-edit--light.png b/frontend/__snapshots__/scenes-app-insights--user-paths-edit--light.png index 6a61fa335f162b..d5de08730ada19 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--user-paths-edit--light.png and b/frontend/__snapshots__/scenes-app-insights--user-paths-edit--light.png differ diff --git a/frontend/__snapshots__/scenes-app-notebooks--recordings-playlist--light.png b/frontend/__snapshots__/scenes-app-notebooks--recordings-playlist--light.png index 10c2fe114e7d62..4e344bfa7e24f4 100644 Binary files a/frontend/__snapshots__/scenes-app-notebooks--recordings-playlist--light.png and b/frontend/__snapshots__/scenes-app-notebooks--recordings-playlist--light.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-destinations-page--dark.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-destinations-page--dark.png index 112ec10aee5196..e98b789ffb2b53 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-destinations-page--dark.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-destinations-page--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-destinations-page--light.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-destinations-page--light.png index ba3df36d9510cf..675e592535157f 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-destinations-page--light.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-destinations-page--light.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-destinations-page-without-pipelines--dark.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-destinations-page-without-pipelines--dark.png index 8879429157b9d8..054ceb2555d605 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-destinations-page-without-pipelines--dark.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-destinations-page-without-pipelines--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-destinations-page-without-pipelines--light.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-destinations-page-without-pipelines--light.png index 786f0f8483897e..3751752e7c0a6c 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-destinations-page-without-pipelines--light.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-destinations-page-without-pipelines--light.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-landing-page--dark.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-landing-page--dark.png index da6bc0f33cff05..5b9a3a52d868d9 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-landing-page--dark.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-landing-page--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-landing-page--light.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-landing-page--light.png index 2f1ae71e816b15..776041c690dd5b 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-landing-page--light.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-landing-page--light.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-landing-page-iff-legacy-sources--dark.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-landing-page-iff-legacy-sources--dark.png index f28e6042ac8031..8bdf26d1681e31 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-landing-page-iff-legacy-sources--dark.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-landing-page-iff-legacy-sources--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-landing-page-iff-legacy-sources--light.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-landing-page-iff-legacy-sources--light.png index e4d8d4fcc377fb..cc810e35c00ec8 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-landing-page-iff-legacy-sources--light.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-landing-page-iff-legacy-sources--light.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-legacy-sources-page--dark.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-legacy-sources-page--dark.png index b81a53b03503d6..f0456aa22b39dd 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-legacy-sources-page--dark.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-legacy-sources-page--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-legacy-sources-page--light.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-legacy-sources-page--light.png index 3e3e6ada1c559b..662bfd3adb4936 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-legacy-sources-page--light.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-legacy-sources-page--light.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-destination--dark.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-destination--dark.png index 3b18041801470b..ab242b19ecd813 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-destination--dark.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-destination--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-destination--light.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-destination--light.png index 0eaa364711ee7a..74d0b791f2e0b4 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-destination--light.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-destination--light.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-destination-without-data-pipelines--dark.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-destination-without-data-pipelines--dark.png index adc7db08089f41..e6c884d50f4f31 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-destination-without-data-pipelines--dark.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-destination-without-data-pipelines--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-destination-without-data-pipelines--light.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-destination-without-data-pipelines--light.png index 946fa6bdb4dc7c..926283d53f5eb7 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-destination-without-data-pipelines--light.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-destination-without-data-pipelines--light.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-nodes-management-page--dark.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-nodes-management-page--dark.png index e03961a84e8aa3..93eaa2a544d545 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-nodes-management-page--dark.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-nodes-management-page--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-nodes-management-page--light.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-nodes-management-page--light.png index fd3675e9061957..8d363c447cec16 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-nodes-management-page--light.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-nodes-management-page--light.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-overview-page--dark.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-overview-page--dark.png index da6bc0f33cff05..5b9a3a52d868d9 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-overview-page--dark.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-overview-page--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-overview-page--light.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-overview-page--light.png index 2f1ae71e816b15..776041c690dd5b 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-overview-page--light.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-overview-page--light.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-site-apps-page--dark.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-site-apps-page--dark.png index f544bc2231949c..84120223f25cbb 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-site-apps-page--dark.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-site-apps-page--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-site-apps-page--light.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-site-apps-page--light.png index f50b0e5089f6d6..09199dd2da4fce 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-site-apps-page--light.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-site-apps-page--light.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-transformations-page--dark.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-transformations-page--dark.png index 1d0c558ef4eaee..aae5f3ff713118 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-transformations-page--dark.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-transformations-page--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-transformations-page--light.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-transformations-page--light.png index 930bcb1a6fba19..d2c18991f44293 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-transformations-page--light.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-transformations-page--light.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-transformations-page-empty--dark.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-transformations-page-empty--dark.png index 89f24f08f4faa1..1ee1c29e8a9381 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-transformations-page-empty--dark.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-transformations-page-empty--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-transformations-page-empty--light.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-transformations-page-empty--light.png index a7c13f9812edf4..a787f5804569a2 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-transformations-page-empty--light.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-transformations-page-empty--light.png differ diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index f1cebb00ab68d1..648ea4aba4478e 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -13,6 +13,9 @@ import { ActionType, ActivityScope, AlertType, + AppMetricsTotalsV2Response, + AppMetricsV2RequestParams, + AppMetricsV2Response, BatchExportConfiguration, BatchExportRun, CohortType, @@ -1626,6 +1629,18 @@ const api = { ): Promise> { return await new ApiRequest().hogFunction(id).withAction('logs').withQueryString(params).get() }, + async metrics( + id: HogFunctionType['id'], + params: AppMetricsV2RequestParams = {} + ): Promise { + return await new ApiRequest().hogFunction(id).withAction('metrics').withQueryString(params).get() + }, + async metricsTotals( + id: HogFunctionType['id'], + params: Partial = {} + ): Promise { + return await new ApiRequest().hogFunction(id).withAction('metrics/totals').withQueryString(params).get() + }, async listTemplates(): Promise> { return await new ApiRequest().hogFunctionTemplates().get() diff --git a/frontend/src/lib/components/PayGateMini/PayGateButton.tsx b/frontend/src/lib/components/PayGateMini/PayGateButton.tsx index ebf911633ac06c..6c36cb8c13b889 100644 --- a/frontend/src/lib/components/PayGateMini/PayGateButton.tsx +++ b/frontend/src/lib/components/PayGateMini/PayGateButton.tsx @@ -1,64 +1,49 @@ -import { LemonButton } from '@posthog/lemon-ui' +import { LemonButton, LemonButtonProps } from '@posthog/lemon-ui' +import { useValues } from 'kea' +import { useMemo } from 'react' import { urls } from 'scenes/urls' -import { BillingFeatureType, BillingProductV2AddonType, BillingProductV2Type } from '~/types' +import { payGateMiniLogic, PayGateMiniLogicProps } from './payGateMiniLogic' -interface PayGateButtonProps { - gateVariant: 'add-card' | 'contact-sales' | 'move-to-cloud' | null - productWithFeature: BillingProductV2AddonType | BillingProductV2Type - featureInfo: BillingFeatureType - onCtaClick: () => void - isAddonProduct?: boolean - scrollToProduct: boolean -} +type PayGateButtonProps = PayGateMiniLogicProps & Partial +export const PayGateButton = ({ feature, currentUsage, ...buttonProps }: PayGateButtonProps): JSX.Element | null => { + const { productWithFeature, featureInfo, gateVariant, isAddonProduct, scrollToProduct } = useValues( + payGateMiniLogic({ feature, currentUsage }) + ) + + const ctaLink = useMemo(() => { + if (gateVariant === 'add-card' && !isAddonProduct) { + return `/api/billing/activate?products=all_products:&redirect_path=${urls.organizationBilling()}&intent_product=${ + productWithFeature?.type + }` + } else if (gateVariant === 'add-card') { + return `/organization/billing${scrollToProduct ? `?products=${productWithFeature?.type}` : ''}` + } else if (gateVariant === 'contact-sales') { + return `mailto:sales@posthog.com?subject=Inquiring about ${featureInfo?.name}` + } else if (gateVariant === 'move-to-cloud') { + return 'https://us.posthog.com/signup?utm_medium=in-product&utm_campaign=move-to-cloud' + } + return undefined + }, [gateVariant, isAddonProduct, productWithFeature, featureInfo, scrollToProduct]) + + const ctaLabel = useMemo(() => { + if (gateVariant === 'add-card') { + return 'Upgrade now' + } else if (gateVariant === 'contact-sales') { + return 'Contact sales' + } + return 'Move to PostHog Cloud' + }, [gateVariant]) -export const PayGateButton = ({ - gateVariant, - productWithFeature, - featureInfo, - onCtaClick, - isAddonProduct, - scrollToProduct = true, -}: PayGateButtonProps): JSX.Element => { return ( - {getCtaLabel(gateVariant)} + {ctaLabel} ) } - -const getCtaLink = ( - gateVariant: 'add-card' | 'contact-sales' | 'move-to-cloud' | null, - productWithFeature: BillingProductV2AddonType | BillingProductV2Type, - featureInfo: BillingFeatureType, - isAddonProduct?: boolean, - scrollToProduct: boolean = true -): string | undefined => { - if (gateVariant === 'add-card' && !isAddonProduct) { - return `/api/billing/activate?products=all_products:&redirect_path=${urls.organizationBilling()}&intent_product=${ - productWithFeature.type - }` - } else if (gateVariant === 'add-card') { - return `/organization/billing${scrollToProduct ? `?products=${productWithFeature.type}` : ''}` - } else if (gateVariant === 'contact-sales') { - return `mailto:sales@posthog.com?subject=Inquiring about ${featureInfo.name}` - } else if (gateVariant === 'move-to-cloud') { - return 'https://us.posthog.com/signup?utm_medium=in-product&utm_campaign=move-to-cloud' - } - return undefined -} - -const getCtaLabel = (gateVariant: 'add-card' | 'contact-sales' | 'move-to-cloud' | null): string => { - if (gateVariant === 'add-card') { - return 'Upgrade now' - } else if (gateVariant === 'contact-sales') { - return 'Contact sales' - } - return 'Move to PostHog Cloud' -} diff --git a/frontend/src/lib/components/PayGateMini/PayGateMini.tsx b/frontend/src/lib/components/PayGateMini/PayGateMini.tsx index 90798ff759c111..cae587b40588fd 100644 --- a/frontend/src/lib/components/PayGateMini/PayGateMini.tsx +++ b/frontend/src/lib/components/PayGateMini/PayGateMini.tsx @@ -12,11 +12,9 @@ import { AvailableFeature, BillingFeatureType, BillingProductV2AddonType, Billin import { upgradeModalLogic } from '../UpgradeModal/upgradeModalLogic' import { PayGateButton } from './PayGateButton' -import { payGateMiniLogic } from './payGateMiniLogic' +import { payGateMiniLogic, PayGateMiniLogicProps } from './payGateMiniLogic' -export interface PayGateMiniProps { - feature: AvailableFeature - currentUsage?: number +export type PayGateMiniProps = PayGateMiniLogicProps & { /** * The content to show when the feature is available. Will show nothing if children is undefined. */ @@ -43,20 +41,11 @@ export function PayGateMini({ isGrandfathered, docsLink, }: PayGateMiniProps): JSX.Element | null { - const { - productWithFeature, - featureInfo, - featureAvailableOnOrg, - gateVariant, - isAddonProduct, - featureInfoOnNextPlan, - } = useValues(payGateMiniLogic({ featureKey: feature, currentUsage })) + const { productWithFeature, featureInfo, gateVariant } = useValues(payGateMiniLogic({ feature, currentUsage })) const { preflight, isCloudOrDev } = useValues(preflightLogic) const { billingLoading } = useValues(billingLogic) const { hideUpgradeModal } = useActions(upgradeModalLogic) - const scrollToProduct = !(featureInfo?.key === AvailableFeature.ORGANIZATIONS_PROJECTS && !isAddonProduct) - useEffect(() => { if (gateVariant) { posthog.capture('pay gate shown', { @@ -87,26 +76,15 @@ export function PayGateMini({ if (gateVariant && productWithFeature && featureInfo && !overrideShouldShowGate) { return (
- + {docsLink && isCloudOrDev && ( {children}
} -interface PayGateContentProps { +interface PayGateContentProps extends PayGateMiniLogicProps { className?: string background: boolean - featureInfo: BillingFeatureType - featureAvailableOnOrg?: BillingFeatureType | null - gateVariant: 'add-card' | 'contact-sales' | 'move-to-cloud' | null - productWithFeature: BillingProductV2AddonType | BillingProductV2Type isGrandfathered?: boolean - isAddonProduct?: boolean - featureInfoOnNextPlan?: BillingFeatureType children: React.ReactNode handleCtaClick: () => void } function PayGateContent({ + feature, + currentUsage, className, background, - featureInfo, - featureAvailableOnOrg, - gateVariant, - productWithFeature, isGrandfathered, - isAddonProduct, - featureInfoOnNextPlan, children, handleCtaClick, -}: PayGateContentProps): JSX.Element { +}: PayGateContentProps): JSX.Element | null { + const { + productWithFeature, + featureInfo, + featureAvailableOnOrg, + gateVariant, + isAddonProduct, + featureInfoOnNextPlan, + } = useValues(payGateMiniLogic({ feature, currentUsage })) + + if (!productWithFeature || !featureInfo) { + return null + } + return (
([ props({} as PayGateMiniLogicProps), path(['lib', 'components', 'payGateMini', 'payGateMiniLogic']), - key((props) => props.featureKey), + key((props) => props.feature), connect(() => ({ values: [ billingLogic, @@ -42,21 +42,21 @@ export const payGateMiniLogic = kea([ let foundProduct: BillingProductV2Type | BillingProductV2AddonType | undefined = undefined - if (checkProductFirst.includes(props.featureKey)) { + if (checkProductFirst.includes(props.feature)) { foundProduct = billing?.products?.find((product) => - product.features?.some((f) => f.key === props.featureKey) + product.features?.some((f) => f.key === props.feature) ) } // Check addons first (if not included in checkProductFirst) since their features are rolled up into the parent const allAddons = billing?.products?.map((product) => product.addons).flat() || [] if (!foundProduct) { - foundProduct = allAddons.find((addon) => addon.features?.some((f) => f.key === props.featureKey)) + foundProduct = allAddons.find((addon) => addon.features?.some((f) => f.key === props.feature)) } if (!foundProduct) { foundProduct = billing?.products?.find((product) => - product.features?.some((f) => f.key === props.featureKey) + product.features?.some((f) => f.key === props.feature) ) } return foundProduct @@ -71,18 +71,18 @@ export const payGateMiniLogic = kea([ ], featureInfo: [ (s) => [s.productWithFeature], - (productWithFeature) => productWithFeature?.features.find((f) => f.key === props.featureKey), + (productWithFeature) => productWithFeature?.features.find((f) => f.key === props.feature), ], featureAvailableOnOrg: [ - (s) => [s.user, (_, props) => props.featureKey], - (_user, featureKey) => { - return values.availableFeature(featureKey) + (s) => [s.user, (_, props) => props.feature], + (_user, feature) => { + return values.availableFeature(feature) }, ], minimumPlanWithFeature: [ (s) => [s.productWithFeature], (productWithFeature) => - productWithFeature?.plans.find((plan) => plan.features?.some((f) => f.key === props.featureKey)), + productWithFeature?.plans.find((plan) => plan.features?.some((f) => f.key === props.feature)), ], nextPlanWithFeature: [ (s) => [s.productWithFeature], @@ -96,18 +96,18 @@ export const payGateMiniLogic = kea([ ], featureInfoOnNextPlan: [ (s) => [s.nextPlanWithFeature], - (nextPlanWithFeature) => nextPlanWithFeature?.features.find((f) => f.key === props.featureKey), + (nextPlanWithFeature) => nextPlanWithFeature?.features.find((f) => f.key === props.feature), ], gateVariant: [ (s) => [ s.billingLoading, s.hasAvailableFeature, s.minimumPlanWithFeature, - (_, props) => props.featureKey, + (_, props) => props.feature, (_, props) => props.currentUsage, ], - (billingLoading, hasAvailableFeature, minimumPlanWithFeature, featureKey, currentUsage) => { - if (hasAvailableFeature(featureKey, currentUsage)) { + (billingLoading, hasAvailableFeature, minimumPlanWithFeature, feature, currentUsage) => { + if (hasAvailableFeature(feature, currentUsage)) { return null } if (billingLoading) { @@ -122,5 +122,12 @@ export const payGateMiniLogic = kea([ return 'move-to-cloud' }, ], + + scrollToProduct: [ + (s) => [s.featureInfo, s.isAddonProduct], + (featureInfo, isAddonProduct) => { + return !(featureInfo?.key === AvailableFeature.ORGANIZATIONS_PROJECTS && !isAddonProduct) + }, + ], })), ]) diff --git a/frontend/src/lib/components/Playlist/Playlist.scss b/frontend/src/lib/components/Playlist/Playlist.scss index 5f6ab75a144b05..250c3acf9997bd 100644 --- a/frontend/src/lib/components/Playlist/Playlist.scss +++ b/frontend/src/lib/components/Playlist/Playlist.scss @@ -51,8 +51,8 @@ .SessionRecordingPlaylistHeightWrapper { // NOTE: Somewhat random way to offset the various headers and tabs above the playlist - height: calc(100vh - 15rem); - min-height: 41rem; + height: calc(100vh - 14rem); + min-height: 25rem; } .SessionRecordingPreview { diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx index f403db126644cc..0fdbccfcb1435c 100644 --- a/frontend/src/lib/constants.tsx +++ b/frontend/src/lib/constants.tsx @@ -186,7 +186,6 @@ export const FEATURE_FLAGS = { REPLAY_SIMILAR_RECORDINGS: 'session-replay-similar-recordings', // owner: #team-replay SAVED_NOT_PINNED: 'saved-not-pinned', // owner: #team-replay NEW_EXPERIMENTS_UI: 'new-experiments-ui', // owner: @jurajmajerik #team-feature-success - SESSION_REPLAY_FILTER_ORDERING: 'session-replay-filter-ordering', // owner: #team-replay REPLAY_ERROR_CLUSTERING: 'session-replay-error-clustering', // owner: #team-replay AUDIT_LOGS_ACCESS: 'audit-logs-access', // owner: #team-growth SUBSCRIBE_FROM_PAYGATE: 'subscribe-from-paygate', // owner: #team-growth @@ -202,7 +201,6 @@ export const FEATURE_FLAGS = { HOG: 'hog', // owner: @mariusandra HOG_FUNCTIONS: 'hog-functions', // owner: #team-cdp PERSONLESS_EVENTS_NOT_SUPPORTED: 'personless-events-not-supported', // owner: @raquelmsmith - SESSION_REPLAY_UNIVERSAL_FILTERS: 'session-replay-universal-filters', // owner: #team-replay ALERTS: 'alerts', // owner: github.com/nikitaevg ERROR_TRACKING: 'error-tracking', // owner: #team-replay SETTINGS_BOUNCE_RATE_PAGE_VIEW_MODE: 'settings-bounce-rate-page-view-mode', // owner: @robbie-c diff --git a/frontend/src/queries/examples.ts b/frontend/src/queries/examples.ts index 5728a1e46fe380..7cbad0eccc4f36 100644 --- a/frontend/src/queries/examples.ts +++ b/frontend/src/queries/examples.ts @@ -287,6 +287,19 @@ limit 100`, }, } +const HogQLForDataWarehouse: HogQLQuery = { + kind: NodeKind.HogQLQuery, + query: `select toDate(timestamp) as timestamp, event as event + from events +limit 100`, + explain: true, +} + +const DataWarehouse: DataVisualizationNode = { + kind: NodeKind.DataVisualizationNode, + source: HogQLForDataWarehouse, +} + const HogQLTable: DataTableNode = { kind: NodeKind.DataTableNode, full: true, @@ -344,6 +357,7 @@ export const examples: Record = { DataVisualization, Hog, Hoggonacci, + DataWarehouse, } export const stringifiedExamples: Record = Object.fromEntries( diff --git a/frontend/src/queries/nodes/DataVisualization/DataVisualization.tsx b/frontend/src/queries/nodes/DataVisualization/DataVisualization.tsx index 757ffb244e9ba7..7c1fcc94e28daf 100644 --- a/frontend/src/queries/nodes/DataVisualization/DataVisualization.tsx +++ b/frontend/src/queries/nodes/DataVisualization/DataVisualization.tsx @@ -1,11 +1,13 @@ import { LemonDivider } from '@posthog/lemon-ui' import { BindLogic, useValues } from 'kea' +import { router } from 'kea-router' import { AnimationType } from 'lib/animations/animations' import { Animation } from 'lib/components/Animation/Animation' import { useCallback, useState } from 'react' import { DatabaseTableTreeWithItems } from 'scenes/data-warehouse/external/DataWarehouseTables' import { insightLogic } from 'scenes/insights/insightLogic' import { HogQLBoldNumber } from 'scenes/insights/views/BoldNumber/BoldNumber' +import { urls } from 'scenes/urls' import { insightVizDataCollectionId, insightVizDataNodeKey } from '~/queries/nodes/InsightViz/InsightViz' import { AnyResponseType, DataVisualizationNode, HogQLQuery, NodeKind } from '~/queries/schema' @@ -123,19 +125,6 @@ function InternalDataTableVisualization(props: DataTableVisualizationProps): JSX {!readOnly && showEditingUI && ( <> - {sourceFeatures.has(QueryFeature.dateRangePicker) && ( -
- { - if (query.kind === NodeKind.HogQLQuery) { - setQuerySource(query) - } - }} - /> -
- )} )} {!readOnly && showResultControls && ( @@ -147,6 +136,20 @@ function InternalDataTableVisualization(props: DataTableVisualizationProps): JSX
+ {sourceFeatures.has(QueryFeature.dateRangePicker) && + !router.values.location.pathname.includes(urls.dataWarehouse()) && ( // decouple this component from insights tab and datawarehouse scene +
+ { + if (query.kind === NodeKind.HogQLQuery) { + setQuerySource(query) + } + }} + /> +
+ )}
diff --git a/frontend/src/queries/nodes/HogQLQuery/HogQLQueryEditor.tsx b/frontend/src/queries/nodes/HogQLQuery/HogQLQueryEditor.tsx index 9b0cc060d17ee4..99ec8572c65521 100644 --- a/frontend/src/queries/nodes/HogQLQuery/HogQLQueryEditor.tsx +++ b/frontend/src/queries/nodes/HogQLQuery/HogQLQueryEditor.tsx @@ -223,29 +223,17 @@ export function HogQLQueryEditor(props: HogQLQueryEditorProps): JSX.Element { : '' } data-attr="hogql-query-editor-save-as-view" - > - Save as view - - )} - } - type="secondary" - size="small" - dropdown={{ - overlay: ( + tooltip={
Save a query as a view that can be referenced in another query. This is useful for modeling data and organizing large queries into readable chunks.{' '} More Info{' '}
- ), - placement: 'right-start', - fallbackPlacements: ['left-start'], - actionable: true, - closeParentPopoverOnClickInside: true, - }} - /> + } + > + Save as view + + )} )} diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index 22351b2189557a..20afa26306c9d4 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -423,6 +423,9 @@ "normalize_url": { "type": "boolean" }, + "property": { + "type": "string" + }, "type": { "anyOf": [ { @@ -432,12 +435,9 @@ "type": "null" } ] - }, - "value": { - "type": "string" } }, - "required": ["value"], + "required": ["property"], "type": "object" }, "BreakdownAttributionType": { diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index 2b7e27639f9140..924bc2bc0ee7ee 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -1604,7 +1604,7 @@ export type MultipleBreakdownType = Extract @@ -57,14 +56,14 @@ export function DataWarehouseExternalScene(): JSX.Element { buttons={ <> {currentTab === DataWarehouseTab.Explore && ( - + saveAs(true)} + loading={insightSaving} + > + Save as insight + )} ([ }, ], }), - listeners(({ actions, values, cache }) => ({ - deleteSelfManagedTable: async ({ tableId }) => { - await api.dataWarehouseTables.delete(tableId) - actions.loadDatabase() - }, - loadSourcesSuccess: () => { + urlToAction(({ cache, actions }) => ({ + '/data-warehouse/managed-sources': () => { clearTimeout(cache.refreshTimeout) cache.refreshTimeout = setTimeout(() => { actions.loadSources(null) }, REFRESH_INTERVAL) }, - loadSourcesFailure: () => { - clearTimeout(cache.refreshTimeout) - - cache.refreshTimeout = setTimeout(() => { - actions.loadSources(null) - }, REFRESH_INTERVAL) + '*': () => { + if (cache.refreshTimeout && router.values.location.pathname !== '/data-warehouse/managed-sources') { + clearTimeout(cache.refreshTimeout) + } + }, + })), + listeners(({ actions, values, cache }) => ({ + deleteSelfManagedTable: async ({ tableId }) => { + await api.dataWarehouseTables.delete(tableId) + actions.loadDatabase() }, deleteSource: async ({ source }) => { await api.externalDataSources.delete(source.id) @@ -214,4 +215,7 @@ export const dataWarehouseSettingsLogic = kea([ afterMount(({ actions }) => { actions.loadSources(null) }), + beforeUnmount(({ cache }) => { + clearTimeout(cache.refreshTimeout) + }), ]) diff --git a/frontend/src/scenes/data-warehouse/settings/dataWarehouseSourcesTableSyncMethodModalLogic.ts b/frontend/src/scenes/data-warehouse/settings/dataWarehouseSourcesTableSyncMethodModalLogic.ts index d76f22a3583ad4..d21813aa509b4e 100644 --- a/frontend/src/scenes/data-warehouse/settings/dataWarehouseSourcesTableSyncMethodModalLogic.ts +++ b/frontend/src/scenes/data-warehouse/settings/dataWarehouseSourcesTableSyncMethodModalLogic.ts @@ -16,7 +16,10 @@ export const dataWarehouseSourcesTableSyncMethodModalLogic = kea props.schema.id), connect(() => ({ - actions: [dataWarehouseSettingsLogic, ['updateSchema', 'updateSchemaSuccess', 'updateSchemaFailure']], + actions: [ + dataWarehouseSettingsLogic, + ['updateSchema', 'updateSchemaSuccess', 'updateSchemaFailure', 'loadSources'], + ], })), actions({ openSyncMethodModal: (schema: ExternalDataSourceSchema) => ({ schema }), @@ -59,6 +62,7 @@ export const dataWarehouseSourcesTableSyncMethodModalLogic = kea ({ updateSchemaSuccess: () => { + actions.loadSources(null) actions.resetSchemaIncrementalFields() actions.closeSyncMethodModal() }, diff --git a/frontend/src/scenes/funnels/funnelDataLogic.ts b/frontend/src/scenes/funnels/funnelDataLogic.ts index 3c50d171db9dca..0a48c272e929bc 100644 --- a/frontend/src/scenes/funnels/funnelDataLogic.ts +++ b/frontend/src/scenes/funnels/funnelDataLogic.ts @@ -179,7 +179,7 @@ export const funnelDataLogic = kea([ if (!isTimeToConvertFunnel && Array.isArray(results)) { if (isBreakdownFunnelResults(results)) { const breakdownProperty = breakdownFilter?.breakdowns - ? breakdownFilter?.breakdowns.map((b) => b.value).join('::') + ? breakdownFilter?.breakdowns.map((b) => b.property).join('::') : breakdownFilter?.breakdown ?? undefined return aggregateBreakdownResult(results, breakdownProperty).sort((a, b) => a.order - b.order) } diff --git a/frontend/src/scenes/groups/Group.tsx b/frontend/src/scenes/groups/Group.tsx index 278132349979c1..1cf23337da773e 100644 --- a/frontend/src/scenes/groups/Group.tsx +++ b/frontend/src/scenes/groups/Group.tsx @@ -5,6 +5,7 @@ import { NotFound } from 'lib/components/NotFound' import { PageHeader } from 'lib/components/PageHeader' import { PropertiesTable } from 'lib/components/PropertiesTable' import { TZLabel } from 'lib/components/TZLabel' +import { isEventFilter } from 'lib/components/UniversalFilters/utils' import { LemonBanner } from 'lib/lemon-ui/LemonBanner' import { LemonTabs } from 'lib/lemon-ui/LemonTabs' import { lemonToast } from 'lib/lemon-ui/LemonToast' @@ -17,6 +18,7 @@ import { NotebookSelectButton } from 'scenes/notebooks/NotebookSelectButton/Note import { RelatedFeatureFlags } from 'scenes/persons/RelatedFeatureFlags' import { SceneExport } from 'scenes/sceneTypes' import { SessionRecordingsPlaylist } from 'scenes/session-recordings/playlist/SessionRecordingsPlaylist' +import { filtersFromUniversalFilterGroups } from 'scenes/session-recordings/utils' import { teamLogic } from 'scenes/teamLogic' import { urls } from 'scenes/urls' @@ -154,7 +156,7 @@ export function Group(): JSX.Element { { - const stillHasGroupFilter = legacyFilters.events?.some((event) => { - return event.properties.some( + onFiltersChange={(filters) => { + const eventFilters = + filtersFromUniversalFilterGroups(filters).filter(isEventFilter) + + const stillHasGroupFilter = eventFilters?.some((event) => { + return event.properties?.some( (prop: Record) => prop.key === `$group_${groupTypeIndex} = '${groupKey}'` ) diff --git a/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownFilter.tsx b/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownFilter.tsx index 20233ac3f74238..f488bfef49aae2 100644 --- a/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownFilter.tsx +++ b/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownFilter.tsx @@ -47,8 +47,8 @@ export function TaxonomicBreakdownFilter({ const tags = breakdownArray.map((breakdown) => typeof breakdown === 'object' ? ( diff --git a/frontend/src/scenes/insights/filters/BreakdownFilter/breakdownTagLogic.ts b/frontend/src/scenes/insights/filters/BreakdownFilter/breakdownTagLogic.ts index 60855f5fdd6feb..e091fc941b7e27 100644 --- a/frontend/src/scenes/insights/filters/BreakdownFilter/breakdownTagLogic.ts +++ b/frontend/src/scenes/insights/filters/BreakdownFilter/breakdownTagLogic.ts @@ -97,7 +97,7 @@ export const breakdownTagLogic = kea([ (s) => [s.breakdownFilter, s.breakdown, s.breakdownType], ({ breakdowns }, breakdown, breakdownType) => breakdowns?.find( - (savedBreakdown) => savedBreakdown.value === breakdown && savedBreakdown.type === breakdownType + (savedBreakdown) => savedBreakdown.property === breakdown && savedBreakdown.type === breakdownType ), ], histogramBinsUsed: [ @@ -147,7 +147,7 @@ export const breakdownTagLogic = kea([ ? filterToTaxonomicFilterType( multipleBreakdown?.type, multipleBreakdown?.group_type_index, - multipleBreakdown?.value + multipleBreakdown?.property ) : breakdownFilterToTaxonomicFilterType(breakdownFilter) diff --git a/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterLogic.test.ts b/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterLogic.test.ts index fb75bb20da8801..1064db47dd9815 100644 --- a/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterLogic.test.ts +++ b/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterLogic.test.ts @@ -237,7 +237,7 @@ describe('taxonomicBreakdownFilterLogic', () => { breakdownFilter: { breakdowns: [ { - value: 'prop1', + property: 'prop1', type: 'event', }, ], @@ -256,11 +256,11 @@ describe('taxonomicBreakdownFilterLogic', () => { breakdownFilter: { breakdowns: [ { - value: 'prop1', + property: 'prop1', type: 'event', }, { - value: 'prop2', + property: 'prop2', type: 'event', }, ], @@ -282,15 +282,15 @@ describe('taxonomicBreakdownFilterLogic', () => { breakdownFilter: { breakdowns: [ { - value: 'prop1', + property: 'prop1', type: 'event', }, { - value: 'prop2', + property: 'prop2', type: 'event', }, { - value: 'prop3', + property: 'prop3', type: 'event', }, ], @@ -397,7 +397,7 @@ describe('taxonomicBreakdownFilterLogic', () => { breakdown_type: undefined, breakdowns: [ { - value: 'c', + property: 'c', type: 'event', }, ], @@ -412,7 +412,7 @@ describe('taxonomicBreakdownFilterLogic', () => { breakdownFilter: { breakdowns: [ { - value: 'c', + property: 'c', type: 'event', }, ], @@ -455,7 +455,7 @@ describe('taxonomicBreakdownFilterLogic', () => { breakdown_type: undefined, breakdowns: [ { - value: 'height', + property: 'height', type: 'person', }, ], @@ -486,7 +486,7 @@ describe('taxonomicBreakdownFilterLogic', () => { breakdowns: [ { type: 'group', - value: '$lib_version', + property: '$lib_version', group_type_index: 0, }, ], @@ -501,7 +501,7 @@ describe('taxonomicBreakdownFilterLogic', () => { breakdownFilter: { breakdowns: [ { - value: 'c', + property: 'c', type: 'event', }, ], @@ -533,7 +533,7 @@ describe('taxonomicBreakdownFilterLogic', () => { breakdowns: [ { type: 'event', - value: 'a', + property: 'a', }, ], breakdown_group_type_index: undefined, @@ -547,11 +547,11 @@ describe('taxonomicBreakdownFilterLogic', () => { breakdownFilter: { breakdowns: [ { - value: 'c', + property: 'c', type: 'event', }, { - value: 'duplicate', + property: 'duplicate', type: 'event', }, ], @@ -562,14 +562,13 @@ describe('taxonomicBreakdownFilterLogic', () => { }) mockFeatureFlag(logic) logic.mount() - const changedBreakdown = 'c' const group: TaxonomicFilterGroup = taxonomicGroupFor(TaxonomicFilterGroupType.EventProperties, undefined) await expectLogic(logic, () => { logic.actions.replaceBreakdown( { type: 'event', - value: changedBreakdown, + value: 'c', }, { group: group, @@ -613,7 +612,7 @@ describe('taxonomicBreakdownFilterLogic', () => { breakdowns: [ { type: 'event', - value: 'prop2', + property: 'prop2', }, ], }) @@ -665,7 +664,7 @@ describe('taxonomicBreakdownFilterLogic', () => { breakdowns: [ { type: 'event', - value: 'prop', + property: 'prop', }, ], }, @@ -736,14 +735,14 @@ describe('taxonomicBreakdownFilterLogic', () => { breakdown_normalize_url: undefined, breakdowns: [ { - value: 'prop', + property: 'prop', type: 'event', normalize_url: true, group_type_index: 0, histogram_bin_count: 10, }, { - value: 'c', + property: 'c', type: 'event', }, ], @@ -893,7 +892,7 @@ describe('taxonomicBreakdownFilterLogic', () => { breakdowns: [ { type: 'person', - value: 'new_prop', + property: 'new_prop', }, ], }) @@ -927,7 +926,7 @@ describe('taxonomicBreakdownFilterLogic', () => { breakdowns: [ { type: 'group', - value: '$lib_version', + property: '$lib_version', group_type_index: 0, }, ], @@ -1002,7 +1001,7 @@ describe('taxonomicBreakdownFilterLogic', () => { breakdown_group_type_index: undefined, breakdowns: [ { - value: 'c', + property: 'c', type: 'person', }, ], diff --git a/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterLogic.ts b/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterLogic.ts index 18ed86670a3b1f..291968d794bf9a 100644 --- a/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterLogic.ts +++ b/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterLogic.ts @@ -236,8 +236,8 @@ export const taxonomicBreakdownFilterLogic = kea !(savedBreakdown.value === breakdown && savedBreakdown.type === breakdownType) + (savedBreakdown) => + !(savedBreakdown.property === breakdown && savedBreakdown.type === breakdownType) ) props.updateBreakdownFilter({ @@ -381,7 +382,7 @@ export const taxonomicBreakdownFilterLogic = kea { if ( - savedBreakdown.value === previousBreakdown.value && + savedBreakdown.property === previousBreakdown.value && savedBreakdown.type === previousBreakdown.type ) { return { ...savedBreakdown, - value: breakdownValue, + property: breakdownValue, type: breakdownType, group_type_index: newBreakdown.group.groupTypeIndex, histogram_bin_count: isHistogramable @@ -505,7 +506,7 @@ function updateNestedBreakdown( lookupType: string ): Breakdown[] | undefined { return breakdowns?.map((savedBreakdown) => - savedBreakdown.value === lookupValue && savedBreakdown.type === lookupType + savedBreakdown.property === lookupValue && savedBreakdown.type === lookupType ? { ...savedBreakdown, ...breakdownUpdate, @@ -520,7 +521,7 @@ function checkBreakdownExists( lookupType: string ): boolean { return !!breakdowns?.find( - (savedBreakdown) => savedBreakdown.value === lookupValue && savedBreakdown.type === lookupType + (savedBreakdown) => savedBreakdown.property === lookupValue && savedBreakdown.type === lookupType ) } diff --git a/frontend/src/scenes/insights/insightDataLogic.tsx b/frontend/src/scenes/insights/insightDataLogic.tsx index 1e8eaf4153423f..668798a9fe49c4 100644 --- a/frontend/src/scenes/insights/insightDataLogic.tsx +++ b/frontend/src/scenes/insights/insightDataLogic.tsx @@ -80,8 +80,8 @@ export const insightDataLogic = kea([ actions({ setQuery: (query: Node | null) => ({ query }), - saveAs: true, - saveAsNamingSuccess: (name: string) => ({ name }), + saveAs: (redirectToViewMode?: boolean) => ({ redirectToViewMode }), + saveAsNamingSuccess: (name: string, redirectToViewMode?: boolean) => ({ name, redirectToViewMode }), saveInsight: (redirectToViewMode = true) => ({ redirectToViewMode }), toggleQueryEditorPanel: true, cancelChanges: true, @@ -237,11 +237,14 @@ export const insightDataLogic = kea([ actions.insightLogicSaveInsight(redirectToViewMode) }, - saveAs: async () => { + saveAs: async ({ redirectToViewMode }) => { LemonDialog.openForm({ title: 'Save as new insight', initialValues: { - insightName: `${values.queryBasedInsight.name || values.queryBasedInsight.derived_name} (copy)`, + insightName: + values.queryBasedInsight.name || values.queryBasedInsight.derived_name + ? `${values.queryBasedInsight.name || values.queryBasedInsight.derived_name} (copy)` + : '', }, content: ( @@ -251,10 +254,10 @@ export const insightDataLogic = kea([ errors: { insightName: (name) => (!name ? 'You must enter a name' : undefined), }, - onSubmit: async ({ insightName }) => actions.saveAsNamingSuccess(insightName), + onSubmit: async ({ insightName }) => actions.saveAsNamingSuccess(insightName, redirectToViewMode), }) }, - saveAsNamingSuccess: ({ name }) => { + saveAsNamingSuccess: ({ name, redirectToViewMode }) => { let filters = values.legacyInsight.filters if (isInsightVizNode(values.query)) { const querySource = values.query.source @@ -277,7 +280,7 @@ export const insightDataLogic = kea([ { overrideFilter: true, fromPersistentApi: false } ) - actions.insightLogicSaveAsNamingSuccess(name) + actions.insightLogicSaveAsNamingSuccess(name, redirectToViewMode) }, cancelChanges: () => { const savedFilters = values.savedInsight.filters diff --git a/frontend/src/scenes/insights/insightLogic.test.ts b/frontend/src/scenes/insights/insightLogic.test.ts index ee8374c6d54dfe..c94ea390e505fb 100644 --- a/frontend/src/scenes/insights/insightLogic.test.ts +++ b/frontend/src/scenes/insights/insightLogic.test.ts @@ -40,6 +40,8 @@ const Insight42 = '42' as InsightShortId const Insight43 = '43' as InsightShortId const Insight44 = '44' as InsightShortId +const MOCK_DASHBOARD_ID = 34 + const partialInsight43 = { id: 43, short_id: Insight43, @@ -61,6 +63,7 @@ const patchResponseFor = ( description: id === '42' ? undefined : 'Lorem ipsum.', tags: id === '42' ? undefined : ['good'], dashboards: payload['dashboards'], + dashboard_tiles: id === '43' ? [{ dashboard_id: MOCK_DASHBOARD_ID }] : undefined, } } @@ -192,6 +195,23 @@ describe('insightLogic', () => { }, ], }, + '/api/projects/:team/dashboards/34/': { + id: 33, + filters: {}, + tiles: [ + { + layouts: {}, + color: null, + insight: { + id: 42, + short_id: Insight43, + result: 'result!', + filters: { insight: InsightType.TRENDS, interval: 'month' }, + tags: ['bla'], + }, + }, + ], + }, }, post: { '/api/projects/:team/insights/funnel/': { result: ['result from api'] }, @@ -513,14 +533,19 @@ describe('insightLogic', () => { }) test('saveInsight updates dashboards', async () => { + const dashLogic = dashboardLogic({ id: MOCK_DASHBOARD_ID }) + dashLogic.mount() + await expectLogic(dashLogic).toDispatchActions(['loadDashboard']) + savedInsightsLogic.mount() + logic = insightLogic({ dashboardItemId: Insight43, }) logic.mount() - logic.actions.saveInsight() - await expectLogic(dashboardsModel).toDispatchActions(['updateDashboardInsight']) + + await expectLogic(dashLogic).toDispatchActions(['loadDashboard']) }) test('updateInsight updates dashboards', async () => { diff --git a/frontend/src/scenes/insights/insightLogic.ts b/frontend/src/scenes/insights/insightLogic.ts index 0aede61c993711..036b501f42f91f 100644 --- a/frontend/src/scenes/insights/insightLogic.ts +++ b/frontend/src/scenes/insights/insightLogic.ts @@ -8,6 +8,7 @@ import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { objectsEqual } from 'lib/utils' import { eventUsageLogic, InsightEventSource } from 'lib/utils/eventUsageLogic' +import { dashboardLogic } from 'scenes/dashboard/dashboardLogic' import { insightSceneLogic } from 'scenes/insights/insightSceneLogic' import { keyForInsightLogicProps } from 'scenes/insights/sharedUtils' import { summarizeInsight } from 'scenes/insights/summarizeInsight' @@ -93,7 +94,7 @@ export const insightLogic = kea([ insight, options, }), - saveAsNamingSuccess: (name: string) => ({ name }), + saveAsNamingSuccess: (name: string, redirectToViewMode?: boolean) => ({ name, redirectToViewMode }), cancelChanges: true, saveInsight: (redirectToViewMode = true) => ({ redirectToViewMode }), saveInsightSuccess: true, @@ -425,6 +426,16 @@ export const insightLogic = kea([ dashboardsModel.actions.updateDashboardInsight(savedInsight) + // reload dashboards with updated insight + // since filters on dashboard might be different from filters on insight + // we need to trigger dashboard reload to pick up results for updated insight + savedInsight.dashboard_tiles?.forEach(({ dashboard_id }) => + dashboardLogic.findMounted({ id: dashboard_id })?.actions.loadDashboard({ + action: 'update', + refresh: 'lazy_async', + }) + ) + const mountedInsightSceneLogic = insightSceneLogic.findMounted() if (redirectToViewMode) { if (!insightNumericId && dashboards?.length === 1) { @@ -441,7 +452,7 @@ export const insightLogic = kea([ router.actions.push(urls.insightEdit(savedInsight.short_id)) } }, - saveAsNamingSuccess: async ({ name }) => { + saveAsNamingSuccess: async ({ name, redirectToViewMode }) => { const { filters, query } = getInsightFilterOrQueryForPersistance( values.queryBasedInsight, values.queryBasedInsightSaving @@ -454,12 +465,17 @@ export const insightLogic = kea([ }) lemonToast.info( `You're now working on a copy of ${ - values.queryBasedInsight.name || values.queryBasedInsight.derived_name + values.queryBasedInsight.name || values.queryBasedInsight.derived_name || name }` ) actions.setInsight(insight, { fromPersistentApi: true, overrideFilter: true }) savedInsightsLogic.findMounted()?.actions.loadInsights() // Load insights afresh - router.actions.push(urls.insightEdit(insight.short_id)) + + if (redirectToViewMode) { + router.actions.push(urls.insightView(insight.short_id)) + } else { + router.actions.push(urls.insightEdit(insight.short_id)) + } }, cancelChanges: () => { actions.setFilters(values.savedInsight.filters || {}) diff --git a/frontend/src/scenes/insights/insightSceneLogic.tsx b/frontend/src/scenes/insights/insightSceneLogic.tsx index 1b06e53060bef0..75e99677715a40 100644 --- a/frontend/src/scenes/insights/insightSceneLogic.tsx +++ b/frontend/src/scenes/insights/insightSceneLogic.tsx @@ -191,7 +191,7 @@ export const insightSceneLogic = kea([ urlToAction(({ actions, values }) => ({ '/data-warehouse/*': (_, __, { q }) => { actions.setSceneState(String('new') as InsightShortId, ItemMode.Edit, undefined) - values.insightDataLogicRef?.logic.actions.setQuery(examples.DataVisualization) + values.insightDataLogicRef?.logic.actions.setQuery(examples.DataWarehouse) values.insightLogicRef?.logic.actions.setInsight( { ...createEmptyInsight('new', false), diff --git a/frontend/src/scenes/insights/summarizeInsight.test.ts b/frontend/src/scenes/insights/summarizeInsight.test.ts index f903c837749681..33f9dd1d31471a 100644 --- a/frontend/src/scenes/insights/summarizeInsight.test.ts +++ b/frontend/src/scenes/insights/summarizeInsight.test.ts @@ -594,15 +594,15 @@ describe('summarizing insights', () => { breakdowns: [ { type: 'event', - value: '$browser', + property: '$browser', }, { type: 'person', - value: 'custom_prop', + property: 'custom_prop', }, { type: 'session', - value: '$session_duration', + property: '$session_duration', }, ], }, diff --git a/frontend/src/scenes/insights/summarizeInsight.ts b/frontend/src/scenes/insights/summarizeInsight.ts index aefdb19a4048c6..f44b3f07e7ed61 100644 --- a/frontend/src/scenes/insights/summarizeInsight.ts +++ b/frontend/src/scenes/insights/summarizeInsight.ts @@ -63,7 +63,7 @@ function summarizeMultipleBreakdown( if (breakdowns && breakdowns.length > 0) { return (breakdowns as Breakdown[]) .map((breakdown) => - summarizeSinglularBreakdown(breakdown.value, breakdown.type, breakdown.group_type_index, context) + summarizeSinglularBreakdown(breakdown.property, breakdown.type, breakdown.group_type_index, context) ) .filter((label): label is string => !!label) .join(', ') diff --git a/frontend/src/scenes/insights/utils.test.ts b/frontend/src/scenes/insights/utils.test.ts index 7d5a6435bb6687..bf6b56a205b560 100644 --- a/frontend/src/scenes/insights/utils.test.ts +++ b/frontend/src/scenes/insights/utils.test.ts @@ -340,11 +340,11 @@ describe('formatBreakdownLabel()', () => { const breakdownFilter: BreakdownFilter = { breakdowns: [ { - value: 'demographic', + property: 'demographic', type: 'event', }, { - value: '$browser', + property: '$browser', type: 'event', }, ], @@ -362,11 +362,11 @@ describe('formatBreakdownLabel()', () => { const breakdownFilter: BreakdownFilter = { breakdowns: [ { - value: 'demographic', + property: 'demographic', type: 'event', }, { - value: '$browser', + property: '$browser', type: 'event', }, ], @@ -390,7 +390,7 @@ describe('formatBreakdownLabel()', () => { const breakdownFilter2: BreakdownFilter = { breakdowns: [ { - value: '$session_duration', + property: '$session_duration', type: 'session', }, ], @@ -410,7 +410,7 @@ describe('formatBreakdownLabel()', () => { const breakdownFilter2: BreakdownFilter = { breakdowns: [ { - value: '$session_duration', + property: '$session_duration', type: 'session', }, ], diff --git a/frontend/src/scenes/insights/utils.tsx b/frontend/src/scenes/insights/utils.tsx index 418821d2329f5e..b0bb7446125edb 100644 --- a/frontend/src/scenes/insights/utils.tsx +++ b/frontend/src/scenes/insights/utils.tsx @@ -242,7 +242,7 @@ function formatNumericBreakdownLabel( return ( formatPropertyValueForDisplay( - nestedBreakdown?.value ?? breakdownFilter?.breakdown, + nestedBreakdown?.property ?? breakdownFilter?.breakdown, breakdown_value, propertyFilterTypeToPropertyDefinitionType(nestedBreakdown?.type ?? breakdownFilter?.breakdown_type) )?.toString() ?? 'None' diff --git a/frontend/src/scenes/insights/utils/cleanFilters.test.ts b/frontend/src/scenes/insights/utils/cleanFilters.test.ts index 4f40cb338f15ff..ea0cb6156f1f69 100644 --- a/frontend/src/scenes/insights/utils/cleanFilters.test.ts +++ b/frontend/src/scenes/insights/utils/cleanFilters.test.ts @@ -64,7 +64,15 @@ describe('cleanFilters', () => { breakdown_type: 'event', } as TrendsFilterType) - expect(cleanedFilters).toHaveProperty('breakdown_normalize_url', true) + expect(cleanedFilters).toMatchObject({ + breakdowns: [ + { + property: '$current_url', + type: 'event', + normalize_url: true, + }, + ], + }) }) it('defaults to normalizing URL for breakdown by $current_url', () => { @@ -82,7 +90,15 @@ describe('cleanFilters', () => { breakdown_type: 'event', } as TrendsFilterType) - expect(cleanedFilters).toHaveProperty('breakdown_normalize_url', true) + expect(cleanedFilters).toMatchObject({ + breakdowns: [ + { + property: '$pathname', + type: 'event', + normalize_url: true, + }, + ], + }) }) it('defaults to normalizing URL for breakdown by $pathname', () => { @@ -172,39 +188,29 @@ describe('cleanFilters', () => { it('keeps multiple breakdowns', () => { const cleanedFilters = cleanFilters({ - breakdowns: [{ value: 'any', type: 'event' }], + breakdowns: [{ property: 'any', type: 'event' }], insight: InsightType.TRENDS, } as TrendsFilterType) - expect(cleanedFilters).toHaveProperty('breakdowns', [{ value: 'any', type: 'event' }]) + expect(cleanedFilters).toHaveProperty('breakdowns', [{ property: 'any', type: 'event' }]) }) it('keeps normalize_url for multiple breakdowns', () => { const cleanedFilters = cleanFilters({ - breakdowns: [{ value: '$current_url', type: 'event', normalize_url: true }], + breakdowns: [{ property: '$current_url', type: 'event', normalize_url: true }], insight: InsightType.TRENDS, } as TrendsFilterType) expect(cleanedFilters).toHaveProperty('breakdowns', [ - { value: '$current_url', type: 'event', normalize_url: true }, + { property: '$current_url', type: 'event', normalize_url: true }, ]) cleanedFilters.breakdowns![0].normalize_url = false expect(cleanedFilters).toHaveProperty('breakdowns', [ - { value: '$current_url', type: 'event', normalize_url: false }, + { property: '$current_url', type: 'event', normalize_url: false }, ]) }) - it('restores legacy multiple breakdowns', () => { - const cleanedFilters = cleanFilters({ - breakdowns: [{ property: 'any', type: 'event' }], - insight: InsightType.TRENDS, - } as TrendsFilterType) - - expect(cleanedFilters).toHaveProperty('breakdown', 'any') - expect(cleanedFilters).toHaveProperty('breakdown_type', 'event') - }) - it('restores a breakdown type for legacy multiple breakdowns', () => { const cleanedFilters = cleanFilters({ breakdowns: [{ property: 'any' }], @@ -212,20 +218,19 @@ describe('cleanFilters', () => { insight: InsightType.TRENDS, } as TrendsFilterType) - expect(cleanedFilters).toHaveProperty('breakdown', 'any') - expect(cleanedFilters).toHaveProperty('breakdown_type', 'event') - expect(cleanedFilters).toHaveProperty('breakdowns', undefined) + expect(cleanedFilters).toHaveProperty('breakdowns', [{ property: 'any', type: 'event' }]) + expect(cleanedFilters.breakdown_type).toBeUndefined() }) it('cleans a breakdown when multiple breakdowns are used', () => { const cleanedFilters = cleanFilters({ - breakdowns: [{ value: 'any', type: 'event' }], + breakdowns: [{ property: 'any', type: 'event' }], breakdown_type: 'event', breakdown: 'test', insight: InsightType.TRENDS, } as TrendsFilterType) - expect(cleanedFilters).toHaveProperty('breakdowns', [{ value: 'any', type: 'event' }]) + expect(cleanedFilters).toHaveProperty('breakdowns', [{ property: 'any', type: 'event' }]) expect(cleanedFilters).toHaveProperty('breakdown_type', undefined) expect(cleanedFilters).toHaveProperty('breakdown', undefined) }) @@ -279,7 +284,7 @@ describe('cleanFilters', () => { expect(cleanedFilters).toHaveProperty('breakdown_group_type_index', undefined) }) - it('keeps the first multi property filter for trends', () => { + it('keeps a multi property breakdown for trends', () => { const cleanedFilters = cleanFilters({ breakdowns: [ { property: 'one thing', type: 'event' }, @@ -289,10 +294,11 @@ describe('cleanFilters', () => { insight: InsightType.TRENDS, }) - expect(cleanedFilters).toHaveProperty('breakdowns', undefined) - expect(cleanedFilters).toHaveProperty('breakdown', 'one thing') - expect(cleanedFilters).toHaveProperty('breakdown_type', 'event') - expect(cleanedFilters).toHaveProperty('breakdown_group_type_index', undefined) + expect(cleanedFilters).toHaveProperty('breakdowns', [ + { property: 'one thing', type: 'event' }, + { property: 'two thing', type: 'event' }, + ]) + expect(cleanedFilters.breakdown_type).toBeUndefined() }) it('reads "smoothing_intervals" and "interval" from URL when viewing and corrects bad pairings', () => { diff --git a/frontend/src/scenes/insights/utils/cleanFilters.ts b/frontend/src/scenes/insights/utils/cleanFilters.ts index 8f6dea7261a6ac..1fc283cf996b25 100644 --- a/frontend/src/scenes/insights/utils/cleanFilters.ts +++ b/frontend/src/scenes/insights/utils/cleanFilters.ts @@ -157,29 +157,20 @@ const cleanBreakdownParams = (cleanedParams: Partial, filters: Parti if (canMultiPropertyBreakdown && filters.breakdowns && filters.breakdowns.length > 0) { cleanedParams['breakdowns'] = filters.breakdowns } else if (isTrends && filters.breakdowns && filters.breakdowns.length > 0) { - // Clean up a legacy breakdown - if (filters.breakdowns[0].property) { - cleanedParams['breakdown'] = filters.breakdowns[0].property - cleanedParams['breakdown_type'] = filters.breakdowns[0].type || filters.breakdown_type - cleanedParams['breakdown_normalize_url'] = cleanBreakdownNormalizeURL( - cleanedParams['breakdown'] as string, - filters.breakdown_normalize_url - ) - } else { - cleanedParams['breakdown_type'] = undefined - cleanedParams['breakdowns'] = filters.breakdowns - .map((b) => ({ - value: b.value, - type: b.type, - histogram_bin_count: b.histogram_bin_count, - group_type_index: b.group_type_index, - normalize_url: - b.normalize_url && b.value - ? cleanBreakdownNormalizeURL(b.value, filters.breakdown_normalize_url) - : b.normalize_url, - })) - .filter((b) => !!b.value) - } + cleanedParams['breakdown_type'] = undefined + cleanedParams['breakdowns'] = filters.breakdowns.map((b) => ({ + property: b.property, + type: b.type || filters.breakdown_type || 'event', + histogram_bin_count: b.histogram_bin_count, + group_type_index: b.group_type_index, + normalize_url: + typeof b.property === 'string' + ? cleanBreakdownNormalizeURL( + b.property, + typeof b.normalize_url === 'boolean' ? b.normalize_url : filters.breakdown_normalize_url + ) + : undefined, + })) } else if (filters.breakdown) { cleanedParams['breakdown'] = filters.breakdown cleanedParams['breakdown_normalize_url'] = cleanBreakdownNormalizeURL( diff --git a/frontend/src/scenes/insights/utils/compareFilters.test.ts b/frontend/src/scenes/insights/utils/compareFilters.test.ts index 9a9a7d933c6325..ff8d3b0fe8c43f 100644 --- a/frontend/src/scenes/insights/utils/compareFilters.test.ts +++ b/frontend/src/scenes/insights/utils/compareFilters.test.ts @@ -129,7 +129,7 @@ describe('compareFilters', () => { insight: InsightType.TRENDS, breakdowns: [ { - value: '$browser', + property: '$browser', type: 'event', }, ], @@ -143,11 +143,11 @@ describe('compareFilters', () => { insight: InsightType.TRENDS, breakdowns: [ { - value: '$browser', + property: '$browser', type: 'event', }, { - value: '$prop', + property: '$prop', type: 'event', }, ], @@ -156,7 +156,7 @@ describe('compareFilters', () => { insight: InsightType.TRENDS, breakdowns: [ { - value: '$browser', + property: '$browser', type: 'event', }, ], @@ -170,11 +170,11 @@ describe('compareFilters', () => { insight: InsightType.TRENDS, breakdowns: [ { - value: '$browser', + property: '$browser', type: 'event', }, { - value: '$prop', + property: '$prop', type: 'event', }, ], @@ -183,11 +183,11 @@ describe('compareFilters', () => { insight: InsightType.TRENDS, breakdowns: [ { - value: '$browser', + property: '$browser', type: 'event', }, { - value: '$prop', + property: '$prop', type: 'event', }, ], @@ -201,13 +201,13 @@ describe('compareFilters', () => { insight: InsightType.TRENDS, breakdowns: [ { - value: '$browser', + property: '$browser', type: 'group', group_type_index: 1, histogram_bin_count: 10, }, { - value: '$prop', + property: '$pathname', type: 'group', normalize_url: true, }, @@ -217,13 +217,13 @@ describe('compareFilters', () => { insight: InsightType.TRENDS, breakdowns: [ { - value: '$browser', + property: '$browser', type: 'group', group_type_index: 1, histogram_bin_count: 10, }, { - value: '$prop', + property: '$pathname', type: 'group', normalize_url: false, }, @@ -236,13 +236,13 @@ describe('compareFilters', () => { insight: InsightType.TRENDS, breakdowns: [ { - value: '$browser', + property: '$browser', type: 'group', group_type_index: 0, histogram_bin_count: 10, }, { - value: '$prop', + property: '$pathname', type: 'group', normalize_url: true, }, diff --git a/frontend/src/scenes/insights/views/InsightsTable/InsightsTable.tsx b/frontend/src/scenes/insights/views/InsightsTable/InsightsTable.tsx index a4f4fb23d72841..0917ebf802747e 100644 --- a/frontend/src/scenes/insights/views/InsightsTable/InsightsTable.tsx +++ b/frontend/src/scenes/insights/views/InsightsTable/InsightsTable.tsx @@ -180,7 +180,7 @@ export function InsightsTable({ ) columns.push({ - title: {breakdown.value?.toString()}, + title: {breakdown.property?.toString()}, render: (_, item) => ( ), - key: `breakdown-${breakdown.value?.toString() || index}`, + key: `breakdown-${breakdown.property?.toString() || index}`, sorter: (a, b) => { const leftValue = Array.isArray(a.breakdown_value) ? a.breakdown_value[index] : a.breakdown_value const rightValue = Array.isArray(b.breakdown_value) ? b.breakdown_value[index] : b.breakdown_value diff --git a/frontend/src/scenes/notebooks/Nodes/NotebookNodePlaylist.tsx b/frontend/src/scenes/notebooks/Nodes/NotebookNodePlaylist.tsx index 555ac064846b36..b97ddaca04e927 100644 --- a/frontend/src/scenes/notebooks/Nodes/NotebookNodePlaylist.tsx +++ b/frontend/src/scenes/notebooks/Nodes/NotebookNodePlaylist.tsx @@ -1,10 +1,8 @@ import { createPostHogWidgetNode } from 'scenes/notebooks/Nodes/NodeWrapper' -import { FilterType, NotebookNodeType, RecordingFilters, RecordingUniversalFilters, ReplayTabs } from '~/types' +import { FilterType, NotebookNodeType, RecordingUniversalFilters, ReplayTabs } from '~/types' import { - DEFAULT_SIMPLE_RECORDING_FILTERS, SessionRecordingPlaylistLogicProps, convertLegacyFiltersToUniversalFilters, - getDefaultFilters, sessionRecordingsPlaylistLogic, } from 'scenes/session-recordings/playlist/sessionRecordingsPlaylistLogic' import { BuiltLogic, useActions, useValues } from 'kea' @@ -12,33 +10,27 @@ import { useEffect, useMemo } from 'react' import { urls } from 'scenes/urls' import { notebookNodeLogic } from './notebookNodeLogic' import { JSONContent, NotebookNodeProps, NotebookNodeAttributeProperties } from '../Notebook/utils' -import { SessionRecordingsFilters } from 'scenes/session-recordings/filters/SessionRecordingsFilters' import { ErrorBoundary } from '@sentry/react' import { SessionRecordingsPlaylist } from 'scenes/session-recordings/playlist/SessionRecordingsPlaylist' import { sessionRecordingPlayerLogic } from 'scenes/session-recordings/player/sessionRecordingPlayerLogic' import { IconComment } from 'lib/lemon-ui/icons' import { sessionRecordingPlayerLogicType } from 'scenes/session-recordings/player/sessionRecordingPlayerLogicType' -import { useFeatureFlag } from 'lib/hooks/useFeatureFlag' import { RecordingsUniversalFilters } from 'scenes/session-recordings/filters/RecordingsUniversalFilters' const Component = ({ attributes, updateAttributes, }: NotebookNodeProps): JSX.Element => { - const { filters, simpleFilters, pinned, nodeId, universalFilters } = attributes + const { pinned, nodeId, universalFilters } = attributes const playerKey = `notebook-${nodeId}` const recordingPlaylistLogicProps: SessionRecordingPlaylistLogicProps = useMemo( () => ({ logicKey: playerKey, - advancedFilters: filters, - simpleFilters, - universalFilters, + filters: universalFilters, updateSearchParams: false, autoPlay: false, - onFiltersChange: (newFilters, legacyFilters) => { - updateAttributes({ universalFilters: newFilters, filters: legacyFilters }) - }, + onFiltersChange: (newFilters) => updateAttributes({ universalFilters: newFilters }), pinnedRecordings: pinned, onPinnedChange(recording, isPinned) { updateAttributes({ @@ -48,7 +40,7 @@ const Component = ({ }) }, }), - [playerKey, filters, pinned] + [playerKey, universalFilters, pinned] ) const { setActions, insertAfter, insertReplayCommentByTimestamp, setMessageListeners, scrollIntoView } = @@ -120,30 +112,15 @@ export const Settings = ({ attributes, updateAttributes, }: NotebookNodeAttributeProperties): JSX.Element => { - const { filters, simpleFilters, universalFilters } = attributes - const defaultFilters = getDefaultFilters() - const hasUniversalFiltering = useFeatureFlag('SESSION_REPLAY_UNIVERSAL_FILTERS') + const { universalFilters: filters } = attributes - const setUniversalFilters = (filters: Partial): void => { - updateAttributes({ universalFilters: { ...universalFilters, ...filters } }) + const setFilters = (newFilters: Partial): void => { + updateAttributes({ universalFilters: { ...filters, ...newFilters } }) } return ( - {hasUniversalFiltering ? ( - - ) : ( - updateAttributes({ filters })} - setSimpleFilters={(simpleFilters) => updateAttributes({ simpleFilters })} - showPropertyFilters - onReset={() => - updateAttributes({ filters: defaultFilters, simpleFilters: DEFAULT_SIMPLE_RECORDING_FILTERS }) - } - /> - )} + ) } @@ -151,9 +128,6 @@ export const Settings = ({ export type NotebookNodePlaylistAttributes = { universalFilters: RecordingUniversalFilters pinned?: string[] - // TODO: these filters are now deprecated and will be removed once we rollout universal filters to everyone - filters: RecordingFilters - simpleFilters?: RecordingFilters } export const NotebookNodePlaylist = createPostHogWidgetNode({ @@ -163,17 +137,11 @@ export const NotebookNodePlaylist = createPostHogWidgetNode { // TODO: Fix parsing of attrs - return urls.replay(undefined, attrs.filters) + return urls.replay(undefined, attrs.universalFilters) }, resizeable: true, expandable: false, attributes: { - filters: { - default: undefined, - }, - simpleFilters: { - default: {}, - }, universalFilters: { default: undefined, }, diff --git a/frontend/src/scenes/notebooks/Notebook/migrations/migrate.ts b/frontend/src/scenes/notebooks/Notebook/migrations/migrate.ts index b48491ffdbb917..b90653f251135c 100644 --- a/frontend/src/scenes/notebooks/Notebook/migrations/migrate.ts +++ b/frontend/src/scenes/notebooks/Notebook/migrations/migrate.ts @@ -33,7 +33,7 @@ import { TrendsFilter, TrendsFilterLegacy, } from '~/queries/schema' -import { FunnelExclusionLegacy, NotebookNodeType, NotebookType } from '~/types' +import { FunnelExclusionLegacy, NotebookNodeType, NotebookType, RecordingFilters } from '~/types' // NOTE: Increment this number when you add a new content migration // It will bust the cache on the localContent in the notebookLogic @@ -61,7 +61,11 @@ function convertPlaylistFiltersToUniversalFilters(content: JSONContent[]): JSONC return node } - const { simpleFilters, filters, universalFilters } = node.attrs as NotebookNodePlaylistAttributes + // Legacy attrs on Notebook playlist nodes + const simpleFilters = node.attrs?.simpleFilters as RecordingFilters + const filters = node.attrs?.filters as RecordingFilters + + const { universalFilters } = node.attrs as NotebookNodePlaylistAttributes if (universalFilters) { return node diff --git a/frontend/src/scenes/pipeline/AppMetricSparkLine.tsx b/frontend/src/scenes/pipeline/AppMetricSparkLine.tsx index e4ef8770e7922e..ed14e30ec0fddb 100644 --- a/frontend/src/scenes/pipeline/AppMetricSparkLine.tsx +++ b/frontend/src/scenes/pipeline/AppMetricSparkLine.tsx @@ -1,7 +1,9 @@ -import { useValues } from 'kea' +import { useActions, useValues } from 'kea' import { Sparkline, SparklineTimeSeries } from 'lib/components/Sparkline' +import { useEffect } from 'react' import { pipelineNodeMetricsLogic } from './pipelineNodeMetricsLogic' +import { pipelineNodeMetricsV2Logic } from './pipelineNodeMetricsV2Logic' import { PipelineBackend, PipelineNode } from './types' export function AppMetricSparkLine({ pipelineNode }: { pipelineNode: PipelineNode }): JSX.Element { @@ -34,3 +36,28 @@ export function AppMetricSparkLine({ pipelineNode }: { pipelineNode: PipelineNod } return } + +export function AppMetricSparkLineV2({ pipelineNode }: { pipelineNode: PipelineNode }): JSX.Element { + const logic = pipelineNodeMetricsV2Logic({ id: `${pipelineNode.id}`.replace('hog-', '') }) + const { appMetrics, appMetricsLoading } = useValues(logic) + const { loadMetrics } = useActions(logic) + + useEffect(() => { + loadMetrics() + }, []) + + const displayData: SparklineTimeSeries[] = [ + { + color: 'success', + name: 'Success', + values: appMetrics?.series.find((s) => s.name === 'succeeded')?.values || [], + }, + { + color: 'danger', + name: 'Failures', + values: appMetrics?.series.find((s) => s.name === 'failed')?.values || [], + }, + ] + + return +} diff --git a/frontend/src/scenes/pipeline/BatchExportBackfill.tsx b/frontend/src/scenes/pipeline/BatchExportBackfillModal.tsx similarity index 97% rename from frontend/src/scenes/pipeline/BatchExportBackfill.tsx rename to frontend/src/scenes/pipeline/BatchExportBackfillModal.tsx index 2608508ac5c2bc..60ecb00763c6cc 100644 --- a/frontend/src/scenes/pipeline/BatchExportBackfill.tsx +++ b/frontend/src/scenes/pipeline/BatchExportBackfillModal.tsx @@ -7,7 +7,7 @@ import { LemonModal } from 'lib/lemon-ui/LemonModal' import { batchExportRunsLogic, BatchExportRunsLogicProps } from './batchExportRunsLogic' -export function BatchExportBackfill({ id }: BatchExportRunsLogicProps): JSX.Element { +export function BatchExportBackfillModal({ id }: BatchExportRunsLogicProps): JSX.Element { const logic = batchExportRunsLogic({ id }) const { batchExportConfig, isBackfillModalOpen, isBackfillFormSubmitting } = useValues(logic) diff --git a/frontend/src/scenes/pipeline/BatchExportRuns.tsx b/frontend/src/scenes/pipeline/BatchExportRuns.tsx index b180f9c105d242..ae8020b866e0e1 100644 --- a/frontend/src/scenes/pipeline/BatchExportRuns.tsx +++ b/frontend/src/scenes/pipeline/BatchExportRuns.tsx @@ -4,13 +4,14 @@ import { LemonButton, LemonDialog, LemonSwitch, LemonTable } from '@posthog/lemo import { useActions, useValues } from 'kea' import { DateFilter } from 'lib/components/DateFilter/DateFilter' import { NotFound } from 'lib/components/NotFound' +import { PageHeader } from 'lib/components/PageHeader' import { IconRefresh } from 'lib/lemon-ui/icons' import { BatchExportRunIcon } from 'scenes/batch_exports/components' import { isRunInProgress } from 'scenes/batch_exports/utils' import { BatchExportConfiguration, GroupedBatchExportRuns } from '~/types' -import { BatchExportBackfill } from './BatchExportBackfill' +import { BatchExportBackfillModal } from './BatchExportBackfillModal' import { batchExportRunsLogic, BatchExportRunsLogicProps } from './batchExportRunsLogic' import { pipelineAccessLogic } from './pipelineAccessLogic' @@ -18,75 +19,76 @@ export function BatchExportRuns({ id }: BatchExportRunsLogicProps): JSX.Element const logic = batchExportRunsLogic({ id }) const { batchExportConfig, groupedRuns, loading, hasMoreRunsToLoad, usingLatestRuns } = useValues(logic) - const { loadOlderRuns, retryRun } = useActions(logic) + const { loadOlderRuns, retryRun, openBackfillModal } = useActions(logic) if (!batchExportConfig) { return } - const dateSelector = - - if (usingLatestRuns) { - return ( - <> - {dateSelector} - - - ) - } - return ( <> - {dateSelector} - openBackfillModal()}> + Backfill batch export + + } /> +
+ + {usingLatestRuns ? ( + + ) : ( + + )} +
+ ) } -export function RunsDateSelection({ id }: { id: string }): JSX.Element { +function BatchExportRunsFilters({ id }: { id: string }): JSX.Element { const logic = batchExportRunsLogic({ id }) const { dateRange, usingLatestRuns, loading } = useValues(logic) const { setDateRange, switchLatestRuns, loadRuns } = useActions(logic) - // TODO: Autoload option similar to frontend/src/queries/nodes/DataNode/AutoLoad.tsx - const latestToggle = ( - - ) - - const dateFilter = ( - setDateRange(from, to)} - allowedRollingDateOptions={['hours', 'days', 'weeks', 'months', 'years']} - makeLabel={(key) => ( - <> - {key} - - )} - /> - ) return ( -
- }> +
+ } size="small"> Refresh - {latestToggle} - {dateFilter} + + setDateRange(from, to)} + allowedRollingDateOptions={['hours', 'days', 'weeks', 'months', 'years']} + makeLabel={(key) => ( + <> + {key} + + )} + />
) } -export function BatchExportLatestRuns({ id }: BatchExportRunsLogicProps): JSX.Element { +function BatchExportLatestRuns({ id }: BatchExportRunsLogicProps): JSX.Element { const logic = batchExportRunsLogic({ id }) const { batchExportConfig, latestRuns, loading, hasMoreRunsToLoad } = useValues(logic) @@ -163,24 +165,24 @@ export function BatchExportLatestRuns({ id }: BatchExportRunsLogicProps): JSX.El width: 0, render: function RenderActions(_, run) { if (canEnableNewDestinations) { - return + return } }, }, ]} emptyState={ - <> - No runs in this time range. Your exporter runs every {batchExportConfig.interval}. -
+
+
+ No runs in this time range. Your exporter runs every {batchExportConfig.interval}. +
{canEnableNewDestinations && ( openBackfillModal()}> Backfill batch export )} - +
} /> - ) } @@ -302,30 +304,29 @@ export function BatchExportRunsGrouped({ width: 0, render: function RenderActions(_, groupedRun) { if (!isRunInProgress(groupedRun.runs[0]) && canEnableNewDestinations) { - return + return } }, }, ]} emptyState={ - <> - No runs in this time range. Your exporter runs every {interval}. -
+
+
+ No runs in this time range. Your exporter runs every {interval}. +
{canEnableNewDestinations && ( openBackfillModal()}> Backfill batch export )} - +
} /> - - ) } -export function RunRetryModal({ run, retryRun }: { run: any; retryRun: any }): JSX.Element { +function RunRetryButton({ run, retryRun }: { run: any; retryRun: any }): JSX.Element { return ( + } + /> - } return ( - : undefined} - /> router.actions.push(urls.pipeline(tab as PipelineTab))} diff --git a/frontend/src/scenes/pipeline/PipelineNode.tsx b/frontend/src/scenes/pipeline/PipelineNode.tsx index 1dcfb5ca63fd50..ea982b38268893 100644 --- a/frontend/src/scenes/pipeline/PipelineNode.tsx +++ b/frontend/src/scenes/pipeline/PipelineNode.tsx @@ -14,6 +14,7 @@ import { BatchExportRuns } from './BatchExportRuns' import { PipelineNodeConfiguration } from './PipelineNodeConfiguration' import { pipelineNodeLogic, PipelineNodeLogicProps } from './pipelineNodeLogic' import { PipelineNodeMetrics } from './PipelineNodeMetrics' +import { PipelineNodeMetricsV2 } from './PipelineNodeMetricsV2' import { PipelineBackend } from './types' export const PIPELINE_TAB_TO_NODE_STAGE: Partial> = { @@ -58,10 +59,8 @@ export function PipelineNode(params: { stage?: string; id?: string } = {}): JSX. [PipelineNodeTab.Configuration]: , } - if ([PipelineBackend.Plugin, PipelineBackend.BatchExport].includes(node.backend)) { - tabToContent[PipelineNodeTab.Metrics] = - } - + tabToContent[PipelineNodeTab.Metrics] = + node.backend === PipelineBackend.HogFunction ? : tabToContent[PipelineNodeTab.Logs] = if (node.backend === PipelineBackend.BatchExport) { diff --git a/frontend/src/scenes/pipeline/PipelineNodeMetricsV2.tsx b/frontend/src/scenes/pipeline/PipelineNodeMetricsV2.tsx new file mode 100644 index 00000000000000..7d16f4ea74329f --- /dev/null +++ b/frontend/src/scenes/pipeline/PipelineNodeMetricsV2.tsx @@ -0,0 +1,251 @@ +import { IconCalendar } from '@posthog/icons' +import { LemonSelect, LemonSkeleton, Popover, SpinnerOverlay, Tooltip } from '@posthog/lemon-ui' +import { BindLogic, useActions, useValues } from 'kea' +import { Chart, ChartDataset, ChartItem } from 'lib/Chart' +import { getColorVar } from 'lib/colors' +import { DateFilter } from 'lib/components/DateFilter/DateFilter' +import { humanFriendlyNumber, inStorybookTestRunner } from 'lib/utils' +import { useEffect, useRef, useState } from 'react' +import { InsightTooltip } from 'scenes/insights/InsightTooltip/InsightTooltip' + +import { pipelineNodeLogic } from './pipelineNodeLogic' +import { pipelineNodeMetricsV2Logic } from './pipelineNodeMetricsV2Logic' +import { PipelineBackend } from './types' + +const METRICS_INFO = { + succeeded: 'Total number of events processed successfully', + failed: 'Total number of events that had errors during processing', + filtered: 'Total number of events that were filtered out', + disabled_temporarily: + 'Total number of events that were skipped due to the destination being temporarily disabled (due to issues such as the destination being down or rate-limited)', + disabled_permanently: + 'Total number of events that were skipped due to the destination being permanently disabled (due to prolonged issues with the destination)', +} + +export function PipelineNodeMetricsV2(): JSX.Element { + const { node } = useValues(pipelineNodeLogic) + + const logic = pipelineNodeMetricsV2Logic({ id: `${node.id}` }) + + const { filters } = useValues(logic) + const { setFilters, loadMetrics, loadMetricsTotals } = useActions(logic) + + useEffect(() => { + loadMetrics() + loadMetricsTotals() + }, []) + + if (node.backend !== PipelineBackend.HogFunction) { + return
Metrics not available for this node
+ } + + return ( + +
+ + +
+

Delivery trends

+
+ setFilters({ interval: value })} + /> + setFilters({ after: from || undefined, before: to || undefined })} + allowedRollingDateOptions={['days', 'weeks', 'months', 'years']} + makeLabel={(key) => ( + <> + {key} + + )} + /> +
+ + +
+ + ) +} + +function AppMetricBigNumber({ + label, + value, + tooltip, +}: { + label: string + value: number | undefined + tooltip: JSX.Element | string +}): JSX.Element { + return ( + +
+
{label.replace(/_/g, ' ')}
+
{humanFriendlyNumber(value ?? 0)}
+
+ + ) +} + +function AppMetricsTotals(): JSX.Element { + const { appMetricsTotals, appMetricsTotalsLoading } = useValues(pipelineNodeMetricsV2Logic) + + return ( +
+
+ {Object.entries(METRICS_INFO).map(([key, value]) => ( +
+ {appMetricsTotalsLoading ? ( + + ) : ( + + )} +
+ ))} +
+
+ ) +} + +function AppMetricsGraph(): JSX.Element { + const { appMetrics, appMetricsLoading } = useValues(pipelineNodeMetricsV2Logic) + const canvasRef = useRef(null) + const [popoverContent, setPopoverContent] = useState(null) + const [tooltipState, setTooltipState] = useState({ x: 0, y: 0, visible: false }) + + useEffect(() => { + let chart: Chart + if (canvasRef.current && appMetrics && !inStorybookTestRunner()) { + chart = new Chart(canvasRef.current?.getContext('2d') as ChartItem, { + type: 'line', + data: { + labels: appMetrics.labels, + datasets: [ + ...appMetrics.series.map((series) => ({ + label: series.name, + data: series.values, + borderColor: '', + ...colorConfig(series.name), + })), + ], + }, + options: { + scales: { + x: { + ticks: { + maxRotation: 0, + }, + grid: { + display: false, + }, + }, + y: { + beginAtZero: true, + }, + }, + plugins: { + // @ts-expect-error Types of library are out of date + crosshair: false, + legend: { + display: false, + }, + tooltip: { + enabled: false, // Using external tooltip + external({ tooltip, chart }) { + setPopoverContent( + ({ + id: i, + dataIndex: 0, + datasetIndex: 0, + label: dp.dataset.label, + color: dp.dataset.borderColor as string, + count: (dp.dataset.data?.[dp.dataIndex] as number) || 0, + }))} + renderSeries={(value) => value} + renderCount={(count) => humanFriendlyNumber(count)} + /> + ) + + const position = chart.canvas.getBoundingClientRect() + setTooltipState({ + x: position.left + tooltip.caretX, + y: position.top + tooltip.caretY, + visible: tooltip.opacity > 0, + }) + }, + }, + }, + maintainAspectRatio: false, + interaction: { + mode: 'index', + axis: 'x', + intersect: false, + }, + }, + }) + + return () => { + chart?.destroy() + } + } + }, [appMetrics]) + + return ( +
+ {appMetricsLoading && } + {!!appMetrics && } + +
+ +
+ ) +} + +function colorConfig(name: string): Partial> { + let color = '' + + switch (name) { + case 'succeeded': + color = getColorVar('success') + break + case 'failed': + color = getColorVar('danger') + break + default: + color = getColorVar('data-color-1') + break + } + + return { + borderColor: color, + hoverBorderColor: color, + hoverBackgroundColor: color, + backgroundColor: color, + fill: false, + borderWidth: 2, + pointRadius: 0, + } +} diff --git a/frontend/src/scenes/pipeline/PipelineNodeNew.tsx b/frontend/src/scenes/pipeline/PipelineNodeNew.tsx index 5b3ed44d1bee49..661012c54b061a 100644 --- a/frontend/src/scenes/pipeline/PipelineNodeNew.tsx +++ b/frontend/src/scenes/pipeline/PipelineNodeNew.tsx @@ -96,21 +96,13 @@ export function PipelineNodeNew(params: { stage?: string; id?: string } = {}): J } if (hogFunctionId) { - const res = - if (stage === PipelineStage.Destination) { - return {res} - } - return res + return } if (stage === PipelineStage.Transformation) { return } else if (stage === PipelineStage.Destination) { - return ( - - - - ) + return } else if (stage === PipelineStage.SiteApp) { return } diff --git a/frontend/src/scenes/pipeline/Transformations.tsx b/frontend/src/scenes/pipeline/Transformations.tsx index 1f728763a19fdb..48f7fecb1d8d5d 100644 --- a/frontend/src/scenes/pipeline/Transformations.tsx +++ b/frontend/src/scenes/pipeline/Transformations.tsx @@ -2,16 +2,19 @@ import { DndContext, DragEndEvent } from '@dnd-kit/core' import { restrictToParentElement, restrictToVerticalAxis } from '@dnd-kit/modifiers' import { arrayMove, SortableContext, useSortable, verticalListSortingStrategy } from '@dnd-kit/sortable' import { CSS } from '@dnd-kit/utilities' -import { LemonBadge, LemonButton, LemonModal, LemonTable, LemonTableColumn } from '@posthog/lemon-ui' +import { LemonBadge, LemonButton, LemonModal, LemonTable, LemonTableColumn, Link } from '@posthog/lemon-ui' import { useActions, useValues } from 'kea' +import { PageHeader } from 'lib/components/PageHeader' import { ProductIntroduction } from 'lib/components/ProductIntroduction/ProductIntroduction' import { More } from 'lib/lemon-ui/LemonButton/More' import { LemonMenuOverlay } from 'lib/lemon-ui/LemonMenu/LemonMenu' import { statusColumn, updatedAtColumn } from 'lib/lemon-ui/LemonTable/columnUtils' import { PluginImage } from 'scenes/plugins/plugin/PluginImage' +import { urls } from 'scenes/urls' -import { PipelineStage, ProductKey } from '~/types' +import { PipelineNodeTab, PipelineStage, ProductKey } from '~/types' +import { AppMetricSparkLine } from './AppMetricSparkLine' import { NewButton } from './NewButton' import { pipelineAccessLogic } from './pipelineAccessLogic' import { pipelineTransformationsLogic } from './transformationsLogic' @@ -27,6 +30,10 @@ export function Transformations(): JSX.Element { return ( <> + } + /> , nameColumn() as LemonTableColumn, + { + title: 'Weekly volume', + render: function RenderSuccessRate(_, transformation) { + return ( + + + + ) + }, + }, updatedAtColumn() as LemonTableColumn, statusColumn() as LemonTableColumn, { diff --git a/frontend/src/scenes/pipeline/destinations/Destinations.tsx b/frontend/src/scenes/pipeline/destinations/Destinations.tsx index cb7c4c365c9988..0dc46d76e3d2c2 100644 --- a/frontend/src/scenes/pipeline/destinations/Destinations.tsx +++ b/frontend/src/scenes/pipeline/destinations/Destinations.tsx @@ -9,6 +9,7 @@ import { Tooltip, } from '@posthog/lemon-ui' import { BindLogic, useActions, useValues } from 'kea' +import { PageHeader } from 'lib/components/PageHeader' import { PayGateMini } from 'lib/components/PayGateMini/PayGateMini' import { ProductIntroduction } from 'lib/components/ProductIntroduction/ProductIntroduction' import { useFeatureFlag } from 'lib/hooks/useFeatureFlag' @@ -20,7 +21,7 @@ import { urls } from 'scenes/urls' import { AvailableFeature, PipelineNodeTab, PipelineStage, ProductKey } from '~/types' -import { AppMetricSparkLine } from '../AppMetricSparkLine' +import { AppMetricSparkLine, AppMetricSparkLineV2 } from '../AppMetricSparkLine' import { HogFunctionIcon } from '../hogfunctions/HogFunctionIcon' import { NewButton } from '../NewButton' import { pipelineAccessLogic } from '../pipelineAccessLogic' @@ -33,7 +34,11 @@ export function Destinations(): JSX.Element { return ( <> - + } + /> + - + {destination.backend === PipelineBackend.HogFunction ? ( + + ) : ( + + )} ) }, diff --git a/frontend/src/scenes/pipeline/destinations/NewDestinations.tsx b/frontend/src/scenes/pipeline/destinations/NewDestinations.tsx index 8f21b9d6a71a6c..ad91ad46e5c95d 100644 --- a/frontend/src/scenes/pipeline/destinations/NewDestinations.tsx +++ b/frontend/src/scenes/pipeline/destinations/NewDestinations.tsx @@ -1,28 +1,38 @@ import { IconPlusSmall } from '@posthog/icons' -import { LemonButton, LemonInput, LemonSelect, LemonTable, LemonTag } from '@posthog/lemon-ui' +import { LemonButton, LemonInput, LemonSelect, LemonTable, LemonTag, Link } from '@posthog/lemon-ui' import { useActions, useValues } from 'kea' +import { capitalizeFirstLetter } from 'kea-forms' +import { PayGateButton } from 'lib/components/PayGateMini/PayGateButton' +import { PayGateMini } from 'lib/components/PayGateMini/PayGateMini' import { useFeatureFlag } from 'lib/hooks/useFeatureFlag' import { LemonTableLink } from 'lib/lemon-ui/LemonTable/LemonTableLink' -import { PipelineStage } from '~/types' +import { AvailableFeature, PipelineStage } from '~/types' +import { pipelineAccessLogic } from '../pipelineAccessLogic' import { PipelineBackend } from '../types' import { newDestinationsLogic } from './newDestinationsLogic' export function DestinationOptionsTable(): JSX.Element { const hogFunctionsEnabled = !!useFeatureFlag('HOG_FUNCTIONS') const { loading, filteredDestinations, filters } = useValues(newDestinationsLogic) - const { setFilters } = useActions(newDestinationsLogic) + const { setFilters, openFeedbackDialog } = useActions(newDestinationsLogic) + const { canEnableNewDestinations } = useValues(pipelineAccessLogic) return ( - <> -
+
+ + +
setFilters({ search: e })} /> + openFeedbackDialog()}> + Can't find what you're looking for? +
{target.name} {target.status === 'alpha' ? ( - Experimental + Experimental ) : target.status === 'beta' ? ( - Beta + Beta ) : target.status === 'stable' ? ( - New // Once Hog Functions are fully released we can remove the new label + New // Once Hog Functions are fully released we can remove the new label + ) : target.status ? ( + + {capitalizeFirstLetter(target.status)} + ) : undefined} } @@ -80,21 +98,26 @@ export function DestinationOptionsTable(): JSX.Element { width: 100, align: 'right', render: function RenderActions(_, target) { - return ( + return canEnableNewDestinations || target.status === 'free' ? ( } // Preserve hash params to pass config in to={target.url} + fullWidth > Create + ) : ( + + + ) }, }, ]} /> - +
) } diff --git a/frontend/src/scenes/pipeline/destinations/newDestinationsLogic.tsx b/frontend/src/scenes/pipeline/destinations/newDestinationsLogic.tsx index 80f554ebf359eb..a0aecfca38babf 100644 --- a/frontend/src/scenes/pipeline/destinations/newDestinationsLogic.tsx +++ b/frontend/src/scenes/pipeline/destinations/newDestinationsLogic.tsx @@ -1,9 +1,12 @@ +import { LemonDialog, LemonInput, LemonTextArea, lemonToast } from '@posthog/lemon-ui' import FuseClass from 'fuse.js' -import { actions, afterMount, connect, kea, path, reducers, selectors } from 'kea' +import { actions, afterMount, connect, kea, listeners, path, reducers, selectors } from 'kea' import { loaders } from 'kea-loaders' import { actionToUrl, combineUrl, router, urlToAction } from 'kea-router' import api from 'lib/api' +import { LemonField } from 'lib/lemon-ui/LemonField' import { objectsEqual } from 'lib/utils' +import posthog from 'posthog-js' import { urls } from 'scenes/urls' import { userLogic } from 'scenes/userLogic' @@ -26,7 +29,7 @@ export type NewDestinationItemType = { name: string description: string backend: PipelineBackend - status?: 'stable' | 'beta' | 'alpha' + status?: 'stable' | 'beta' | 'alpha' | 'free' } export type NewDestinationFilters = { @@ -45,6 +48,7 @@ export const newDestinationsLogic = kea([ actions({ setFilters: (filters: Partial) => ({ filters }), resetFilters: true, + openFeedbackDialog: true, }), reducers({ filters: [ @@ -159,6 +163,41 @@ export const newDestinationsLogic = kea([ ], })), + listeners(({ values }) => ({ + setFilters: async ({ filters }, breakpoint) => { + if (filters.search && filters.search.length > 2) { + await breakpoint(1000) + posthog.capture('cdp destination search', { search: filters.search }) + } + }, + + openFeedbackDialog: async (_, breakpoint) => { + await breakpoint(100) + LemonDialog.openForm({ + title: 'What destination would you like to see?', + initialValues: { destination_name: values.filters.search }, + errors: { + destination_name: (x) => (!x ? 'Required' : undefined), + }, + description: undefined, + content: ( +
+ + + + + + +
+ ), + onSubmit: async (values) => { + posthog.capture('cdp destination feedback', { ...values }) + lemonToast.success('Thank you for your feedback!') + }, + }) + }, + })), + actionToUrl(({ values }) => { const urlFromFilters = (): [ string, diff --git a/frontend/src/scenes/pipeline/hogfunctions/HogFunctionConfiguration.tsx b/frontend/src/scenes/pipeline/hogfunctions/HogFunctionConfiguration.tsx index e1a4d29fc00293..a3122378587c3c 100644 --- a/frontend/src/scenes/pipeline/hogfunctions/HogFunctionConfiguration.tsx +++ b/frontend/src/scenes/pipeline/hogfunctions/HogFunctionConfiguration.tsx @@ -15,6 +15,7 @@ import { BindLogic, useActions, useValues } from 'kea' import { Form } from 'kea-forms' import { NotFound } from 'lib/components/NotFound' import { PageHeader } from 'lib/components/PageHeader' +import { PayGateMini } from 'lib/components/PayGateMini/PayGateMini' import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types' import { TestAccountFilterSwitch } from 'lib/components/TestAccountFiltersSwitch' import { useFeatureFlag } from 'lib/hooks/useFeatureFlag' @@ -25,7 +26,7 @@ import { ActionFilter } from 'scenes/insights/filters/ActionFilter/ActionFilter' import { MathAvailability } from 'scenes/insights/filters/ActionFilter/ActionFilterRow/ActionFilterRow' import { groupsModel } from '~/models/groupsModel' -import { EntityTypes } from '~/types' +import { AvailableFeature, EntityTypes } from '~/types' import { hogFunctionConfigurationLogic } from './hogFunctionConfigurationLogic' import { HogFunctionIconEditable } from './HogFunctionIcon' @@ -46,6 +47,8 @@ export function HogFunctionConfiguration({ templateId, id }: { templateId?: stri hogFunction, willReEnableOnSave, exampleInvocationGlobalsWithInputs, + showPaygate, + hasAddon, } = useValues(logic) const { submitConfiguration, @@ -126,6 +129,10 @@ export function HogFunctionConfiguration({ templateId, id }: { templateId?: stri ) + if (showPaygate) { + return + } + return (
@@ -359,6 +366,11 @@ export function HogFunctionConfiguration({ templateId, id }: { templateId?: stri size="xsmall" type="secondary" onClick={() => setShowSource(true)} + disabledReason={ + !hasAddon + ? 'Editing the source code requires the Data Pipelines addon' + : undefined + } > Show function source code diff --git a/frontend/src/scenes/pipeline/hogfunctions/hogFunctionConfigurationLogic.tsx b/frontend/src/scenes/pipeline/hogfunctions/hogFunctionConfigurationLogic.tsx index 9b8f3d58e67ffa..5b66a92a708b13 100644 --- a/frontend/src/scenes/pipeline/hogfunctions/hogFunctionConfigurationLogic.tsx +++ b/frontend/src/scenes/pipeline/hogfunctions/hogFunctionConfigurationLogic.tsx @@ -10,9 +10,11 @@ import { uuid } from 'lib/utils' import { deleteWithUndo } from 'lib/utils/deleteWithUndo' import { teamLogic } from 'scenes/teamLogic' import { urls } from 'scenes/urls' +import { userLogic } from 'scenes/userLogic' import { groupsModel } from '~/models/groupsModel' import { + AvailableFeature, FilterType, HogFunctionConfigurationType, HogFunctionInputType, @@ -121,12 +123,9 @@ export const hogFunctionConfigurationLogic = kea ['scenes', 'pipeline', 'hogFunctionConfigurationLogic', id]), - connect({ - values: [teamLogic, ['currentTeam'], groupsModel, ['groupTypes']], - }), actions({ setShowSource: (showSource: boolean) => ({ showSource }), resetForm: (configuration?: HogFunctionConfigurationType) => ({ configuration }), @@ -205,9 +204,13 @@ export const hogFunctionConfigurationLogic = kea { const payload = sanitizeConfiguration(data) - if (props.templateId) { - // Only sent on create - ;(payload as any).template_id = props.templateId + // Only sent on create + ;(payload as any).template_id = props.templateId || values.hogFunction?.template?.id + + if (!values.hasAddon) { + // Remove the source field if the user doesn't have the addon + delete payload.hog + delete payload.inputs_schema } await asyncActions.upsertHogFunction(payload) @@ -215,6 +218,18 @@ export const hogFunctionConfigurationLogic = kea ({ + hasAddon: [ + (s) => [s.hasAvailableFeature], + (hasAvailableFeature) => { + return hasAvailableFeature(AvailableFeature.DATA_PIPELINES) + }, + ], + showPaygate: [ + (s) => [s.template, s.hasAddon], + (template, hasAddon) => { + return template && template.status !== 'free' && !hasAddon + }, + ], defaultFormState: [ (s) => [s.template, s.hogFunction], (template, hogFunction): HogFunctionConfigurationType => { diff --git a/frontend/src/scenes/pipeline/pipelineNodeLogic.tsx b/frontend/src/scenes/pipeline/pipelineNodeLogic.tsx index 67a753a428f597..a96681205f7782 100644 --- a/frontend/src/scenes/pipeline/pipelineNodeLogic.tsx +++ b/frontend/src/scenes/pipeline/pipelineNodeLogic.tsx @@ -107,8 +107,8 @@ export const pipelineNodeLogic = kea([ setCurrentTab: () => [urls.pipelineNode(props.stage as PipelineStage, props.id, values.currentTab)], } }), - urlToAction(({ actions, values }) => ({ - '/pipeline/:stage/:id/:nodeTab': ({ nodeTab }) => { + urlToAction(({ props, actions, values }) => ({ + [urls.pipelineNode(props.stage as PipelineStage, props.id, ':nodeTab')]: ({ nodeTab }) => { if (nodeTab !== values.currentTab && Object.values(PipelineNodeTab).includes(nodeTab as PipelineNodeTab)) { actions.setCurrentTab(nodeTab as PipelineNodeTab) } diff --git a/frontend/src/scenes/pipeline/pipelineNodeMetricsV2Logic.tsx b/frontend/src/scenes/pipeline/pipelineNodeMetricsV2Logic.tsx new file mode 100644 index 00000000000000..2564a6562ed123 --- /dev/null +++ b/frontend/src/scenes/pipeline/pipelineNodeMetricsV2Logic.tsx @@ -0,0 +1,68 @@ +import { actions, kea, key, listeners, path, props, reducers } from 'kea' +import { loaders } from 'kea-loaders' +import api from 'lib/api' + +import { AppMetricsTotalsV2Response, AppMetricsV2RequestParams, AppMetricsV2Response } from '~/types' + +import type { pipelineNodeMetricsV2LogicType } from './pipelineNodeMetricsV2LogicType' + +export type PipelineNodeMetricsProps = { + id: string +} + +export type MetricsFilters = Pick + +const DEFAULT_FILTERS: MetricsFilters = { + before: undefined, + after: '-7d', + interval: 'day', +} + +export const pipelineNodeMetricsV2Logic = kea([ + props({} as PipelineNodeMetricsProps), + key(({ id }: PipelineNodeMetricsProps) => id), + path((id) => ['scenes', 'pipeline', 'appMetricsLogic', id]), + actions({ + setFilters: (filters: Partial) => ({ filters }), + }), + loaders(({ values, props }) => ({ + appMetrics: [ + null as AppMetricsV2Response | null, + { + loadMetrics: async () => { + const params: AppMetricsV2RequestParams = { + ...values.filters, + breakdown_by: 'name', + } + return await api.hogFunctions.metrics(props.id, params) + }, + }, + ], + + appMetricsTotals: [ + null as AppMetricsTotalsV2Response | null, + { + loadMetricsTotals: async () => { + const params: AppMetricsV2RequestParams = { + breakdown_by: 'name', + } + return await api.hogFunctions.metricsTotals(props.id, params) + }, + }, + ], + })), + reducers({ + filters: [ + DEFAULT_FILTERS, + { + setFilters: (state, { filters }) => ({ ...state, ...filters }), + }, + ], + }), + listeners(({ actions }) => ({ + setFilters: async (_, breakpoint) => { + await breakpoint(100) + actions.loadMetrics() + }, + })), +]) diff --git a/frontend/src/scenes/pipeline/pipelinePluginConfigurationLogic.tsx b/frontend/src/scenes/pipeline/pipelinePluginConfigurationLogic.tsx index b8fded3120c81e..9e7d981624e254 100644 --- a/frontend/src/scenes/pipeline/pipelinePluginConfigurationLogic.tsx +++ b/frontend/src/scenes/pipeline/pipelinePluginConfigurationLogic.tsx @@ -148,7 +148,8 @@ export const pipelinePluginConfigurationLogic = kea = { [urls.dataWarehouse()]: Scene.DataWarehouse, [urls.dataWarehouseView(':id')]: Scene.DataWarehouse, [urls.dataWarehouseTable()]: Scene.DataWarehouseTable, - [urls.dataWarehouseSettings(':tab')]: Scene.DataWarehouseSettings, + [urls.dataWarehouseSettings(':tab')]: Scene.DataWarehouse, [urls.dataWarehouseRedirect(':kind')]: Scene.DataWarehouseRedirect, [urls.dataWarehouseSourceSettings(':id', ':tab')]: Scene.dataWarehouseSourceSettings, [urls.featureFlags()]: Scene.FeatureFlags, diff --git a/frontend/src/scenes/session-recordings/SessionRecordings.tsx b/frontend/src/scenes/session-recordings/SessionRecordings.tsx index 890a81d1cb0593..b82171522363b1 100644 --- a/frontend/src/scenes/session-recordings/SessionRecordings.tsx +++ b/frontend/src/scenes/session-recordings/SessionRecordings.tsx @@ -7,17 +7,13 @@ import { PageHeader } from 'lib/components/PageHeader' import { upgradeModalLogic } from 'lib/components/UpgradeModal/upgradeModalLogic' import { VersionCheckerBanner } from 'lib/components/VersionChecker/VersionCheckerBanner' import { useAsyncHandler } from 'lib/hooks/useAsyncHandler' -import { useFeatureFlag } from 'lib/hooks/useFeatureFlag' import { LemonBanner } from 'lib/lemon-ui/LemonBanner' import { LemonTabs } from 'lib/lemon-ui/LemonTabs' import { Spinner } from 'lib/lemon-ui/Spinner/Spinner' import { eventUsageLogic } from 'lib/utils/eventUsageLogic' import { NotebookSelectButton } from 'scenes/notebooks/NotebookSelectButton/NotebookSelectButton' import { SceneExport } from 'scenes/sceneTypes' -import { - convertUniversalFiltersToLegacyFilters, - sessionRecordingsPlaylistLogic, -} from 'scenes/session-recordings/playlist/sessionRecordingsPlaylistLogic' +import { sessionRecordingsPlaylistLogic } from 'scenes/session-recordings/playlist/sessionRecordingsPlaylistLogic' import { teamLogic } from 'scenes/teamLogic' import { urls } from 'scenes/urls' @@ -39,15 +35,13 @@ function Header(): JSX.Element { const { currentTeam } = useValues(teamLogic) const recordingsDisabled = currentTeam && !currentTeam?.session_recording_opt_in const { reportRecordingPlaylistCreated } = useActions(eventUsageLogic) - const hasUniversalFiltering = useFeatureFlag('SESSION_REPLAY_UNIVERSAL_FILTERS') const { openSettingsPanel } = useActions(sidePanelSettingsLogic) // NB this relies on `updateSearchParams` being the only prop needed to pick the correct "Recent" tab list logic const { filters, totalFiltersCount } = useValues(sessionRecordingsPlaylistLogic({ updateSearchParams: true })) const saveFiltersPlaylistHandler = useAsyncHandler(async () => { - const existingFilters = hasUniversalFiltering ? filters : convertUniversalFiltersToLegacyFilters(filters) - await createPlaylist({ filters: existingFilters }, true) + await createPlaylist({ filters }, true) reportRecordingPlaylistCreated('filters') }) diff --git a/frontend/src/scenes/session-recordings/detail/SessionRecordingScene.scss b/frontend/src/scenes/session-recordings/detail/SessionRecordingScene.scss index a5928a38658ea4..0973ee37587bbd 100644 --- a/frontend/src/scenes/session-recordings/detail/SessionRecordingScene.scss +++ b/frontend/src/scenes/session-recordings/detail/SessionRecordingScene.scss @@ -1,6 +1,6 @@ .SessionRecordingScene { .SessionRecordingPlayer { - height: calc(100vh - 12rem); - min-height: 30rem; + height: calc(100vh - 6rem); + min-height: 25rem; } } diff --git a/frontend/src/scenes/session-recordings/filters/AdvancedSessionRecordingsFilters.tsx b/frontend/src/scenes/session-recordings/filters/AdvancedSessionRecordingsFilters.tsx deleted file mode 100644 index 2dc0a86c7fc635..00000000000000 --- a/frontend/src/scenes/session-recordings/filters/AdvancedSessionRecordingsFilters.tsx +++ /dev/null @@ -1,224 +0,0 @@ -import { LemonButtonWithDropdown, LemonCheckbox, LemonInput } from '@posthog/lemon-ui' -import { useValues } from 'kea' -import { DateFilter } from 'lib/components/DateFilter/DateFilter' -import { PropertyFilters } from 'lib/components/PropertyFilters/PropertyFilters' -import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types' -import { TestAccountFilterSwitch } from 'lib/components/TestAccountFiltersSwitch' -import { LemonLabel } from 'lib/lemon-ui/LemonLabel/LemonLabel' -import { ActionFilter } from 'scenes/insights/filters/ActionFilter/ActionFilter' -import { MathAvailability } from 'scenes/insights/filters/ActionFilter/ActionFilterRow/ActionFilterRow' -import { defaultRecordingDurationFilter } from 'scenes/session-recordings/playlist/sessionRecordingsPlaylistLogic' - -import { groupsModel } from '~/models/groupsModel' -import { EntityTypes, FilterableLogLevel, RecordingFilters } from '~/types' - -import { DurationFilter } from './DurationFilter' - -function DateAndDurationFilters({ - filters, - setFilters, -}: { - filters: RecordingFilters - setFilters: (filters: RecordingFilters) => void -}): JSX.Element { - return ( -
- Time and duration -
- { - setFilters({ - date_from: changedDateFrom, - date_to: changedDateTo, - }) - }} - dateOptions={[ - { key: 'Custom', values: [] }, - { key: 'Last 24 hours', values: ['-24h'] }, - { key: 'Last 3 days', values: ['-3d'] }, - { key: 'Last 7 days', values: ['-7d'] }, - { key: 'Last 30 days', values: ['-30d'] }, - { key: 'All time', values: ['-90d'] }, - ]} - dropdownPlacement="bottom-start" - /> - { - setFilters({ - session_recording_duration: newRecordingDurationFilter, - duration_type_filter: newDurationType, - }) - }} - recordingDurationFilter={filters.session_recording_duration || defaultRecordingDurationFilter} - durationTypeFilter={filters.duration_type_filter || 'duration'} - pageKey="session-recordings" - /> -
-
- ) -} - -export const AdvancedSessionRecordingsFilters = ({ - filters, - setFilters, - showPropertyFilters, -}: { - filters: RecordingFilters - setFilters: (filters: RecordingFilters) => void - showPropertyFilters?: boolean -}): JSX.Element => { - const { groupsTaxonomicTypes } = useValues(groupsModel) - - const allowedPropertyTaxonomyTypes = [ - TaxonomicFilterGroupType.EventProperties, - TaxonomicFilterGroupType.EventFeatureFlags, - TaxonomicFilterGroupType.Elements, - TaxonomicFilterGroupType.HogQLExpression, - ...groupsTaxonomicTypes, - ] - - allowedPropertyTaxonomyTypes.push(TaxonomicFilterGroupType.SessionProperties) - - const addFilterTaxonomyTypes = [TaxonomicFilterGroupType.PersonProperties, TaxonomicFilterGroupType.Cohorts] - addFilterTaxonomyTypes.push(TaxonomicFilterGroupType.SessionProperties) - - return ( -
- - Events and actions - - - { - setFilters({ - events: payload.events || [], - actions: payload.actions || [], - }) - }} - typeKey="session-recordings" - mathAvailability={MathAvailability.None} - hideRename - hideDuplicate - showNestedArrow={false} - actionsTaxonomicGroupTypes={[TaxonomicFilterGroupType.Actions, TaxonomicFilterGroupType.Events]} - propertiesTaxonomicGroupTypes={allowedPropertyTaxonomyTypes} - propertyFiltersPopover - addFilterDefaultOptions={{ - id: '$pageview', - name: '$pageview', - type: EntityTypes.EVENTS, - }} - buttonProps={{ type: 'secondary', size: 'small' }} - /> - - - Properties - - - setFilters({ filter_test_accounts: val })} - fullWidth - /> - - {showPropertyFilters && ( - { - setFilters({ properties }) - }} - /> - )} - - - - -
- ) -} - -function ConsoleFilters({ - filters, - setFilters, -}: { - filters: RecordingFilters - setFilters: (filters: RecordingFilters) => void -}): JSX.Element { - function updateLevelChoice(checked: boolean, level: FilterableLogLevel): void { - const newChoice = filters.console_logs?.filter((c) => c !== level) || [] - if (checked) { - setFilters({ - console_logs: [...newChoice, level], - }) - } else { - setFilters({ - console_logs: newChoice, - }) - } - } - - return ( - <> - Console logs -
- { - setFilters({ - console_search_query: s, - }) - }} - /> -
- - { - updateLevelChoice(checked, 'info') - }} - label="info" - /> - updateLevelChoice(checked, 'warn')} - label="warn" - /> - updateLevelChoice(checked, 'error')} - label="error" - /> - , - ], - actionable: true, - }} - > - {filters.console_logs?.map((x) => `console.${x}`).join(' or ') || ( - Console types to filter for... - )} - - - ) -} diff --git a/frontend/src/scenes/session-recordings/filters/SessionRecordingsFilters.tsx b/frontend/src/scenes/session-recordings/filters/SessionRecordingsFilters.tsx deleted file mode 100644 index 2af4cfe621c332..00000000000000 --- a/frontend/src/scenes/session-recordings/filters/SessionRecordingsFilters.tsx +++ /dev/null @@ -1,106 +0,0 @@ -import { LemonButton, LemonCollapse } from '@posthog/lemon-ui' -import equal from 'fast-deep-equal' -import { useActions, useValues } from 'kea' -import { useFeatureFlag } from 'lib/hooks/useFeatureFlag' -import { LemonLabel } from 'lib/lemon-ui/LemonLabel/LemonLabel' -import { useMemo } from 'react' - -import { RecordingFilters } from '~/types' - -import { playerSettingsLogic } from '../player/playerSettingsLogic' -import { getDefaultFilters } from '../playlist/sessionRecordingsPlaylistLogic' -import { AdvancedSessionRecordingsFilters } from './AdvancedSessionRecordingsFilters' -import { OrderingFilters } from './OrderingFilters' -import { SimpleSessionRecordingsFilters } from './SimpleSessionRecordingsFilters' - -interface SessionRecordingsFiltersProps { - advancedFilters: RecordingFilters - simpleFilters: RecordingFilters - setAdvancedFilters: (filters: RecordingFilters) => void - setSimpleFilters: (filters: RecordingFilters) => void - showPropertyFilters?: boolean - hideSimpleFilters?: boolean - onReset?: () => void -} - -export function SessionRecordingsFilters({ - advancedFilters, - simpleFilters, - setAdvancedFilters, - setSimpleFilters, - showPropertyFilters, - hideSimpleFilters, - onReset, -}: SessionRecordingsFiltersProps): JSX.Element { - const { prefersAdvancedFilters } = useValues(playerSettingsLogic) - const { setPrefersAdvancedFilters } = useActions(playerSettingsLogic) - - const initiallyOpen = useMemo(() => { - // advanced always open if not showing simple filters, saves computation - if (hideSimpleFilters) { - return true - } - const defaultFilters = getDefaultFilters() - return prefersAdvancedFilters || !equal(advancedFilters, defaultFilters) - }, []) - const hasFilterOrdering = useFeatureFlag('SESSION_REPLAY_FILTER_ORDERING') - - const AdvancedFilters = ( - - ) - - const panels = [ - !hideSimpleFilters && { - key: 'advanced-filters', - header: 'Advanced filters', - className: 'p-0', - content: AdvancedFilters, - }, - hasFilterOrdering && { - key: 'ordering', - header: 'Ordering', - content: , - }, - ].filter(Boolean) - - return ( -
-
-
- Find sessions: - - {onReset && ( - - - Reset - - - )} -
- - {!hideSimpleFilters && ( - - )} -
- - {hideSimpleFilters && AdvancedFilters} - - {panels.length > 0 && ( - { - setPrefersAdvancedFilters(activeKeys.includes('advanced-filters')) - }} - /> - )} -
- ) -} diff --git a/frontend/src/scenes/session-recordings/filters/SimpleSessionRecordingsFilters.tsx b/frontend/src/scenes/session-recordings/filters/SimpleSessionRecordingsFilters.tsx deleted file mode 100644 index f03ed52112f445..00000000000000 --- a/frontend/src/scenes/session-recordings/filters/SimpleSessionRecordingsFilters.tsx +++ /dev/null @@ -1,218 +0,0 @@ -import { IconGear, IconTrash } from '@posthog/icons' -import { LemonButton, LemonMenu, Popover } from '@posthog/lemon-ui' -import { useActions, useValues } from 'kea' -import { PropertyFilters } from 'lib/components/PropertyFilters/PropertyFilters' -import { PropertyKeyInfo } from 'lib/components/PropertyKeyInfo' -import { TaxonomicFilter } from 'lib/components/TaxonomicFilter/TaxonomicFilter' -import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types' -import { useMemo, useRef, useState } from 'react' - -import { EntityTypes, EventPropertyFilter, PropertyFilterType, PropertyOperator, RecordingFilters } from '~/types' - -import { playerSettingsLogic } from '../player/playerSettingsLogic' - -export const SimpleSessionRecordingsFilters = ({ - filters, - setFilters, -}: { - filters: RecordingFilters - setFilters: (filters: RecordingFilters) => void -}): JSX.Element => { - const { quickFilterProperties } = useValues(playerSettingsLogic) - const { setQuickFilterProperties } = useActions(playerSettingsLogic) - const [showPropertySelector, setShowPropertySelector] = useState(false) - const buttonRef = useRef(null) - - const pageviewEvent = filters.events?.find((event) => event.id === '$pageview') - - const personProperties = filters.properties || [] - const eventProperties = pageviewEvent?.properties || [] - - const onClickPersonProperty = (key: string): void => { - setFilters({ - ...filters, - properties: [ - ...personProperties, - { type: PropertyFilterType.Person, key: key, value: null, operator: PropertyOperator.Exact }, - ], - }) - } - - const onClickCurrentUrl = (): void => { - const events = filters.events || [] - setFilters({ - ...filters, - events: [ - ...events, - { - id: '$pageview', - name: '$pageview', - type: EntityTypes.EVENTS, - properties: [ - { - type: PropertyFilterType.Event, - key: '$current_url', - value: null, - operator: PropertyOperator.Exact, - }, - ], - }, - ], - }) - } - - const defaultItems = useMemo(() => { - const eventKeys = eventProperties.map((p: EventPropertyFilter) => p.key) - - return [ - !eventKeys.includes('$current_url') && { - label: , - key: '$current_url', - onClick: onClickCurrentUrl, - }, - ].filter(Boolean) - }, [eventProperties]) - - const personPropertyItems = useMemo(() => { - const personKeys = personProperties.map((p) => p.key) - - return quickFilterProperties - .map((property) => { - return ( - !personKeys.includes(property) && { - label: , - key: property, - onClick: () => onClickPersonProperty(property), - } - ) - }) - .filter(Boolean) - }, [quickFilterProperties, personProperties]) - - return ( -
-
- setFilters({ properties })} - allowNew={false} - openOnInsert - /> - { - setFilters({ - ...filters, - events: - properties.length > 0 - ? [ - { - id: '$pageview', - name: '$pageview', - type: EntityTypes.EVENTS, - properties: properties, - }, - ] - : [], - }) - }} - allowNew={false} - openOnInsert - /> - setShowPropertySelector(false)} - overlay={ - { - setQuickFilterProperties(value) - }} - /> - } - referenceElement={buttonRef.current} - placement="right-start" - > - 0 && { items: defaultItems }, - personPropertyItems.length > 0 && { - title: 'Person properties', - items: personPropertyItems, - }, - ]} - onVisibilityChange={() => setShowPropertySelector(false)} - > - , - tooltip: 'Edit properties', - onClick: () => setShowPropertySelector(true), - }} - > - Choose quick filter - - - -
-
- ) -} - -const Configuration = ({ - properties, - onChange, -}: { - properties: string[] - onChange: (properties: string[]) => void -}): JSX.Element => { - const [showPropertySelector, setShowPropertySelector] = useState(false) - - return ( -
- {properties.map((property) => ( -
- - - - } - onClick={() => { - const newProperties = properties.filter((p) => p != property) - onChange(newProperties) - }} - /> -
- ))} - setShowPropertySelector(false)} - placement="right-start" - overlay={ - { - properties.push(value as string) - onChange([...properties]) - setShowPropertySelector(false) - }} - taxonomicGroupTypes={[TaxonomicFilterGroupType.PersonProperties]} - excludedProperties={{ [TaxonomicFilterGroupType.PersonProperties]: properties }} - /> - } - > - setShowPropertySelector(!showPropertySelector)} fullWidth> - Add person properties - - -
- ) -} diff --git a/frontend/src/scenes/session-recordings/player/playerSettingsLogic.ts b/frontend/src/scenes/session-recordings/player/playerSettingsLogic.ts index 89d3d41e7d0cb5..7de31840ebca27 100644 --- a/frontend/src/scenes/session-recordings/player/playerSettingsLogic.ts +++ b/frontend/src/scenes/session-recordings/player/playerSettingsLogic.ts @@ -199,7 +199,6 @@ export const playerSettingsLogic = kea([ setSearchQuery: (search: string) => ({ search }), setDurationTypeToShow: (type: DurationType) => ({ type }), setShowFilters: (showFilters: boolean) => ({ showFilters }), - setPrefersAdvancedFilters: (prefersAdvancedFilters: boolean) => ({ prefersAdvancedFilters }), setQuickFilterProperties: (properties: string[]) => ({ properties }), setTimestampFormat: (format: TimestampFormat) => ({ format }), setPreferredInspectorStacking: (stacking: InspectorStacking) => ({ stacking }), @@ -232,15 +231,6 @@ export const playerSettingsLogic = kea([ setPlaybackViewMode: (_, { mode }) => mode, }, ], - prefersAdvancedFilters: [ - true, - { - persist: true, - }, - { - setPrefersAdvancedFilters: (_, { prefersAdvancedFilters }) => prefersAdvancedFilters, - }, - ], quickFilterProperties: [ [...(values.currentTeam?.person_display_name_properties || [])] as string[], { diff --git a/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx b/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx index 8dee5b119caf91..b0b9d18313f3be 100644 --- a/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx +++ b/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx @@ -1,11 +1,10 @@ -import { IconFilter, IconGear } from '@posthog/icons' +import { IconGear } from '@posthog/icons' import { LemonButton, Link, Spinner } from '@posthog/lemon-ui' import { BindLogic, useActions, useValues } from 'kea' import { EmptyMessage } from 'lib/components/EmptyMessage/EmptyMessage' import { Playlist, PlaylistSection } from 'lib/components/Playlist/Playlist' import { PropertyKeyInfo } from 'lib/components/PropertyKeyInfo' import { FEATURE_FLAGS } from 'lib/constants' -import { IconWithCount } from 'lib/lemon-ui/icons' import { LemonBanner } from 'lib/lemon-ui/LemonBanner' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { useNotebookNode } from 'scenes/notebooks/Nodes/NotebookNodeContext' @@ -14,7 +13,6 @@ import { urls } from 'scenes/urls' import { ReplayTabs, SessionRecordingType } from '~/types' import { RecordingsUniversalFilters } from '../filters/RecordingsUniversalFilters' -import { SessionRecordingsFilters } from '../filters/SessionRecordingsFilters' import { SessionRecordingPlayer } from '../player/SessionRecordingPlayer' import { SessionRecordingPreview } from './SessionRecordingPreview' import { @@ -34,27 +32,14 @@ export function SessionRecordingsPlaylist(props: SessionRecordingPlaylistLogicPr const { filters, pinnedRecordings, - totalFiltersCount, - useUniversalFiltering, matchingEventsMatchType, sessionRecordingsResponseLoading, otherRecordings, sessionSummaryLoading, - advancedFilters, - simpleFilters, activeSessionRecordingId, hasNext, - universalFilters, } = useValues(logic) - const { - maybeLoadSessionRecordings, - summarizeSession, - setSelectedRecordingId, - setAdvancedFilters, - setSimpleFilters, - resetFilters, - setUniversalFilters, - } = useActions(logic) + const { maybeLoadSessionRecordings, summarizeSession, setSelectedRecordingId, setFilters } = useActions(logic) const { featureFlags } = useValues(featureFlagLogic) const isTestingSaved = featureFlags[FEATURE_FLAGS.SAVED_NOT_PINNED] === 'test' @@ -70,30 +55,6 @@ export function SessionRecordingsPlaylist(props: SessionRecordingPlaylistLogicPr summarizeSession(recording.id) } - if (!useUniversalFiltering) { - headerActions.push({ - key: 'filters', - tooltip: 'Filter recordings', - content: ( - - ), - icon: ( - - - - ), - children: 'Filter', - }) - } - headerActions.push({ key: 'settings', tooltip: 'Playlist settings', @@ -146,13 +107,9 @@ export function SessionRecordingsPlaylist(props: SessionRecordingPlaylistLogicPr return (
- {useUniversalFiltering && !notebookNode ? ( - - ) : null} + {!notebookNode && ( + + )} { const { filters, sessionRecordingsAPIErrored, unusableEventsInFilter } = useValues(sessionRecordingsPlaylistLogic) - const { setAdvancedFilters } = useActions(sessionRecordingsPlaylistLogic) + const { setFilters } = useActions(sessionRecordingsPlaylistLogic) return (
@@ -226,11 +183,7 @@ const ListEmptyState = (): JSX.Element => { { - setAdvancedFilters({ - date_from: '-30d', - }) - }} + onClick={() => setFilters({ date_from: '-30d' })} > Search over the last 30 days diff --git a/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylistScene.tsx b/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylistScene.tsx index 02cfdeadecc84d..64a6c62143bad0 100644 --- a/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylistScene.tsx +++ b/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylistScene.tsx @@ -4,7 +4,6 @@ import { EditableField } from 'lib/components/EditableField/EditableField' import { NotFound } from 'lib/components/NotFound' import { PageHeader } from 'lib/components/PageHeader' import { UserActivityIndicator } from 'lib/components/UserActivityIndicator/UserActivityIndicator' -import { useFeatureFlag } from 'lib/hooks/useFeatureFlag' import { More } from 'lib/lemon-ui/LemonButton/More' import { LemonSkeleton } from 'lib/lemon-ui/LemonSkeleton' import { SceneExport } from 'scenes/sceneTypes' @@ -31,7 +30,6 @@ export function SessionRecordingsPlaylistScene(): JSX.Element { const { showFilters } = useValues(playerSettingsLogic) const { setShowFilters } = useActions(playerSettingsLogic) - const hasUniversalFiltering = useFeatureFlag('SESSION_REPLAY_UNIVERSAL_FILTERS') if (!playlist && playlistLoading) { return ( @@ -136,16 +134,13 @@ export function SessionRecordingsPlaylistScene(): JSX.Element { {playlist.short_id && pinnedRecordings !== null ? (
- setFilters(hasUniversalFiltering ? universalFilters : legacyFilters) - } + onFiltersChange={setFilters} onPinnedChange={onPinnedChange} pinnedRecordings={pinnedRecordings ?? []} /> diff --git a/frontend/src/scenes/session-recordings/playlist/sessionRecordingsPlaylistLogic.test.ts b/frontend/src/scenes/session-recordings/playlist/sessionRecordingsPlaylistLogic.test.ts index 3bd970487e15b4..d0104691cfa886 100644 --- a/frontend/src/scenes/session-recordings/playlist/sessionRecordingsPlaylistLogic.test.ts +++ b/frontend/src/scenes/session-recordings/playlist/sessionRecordingsPlaylistLogic.test.ts @@ -3,16 +3,13 @@ import { expectLogic } from 'kea-test-utils' import { useMocks } from '~/mocks/jest' import { initKeaTests } from '~/test/init' -import { FilterLogicalOperator, PropertyFilterType, PropertyOperator, RecordingFilters } from '~/types' +import { FilterLogicalOperator, PropertyFilterType, PropertyOperator } from '~/types' import { sessionRecordingDataLogic } from '../player/sessionRecordingDataLogic' import { convertLegacyFiltersToUniversalFilters, convertUniversalFiltersToLegacyFilters, DEFAULT_RECORDING_FILTERS, - DEFAULT_RECORDING_UNIVERSAL_FILTERS, - DEFAULT_SIMPLE_RECORDING_FILTERS, - defaultRecordingDurationFilter, sessionRecordingsPlaylistLogic, } from './sessionRecordingsPlaylistLogic' @@ -201,120 +198,140 @@ describe('sessionRecordingsPlaylistLogic', () => { describe('entityFilters', () => { it('starts with default values', () => { expectLogic(logic).toMatchValues({ - advancedFilters: DEFAULT_RECORDING_FILTERS, - simpleFilters: DEFAULT_SIMPLE_RECORDING_FILTERS, - universalFilters: DEFAULT_RECORDING_UNIVERSAL_FILTERS, + filters: DEFAULT_RECORDING_FILTERS, }) }) - it('is set by setAdvancedFilters and loads filtered results and sets the url', async () => { + it('is set by setFilters and loads filtered results and sets the url', async () => { await expectLogic(logic, () => { - logic.actions.setAdvancedFilters({ - events: [{ id: '$autocapture', type: 'events', order: 0, name: '$autocapture' }], + logic.actions.setFilters({ + filter_group: { + type: FilterLogicalOperator.And, + values: [ + { + type: FilterLogicalOperator.And, + values: [{ id: '$autocapture', type: 'events', order: 0, name: '$autocapture' }], + }, + ], + }, }) }) - .toDispatchActions(['setAdvancedFilters', 'loadSessionRecordings', 'loadSessionRecordingsSuccess']) + .toDispatchActions(['setFilters', 'loadSessionRecordings', 'loadSessionRecordingsSuccess']) .toMatchValues({ sessionRecordings: ['List of recordings filtered by events'], }) - expect(router.values.searchParams.advancedFilters).toHaveProperty('events', [ - { id: '$autocapture', type: 'events', order: 0, name: '$autocapture' }, - ]) + expect(router.values.searchParams.filters).toHaveProperty('filter_group', { + type: FilterLogicalOperator.And, + values: [ + { + type: FilterLogicalOperator.And, + values: [{ id: '$autocapture', type: 'events', order: 0, name: '$autocapture' }], + }, + ], + }) }) it('reads filters from the logic props', async () => { logic = sessionRecordingsPlaylistLogic({ key: 'tests-with-props', - advancedFilters: { - events: [{ id: '$autocapture', type: 'events', order: 0, name: '$autocapture' }], - }, - simpleFilters: { - properties: [ - { - key: '$geoip_country_name', - value: ['Australia'], - operator: PropertyOperator.Exact, - type: PropertyFilterType.Person, - }, - ], + filters: { + duration: [], + filter_group: { + type: FilterLogicalOperator.And, + values: [ + { + type: FilterLogicalOperator.And, + values: [ + { id: '$autocapture', type: 'events', order: 0, name: '$autocapture' }, + { + key: '$geoip_country_name', + value: ['Australia'], + operator: PropertyOperator.Exact, + type: PropertyFilterType.Person, + }, + ], + }, + ], + }, }, }) logic.mount() await expectLogic(logic).toMatchValues({ - advancedFilters: { - events: [{ id: '$autocapture', type: 'events', order: 0, name: '$autocapture' }], - }, - simpleFilters: { - properties: [ - { key: '$geoip_country_name', value: ['Australia'], operator: 'exact', type: 'person' }, - ], + filters: { + duration: [], + filter_group: { + type: FilterLogicalOperator.And, + values: [ + { + type: FilterLogicalOperator.And, + values: [ + { id: '$autocapture', type: 'events', order: 0, name: '$autocapture' }, + { + key: '$geoip_country_name', + value: ['Australia'], + operator: PropertyOperator.Exact, + type: PropertyFilterType.Person, + }, + ], + }, + ], + }, }, }) }) }) describe('date range', () => { - it('is set by setAdvancedFilters and fetches results from server and sets the url', async () => { + it('is set by setFilters and fetches results from server and sets the url', async () => { await expectLogic(logic, () => { - logic.actions.setAdvancedFilters({ + logic.actions.setFilters({ date_from: '2021-10-05', date_to: '2021-10-20', }) }) .toMatchValues({ - legacyFilters: expect.objectContaining({ - date_from: '2021-10-05', - date_to: '2021-10-20', - }), filters: expect.objectContaining({ date_from: '2021-10-05', date_to: '2021-10-20', }), }) - .toDispatchActions(['setAdvancedFilters', 'loadSessionRecordingsSuccess']) + .toDispatchActions(['setFilters', 'loadSessionRecordingsSuccess']) .toMatchValues({ sessionRecordings: ['Recordings filtered by date'] }) - expect(router.values.searchParams.advancedFilters).toHaveProperty('date_from', '2021-10-05') expect(router.values.searchParams.filters).toHaveProperty('date_from', '2021-10-05') - expect(router.values.searchParams.advancedFilters).toHaveProperty('date_to', '2021-10-20') expect(router.values.searchParams.filters).toHaveProperty('date_to', '2021-10-20') }) }) describe('duration filter', () => { - it('is set by setAdvancedFilters and fetches results from server and sets the url', async () => { + it('is set by setFilters and fetches results from server and sets the url', async () => { await expectLogic(logic, () => { - logic.actions.setAdvancedFilters({ - session_recording_duration: { - type: PropertyFilterType.Recording, - key: 'duration', - value: 600, - operator: PropertyOperator.LessThan, - }, - }) - }) - .toMatchValues({ - legacyFilters: expect.objectContaining({ - session_recording_duration: { + logic.actions.setFilters({ + duration: [ + { type: PropertyFilterType.Recording, key: 'duration', value: 600, operator: PropertyOperator.LessThan, }, - }), + ], + }) + }) + .toMatchValues({ filters: expect.objectContaining({ - duration: [{ key: 'duration', operator: 'lt', type: 'recording', value: 600 }], + duration: [ + { + key: 'duration', + operator: PropertyOperator.LessThan, + type: PropertyFilterType.Recording, + value: 600, + }, + ], }), }) - .toDispatchActions(['setAdvancedFilters', 'loadSessionRecordingsSuccess']) + .toDispatchActions(['setFilters', 'loadSessionRecordingsSuccess']) .toMatchValues({ sessionRecordings: ['Recordings filtered by duration'] }) - expect(router.values.searchParams.advancedFilters).toHaveProperty('session_recording_duration', { - type: PropertyFilterType.Recording, - key: 'duration', - value: 600, - operator: PropertyOperator.LessThan, - }) expect(router.values.searchParams.filters).toHaveProperty('duration', [ { type: PropertyFilterType.Recording, @@ -381,13 +398,21 @@ describe('sessionRecordingsPlaylistLogic', () => { }) }) - it('is set by setAdvancedFilters and loads filtered results', async () => { + it('is set by setFilters and loads filtered results', async () => { await expectLogic(logic, () => { - logic.actions.setAdvancedFilters({ - events: [{ id: '$autocapture', type: 'events', order: 0, name: '$autocapture' }], + logic.actions.setFilters({ + filter_group: { + type: FilterLogicalOperator.And, + values: [ + { + type: FilterLogicalOperator.And, + values: [{ id: '$autocapture', type: 'events', order: 0, name: '$autocapture' }], + }, + ], + }, }) }) - .toDispatchActions(['setAdvancedFilters', 'loadSessionRecordings', 'loadSessionRecordingsSuccess']) + .toDispatchActions(['setFilters', 'loadSessionRecordings', 'loadSessionRecordingsSuccess']) .toMatchValues({ sessionRecordings: ['List of recordings filtered by events'], }) @@ -396,43 +421,29 @@ describe('sessionRecordingsPlaylistLogic', () => { it('reads filters from the URL', async () => { router.actions.push('/replay', { - advancedFilters: { - actions: [{ id: '1', type: 'actions', order: 0, name: 'View Recording' }], - events: [{ id: '$autocapture', type: 'events', order: 0, name: '$autocapture' }], + filters: { date_from: '2021-10-01', date_to: '2021-10-10', - offset: 50, - session_recording_duration: { - type: PropertyFilterType.Recording, - key: 'duration', - value: 600, - operator: PropertyOperator.LessThan, + duration: [{ key: 'duration', operator: 'lt', type: 'recording', value: 600 }], + filter_group: { + type: FilterLogicalOperator.And, + values: [ + { + type: FilterLogicalOperator.And, + values: [ + { id: '$autocapture', name: '$autocapture', order: 0, type: 'events' }, + { id: '1', name: 'View Recording', order: 0, type: 'actions' }, + ], + }, + ], }, - operand: FilterLogicalOperator.And, + filter_test_accounts: false, }, }) await expectLogic(logic) - .toDispatchActions(['setAdvancedFilters']) + .toDispatchActions(['setFilters']) .toMatchValues({ - legacyFilters: { - events: [{ id: '$autocapture', type: 'events', order: 0, name: '$autocapture' }], - actions: [{ id: '1', type: 'actions', order: 0, name: 'View Recording' }], - date_from: '2021-10-01', - date_to: '2021-10-10', - offset: 50, - console_logs: [], - console_search_query: '', - properties: [], - session_recording_duration: { - type: PropertyFilterType.Recording, - key: 'duration', - value: 600, - operator: PropertyOperator.LessThan, - }, - snapshot_source: null, - operand: FilterLogicalOperator.And, - }, filters: { date_from: '2021-10-01', date_to: '2021-10-10', @@ -456,38 +467,31 @@ describe('sessionRecordingsPlaylistLogic', () => { it('reads filters from the URL and defaults the duration filter', async () => { router.actions.push('/replay', { - advancedFilters: { - actions: [{ id: '1', type: 'actions', order: 0, name: 'View Recording' }], + filters: { + filter_group: { + type: FilterLogicalOperator.And, + values: [ + { + type: FilterLogicalOperator.And, + values: [{ id: '1', type: 'actions', order: 0, name: 'View Recording' }], + }, + ], + }, }, }) await expectLogic(logic) - .toDispatchActions(['setAdvancedFilters']) + .toDispatchActions(['setFilters']) .toMatchValues({ - advancedFilters: expect.objectContaining({ - actions: [{ id: '1', type: 'actions', order: 0, name: 'View Recording' }], - }), - legacyFilters: { - actions: [{ id: '1', type: 'actions', order: 0, name: 'View Recording' }], - session_recording_duration: defaultRecordingDurationFilter, - console_logs: [], - console_search_query: '', - date_from: '-3d', - date_to: null, - events: [], - properties: [], - operand: FilterLogicalOperator.And, - snapshot_source: null, - }, filters: { date_from: '-3d', date_to: null, duration: [{ key: 'duration', operator: 'gt', type: 'recording', value: 1 }], filter_group: { - type: 'AND', + type: FilterLogicalOperator.And, values: [ { - type: 'AND', + type: FilterLogicalOperator.And, values: [{ id: '1', name: 'View Recording', order: 0, type: 'actions' }], }, ], @@ -505,10 +509,18 @@ describe('sessionRecordingsPlaylistLogic', () => { }) await expectLogic(logic) - .toDispatchActions(['setAdvancedFilters']) + .toDispatchActions(['setFilters']) .toMatchValues({ - advancedFilters: expect.objectContaining({ - events: [{ id: '$autocapture', type: 'events', order: 0, name: '$autocapture' }], + filters: expect.objectContaining({ + filter_group: { + type: FilterLogicalOperator.And, + values: [ + { + type: FilterLogicalOperator.And, + values: [{ id: '$autocapture', type: 'events', order: 0, name: '$autocapture' }], + }, + ], + }, }), }) }) @@ -528,19 +540,26 @@ describe('sessionRecordingsPlaylistLogic', () => { }) await expectLogic(logic) - .toDispatchActions(['setSimpleFilters']) + .toDispatchActions(['setFilters']) .toMatchValues({ - simpleFilters: { - events: [], - properties: [ - { - key: '$geoip_country_name', - value: ['Australia'], - operator: PropertyOperator.Exact, - type: PropertyFilterType.Person, - }, - ], - }, + filters: expect.objectContaining({ + filter_group: { + type: FilterLogicalOperator.And, + values: [ + { + type: FilterLogicalOperator.And, + values: [ + { + key: '$geoip_country_name', + value: ['Australia'], + operator: PropertyOperator.Exact, + type: PropertyFilterType.Person, + }, + ], + }, + ], + }, + }), }) }) }) @@ -584,17 +603,47 @@ describe('sessionRecordingsPlaylistLogic', () => { it('counts console log filters', async () => { await expectLogic(logic, () => { - logic.actions.setAdvancedFilters({ - console_logs: ['warn', 'error'], - } satisfies Partial) + logic.actions.setFilters({ + filter_group: { + type: FilterLogicalOperator.And, + values: [ + { + type: FilterLogicalOperator.And, + values: [ + { + type: PropertyFilterType.Recording, + key: 'console_log_level', + operator: PropertyOperator.IContains, + value: ['warn', 'error'], + }, + ], + }, + ], + }, + }) }).toMatchValues({ totalFiltersCount: 1 }) }) it('counts console log search query', async () => { await expectLogic(logic, () => { - logic.actions.setAdvancedFilters({ - console_search_query: 'this is a test', - } satisfies Partial) + logic.actions.setFilters({ + filter_group: { + type: FilterLogicalOperator.And, + values: [ + { + type: FilterLogicalOperator.And, + values: [ + { + type: PropertyFilterType.Recording, + key: 'console_log_query', + operator: PropertyOperator.Exact, + value: 'this is a test', + }, + ], + }, + ], + }, + }) }).toMatchValues({ totalFiltersCount: 1 }) }) }) @@ -611,9 +660,24 @@ describe('sessionRecordingsPlaylistLogic', () => { it('resets console log filters', async () => { await expectLogic(logic, () => { - logic.actions.setAdvancedFilters({ - console_logs: ['warn', 'error'], - } satisfies Partial) + logic.actions.setFilters({ + filter_group: { + type: FilterLogicalOperator.And, + values: [ + { + type: FilterLogicalOperator.And, + values: [ + { + type: PropertyFilterType.Recording, + key: 'console_log_level', + operator: PropertyOperator.IContains, + value: ['warn', 'error'], + }, + ], + }, + ], + }, + }) logic.actions.resetFilters() }).toMatchValues({ totalFiltersCount: 0 }) }) @@ -646,7 +710,7 @@ describe('sessionRecordingsPlaylistLogic', () => { describe('convertUniversalFiltersToLegacyFilters', () => { it('expands the visited_page filter to a pageview with $current_url property', () => { const result = convertUniversalFiltersToLegacyFilters({ - ...DEFAULT_RECORDING_UNIVERSAL_FILTERS, + ...DEFAULT_RECORDING_FILTERS, filter_group: { type: FilterLogicalOperator.And, values: [ diff --git a/frontend/src/scenes/session-recordings/playlist/sessionRecordingsPlaylistLogic.ts b/frontend/src/scenes/session-recordings/playlist/sessionRecordingsPlaylistLogic.ts index 476638ee412a94..83105f0c7f8eed 100644 --- a/frontend/src/scenes/session-recordings/playlist/sessionRecordingsPlaylistLogic.ts +++ b/frontend/src/scenes/session-recordings/playlist/sessionRecordingsPlaylistLogic.ts @@ -7,9 +7,8 @@ import api from 'lib/api' import { isAnyPropertyfilter } from 'lib/components/PropertyFilters/utils' import { DEFAULT_UNIVERSAL_GROUP_FILTER } from 'lib/components/UniversalFilters/universalFiltersLogic' import { isActionFilter, isEventFilter, isRecordingPropertyFilter } from 'lib/components/UniversalFilters/utils' -import { FEATURE_FLAGS } from 'lib/constants' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' -import { objectClean, objectsEqual } from 'lib/utils' +import { objectClean } from 'lib/utils' import { eventUsageLogic } from 'lib/utils/eventUsageLogic' import posthog from 'posthog-js' @@ -78,25 +77,7 @@ export const defaultRecordingDurationFilter: RecordingDurationFilter = { operator: PropertyOperator.GreaterThan, } -export const DEFAULT_SIMPLE_RECORDING_FILTERS: SimpleFiltersType = { - events: [], - properties: [], -} - -export const DEFAULT_RECORDING_FILTERS: RecordingFilters = { - session_recording_duration: defaultRecordingDurationFilter, - properties: [], - events: [], - actions: [], - date_from: '-3d', - date_to: null, - console_logs: [], - snapshot_source: null, - console_search_query: '', - operand: FilterLogicalOperator.And, -} - -export const DEFAULT_RECORDING_UNIVERSAL_FILTERS: RecordingUniversalFilters = { +export const DEFAULT_RECORDING_FILTERS: RecordingUniversalFilters = { filter_test_accounts: false, date_from: '-3d', date_to: null, @@ -104,12 +85,12 @@ export const DEFAULT_RECORDING_UNIVERSAL_FILTERS: RecordingUniversalFilters = { duration: [defaultRecordingDurationFilter], } -const DEFAULT_PERSON_RECORDING_FILTERS: RecordingFilters = { +const DEFAULT_PERSON_RECORDING_FILTERS: RecordingUniversalFilters = { ...DEFAULT_RECORDING_FILTERS, date_from: '-30d', } -export const getDefaultFilters = (personUUID?: PersonUUID): RecordingFilters => { +export const getDefaultFilters = (personUUID?: PersonUUID): RecordingUniversalFilters => { return personUUID ? DEFAULT_PERSON_RECORDING_FILTERS : DEFAULT_RECORDING_FILTERS } @@ -221,15 +202,15 @@ export function convertLegacyFiltersToUniversalFilters( : [] return { - date_from: filters.date_from || DEFAULT_RECORDING_UNIVERSAL_FILTERS['date_from'], - date_to: filters.date_to || DEFAULT_RECORDING_UNIVERSAL_FILTERS['date_to'], + date_from: filters.date_from || DEFAULT_RECORDING_FILTERS['date_from'], + date_to: filters.date_to || DEFAULT_RECORDING_FILTERS['date_to'], filter_test_accounts: filters.filter_test_accounts == undefined - ? DEFAULT_RECORDING_UNIVERSAL_FILTERS['filter_test_accounts'] + ? DEFAULT_RECORDING_FILTERS['filter_test_accounts'] : filters.filter_test_accounts, duration: filters.session_recording_duration ? [{ ...filters.session_recording_duration, key: filters.duration_type_filter || 'duration' }] - : DEFAULT_RECORDING_UNIVERSAL_FILTERS['duration'], + : DEFAULT_RECORDING_FILTERS['duration'], filter_group: { type: FilterLogicalOperator.And, values: [ @@ -255,11 +236,8 @@ export interface SessionRecordingPlaylistLogicProps { personUUID?: PersonUUID updateSearchParams?: boolean autoPlay?: boolean - hideSimpleFilters?: boolean - universalFilters?: RecordingUniversalFilters - advancedFilters?: RecordingFilters - simpleFilters?: RecordingFilters - onFiltersChange?: (filters: RecordingUniversalFilters, legacyFilters: RecordingFilters) => void + filters?: RecordingUniversalFilters + onFiltersChange?: (filters: RecordingUniversalFilters) => void pinnedRecordings?: (SessionRecordingType | string)[] onPinnedChange?: (recording: SessionRecordingType, pinned: boolean) => void } @@ -293,9 +271,7 @@ export const sessionRecordingsPlaylistLogic = kea) => ({ filters }), - setAdvancedFilters: (filters: Partial) => ({ filters }), - setSimpleFilters: (filters: SimpleFiltersType) => ({ filters }), + setFilters: (filters: Partial) => ({ filters }), setShowFilters: (showFilters: boolean) => ({ showFilters }), setShowSettings: (showSettings: boolean) => ({ showSettings }), setOrderBy: (orderBy: SessionOrderingType) => ({ orderBy }), @@ -313,13 +289,6 @@ export const sessionRecordingsPlaylistLogic = kea ({ show }), }), propsChanged(({ actions, props }, oldProps) => { - if (!objectsEqual(props.advancedFilters, oldProps.advancedFilters)) { - actions.setAdvancedFilters(props.advancedFilters || {}) - } - if (!objectsEqual(props.simpleFilters, oldProps.simpleFilters)) { - actions.setSimpleFilters(props.simpleFilters || {}) - } - // If the defined list changes, we need to call the loader to either load the new items or change the list if (props.pinnedRecordings !== oldProps.pinnedRecordings) { actions.loadPinnedRecordings() @@ -471,20 +440,10 @@ export const sessionRecordingsPlaylistLogic = kea ({ - ...state, - ...filters, - }), - resetFilters: () => DEFAULT_SIMPLE_RECORDING_FILTERS, - }, - ], - advancedFilters: [ - props.advancedFilters ?? getDefaultFilters(props.personUUID), + filters: [ + props.filters ?? getDefaultFilters(props.personUUID), { - setAdvancedFilters: (state, { filters }) => { + setFilters: (state, { filters }) => { return { ...state, ...filters, @@ -493,18 +452,6 @@ export const sessionRecordingsPlaylistLogic = kea getDefaultFilters(props.personUUID), }, ], - universalFilters: [ - props.universalFilters ?? DEFAULT_RECORDING_UNIVERSAL_FILTERS, - { - setUniversalFilters: (state, { filters }) => { - return { - ...state, - ...filters, - } - }, - resetFilters: () => DEFAULT_RECORDING_UNIVERSAL_FILTERS, - }, - ], showFilters: [ true, { @@ -584,9 +531,8 @@ export const sessionRecordingsPlaylistLogic = kea true, loadSessionRecordingSuccess: () => false, - setUniversalFilters: () => false, + setFilters: () => false, setAdvancedFilters: () => false, - setSimpleFilters: () => false, loadNext: () => false, loadPrev: () => false, }, @@ -597,21 +543,9 @@ export const sessionRecordingsPlaylistLogic = kea { - actions.loadSessionRecordings() - props.onFiltersChange?.(values.filters, values.legacyFilters) - capturePartialFilters(filters) - actions.loadEventsHaveSessionId() - }, - setAdvancedFilters: ({ filters }) => { + setFilters: ({ filters }) => { actions.loadSessionRecordings() - props.onFiltersChange?.(values.filters, values.legacyFilters) - capturePartialFilters(filters) - actions.loadEventsHaveSessionId() - }, - setUniversalFilters: ({ filters }) => { - actions.loadSessionRecordings() - props.onFiltersChange?.(values.filters, values.legacyFilters) + props.onFiltersChange?.(values.filters) capturePartialFilters(filters) actions.loadEventsHaveSessionId() }, @@ -622,7 +556,7 @@ export const sessionRecordingsPlaylistLogic = kea { actions.loadSessionRecordings() - props.onFiltersChange?.(values.filters, values.legacyFilters) + props.onFiltersChange?.(values.filters) }, maybeLoadSessionRecordings: ({ direction }) => { @@ -648,29 +582,8 @@ export const sessionRecordingsPlaylistLogic = kea [s.featureFlags], - (featureFlags) => !!featureFlags[FEATURE_FLAGS.SESSION_REPLAY_UNIVERSAL_FILTERS], - ], - logicProps: [() => [(_, props) => props], (props): SessionRecordingPlaylistLogicProps => props], - filters: [ - (s) => [s.simpleFilters, s.advancedFilters, s.universalFilters, s.featureFlags], - (simpleFilters, advancedFilters, universalFilters, featureFlags): RecordingUniversalFilters => { - if (featureFlags[FEATURE_FLAGS.SESSION_REPLAY_UNIVERSAL_FILTERS]) { - return universalFilters - } - - return convertLegacyFiltersToUniversalFilters(simpleFilters, advancedFilters) - }, - ], - legacyFilters: [ - (s) => [s.simpleFilters, s.advancedFilters], - (simpleFilters, advancedFilters): RecordingFilters => - combineRecordingFilters(simpleFilters, advancedFilters), - ], - matchingEventsMatchType: [ (s) => [s.filters], (filters): MatchingEventsMatchType => { @@ -754,7 +667,7 @@ export const sessionRecordingsPlaylistLogic = kea buildURL(false), - setUniversalFilters: () => buildURL(true), - setAdvancedFilters: () => buildURL(true), - setSimpleFilters: () => buildURL(true), + setFilters: () => buildURL(true), resetFilters: () => buildURL(true), } }), @@ -847,23 +756,17 @@ export const sessionRecordingsPlaylistLogic = kea Type - - {survey.questions.length > 1 - ? 'Multiple questions' - : capitalizeFirstLetter(survey.questions[0].type)} - + {SurveyQuestionLabel[survey.questions[0].type]} {pluralize( survey.questions.length, diff --git a/frontend/src/scenes/surveys/surveyActivityDescriber.test.tsx b/frontend/src/scenes/surveys/surveyActivityDescriber.test.tsx index cc9dca12c7a6c0..5d56642390a89b 100644 --- a/frontend/src/scenes/surveys/surveyActivityDescriber.test.tsx +++ b/frontend/src/scenes/surveys/surveyActivityDescriber.test.tsx @@ -255,7 +255,9 @@ describe('describeQuestionChanges', () => { ) expect(getTextContent(changes[1])).toBe('made question optional') expect(getTextContent(changes[2])).toBe('changed button text from "Next" to "Continue"') - expect(getTextContent(changes[3])).toBe('changed question type from single_choice to multiple_choice') + expect(getTextContent(changes[3])).toBe( + 'changed question type from Single choice select to Multiple choice select' + ) expect(getTextContent(changes[4])).toBe('added choices: Maybe') expect(getTextContent(changes[5])).toBe('updated branching logic') }) diff --git a/frontend/src/scenes/surveys/surveyActivityDescriber.tsx b/frontend/src/scenes/surveys/surveyActivityDescriber.tsx index 4a355ca7b2eb7d..8b538b9ae35cb7 100644 --- a/frontend/src/scenes/surveys/surveyActivityDescriber.tsx +++ b/frontend/src/scenes/surveys/surveyActivityDescriber.tsx @@ -25,6 +25,8 @@ import { SurveyQuestionType, } from '~/types' +import { SurveyQuestionLabel } from './constants' + const isEmptyOrUndefined = (value: any): boolean => value === undefined || value === null || value === '' const nameOrLinkToSurvey = ( @@ -440,7 +442,8 @@ export function describeQuestionChanges(before: SurveyQuestion, after: SurveyQue before.type !== after.type ? [ <> - changed question type from {before.type} to {after.type} + changed question type from {SurveyQuestionLabel[before.type]} to{' '} + {SurveyQuestionLabel[after.type]} , ] : [] diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 5a263adbcbcb2d..ceede7bd84355e 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -2082,15 +2082,10 @@ export enum RetentionPeriod { export type BreakdownKeyType = string | number | (string | number)[] | null /** - * Legacy multiple breakdowns had `property` and `type` fields. - * Mirroring the legacy fields here for backwards compatibility with multiple breakdowns. + * Legacy breakdown. */ export interface Breakdown { - value?: string - /** - * Legacy breakdown has a `property` field that is `value` now. - */ - property?: string | number + property: string | number type: BreakdownType normalize_url?: boolean histogram_bin_count?: number @@ -4289,13 +4284,18 @@ export type HogFunctionType = { status?: HogFunctionStatus } -export type HogFunctionConfigurationType = Omit +export type HogFunctionConfigurationType = Omit< + HogFunctionType, + 'created_at' | 'created_by' | 'updated_at' | 'status' | 'hog' +> & { + hog?: HogFunctionType['hog'] // In the config it can be empty if using a template +} export type HogFunctionTemplateType = Pick< HogFunctionType, 'id' | 'name' | 'description' | 'hog' | 'inputs_schema' | 'filters' | 'icon_url' > & { - status: 'alpha' | 'beta' | 'stable' + status: 'alpha' | 'beta' | 'stable' | 'free' } export type HogFunctionIconResponse = { @@ -4374,6 +4374,28 @@ export interface AlertType { anomaly_condition: AnomalyCondition } +export type AppMetricsV2Response = { + labels: string[] + series: { + name: string + values: number[] + }[] +} + +export type AppMetricsTotalsV2Response = { + totals: Record +} + +export type AppMetricsV2RequestParams = { + after?: string + before?: string + // Comma separated list of log levels + name?: string + kind?: string + interval?: 'hour' | 'day' | 'week' + breakdown_by?: 'name' | 'kind' +} + export enum DataWarehouseTab { Explore = 'explore', ManagedSources = 'managed-sources', diff --git a/package.json b/package.json index 1e34b62df4af54..6fe9d5d70e712d 100644 --- a/package.json +++ b/package.json @@ -147,7 +147,7 @@ "pmtiles": "^2.11.0", "postcss": "^8.4.31", "postcss-preset-env": "^9.3.0", - "posthog-js": "1.148.2", + "posthog-js": "1.149.0", "posthog-js-lite": "3.0.0", "prettier": "^2.8.8", "prop-types": "^15.7.2", diff --git a/plugin-server/package.json b/plugin-server/package.json index cea586fc0d38d5..28c9bb1a1351ec 100644 --- a/plugin-server/package.json +++ b/plugin-server/package.json @@ -73,7 +73,7 @@ "lru-cache": "^6.0.0", "luxon": "^3.4.4", "node-fetch": "^2.6.1", - "node-rdkafka": "^3.1.0", + "node-rdkafka": "^2.17.0", "node-schedule": "^2.1.0", "pg": "^8.6.0", "pino": "^8.6.0", @@ -137,7 +137,7 @@ }, "pnpm": { "patchedDependencies": { - "node-rdkafka@3.1.0": "patches/node-rdkafka@3.1.0.patch" + "node-rdkafka@2.17.0": "patches/node-rdkafka@2.17.0.patch" } } } diff --git a/plugin-server/patches/node-rdkafka@3.1.0.patch b/plugin-server/patches/node-rdkafka@2.17.0.patch similarity index 91% rename from plugin-server/patches/node-rdkafka@3.1.0.patch rename to plugin-server/patches/node-rdkafka@2.17.0.patch index 36c94f66aef381..a83bf425bd5e33 100644 --- a/plugin-server/patches/node-rdkafka@3.1.0.patch +++ b/plugin-server/patches/node-rdkafka@2.17.0.patch @@ -1,3 +1,35 @@ +diff --git a/Oops.rej b/Oops.rej +new file mode 100644 +index 0000000000000000000000000000000000000000..328fc546fcb400745783b3562f1cb1cb055e1804 +--- /dev/null ++++ b/Oops.rej +@@ -0,0 +1,26 @@ ++@@ -1,25 +0,0 @@ ++-# This workflow will run tests using node and then publish a package to GitHub Packages when a release is created ++-# For more information see: https://help.github.com/actions/language-and-framework-guides/publishing-nodejs-packages ++- ++-name: Publish node-rdkafka ++- ++-on: ++- release: ++- types: [created] ++- ++-jobs: ++- publish-npm: ++- runs-on: ubuntu-latest ++- steps: ++- - uses: actions/checkout@v3 ++- with: ++- submodules: recursive ++- - uses: actions/setup-node@v3 ++- with: ++- node-version: 18 ++- registry-url: https://registry.npmjs.org/ ++- cache: "npm" ++- - run: npm ci ++- - run: npm publish ++- env: ++- NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} diff --git a/docker-compose.yml b/docker-compose.yml index abe29df25c7312382074b3e15289cb862a340247..8a12f135b4f96e5a0dd25e7c21adb2b3b0e644fa 100644 --- a/docker-compose.yml @@ -519,10 +551,10 @@ index a167483f1e0ea15c4edcb368e36640b4349574e8..38fcfd7464afb7df682b7b5f1fdb228b it('should happen gracefully', function(cb) { diff --git a/index.d.ts b/index.d.ts -index 43bedfc97223ee938c6d0f2c5b2c4ecec7dfa800..e10a6d7258613b8ff78b21e9da3d2370f042c5cf 100644 +index d7ce7e61e985ce46ceae2c10329d6448cc487dca..2c7b9a3d40b0547209c2cffe1f4e62d9573ab617 100644 --- a/index.d.ts +++ b/index.d.ts -@@ -227,6 +227,12 @@ export class KafkaConsumer extends Client { +@@ -223,6 +223,12 @@ export class KafkaConsumer extends Client { consume(cb: (err: LibrdKafkaError, messages: Message[]) => void): void; consume(): void; @@ -749,10 +781,10 @@ index a6aadbd64609e5d5ae1a80205aac7ce3a49d9345..f817aa976c83b74670c7464099679eb3 topic_result="$?" if [ "$topic_result" == "1" ]; then diff --git a/src/kafka-consumer.cc b/src/kafka-consumer.cc -index 0f5e32eda8968a26733d7c76cbdce71bfe7d7085..eec338eb06e5502960f12e826b1a48bdc146361f 100644 +index 019b0cb6478756120efe9a5f6f1bb4182b4af4ea..3895407788ae31ae38d7707eb63528ebac6e3b24 100644 --- a/src/kafka-consumer.cc +++ b/src/kafka-consumer.cc -@@ -197,6 +197,32 @@ Baton KafkaConsumer::Assign(std::vector partitions) { +@@ -179,6 +179,32 @@ Baton KafkaConsumer::Assign(std::vector partitions) { return Baton(errcode); } @@ -785,7 +817,7 @@ index 0f5e32eda8968a26733d7c76cbdce71bfe7d7085..eec338eb06e5502960f12e826b1a48bd Baton KafkaConsumer::Unassign() { if (!IsClosing() && !IsConnected()) { return Baton(RdKafka::ERR__STATE); -@@ -213,12 +239,46 @@ Baton KafkaConsumer::Unassign() { +@@ -195,12 +221,46 @@ Baton KafkaConsumer::Unassign() { // Destroy the old list of partitions since we are no longer using it RdKafka::TopicPartition::destroy(m_partitions); @@ -832,7 +864,7 @@ index 0f5e32eda8968a26733d7c76cbdce71bfe7d7085..eec338eb06e5502960f12e826b1a48bd Baton KafkaConsumer::Commit(std::vector toppars) { if (!IsConnected()) { return Baton(RdKafka::ERR__STATE); -@@ -487,6 +547,12 @@ Baton KafkaConsumer::RefreshAssignments() { +@@ -469,6 +529,12 @@ Baton KafkaConsumer::RefreshAssignments() { } } @@ -845,7 +877,7 @@ index 0f5e32eda8968a26733d7c76cbdce71bfe7d7085..eec338eb06e5502960f12e826b1a48bd std::string KafkaConsumer::Name() { if (!IsConnected()) { return std::string(""); -@@ -546,8 +612,11 @@ void KafkaConsumer::Init(v8::Local exports) { +@@ -527,8 +593,11 @@ void KafkaConsumer::Init(v8::Local exports) { Nan::SetPrototypeMethod(tpl, "committed", NodeCommitted); Nan::SetPrototypeMethod(tpl, "position", NodePosition); Nan::SetPrototypeMethod(tpl, "assign", NodeAssign); @@ -857,7 +889,7 @@ index 0f5e32eda8968a26733d7c76cbdce71bfe7d7085..eec338eb06e5502960f12e826b1a48bd Nan::SetPrototypeMethod(tpl, "commit", NodeCommit); Nan::SetPrototypeMethod(tpl, "commitSync", NodeCommitSync); -@@ -778,6 +847,64 @@ NAN_METHOD(KafkaConsumer::NodeAssign) { +@@ -759,6 +828,64 @@ NAN_METHOD(KafkaConsumer::NodeAssign) { info.GetReturnValue().Set(Nan::True()); } @@ -922,7 +954,7 @@ index 0f5e32eda8968a26733d7c76cbdce71bfe7d7085..eec338eb06e5502960f12e826b1a48bd NAN_METHOD(KafkaConsumer::NodeUnassign) { Nan::HandleScope scope; -@@ -798,6 +925,71 @@ NAN_METHOD(KafkaConsumer::NodeUnassign) { +@@ -779,6 +906,71 @@ NAN_METHOD(KafkaConsumer::NodeUnassign) { info.GetReturnValue().Set(Nan::True()); } @@ -1021,7 +1053,7 @@ index c91590ecc5d47c1d7a2a93c3e46b4b4657525df0..43e016db4ec47121051cb282f718a2b3 static NAN_METHOD(NodeUnsubscribe); static NAN_METHOD(NodeCommit); diff --git a/test/consumer.spec.js b/test/consumer.spec.js -index 4fc3bd53208bdaafa3df2df1e5bd9184387b8859..2d32c854d13f99531b9b6a36765c65fa42dd8d62 100644 +index 40b52ee4e1c718890f43b91adfb543319d5cc342..5e1a5655be0d2598163478aaaae936213c3bf27c 100644 --- a/test/consumer.spec.js +++ b/test/consumer.spec.js @@ -77,7 +77,7 @@ module.exports = { @@ -1046,4 +1078,50 @@ index 0f4de520ed6b8a06dfe355e0bb9091273def98a5..ada72a7e621ea5433f194ab3d22eef32 + var t = require('assert'); - var client; \ No newline at end of file + var client; +diff --git a/deps/librdkafka/src/rdkafka_partition.h b/deps/librdkafka/src/rdkafka_partition.h +index f9dd686423..aef704b95f 100644 +--- a/deps/librdkafka/src/rdkafka_partition.h ++++ b/deps/librdkafka/src/rdkafka_partition.h +@@ -68,24 +68,35 @@ struct rd_kafka_toppar_err { + * last msg sequence */ + }; + +- ++/** ++ * @brief Fetchpos comparator, only offset is compared. ++ */ ++static RD_UNUSED RD_INLINE int ++rd_kafka_fetch_pos_cmp_offset(const rd_kafka_fetch_pos_t *a, ++ const rd_kafka_fetch_pos_t *b) { ++ if (a->offset < b->offset) ++ return -1; ++ else if (a->offset > b->offset) ++ return 1; ++ else ++ return 0; ++} + + /** + * @brief Fetchpos comparator, leader epoch has precedence. ++ * iff both values are not null. + */ + static RD_UNUSED RD_INLINE int + rd_kafka_fetch_pos_cmp(const rd_kafka_fetch_pos_t *a, + const rd_kafka_fetch_pos_t *b) { ++ if (a->leader_epoch == -1 || b->leader_epoch == -1) ++ return rd_kafka_fetch_pos_cmp_offset(a, b); + if (a->leader_epoch < b->leader_epoch) + return -1; + else if (a->leader_epoch > b->leader_epoch) + return 1; +- else if (a->offset < b->offset) +- return -1; +- else if (a->offset > b->offset) +- return 1; + else +- return 0; ++ return rd_kafka_fetch_pos_cmp_offset(a, b); + } + + diff --git a/plugin-server/pnpm-lock.yaml b/plugin-server/pnpm-lock.yaml index 3e91a8bd187aa9..ea69c6b1b7c6ad 100644 --- a/plugin-server/pnpm-lock.yaml +++ b/plugin-server/pnpm-lock.yaml @@ -1,13 +1,13 @@ -lockfileVersion: '6.1' +lockfileVersion: '6.0' settings: autoInstallPeers: true excludeLinksFromLockfile: false patchedDependencies: - node-rdkafka@3.1.0: - hash: 2blh4g6m4osumqx2eqyegyjb7q - path: patches/node-rdkafka@3.1.0.patch + node-rdkafka@2.17.0: + hash: bugorwxdhlhl2utknko2c5ibqm + path: patches/node-rdkafka@2.17.0.patch dependencies: '@aws-sdk/client-s3': @@ -113,8 +113,8 @@ dependencies: specifier: ^2.6.1 version: 2.6.9 node-rdkafka: - specifier: ^3.1.0 - version: 3.1.0(patch_hash=2blh4g6m4osumqx2eqyegyjb7q) + specifier: ^2.17.0 + version: 2.17.0(patch_hash=bugorwxdhlhl2utknko2c5ibqm) node-schedule: specifier: ^2.1.0 version: 2.1.1 @@ -8318,10 +8318,6 @@ packages: resolution: {integrity: sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==} dev: false - /nan@2.20.0: - resolution: {integrity: sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==} - dev: false - /nanoassert@1.1.0: resolution: {integrity: sha512-C40jQ3NzfkP53NsO8kEOFd79p4b9kDXQMwgiY1z8ZwrDZgUyom0AHwGegF4Dm99L+YoYhuaB0ceerUcXmqr1rQ==} dev: true @@ -8435,13 +8431,13 @@ packages: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} dev: true - /node-rdkafka@3.1.0(patch_hash=2blh4g6m4osumqx2eqyegyjb7q): - resolution: {integrity: sha512-H0FeV6cgkFX/NHKPyWsUHoC4l645/2vfKVdTyvKmwX+nafHuZzT6xVWl2kJQBNVJcRVgtQA1Loz6iXWhq03RKw==} - engines: {node: '>=16'} + /node-rdkafka@2.17.0(patch_hash=bugorwxdhlhl2utknko2c5ibqm): + resolution: {integrity: sha512-vFABzRcE5FaH0WqfqJRxDoqeG6P8UEB3M4qFQ7SkwMgQueMMO78+fm8MYfl5hLW3bBYfBekK2BXIIr0lDQtSEQ==} + engines: {node: '>=6.0.0'} requiresBuild: true dependencies: bindings: 1.5.0 - nan: 2.20.0 + nan: 2.17.0 dev: false patched: true diff --git a/plugin-server/src/cdp/cdp-consumers.ts b/plugin-server/src/cdp/cdp-consumers.ts index 1d58275228eee9..8463946bbf672a 100644 --- a/plugin-server/src/cdp/cdp-consumers.ts +++ b/plugin-server/src/cdp/cdp-consumers.ts @@ -2,6 +2,7 @@ import { features, librdkafkaVersion, Message } from 'node-rdkafka' import { Counter, Histogram } from 'prom-client' import { + KAFKA_APP_METRICS_2, KAFKA_CDP_FUNCTION_CALLBACKS, KAFKA_CDP_FUNCTION_OVERFLOW, KAFKA_EVENTS_JSON, @@ -13,7 +14,7 @@ import { createRdConnectionConfigFromEnvVars, createRdProducerConfigFromEnvVars import { createKafkaProducer } from '../kafka/producer' import { addSentryBreadcrumbsEventListeners } from '../main/ingestion-queues/kafka-metrics' import { runInstrumentedFunction } from '../main/utils' -import { GroupTypeToColumnIndex, Hub, RawClickHouseEvent, TeamId, TimestampFormat } from '../types' +import { AppMetric2Type, GroupTypeToColumnIndex, Hub, RawClickHouseEvent, TeamId, TimestampFormat } from '../types' import { KafkaProducerWrapper } from '../utils/db/kafka-producer-wrapper' import { status } from '../utils/status' import { castTimestampOrNow } from '../utils/utils' @@ -151,6 +152,24 @@ abstract class CdpConsumerBase { ) } + protected logAppMetrics( + metric: Pick + ) { + const appMetric: AppMetric2Type = { + app_source: 'hog_function', + ...metric, + timestamp: castTimestampOrNow(null, TimestampFormat.ClickHouse), + } + + this.messagesToProduce.push({ + topic: KAFKA_APP_METRICS_2, + value: appMetric, + key: appMetric.app_source_id, + }) + + counterFunctionInvocation.inc({ outcome: appMetric.metric_name }, appMetric.count) + } + protected async processInvocationResults(results: HogFunctionInvocationResult[]): Promise { await runInstrumentedFunction({ statsKey: `cdpConsumer.handleEachBatch.produceResults`, @@ -161,8 +180,12 @@ abstract class CdpConsumerBase { const logs = result.logs result.logs = [] - counterFunctionInvocation.inc({ - outcome: result.error ? 'failed' : 'succeeded', + this.logAppMetrics({ + team_id: result.teamId, + app_source_id: result.hogFunctionId, + metric_kind: result.error ? 'failure' : 'success', + metric_name: result.error ? 'failed' : 'succeeded', + count: 1, }) logs.forEach((x) => { @@ -242,10 +265,19 @@ abstract class CdpConsumerBase { }, key: item.id, }) + // We don't report overflowed metric to appmetrics as it is sort of a meta-metric counterFunctionInvocation.inc({ outcome: 'overflowed' }) } else if (functionState > HogWatcherState.disabledForPeriod) { - // TODO: Report to AppMetrics 2 when it is ready - counterFunctionInvocation.inc({ outcome: 'disabled' }) + this.logAppMetrics({ + team_id: item.teamId, + app_source_id: item.hogFunctionId, + metric_kind: 'failure', + metric_name: + functionState === HogWatcherState.disabledForPeriod + ? 'disabled_temporarily' + : 'disabled_permanently', + count: 1, + }) continue } else { asyncResponsesToRun.push(item) @@ -271,34 +303,32 @@ abstract class CdpConsumerBase { const invocations: { globals: HogFunctionInvocationGlobals; hogFunction: HogFunctionType }[] = [] invocationGlobals.forEach((globals) => { - const { functions, total, matching } = this.hogExecutor.findMatchingFunctions(globals) - - counterFunctionInvocation.inc({ outcome: 'filtered' }, total - matching) + const { matchingFunctions, nonMatchingFunctions } = this.hogExecutor.findMatchingFunctions(globals) + + nonMatchingFunctions.forEach((item) => + this.logAppMetrics({ + team_id: item.team_id, + app_source_id: item.id, + metric_kind: 'other', + metric_name: 'filtered', + count: 1, + }) + ) // Filter for overflowed and disabled functions - const [healthy, overflowed, disabled] = functions.reduce( - (acc, item) => { - const state = this.hogWatcher.getFunctionState(item.id) - if (state >= HogWatcherState.disabledForPeriod) { - acc[2].push(item) - } else if (state >= HogWatcherState.overflowed) { - acc[1].push(item) - } else { - acc[0].push(item) - } - - return acc - }, - [[], [], []] as [HogFunctionType[], HogFunctionType[], HogFunctionType[]] - ) + const hogFunctionsByState = matchingFunctions.reduce((acc, item) => { + const state = this.hogWatcher.getFunctionState(item.id) + return { + ...acc, + [state]: [...(acc[state] ?? []), item], + } + return acc + }, {} as Record) - if (overflowed.length) { + if (hogFunctionsByState[HogWatcherState.overflowed]?.length) { + const overflowed = hogFunctionsByState[HogWatcherState.overflowed]! // Group all overflowed functions into one event counterFunctionInvocation.inc({ outcome: 'overflowed' }, overflowed.length) - // TODO: Report to AppMetrics 2 when it is ready - status.debug('🔁', `Oveflowing functions`, { - count: overflowed.length, - }) this.messagesToProduce.push({ topic: KAFKA_CDP_FUNCTION_OVERFLOW, @@ -313,18 +343,30 @@ abstract class CdpConsumerBase { }) } - if (disabled.length) { - counterFunctionInvocation.inc({ outcome: 'disabled' }, disabled.length) - // TODO: Report to AppMetrics 2 when it is ready - status.debug('🔁', `Disabled functions skipped`, { - count: disabled.length, + hogFunctionsByState[HogWatcherState.disabledForPeriod]?.forEach((item) => { + this.logAppMetrics({ + team_id: item.team_id, + app_source_id: item.id, + metric_kind: 'failure', + metric_name: 'disabled_temporarily', + count: 1, }) - } + }) - healthy.forEach((x) => { + hogFunctionsByState[HogWatcherState.disabledIndefinitely]?.forEach((item) => { + this.logAppMetrics({ + team_id: item.team_id, + app_source_id: item.id, + metric_kind: 'failure', + metric_name: 'disabled_permanently', + count: 1, + }) + }) + + hogFunctionsByState[HogWatcherState.healthy]?.forEach((item) => { invocations.push({ globals, - hogFunction: x, + hogFunction: item, }) }) }) @@ -583,7 +625,16 @@ export class CdpOverflowConsumer extends CdpConsumerBase { await this.runManyWithHeartbeat(invocations, (item) => { const state = this.hogWatcher.getFunctionState(item.hogFunctionId) if (state >= HogWatcherState.disabledForPeriod) { - counterFunctionInvocation.inc({ outcome: 'disabled' }) + this.logAppMetrics({ + team_id: item.globals.project.id, + app_source_id: item.hogFunctionId, + metric_kind: 'failure', + metric_name: + state === HogWatcherState.disabledForPeriod + ? 'disabled_temporarily' + : 'disabled_permanently', + count: 1, + }) return } return this.hogExecutor.executeFunction(item.globals, item.hogFunctionId) diff --git a/plugin-server/src/cdp/hog-executor.ts b/plugin-server/src/cdp/hog-executor.ts index 9bee644c3d80df..debab8c702561a 100644 --- a/plugin-server/src/cdp/hog-executor.ts +++ b/plugin-server/src/cdp/hog-executor.ts @@ -96,36 +96,30 @@ export class HogExecutor { constructor(private hogFunctionManager: HogFunctionManager) {} findMatchingFunctions(event: HogFunctionInvocationGlobals): { - total: number - matching: number - functions: HogFunctionType[] + matchingFunctions: HogFunctionType[] + nonMatchingFunctions: HogFunctionType[] } { const allFunctionsForTeam = this.hogFunctionManager.getTeamHogFunctions(event.project.id) const filtersGlobals = convertToHogFunctionFilterGlobal(event) + const nonMatchingFunctions: HogFunctionType[] = [] + const matchingFunctions: HogFunctionType[] = [] + // Filter all functions based on the invocation - const functions = allFunctionsForTeam.filter((hogFunction) => { + allFunctionsForTeam.forEach((hogFunction) => { try { - const filters = hogFunction.filters - - if (!filters?.bytecode) { - // NOTE: If we don't have bytecode this indicates something went wrong. - // The model will always save a bytecode if it was compiled correctly - return false - } - - const filterResult = exec(filters.bytecode, { - globals: filtersGlobals, - timeout: DEFAULT_TIMEOUT_MS, - maxAsyncSteps: 0, - }) + if (hogFunction.filters?.bytecode) { + const filterResult = exec(hogFunction.filters.bytecode, { + globals: filtersGlobals, + timeout: DEFAULT_TIMEOUT_MS, + maxAsyncSteps: 0, + }) - if (typeof filterResult.result !== 'boolean') { - // NOTE: If the result is not a boolean we should not execute the function - return false + if (typeof filterResult.result === 'boolean' && filterResult.result) { + matchingFunctions.push(hogFunction) + return + } } - - return filterResult.result } catch (error) { status.error('🦔', `[HogExecutor] Error filtering function`, { hogFunctionId: hogFunction.id, @@ -134,20 +128,19 @@ export class HogExecutor { }) } - return false + nonMatchingFunctions.push(hogFunction) }) status.debug( '🦔', - `[HogExecutor] Found ${Object.keys(functions).length} matching functions out of ${ + `[HogExecutor] Found ${Object.keys(matchingFunctions).length} matching functions out of ${ Object.keys(allFunctionsForTeam).length } for team` ) return { - total: allFunctionsForTeam.length, - matching: functions.length, - functions, + nonMatchingFunctions, + matchingFunctions, } } @@ -243,7 +236,7 @@ export class HogExecutor { hogFunctionUrl: invocation.globals.source?.url, } - status.info('🦔', `[HogExecutor] Executing function`, loggingContext) + status.debug('🦔', `[HogExecutor] Executing function`, loggingContext) const result: HogFunctionInvocationResult = { ...invocation, diff --git a/plugin-server/src/cdp/types.ts b/plugin-server/src/cdp/types.ts index 5b3e75fc028a10..ef33a540df134c 100644 --- a/plugin-server/src/cdp/types.ts +++ b/plugin-server/src/cdp/types.ts @@ -1,7 +1,13 @@ import { VMState } from '@posthog/hogvm' import { DateTime } from 'luxon' -import { ClickHouseTimestamp, ElementPropertyFilter, EventPropertyFilter, PersonPropertyFilter } from '../types' +import { + AppMetric2Type, + ClickHouseTimestamp, + ElementPropertyFilter, + EventPropertyFilter, + PersonPropertyFilter, +} from '../types' export type HogBytecode = any[] @@ -247,7 +253,7 @@ export type CdpOverflowMessage = CdpOverflowMessageInvocations | CdpOverflowMess export type HogFunctionMessageToProduce = { topic: string - value: CdpOverflowMessage | HogFunctionLogEntrySerialized | HogFunctionInvocationAsyncResponse + value: CdpOverflowMessage | HogFunctionLogEntrySerialized | HogFunctionInvocationAsyncResponse | AppMetric2Type key: string } diff --git a/plugin-server/src/config/kafka-topics.ts b/plugin-server/src/config/kafka-topics.ts index 2f095e117f7ba2..5ff55c487524a3 100644 --- a/plugin-server/src/config/kafka-topics.ts +++ b/plugin-server/src/config/kafka-topics.ts @@ -20,6 +20,7 @@ export const KAFKA_GROUPS = `${prefix}clickhouse_groups${suffix}` export const KAFKA_BUFFER = `${prefix}conversion_events_buffer${suffix}` export const KAFKA_INGESTION_WARNINGS = `${prefix}clickhouse_ingestion_warnings${suffix}` export const KAFKA_APP_METRICS = `${prefix}clickhouse_app_metrics${suffix}` +export const KAFKA_APP_METRICS_2 = `${prefix}clickhouse_app_metrics2${suffix}` export const KAFKA_JOBS = `${prefix}jobs${suffix}` export const KAFKA_JOBS_DLQ = `${prefix}jobs_dlq${suffix}` export const KAFKA_SCHEDULED_TASKS = `${prefix}scheduled_tasks${suffix}` diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts index e73945c6f1f233..c4df28fa9e7988 100644 --- a/plugin-server/src/types.ts +++ b/plugin-server/src/types.ts @@ -1224,3 +1224,14 @@ export interface HookPayload { } } } + +export type AppMetric2Type = { + team_id: number + timestamp: ClickHouseTimestamp + app_source: string + app_source_id: string + instance_id?: string + metric_kind: 'failure' | 'success' | 'other' + metric_name: 'succeeded' | 'failed' | 'filtered' | 'disabled_temporarily' | 'disabled_permanently' + count: number +} diff --git a/plugin-server/tests/cdp/cdp-processed-events-consumer.test.ts b/plugin-server/tests/cdp/cdp-processed-events-consumer.test.ts index 0050446c026a9e..a5710a620cd9df 100644 --- a/plugin-server/tests/cdp/cdp-processed-events-consumer.test.ts +++ b/plugin-server/tests/cdp/cdp-processed-events-consumer.test.ts @@ -148,8 +148,8 @@ describe('CDP Processed Events Consuner', () => { `) }) - it('generates logs and produces them to kafka', async () => { - await insertHogFunction({ + it('generates logs and metrics and produces them to kafka', async () => { + const hogFunction = await insertHogFunction({ ...HOG_EXAMPLES.simple_fetch, ...HOG_INPUTS_EXAMPLES.simple_fetch, ...HOG_FILTERS_EXAMPLES.no_filters, @@ -173,10 +173,25 @@ describe('CDP Processed Events Consuner', () => { ) expect(mockFetch).toHaveBeenCalledTimes(1) - // Once for the async callback, twice for the logs - expect(mockProducer.produce).toHaveBeenCalledTimes(3) + // Once for the async callback, twice for the logs, once for metrics + expect(mockProducer.produce).toHaveBeenCalledTimes(4) + + expect(decodeKafkaMessage(mockProducer.produce.mock.calls[0][0])).toEqual({ + key: expect.any(String), + topic: 'clickhouse_app_metrics2_test', + value: { + app_source: 'hog_function', + team_id: 2, + app_source_id: hogFunction.id, + metric_kind: 'success', + metric_name: 'succeeded', + count: 1, + timestamp: expect.any(String), + }, + waitForAck: true, + }) - expect(decodeKafkaMessage(mockProducer.produce.mock.calls[0][0])).toMatchObject({ + expect(decodeKafkaMessage(mockProducer.produce.mock.calls[1][0])).toEqual({ key: expect.any(String), topic: 'log_entries_test', value: { @@ -188,10 +203,11 @@ describe('CDP Processed Events Consuner', () => { team_id: 2, timestamp: expect.any(String), }, + waitForAck: true, }) - expect(decodeKafkaMessage(mockProducer.produce.mock.calls[1][0])).toMatchObject({ + expect(decodeKafkaMessage(mockProducer.produce.mock.calls[2][0])).toMatchObject({ topic: 'log_entries_test', value: { log_source: 'hog_function', @@ -200,7 +216,7 @@ describe('CDP Processed Events Consuner', () => { }, }) - const msg = decodeKafkaMessage(mockProducer.produce.mock.calls[2][0]) + const msg = decodeKafkaMessage(mockProducer.produce.mock.calls[3][0]) // Parse body so it can match by object equality rather than exact string equality msg.value.asyncFunctionRequest.args[1].body = JSON.parse(msg.value.asyncFunctionRequest.args[1].body) expect(msg).toEqual({ diff --git a/plugin-server/tests/cdp/hog-executor.test.ts b/plugin-server/tests/cdp/hog-executor.test.ts index 4b8074a5ccc3cf..d1cd558e925216 100644 --- a/plugin-server/tests/cdp/hog-executor.test.ts +++ b/plugin-server/tests/cdp/hog-executor.test.ts @@ -65,7 +65,7 @@ describe('Hog Executor', () => { const globals = createHogExecutionGlobals() const results = executor .findMatchingFunctions(createHogExecutionGlobals()) - .functions.map((x) => executor.executeFunction(globals, x) as HogFunctionInvocationResult) + .matchingFunctions.map((x) => executor.executeFunction(globals, x) as HogFunctionInvocationResult) expect(results).toHaveLength(1) expect(results[0]).toMatchObject({ id: expect.any(String), @@ -77,7 +77,7 @@ describe('Hog Executor', () => { const globals = createHogExecutionGlobals() const results = executor .findMatchingFunctions(createHogExecutionGlobals()) - .functions.map((x) => executor.executeFunction(globals, x) as HogFunctionInvocationResult) + .matchingFunctions.map((x) => executor.executeFunction(globals, x) as HogFunctionInvocationResult) expect(results[0].logs).toMatchObject([ { team_id: 1, @@ -134,7 +134,7 @@ describe('Hog Executor', () => { const globals = createHogExecutionGlobals() const results = executor .findMatchingFunctions(createHogExecutionGlobals()) - .functions.map((x) => executor.executeFunction(globals, x) as HogFunctionInvocationResult) + .matchingFunctions.map((x) => executor.executeFunction(globals, x) as HogFunctionInvocationResult) expect(results[0]).toMatchObject({ id: results[0].id, globals: { @@ -193,7 +193,7 @@ describe('Hog Executor', () => { const globals = createHogExecutionGlobals() const results = executor .findMatchingFunctions(createHogExecutionGlobals()) - .functions.map((x) => executor.executeFunction(globals, x) as HogFunctionInvocationResult) + .matchingFunctions.map((x) => executor.executeFunction(globals, x) as HogFunctionInvocationResult) const splicedLogs = results[0].logs.splice(0, 100) logs.push(...splicedLogs) @@ -223,9 +223,8 @@ describe('Hog Executor', () => { mockFunctionManager.getTeamHogFunctions.mockReturnValue([fn]) const resultsShouldntMatch = executor.findMatchingFunctions(createHogExecutionGlobals()) - expect(resultsShouldntMatch.functions).toHaveLength(0) - expect(resultsShouldntMatch.total).toBe(1) - expect(resultsShouldntMatch.matching).toBe(0) + expect(resultsShouldntMatch.matchingFunctions).toHaveLength(0) + expect(resultsShouldntMatch.nonMatchingFunctions).toHaveLength(1) const resultsShouldMatch = executor.findMatchingFunctions( createHogExecutionGlobals({ @@ -237,9 +236,8 @@ describe('Hog Executor', () => { } as any, }) ) - expect(resultsShouldMatch.functions).toHaveLength(1) - expect(resultsShouldMatch.total).toBe(1) - expect(resultsShouldMatch.matching).toBe(1) + expect(resultsShouldMatch.matchingFunctions).toHaveLength(1) + expect(resultsShouldMatch.nonMatchingFunctions).toHaveLength(0) }) }) @@ -257,7 +255,7 @@ describe('Hog Executor', () => { const globals = createHogExecutionGlobals() const results = executor .findMatchingFunctions(createHogExecutionGlobals()) - .functions.map((x) => executor.executeFunction(globals, x) as HogFunctionInvocationResult) + .matchingFunctions.map((x) => executor.executeFunction(globals, x) as HogFunctionInvocationResult) expect(results).toHaveLength(1) // Run the result one time simulating a successful fetch @@ -295,7 +293,7 @@ describe('Hog Executor', () => { const globals = createHogExecutionGlobals() const results = executor .findMatchingFunctions(createHogExecutionGlobals()) - .functions.map((x) => executor.executeFunction(globals, x) as HogFunctionInvocationResult) + .matchingFunctions.map((x) => executor.executeFunction(globals, x) as HogFunctionInvocationResult) expect(results).toHaveLength(1) expect(results[0].error).toContain('Execution timed out after 0.1 seconds. Performed ') diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8489d56f6dc976..13add50e919ca3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -263,8 +263,8 @@ dependencies: specifier: ^9.3.0 version: 9.3.0(postcss@8.4.31) posthog-js: - specifier: 1.148.2 - version: 1.148.2 + specifier: 1.149.0 + version: 1.149.0 posthog-js-lite: specifier: 3.0.0 version: 3.0.0 @@ -17721,8 +17721,8 @@ packages: resolution: {integrity: sha512-dyajjnfzZD1tht4N7p7iwf7nBnR1MjVaVu+MKr+7gBgA39bn28wizCIJZztZPtHy4PY0YwtSGgwfBCuG/hnHgA==} dev: false - /posthog-js@1.148.2: - resolution: {integrity: sha512-YQt8D+RS1a56ykLPLsqSq73vLYLIFQwlvY40ncgOM/uhxAV4FmbnYv85ZtwnB2SAZD/gChWXI/fprYQA9lhc1w==} + /posthog-js@1.149.0: + resolution: {integrity: sha512-uIknyqxv5uDAToPaYVBzGqWwTiuga56cHs+3OeiXKZgjkm97yWh9VA5/gRD/3LEq3iszxHEOU4I5pVIaUrMNtg==} dependencies: fflate: 0.4.8 preact: 10.22.1 diff --git a/posthog/api/app_metrics2.py b/posthog/api/app_metrics2.py new file mode 100644 index 00000000000000..a6fc2c6daef544 --- /dev/null +++ b/posthog/api/app_metrics2.py @@ -0,0 +1,290 @@ +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Any, Optional, cast +from rest_framework import serializers, viewsets +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.decorators import action +from rest_framework.exceptions import ValidationError +from rest_framework_dataclasses.serializers import DataclassSerializer + +from posthog.clickhouse.client.execute import sync_execute +from posthog.models.team.team import Team +from posthog.utils import relative_date_parse_with_delta_mapping + + +@dataclass +class AppMetricSeries: + name: str + values: list[int] + + +@dataclass +class AppMetricsResponse: + labels: list[str] + series: list[AppMetricSeries] + + +class AppMetricResponseSerializer(DataclassSerializer): + class Meta: + dataclass = AppMetricsResponse + + +@dataclass +class AppMetricsTotalsResponse: + totals: dict[str, int] + + +class AppMetricsTotalsResponseSerializer(DataclassSerializer): + class Meta: + dataclass = AppMetricsTotalsResponse + + +class AppMetricsRequestSerializer(serializers.Serializer): + after = serializers.CharField(required=False, default="-7d") + before = serializers.CharField(required=False) + instance_id = serializers.CharField(required=False) + interval = serializers.ChoiceField(choices=["hour", "day", "week"], required=False, default="day") + name = serializers.CharField(required=False) + kind = serializers.CharField(required=False) + breakdown_by = serializers.ChoiceField(choices=["name", "kind"], required=False, default="kind") + + +def fetch_app_metrics_trends( + team_id: int, + app_source: str, + app_source_id: str, + after: datetime, + before: datetime, + breakdown_by: str = "kind", + interval: str = "day", + instance_id: Optional[str] = None, + name: Optional[list[str]] = None, + kind: Optional[list[str]] = None, +) -> AppMetricsResponse: + """Fetch a list of batch export log entries from ClickHouse.""" + + name = name or [] + kind = kind or [] + + clickhouse_kwargs: dict[str, Any] = {} + + clickhouse_query = f""" + SELECT + toStartOfInterval(timestamp, INTERVAL 1 {interval}) as timestamp, + metric_{breakdown_by} as breakdown, + count(breakdown) as count + FROM app_metrics2 + WHERE team_id = %(team_id)s + AND app_source = %(app_source)s + AND app_source_id = %(app_source_id)s + AND timestamp >= toDateTime64(%(after)s, 6) + AND timestamp <= toDateTime64(%(before)s, 6) + {'AND instance_id = %(instance_id)s' if instance_id else ''} + {'AND metric_name IN %(name)s' if name else ''} + {'AND metric_kind IN %(kind)s' if kind else ''} + GROUP BY timestamp, breakdown + ORDER BY timestamp ASC + """ + + clickhouse_kwargs["team_id"] = team_id + clickhouse_kwargs["app_source"] = app_source + clickhouse_kwargs["app_source_id"] = app_source_id + clickhouse_kwargs["after"] = after.strftime("%Y-%m-%dT%H:%M:%S") + clickhouse_kwargs["before"] = before.strftime("%Y-%m-%dT%H:%M:%S") + clickhouse_kwargs["instance_id"] = instance_id + clickhouse_kwargs["name"] = name + clickhouse_kwargs["kind"] = kind + clickhouse_kwargs["interval"] = interval.upper() + + results = sync_execute(clickhouse_query, clickhouse_kwargs) + + if not isinstance(results, list): + raise ValueError("Unexpected results from ClickHouse") + + # We create the x values based on the date range and interval + labels: list[str] = [] + label_format = "%Y-%m-%dT%H:%M" if interval == "hour" else "%Y-%m-%d" + + range_date = after + # Normalize the start of the range to the start of the interval + if interval == "hour": + range_date = range_date.replace(minute=0, second=0, microsecond=0) + elif interval == "day": + range_date = range_date.replace(hour=0, minute=0, second=0, microsecond=0) + elif interval == "week": + range_date = range_date.replace(hour=0, minute=0, second=0, microsecond=0) + range_date -= timedelta(days=range_date.weekday()) + + while range_date <= before: + labels.append(range_date.strftime(label_format)) + if interval == "hour": + range_date += timedelta(hours=1) + elif interval == "day": + range_date += timedelta(days=1) + elif interval == "week": + range_date += timedelta(weeks=1) + + response = AppMetricsResponse(labels=[], series=[]) + data_by_breakdown: dict[str, dict[str, int]] = {} + + breakdown_names = {row[1] for row in results} + + for result in results: + timestamp, breakdown, count = result + if breakdown not in data_by_breakdown: + data_by_breakdown[breakdown] = {} + + data_by_breakdown[breakdown][timestamp.strftime(label_format)] = count + + # Now we can construct the response object + + response.labels = labels + + for breakdown in breakdown_names: + series = AppMetricSeries(name=breakdown, values=[]) + for x in labels: + series.values.append(data_by_breakdown.get(breakdown, {}).get(x, 0)) + response.series.append(series) + + return response + + +def fetch_app_metric_totals( + team_id: int, + app_source: str, + app_source_id: str, + breakdown_by: str = "kind", + after: Optional[datetime] = None, + before: Optional[datetime] = None, + instance_id: Optional[str] = None, + name: Optional[list[str]] = None, + kind: Optional[list[str]] = None, +) -> AppMetricsTotalsResponse: + """ + Calculate the totals for the app metrics over the given period. + """ + + name = name or [] + kind = kind or [] + + clickhouse_kwargs: dict[str, Any] = { + "team_id": team_id, + "app_source": app_source, + "app_source_id": app_source_id, + "after": after.strftime("%Y-%m-%dT%H:%M:%S") if after else None, + "before": before.strftime("%Y-%m-%dT%H:%M:%S") if before else None, + } + + clickhouse_query = f""" + SELECT + metric_{breakdown_by} as breakdown, + count(breakdown) as count + FROM app_metrics2 + WHERE team_id = %(team_id)s + AND app_source = %(app_source)s + AND app_source_id = %(app_source_id)s + {'AND timestamp >= toDateTime64(%(after)s, 6)' if after else ''} + {'AND timestamp <= toDateTime64(%(before)s, 6)' if before else ''} + {'AND instance_id = %(instance_id)s' if instance_id else ''} + {'AND metric_name IN %(name)s' if name else ''} + {'AND metric_kind IN %(kind)s' if kind else ''} + GROUP BY breakdown + """ + + results = sync_execute(clickhouse_query, clickhouse_kwargs) + + if not isinstance(results, list): + raise ValueError("Unexpected results from ClickHouse") + + totals = {row[0]: row[1] for row in results} + return AppMetricsTotalsResponse(totals=totals) + + +class AppMetricsMixin(viewsets.GenericViewSet): + app_source: str # Should be set by the inheriting class + + def get_app_metrics_instance_id(self) -> Optional[str]: + """ + Can be used overridden to help with getting the instance_id for the app metrics. + Otherwise it defaults to null or the query param if given + """ + raise NotImplementedError() + + @action(detail=True, methods=["GET"]) + def metrics(self, request: Request, *args, **kwargs): + obj = self.get_object() + param_serializer = AppMetricsRequestSerializer(data=request.query_params) + + if not self.app_source: + raise ValidationError("app_source not set on the viewset") + + if not param_serializer.is_valid(): + raise ValidationError(param_serializer.errors) + + params = param_serializer.validated_data + + try: + instance_id = self.get_app_metrics_instance_id() + except NotImplementedError: + instance_id = params.get("instance_id") + + team = cast(Team, self.team) # type: ignore + + after_date, _, _ = relative_date_parse_with_delta_mapping(params.get("after", "-7d"), team.timezone_info) + before_date, _, _ = relative_date_parse_with_delta_mapping(params.get("before", "-0d"), team.timezone_info) + + data = fetch_app_metrics_trends( + team_id=self.team_id, # type: ignore + app_source=self.app_source, + app_source_id=str(obj.id), + # From request params + instance_id=instance_id, + interval=params.get("interval", "day"), + after=after_date, + before=before_date, + breakdown_by=params.get("breakdown_by"), + name=params["name"].split(",") if params.get("name") else None, + kind=params["kind"].split(",") if params.get("kind") else None, + ) + + serializer = AppMetricResponseSerializer(instance=data) + return Response(serializer.data) + + @action(detail=True, methods=["GET"], url_path="metrics/totals") + def metrics_totals(self, request: Request, *args, **kwargs): + obj = self.get_object() + param_serializer = AppMetricsRequestSerializer(data=request.query_params) + + if not self.app_source: + raise ValidationError("app_source not set on the viewset") + + if not param_serializer.is_valid(): + raise ValidationError(param_serializer.errors) + + params = param_serializer.validated_data + team = cast(Team, self.team) # type: ignore + + after_date = None + before_date = None + + if params.get("after"): + after_date, _, _ = relative_date_parse_with_delta_mapping(params["after"], team.timezone_info) + + if params.get("before"): + before_date, _, _ = relative_date_parse_with_delta_mapping(params["before"], team.timezone_info) + + data = fetch_app_metric_totals( + team_id=self.team_id, # type: ignore + app_source=self.app_source, + app_source_id=str(obj.id), + # From request params + after=after_date, + before=before_date, + breakdown_by=params.get("breakdown_by"), + name=params["name"].split(",") if params.get("name") else None, + kind=params["kind"].split(",") if params.get("kind") else None, + ) + + serializer = AppMetricsTotalsResponseSerializer(instance=data) + return Response(serializer.data) diff --git a/posthog/api/hog_function.py b/posthog/api/hog_function.py index f754a97e54822d..972178a875401a 100644 --- a/posthog/api/hog_function.py +++ b/posthog/api/hog_function.py @@ -9,6 +9,7 @@ from rest_framework.request import Request from rest_framework.response import Response +from posthog.api.app_metrics2 import AppMetricsMixin from posthog.api.forbid_destroy_model import ForbidDestroyModel from posthog.api.hog_function_template import HogFunctionTemplateSerializer from posthog.api.log_entries import LogEntryMixin @@ -16,7 +17,9 @@ from posthog.api.shared import UserBasicSerializer from posthog.cdp.services.icons import CDPIconsService +from posthog.cdp.templates import HOG_FUNCTION_TEMPLATES_BY_ID from posthog.cdp.validation import compile_hog, validate_inputs, validate_inputs_schema +from posthog.constants import AvailableFeature from posthog.models.hog_functions.hog_function import HogFunction, HogFunctionState from posthog.permissions import PostHogFeatureFlagPermission from posthog.plugins.plugin_server_api import create_hog_invocation_test @@ -86,18 +89,52 @@ class Meta: "status", ] extra_kwargs = { + "hog": {"required": False}, + "inputs_schema": {"required": False}, "template_id": {"write_only": True}, "deleted": {"write_only": True}, } - def validate_inputs_schema(self, value): - return validate_inputs_schema(value) - def validate(self, attrs): team = self.context["get_team"]() attrs["team"] = team + + has_addon = team.organization.is_feature_available(AvailableFeature.DATA_PIPELINES) + + if not has_addon: + template_id = attrs.get("template_id") + template = HOG_FUNCTION_TEMPLATES_BY_ID.get(template_id, None) + + # In this case they are only allowed to create or update the function with free templates + if not template: + raise serializers.ValidationError( + {"template_id": "The Data Pipelines addon is required to create custom functions."} + ) + + if template.status != "free": + raise serializers.ValidationError( + {"template_id": "The Data Pipelines addon is required for this template."} + ) + + if attrs.get("hog"): + raise serializers.ValidationError( + {"hog": "The Data Pipelines addon is required to create custom functions."} + ) + + if attrs.get("inputs_schema"): + raise serializers.ValidationError( + {"inputs_schema": "The Data Pipelines addon is required to create custom functions."} + ) + + # Without the addon, they cannot deviate from the template + attrs["inputs_schema"] = template.inputs_schema + attrs["hog"] = template.hog + instance = cast(Optional[HogFunction], self.context.get("instance", self.instance)) + if "inputs_schema" in attrs: + attrs["inputs_schema"] = validate_inputs_schema(attrs["inputs_schema"]) + if self.context["view"].action == "create": # Ensure we have sensible defaults when created attrs["filters"] = attrs.get("filters", {}) @@ -156,7 +193,9 @@ class HogFunctionInvocationSerializer(serializers.Serializer): logs = serializers.ListField(read_only=True) -class HogFunctionViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, ForbidDestroyModel, viewsets.ModelViewSet): +class HogFunctionViewSet( + TeamAndOrgViewSetMixin, LogEntryMixin, AppMetricsMixin, ForbidDestroyModel, viewsets.ModelViewSet +): scope_object = "INTERNAL" # Keep internal until we are happy to release this GA queryset = HogFunction.objects.all() filter_backends = [DjangoFilterBackend] @@ -165,6 +204,7 @@ class HogFunctionViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, ForbidDestroyMod permission_classes = [PostHogFeatureFlagPermission] posthog_feature_flag = {"hog-functions": ["create", "partial_update", "update"]} log_source = "hog_function" + app_source = "hog_function" def get_serializer_class(self) -> type[BaseSerializer]: return HogFunctionMinimalSerializer if self.action == "list" else HogFunctionSerializer diff --git a/posthog/api/test/test_hog_function.py b/posthog/api/test/test_hog_function.py index f3e606d4a6c15b..23b33ff5f91a4b 100644 --- a/posthog/api/test/test_hog_function.py +++ b/posthog/api/test/test_hog_function.py @@ -1,12 +1,15 @@ import json +from typing import Optional from unittest.mock import ANY, patch from rest_framework import status +from posthog.constants import AvailableFeature from posthog.models.action.action import Action from posthog.models.hog_functions.hog_function import HogFunction from posthog.test.base import APIBaseTest, ClickhouseTestMixin, QueryMatchingTest from posthog.cdp.templates.webhook.template_webhook import template as template_webhook +from posthog.cdp.templates.slack.template_slack import template as template_slack EXAMPLE_FULL = { @@ -55,7 +58,75 @@ } +class TestHogFunctionAPIWithoutAvailableFeature(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest): + def create_slack_function(self, data: Optional[dict] = None): + payload = { + "name": "Slack", + "template_id": template_slack.id, + "inputs": { + "slack_workspace": {"value": 1}, + "channel": {"value": "#general"}, + }, + } + + payload.update(data or {}) + + return self.client.post( + f"/api/projects/{self.team.id}/hog_functions/", + data=payload, + ) + + @patch("posthog.permissions.posthoganalytics.feature_enabled") + def test_create_hog_function_works_for_free_template(self, mock_feature_enabled): + response = self.create_slack_function() + + assert response.status_code == status.HTTP_201_CREATED, response.json() + assert response.json()["created_by"]["id"] == self.user.id + assert response.json()["hog"] == template_slack.hog + assert response.json()["inputs_schema"] == template_slack.inputs_schema + + @patch("posthog.permissions.posthoganalytics.feature_enabled") + def test_free_users_cannot_override_hog_or_schema(self, mock_feature_enabled): + response = self.create_slack_function( + { + "hog": "fetch(inputs.url);", + "inputs_schema": [ + {"key": "url", "type": "string", "label": "Webhook URL", "required": True}, + ], + } + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json() + assert response.json()["detail"] == "The Data Pipelines addon is required to create custom functions." + + @patch("posthog.permissions.posthoganalytics.feature_enabled") + def test_free_users_cannot_use_without_template(self, mock_feature_enabled): + response = self.create_slack_function({"template_id": None}) + + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json() + assert response.json()["detail"] == "The Data Pipelines addon is required to create custom functions." + + @patch("posthog.permissions.posthoganalytics.feature_enabled") + def test_free_users_cannot_use_non_free_templates(self, mock_feature_enabled): + response = self.create_slack_function( + { + "template_id": template_webhook.id, + } + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json() + assert response.json()["detail"] == "The Data Pipelines addon is required for this template." + + class TestHogFunctionAPI(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest): + def setUp(self): + super().setUp() + + self.organization.available_product_features = [ + {"key": AvailableFeature.DATA_PIPELINES, "name": AvailableFeature.DATA_PIPELINES} + ] + self.organization.save() + @patch("posthog.permissions.posthoganalytics.feature_enabled") def test_create_hog_function_forbidden_if_not_in_flag(self, mock_feature_enabled): mock_feature_enabled.return_value = False diff --git a/posthog/cdp/templates/__init__.py b/posthog/cdp/templates/__init__.py index 6cd1040e6116dc..0e39e711091907 100644 --- a/posthog/cdp/templates/__init__.py +++ b/posthog/cdp/templates/__init__.py @@ -1,5 +1,4 @@ from .webhook.template_webhook import template as webhook -from .helloworld.template_helloworld import template as hello_world from .slack.template_slack import template as slack from .hubspot.template_hubspot import template as hubspot from .customerio.template_customerio import template as customerio @@ -12,9 +11,8 @@ HOG_FUNCTION_TEMPLATES = [ - webhook, - hello_world, slack, + webhook, hubspot, customerio, intercom, diff --git a/posthog/cdp/templates/helloworld/template_helloworld.py b/posthog/cdp/templates/helloworld/template_helloworld.py deleted file mode 100644 index fde1f727a92cf2..00000000000000 --- a/posthog/cdp/templates/helloworld/template_helloworld.py +++ /dev/null @@ -1,22 +0,0 @@ -from posthog.cdp.templates.hog_function_template import HogFunctionTemplate - - -template: HogFunctionTemplate = HogFunctionTemplate( - status="alpha", - id="template-hello-world", - name="Hello world", - description="Prints your message or hello world!", - icon_url="/static/posthog-icon.svg?temp=true", - hog=""" -print(inputs.message ?? 'hello world!'); -""".strip(), - inputs_schema=[ - { - "key": "message", - "type": "string", - "label": "Message to print", - "secret": False, - "required": False, - } - ], -) diff --git a/posthog/cdp/templates/hog_function_template.py b/posthog/cdp/templates/hog_function_template.py index ac4f36e58f27ca..d01a0a30212eb9 100644 --- a/posthog/cdp/templates/hog_function_template.py +++ b/posthog/cdp/templates/hog_function_template.py @@ -4,7 +4,7 @@ @dataclasses.dataclass(frozen=True) class HogFunctionTemplate: - status: Literal["alpha", "beta", "stable"] + status: Literal["alpha", "beta", "stable", "free"] id: str name: str description: str diff --git a/posthog/cdp/templates/slack/template_slack.py b/posthog/cdp/templates/slack/template_slack.py index 12e4c430d7bce7..dce9872bc3b891 100644 --- a/posthog/cdp/templates/slack/template_slack.py +++ b/posthog/cdp/templates/slack/template_slack.py @@ -1,7 +1,7 @@ from posthog.cdp.templates.hog_function_template import HogFunctionTemplate template: HogFunctionTemplate = HogFunctionTemplate( - status="beta", + status="free", id="template-slack", name="Post a Slack message", description="Sends a message to a slack channel", @@ -45,8 +45,22 @@ "secret": False, "required": True, }, - {"key": "icon_emoji", "type": "string", "label": "Emoji icon", "default": ":hedgehog:", "required": False}, - {"key": "username", "type": "string", "label": "Bot name", "defaukt": "PostHog", "required": False}, + { + "key": "icon_emoji", + "type": "string", + "label": "Emoji icon", + "default": ":hedgehog:", + "required": False, + "secret": False, + }, + { + "key": "username", + "type": "string", + "label": "Bot name", + "default": "PostHog", + "required": False, + "secret": False, + }, { "key": "blocks", "type": "json", @@ -84,6 +98,8 @@ "type": "string", "label": "Plain text message", "description": "Optional fallback message if blocks are not provided or supported", + "secret": False, + "required": False, }, ], ) diff --git a/posthog/constants.py b/posthog/constants.py index 5c135dbcae3b7b..d90f91c359d752 100644 --- a/posthog/constants.py +++ b/posthog/constants.py @@ -36,6 +36,7 @@ class AvailableFeature(StrEnum): SURVEYS_MULTIPLE_QUESTIONS = "surveys_multiple_questions" AUTOMATIC_PROVISIONING = "automatic_provisioning" MANAGED_REVERSE_PROXY = "managed_reverse_proxy" + DATA_PIPELINES = "data_pipelines" TREND_FILTER_TYPE_ACTIONS = "actions" diff --git a/posthog/hogql_queries/insights/trends/breakdown.py b/posthog/hogql_queries/insights/trends/breakdown.py index a4975ae1e4a9db..24d281d6b2bf22 100644 --- a/posthog/hogql_queries/insights/trends/breakdown.py +++ b/posthog/hogql_queries/insights/trends/breakdown.py @@ -120,7 +120,7 @@ def column_exprs(self) -> list[ast.Alias]: breakdowns.append( self._get_breakdown_col_expr( self._get_multiple_breakdown_alias_name(idx + 1), - value=breakdown.value, + value=breakdown.property, breakdown_type=breakdown.type, normalize_url=breakdown.normalize_url, histogram_bin_count=breakdown.histogram_bin_count, @@ -233,7 +233,7 @@ def get_actors_query_where_filter(self, lookup_values: str | int | list[int | st cast(list[BreakdownSchema], self._breakdown_filter.breakdowns), lookup_values ): actors_filter = self._get_actors_query_where_expr( - breakdown_value=breakdown.value, + breakdown_value=breakdown.property, breakdown_type=breakdown.type, lookup_value=str( lookup_value diff --git a/posthog/hogql_queries/insights/trends/test/test_trends.py b/posthog/hogql_queries/insights/trends/test/test_trends.py index 7cb7980cf68b39..7f7977f406bfbd 100644 --- a/posthog/hogql_queries/insights/trends/test/test_trends.py +++ b/posthog/hogql_queries/insights/trends/test/test_trends.py @@ -534,7 +534,7 @@ def test_no_props_string(self): team=self.team, data={ "date_from": "-14d", - "breakdowns": [{"value": "$some_property"}], + "breakdowns": [{"property": "$some_property"}], "events": [ { "id": "sign up", @@ -582,7 +582,7 @@ def test_no_props_numeric(self): team=self.team, data={ "date_from": "-14d", - "breakdowns": [{"value": "$some_property"}], + "breakdowns": [{"property": "$some_property"}], "events": [ { "id": "sign up", @@ -630,7 +630,7 @@ def test_no_props_boolean(self): team=self.team, data={ "date_from": "-14d", - "breakdowns": [{"value": "$some_property"}], + "breakdowns": [{"property": "$some_property"}], "events": [ { "id": "sign up", @@ -1406,8 +1406,8 @@ def test_trends_multiple_breakdowns_single_aggregate(self): data={ "display": TRENDS_TABLE, "breakdowns": [ - {"value": "$browser"}, - {"value": "$variant"}, + {"property": "$browser"}, + {"property": "$variant"}, ], "events": [{"id": "sign up"}], }, @@ -1544,7 +1544,7 @@ def test_trends_breakdown_single_aggregate_with_zero_person_ids(self): team=self.team, data={ "display": TRENDS_TABLE, - "breakdowns": [{"value": "$browser"}], + "breakdowns": [{"property": "$browser"}], "events": [{"id": "sign up"}], }, ), @@ -1611,7 +1611,7 @@ def test_trends_breakdown_single_aggregate_math(self): filters: list[dict[str, Any]] = [ {"breakdown": "$some_property"}, - {"breakdowns": [{"value": "$some_property"}]}, + {"breakdowns": [{"property": "$some_property"}]}, ] for breakdown_filter in filters: with freeze_time("2020-01-04T13:00:01Z"): @@ -1821,7 +1821,7 @@ def test_trends_breakdown_with_session_property_single_aggregate_math_and_breakd data={ "display": TRENDS_TABLE, "interval": "week", - "breakdowns": [{"value": "$some_property"}], + "breakdowns": [{"property": "$some_property"}], "events": [ { "id": "sign up", @@ -1850,7 +1850,7 @@ def test_trends_breakdown_with_session_property_single_aggregate_math_and_breakd data={ "display": TRENDS_TABLE, "interval": "day", - "breakdowns": [{"value": "$some_property"}], + "breakdowns": [{"property": "$some_property"}], "events": [ { "id": "sign up", @@ -2002,7 +2002,7 @@ def test_trends_person_breakdown_with_session_property_single_aggregate_math_and data={ "display": TRENDS_TABLE, "interval": "week", - "breakdowns": [{"type": "person", "value": "$some_prop"}], + "breakdowns": [{"type": "person", "property": "$some_prop"}], "events": [ { "id": "sign up", @@ -2110,7 +2110,7 @@ def test_trends_breakdown_with_math_func(self): data={ "display": TRENDS_TABLE, "interval": "day", - "breakdowns": [{"value": "$some_property"}], + "breakdowns": [{"property": "$some_property"}], "events": [ { "id": "sign up", @@ -3392,7 +3392,7 @@ def test_trends_with_session_property_total_volume_math_with_breakdowns(self): breakdown_filter: dict[str, Any] = ( {"breakdown": "$some_property"} if breakdown_type == "single" - else {"breakdowns": [{"value": "$some_property"}]} + else {"breakdowns": [{"property": "$some_property"}]} ) with freeze_time("2020-01-04T13:00:01Z"): @@ -4554,7 +4554,7 @@ def test_breakdown_by_person_property(self): "breakdowns": [ { "type": "person", - "value": "name", + "property": "name", } ] } @@ -4673,7 +4673,7 @@ def test_breakdown_by_person_property_for_person_on_events(self): "date_from": "-14d", "breakdowns": [ { - "value": "name", + "property": "name", "type": "person", } ], @@ -4779,7 +4779,7 @@ def test_breakdown_by_person_property_for_person_on_events_with_zero_person_ids( "date_from": "-14d", "breakdowns": [ { - "value": "name", + "property": "name", "type": "person", } ], @@ -4934,7 +4934,7 @@ def test_breakdown_by_person_property_pie(self): team=self.team, data={ "date_from": "-14d", - "breakdowns": [{"type": "person", "value": "name"}], + "breakdowns": [{"type": "person", "property": "name"}], "display": "ActionsPie", "events": [ { @@ -5006,7 +5006,7 @@ def test_breakdown_by_person_property_pie_with_event_dau_filter(self): "breakdowns": [ { "type": "person", - "value": "name", + "property": "name", } ], } @@ -5121,7 +5121,7 @@ def test_breakdown_filter_by_precalculated_cohort(self): "breakdowns": [ { "type": "person", - "value": "name", + "property": "name", }, ], }, @@ -5233,7 +5233,7 @@ def test_trends_aggregate_by_distinct_id(self): data={ "interval": "day", "events": [{"id": "sign up", "math": "dau"}], - "breakdowns": [{"type": "person", "value": "$some_prop"}], + "breakdowns": [{"type": "person", "property": "$some_prop"}], }, ), self.team, @@ -5318,7 +5318,7 @@ def test_breakdown_filtering_limit(self): team=self.team, data={ "date_from": "-14d", - "breakdowns": [{"value": "$some_property"}], + "breakdowns": [{"property": "$some_property"}], "events": [ { "id": "sign up", @@ -5396,7 +5396,7 @@ def test_breakdown_with_person_property_filter(self): team=self.team, data={ **action_filter, - "breakdowns": [{"value": "order"}], + "breakdowns": [{"property": "order"}], }, ), self.team, @@ -5406,7 +5406,7 @@ def test_breakdown_with_person_property_filter(self): team=self.team, data={ **event_filter, - "breakdowns": [{"value": "order"}], + "breakdowns": [{"property": "order"}], }, ), self.team, @@ -5462,7 +5462,7 @@ def test_breakdown_filtering(self): team=self.team, data={ **filter, - "breakdowns": [{"value": "$some_property"}], + "breakdowns": [{"property": "$some_property"}], }, ), self.team, @@ -5561,7 +5561,7 @@ def test_multiple_breakdowns_label_formatting(self): team=self.team, data={ **filter, - "breakdowns": [{"value": "$browser"}, {"value": "$variant"}], + "breakdowns": [{"property": "$browser"}, {"property": "$variant"}], }, ), self.team, @@ -5582,7 +5582,7 @@ def test_multiple_breakdowns_label_formatting(self): team=self.team, data={ **filter, - "breakdowns": [{"value": "$browser"}, {"value": "$variant"}], + "breakdowns": [{"property": "$browser"}, {"property": "$variant"}], "breakdown_limit": 1, }, ), @@ -5627,7 +5627,7 @@ def test_breakdown_filtering_persons(self): filters: list[dict[str, Any]] = [ {"breakdown": "email", "breakdown_type": "person"}, - {"breakdowns": [{"type": "person", "value": "email"}]}, + {"breakdowns": [{"type": "person", "property": "email"}]}, ] for breakdown_filter in filters: response = self._run( @@ -5697,7 +5697,7 @@ def test_breakdown_filtering_persons_with_action_props(self): filters: list[dict[str, Any]] = [ {"breakdown": "email", "breakdown_type": "person"}, - {"breakdowns": [{"value": "email", "type": "person"}]}, + {"breakdowns": [{"property": "email", "type": "person"}]}, ] for breakdown_filter in filters: response = self._run( @@ -5764,7 +5764,7 @@ def test_breakdown_filtering_with_properties(self): }, ) - filters: list[dict[str, Any]] = [{"breakdown": "$current_url"}, {"breakdowns": [{"value": "$current_url"}]}] + filters: list[dict[str, Any]] = [{"breakdown": "$current_url"}, {"breakdowns": [{"property": "$current_url"}]}] for breakdown_filter in filters: with freeze_time("2020-01-05T13:01:01Z"): response = self._run( @@ -5851,7 +5851,7 @@ def test_breakdown_filtering_with_properties_in_new_format(self): filters: list[dict[str, Any]] = [ {"breakdown": "$current_url"}, - {"breakdowns": [{"value": "$current_url"}]}, + {"breakdowns": [{"property": "$current_url"}]}, ] for breakdown_filter in filters: with freeze_time("2020-01-05T13:01:01Z"): @@ -5952,7 +5952,7 @@ def test_mau_with_breakdown_filtering_and_prop_filter(self): filters: list[dict[str, Any]] = [ {"breakdown": "$some_prop", "breakdown_type": "person"}, - {"breakdowns": [{"value": "$some_prop", "type": "person"}]}, + {"breakdowns": [{"property": "$some_prop", "type": "person"}]}, ] for breakdown_filter in filters: with freeze_time("2020-01-04T13:01:01Z"): @@ -5997,7 +5997,7 @@ def test_dau_with_breakdown_filtering(self): filters: list[dict[str, Any]] = [ {"breakdown": "$some_property"}, - {"breakdowns": [{"value": "$some_property"}]}, + {"breakdowns": [{"property": "$some_property"}]}, ] for breakdown_filter in filters: with freeze_time("2020-01-04T13:01:01Z"): @@ -6047,7 +6047,7 @@ def test_dau_with_breakdown_filtering_with_sampling(self): filters: list[dict[str, Any]] = [ {"breakdown": "$some_property"}, - {"breakdowns": [{"value": "$some_property"}]}, + {"breakdowns": [{"property": "$some_property"}]}, ] for breakdown_filter in filters: with freeze_time("2020-01-04T13:01:01Z"): @@ -6099,7 +6099,7 @@ def test_dau_with_breakdown_filtering_with_prop_filter(self): filters: list[dict[str, Any]] = [ {"breakdown": "$some_property"}, - {"breakdowns": [{"value": "$some_property"}]}, + {"breakdowns": [{"property": "$some_property"}]}, ] for breakdown_filter in filters: with freeze_time("2020-01-04T13:01:01Z"): @@ -6151,7 +6151,7 @@ def test_against_clashing_entity_and_property_filter_naming(self): filters: list[dict[str, Any]] = [ {"breakdown": "$some_prop", "breakdown_type": "person"}, - {"breakdowns": [{"value": "$some_prop", "type": "person"}]}, + {"breakdowns": [{"property": "$some_prop", "type": "person"}]}, ] for breakdown_filter in filters: with freeze_time("2020-01-04T13:01:01Z"): diff --git a/posthog/hogql_queries/insights/trends/test/test_trends_persons.py b/posthog/hogql_queries/insights/trends/test/test_trends_persons.py index 7a235302337f02..ae1ba004042913 100644 --- a/posthog/hogql_queries/insights/trends/test/test_trends_persons.py +++ b/posthog/hogql_queries/insights/trends/test/test_trends_persons.py @@ -401,11 +401,7 @@ def test_trends_multiple_breakdowns_others_persons(self): series=[EventsNode(event="$pageview")], dateRange=InsightDateRange(date_from="-7d"), breakdownFilter=BreakdownFilter( - breakdowns=[ - Breakdown( - value="$browser", - ) - ], + breakdowns=[Breakdown(property="$browser")], breakdown_limit=1, ), ) @@ -431,7 +427,7 @@ def test_trends_filter_by_other(self): series=[EventsNode(event="$pageview")], dateRange=InsightDateRange(date_from="-7d"), breakdownFilter=BreakdownFilter( - breakdowns=[Breakdown(value="some_property", type=MultipleBreakdownType.EVENT)], + breakdowns=[Breakdown(property="some_property", type=MultipleBreakdownType.EVENT)], breakdown_limit=1, ), ) @@ -444,8 +440,8 @@ def test_trends_filter_by_other(self): dateRange=InsightDateRange(date_from="-7d"), breakdownFilter=BreakdownFilter( breakdowns=[ - Breakdown(value="some_property", type=MultipleBreakdownType.EVENT), - Breakdown(value="$browser", type=MultipleBreakdownType.EVENT), + Breakdown(property="some_property", type=MultipleBreakdownType.EVENT), + Breakdown(property="$browser", type=MultipleBreakdownType.EVENT), ], breakdown_limit=1, ), @@ -769,7 +765,7 @@ def test_trends_event_multiple_breakdowns_persons(self): source_query = TrendsQuery( series=[EventsNode(event="$pageview")], dateRange=InsightDateRange(date_from="-7d"), - breakdownFilter=BreakdownFilter(breakdowns=[Breakdown(value="$browser")]), + breakdownFilter=BreakdownFilter(breakdowns=[Breakdown(property="$browser")]), ) result = self._get_actors(trends_query=source_query, day="2023-05-01", breakdown=["Safari"]) @@ -790,7 +786,7 @@ def test_trends_person_multiple_breakdown_persons(self): series=[EventsNode(event="$pageview")], dateRange=InsightDateRange(date_from="-7d"), breakdownFilter=BreakdownFilter( - breakdowns=[Breakdown(value="$geoip_country_code", type=BreakdownType.PERSON)] + breakdowns=[Breakdown(property="$geoip_country_code", type=BreakdownType.PERSON)] ), ) @@ -812,11 +808,7 @@ def test_trends_multiple_breakdown_null_persons(self): series=[EventsNode(event="$pageview")], dateRange=InsightDateRange(date_from="-7d"), breakdownFilter=BreakdownFilter( - breakdowns=[ - Breakdown( - value="$browser", - ) - ], + breakdowns=[Breakdown(property="$browser")], breakdown_limit=1, ), ) @@ -839,7 +831,7 @@ def test_trends_multiple_breakdowns_hogql_persons(self): series=[EventsNode(event="$pageview")], dateRange=InsightDateRange(date_from="-7d"), breakdownFilter=BreakdownFilter( - breakdowns=[Breakdown(value="properties.some_property", type=BreakdownType.HOGQL)], + breakdowns=[Breakdown(property="properties.some_property", type=BreakdownType.HOGQL)], breakdown_limit=1, ), ) @@ -894,7 +886,7 @@ def test_trends_multiple_breakdowns_filter_by_range(self): source_query = TrendsQuery( series=[EventsNode(event="$pageview")], dateRange=InsightDateRange(date_from="-7d"), - breakdownFilter=BreakdownFilter(breakdowns=[Breakdown(value="some_property", histogram_bin_count=4)]), + breakdownFilter=BreakdownFilter(breakdowns=[Breakdown(property="some_property", histogram_bin_count=4)]), ) # should not include 20 @@ -962,11 +954,7 @@ def test_trends_breakdown_by_boolean(self): series=[EventsNode(event="$pageview")], dateRange=InsightDateRange(date_from="-7d"), breakdownFilter=BreakdownFilter( - breakdowns=[ - Breakdown( - value="bool", - ) - ], + breakdowns=[Breakdown(property="bool")], breakdown_limit=1, ), ) diff --git a/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py b/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py index 192e8bcd4fb0a0..f0deb2b73dab12 100644 --- a/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py +++ b/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py @@ -1782,7 +1782,7 @@ def test_multiple_breakdowns_values_limit(self): IntervalType.DAY, [EventsNode(event="$pageview")], TrendsFilter(display=ChartDisplayType.ACTIONS_LINE_GRAPH), - BreakdownFilter(breakdowns=[Breakdown(value="breakdown_value", type=MultipleBreakdownType.EVENT)]), + BreakdownFilter(breakdowns=[Breakdown(property="breakdown_value", type=MultipleBreakdownType.EVENT)]), ) self.assertEqual(len(response.results), 26) @@ -1794,7 +1794,7 @@ def test_multiple_breakdowns_values_limit(self): [EventsNode(event="$pageview")], TrendsFilter(display=ChartDisplayType.ACTIONS_LINE_GRAPH), BreakdownFilter( - breakdowns=[Breakdown(value="breakdown_value", type=MultipleBreakdownType.EVENT)], breakdown_limit=10 + breakdowns=[Breakdown(property="breakdown_value", type=MultipleBreakdownType.EVENT)], breakdown_limit=10 ), ) self.assertEqual(len(response.results), 11) @@ -1807,7 +1807,7 @@ def test_multiple_breakdowns_values_limit(self): [EventsNode(event="$pageview")], TrendsFilter(display=ChartDisplayType.ACTIONS_LINE_GRAPH), BreakdownFilter( - breakdowns=[Breakdown(value="breakdown_value", type=MultipleBreakdownType.EVENT)], + breakdowns=[Breakdown(property="breakdown_value", type=MultipleBreakdownType.EVENT)], breakdown_limit=10, breakdown_hide_other_aggregation=True, ), @@ -1820,7 +1820,7 @@ def test_multiple_breakdowns_values_limit(self): IntervalType.DAY, [EventsNode(event="$pageview")], TrendsFilter(display=ChartDisplayType.ACTIONS_LINE_GRAPH), - BreakdownFilter(breakdowns=[Breakdown(value="breakdown_value", type=MultipleBreakdownType.EVENT)]), + BreakdownFilter(breakdowns=[Breakdown(property="breakdown_value", type=MultipleBreakdownType.EVENT)]), limit_context=LimitContext.EXPORT, ) self.assertEqual(len(response.results), 30) @@ -1834,7 +1834,7 @@ def test_multiple_breakdowns_values_limit(self): [EventsNode(event="$pageview")], TrendsFilter(display=ChartDisplayType.ACTIONS_TABLE), BreakdownFilter( - breakdowns=[Breakdown(value="breakdown_value", type=MultipleBreakdownType.EVENT)], breakdown_limit=10 + breakdowns=[Breakdown(property="breakdown_value", type=MultipleBreakdownType.EVENT)], breakdown_limit=10 ), ) self.assertEqual(len(response.results), 11) @@ -1847,7 +1847,7 @@ def test_multiple_breakdowns_values_limit(self): [EventsNode(event="$pageview")], TrendsFilter(display=ChartDisplayType.ACTIONS_TABLE), BreakdownFilter( - breakdowns=[Breakdown(value="breakdown_value", type=MultipleBreakdownType.EVENT)], + breakdowns=[Breakdown(property="breakdown_value", type=MultipleBreakdownType.EVENT)], breakdown_limit=10, breakdown_hide_other_aggregation=True, ), @@ -2356,7 +2356,7 @@ def test_to_actors_query_options_breakdowns(self): IntervalType.DAY, [EventsNode(event="$pageview")], None, - BreakdownFilter(breakdowns=[Breakdown(type=BreakdownType.EVENT, value="$browser")], breakdown_limit=3), + BreakdownFilter(breakdowns=[Breakdown(type=BreakdownType.EVENT, property="$browser")], breakdown_limit=3), ) response = runner.to_actors_query_options() @@ -2398,7 +2398,7 @@ def test_to_actors_query_options_breakdowns_boolean(self): IntervalType.DAY, [EventsNode(event="$pageview")], None, - BreakdownFilter(breakdowns=[Breakdown(type=BreakdownType.EVENT, value="bool_field")]), + BreakdownFilter(breakdowns=[Breakdown(type=BreakdownType.EVENT, property="bool_field")]), ) response = runner.to_actors_query_options() @@ -2445,15 +2445,7 @@ def test_to_actors_query_options_breakdowns_histogram(self): IntervalType.DAY, [EventsNode(event="$pageview")], None, - BreakdownFilter( - breakdowns=[ - Breakdown( - type=BreakdownType.EVENT, - value="prop", - histogram_bin_count=4, - ) - ] - ), + BreakdownFilter(breakdowns=[Breakdown(type=BreakdownType.EVENT, property="prop", histogram_bin_count=4)]), ) response = runner.to_actors_query_options() @@ -2533,7 +2525,7 @@ def test_to_actors_query_options_breakdowns_hogql(self): IntervalType.DAY, [EventsNode(event="$pageview")], None, - BreakdownFilter(breakdowns=[Breakdown(type=BreakdownType.HOGQL, value="properties.$browser")]), + BreakdownFilter(breakdowns=[Breakdown(type=BreakdownType.HOGQL, property="properties.$browser")]), ) response = runner.to_actors_query_options() @@ -2577,7 +2569,7 @@ def test_to_actors_query_options_bar_value(self): IntervalType.DAY, [EventsNode(event="$pageview")], TrendsFilter(display=ChartDisplayType.ACTIONS_BAR_VALUE), - BreakdownFilter(breakdowns=[Breakdown(type=BreakdownType.EVENT, value="$browser")]), + BreakdownFilter(breakdowns=[Breakdown(type=BreakdownType.EVENT, property="$browser")]), ) response = runner.to_actors_query_options() @@ -2604,9 +2596,9 @@ def test_to_actors_query_options_multiple_breakdowns(self): TrendsFilter(display=ChartDisplayType.ACTIONS_BAR_VALUE), BreakdownFilter( breakdowns=[ - Breakdown(type=BreakdownType.EVENT, value="$browser"), - Breakdown(type=BreakdownType.EVENT, value="prop", histogram_bin_count=2), - Breakdown(type=BreakdownType.EVENT, value="bool_field"), + Breakdown(type=BreakdownType.EVENT, property="$browser"), + Breakdown(type=BreakdownType.EVENT, property="prop", histogram_bin_count=2), + Breakdown(type=BreakdownType.EVENT, property="bool_field"), ] ), ) @@ -2731,7 +2723,7 @@ def test_trends_multiple_event_breakdowns(self): [EventsNode(event="$pageview")], None, BreakdownFilter( - breakdowns=[Breakdown(type="event", value="$browser"), Breakdown(type="event", value="prop")] + breakdowns=[Breakdown(type="event", property="$browser"), Breakdown(type="event", property="prop")] ), ) @@ -2758,9 +2750,9 @@ def test_trends_multiple_event_breakdowns(self): None, BreakdownFilter( breakdowns=[ - Breakdown(type="event", value="$browser"), - Breakdown(type="event", value="prop"), - Breakdown(type="event", value="bool_field"), + Breakdown(type="event", property="$browser"), + Breakdown(type="event", property="prop"), + Breakdown(type="event", property="bool_field"), ] ), ) @@ -2789,10 +2781,10 @@ def test_trends_multiple_breakdowns_have_max_limit(self): with pytest.raises(ValidationError, match=".*at most 3.*"): BreakdownFilter( breakdowns=[ - Breakdown(type="event", value="$browser"), - Breakdown(type="event", value="prop"), - Breakdown(type="event", value="bool_field"), - Breakdown(type="event", value="bool_field"), + Breakdown(type="event", property="$browser"), + Breakdown(type="event", property="prop"), + Breakdown(type="event", property="bool_field"), + Breakdown(type="event", property="bool_field"), ] ) @@ -2808,7 +2800,7 @@ def test_trends_event_and_person_breakdowns(self): [EventsNode(event="$pageview")], None, BreakdownFilter( - breakdowns=[Breakdown(type="event", value="$browser"), Breakdown(type="person", value="name")] + breakdowns=[Breakdown(type="event", property="$browser"), Breakdown(type="person", property="name")] ), ) @@ -2840,8 +2832,8 @@ def test_trends_event_person_group_breakdowns(self): None, BreakdownFilter( breakdowns=[ - Breakdown(type="event", value="$browser"), - Breakdown(type="group", group_type_index=0, value="industry"), + Breakdown(type="event", property="$browser"), + Breakdown(type="group", group_type_index=0, property="industry"), ] ), ) @@ -2879,8 +2871,8 @@ def test_trends_event_with_two_group_breakdowns(self): None, BreakdownFilter( breakdowns=[ - Breakdown(type="group", group_type_index=1, value="employee_count"), - Breakdown(type="group", group_type_index=0, value="industry"), + Breakdown(type="group", group_type_index=1, property="employee_count"), + Breakdown(type="group", group_type_index=0, property="industry"), ] ), ) @@ -2918,9 +2910,9 @@ def test_trends_event_with_three_group_breakdowns(self): None, BreakdownFilter( breakdowns=[ - Breakdown(type="group", group_type_index=0, value="industry"), - Breakdown(type="group", group_type_index=0, value="name"), - Breakdown(type="group", group_type_index=1, value="employee_count"), + Breakdown(type="group", group_type_index=0, property="industry"), + Breakdown(type="group", group_type_index=0, property="name"), + Breakdown(type="group", group_type_index=1, property="employee_count"), ] ), ) @@ -3019,7 +3011,7 @@ def test_trends_event_multiple_breakdowns_normalizes_url(self): None, BreakdownFilter( breakdowns=[ - Breakdown(value="$url", normalize_url=True), + Breakdown(property="$url", normalize_url=True), ] ), ) @@ -3041,7 +3033,7 @@ def test_trends_event_multiple_breakdowns_normalizes_url(self): None, BreakdownFilter( breakdowns=[ - Breakdown(value="$url", normalize_url=normalize_url), + Breakdown(property="$url", normalize_url=normalize_url), ] ), ) @@ -3138,7 +3130,7 @@ def test_trends_event_multiple_numeric_breakdowns(self): None, BreakdownFilter( breakdowns=[ - Breakdown(value="$bin"), + Breakdown(property="$bin"), ], ), ) @@ -3246,7 +3238,7 @@ def test_trends_event_multiple_numeric_breakdowns_into_bins(self): None, BreakdownFilter( breakdowns=[ - Breakdown(value="$bin", histogram_bin_count=5), + Breakdown(property="$bin", histogram_bin_count=5), ], ), ) @@ -3271,7 +3263,7 @@ def test_trends_event_multiple_numeric_breakdowns_into_bins(self): None, BreakdownFilter( breakdowns=[ - Breakdown(value="$bin", histogram_bin_count=5), + Breakdown(property="$bin", histogram_bin_count=5), ], breakdown_limit=2, ), @@ -3367,7 +3359,7 @@ def test_trends_event_histogram_breakdowns_return_equal_result(self): None, BreakdownFilter( breakdowns=[ - Breakdown(value="$bin", histogram_bin_count=5), + Breakdown(property="$bin", histogram_bin_count=5), ], ), ) @@ -3485,7 +3477,7 @@ def test_trends_event_breakdowns_handle_null(self): IntervalType.DAY, [EventsNode(event="$pageview")], None, - BreakdownFilter(breakdowns=[Breakdown(value="$bin", histogram_bin_count=10)]), + BreakdownFilter(breakdowns=[Breakdown(property="$bin", histogram_bin_count=10)]), ) breakdown_labels = [result["breakdown_value"] for result in response.results] @@ -3502,8 +3494,8 @@ def test_trends_event_breakdowns_handle_null(self): None, BreakdownFilter( breakdowns=[ - Breakdown(value="$bin", histogram_bin_count=10), - Breakdown(value="$second_bin", histogram_bin_count=10), + Breakdown(property="$bin", histogram_bin_count=10), + Breakdown(property="$second_bin", histogram_bin_count=10), ] ), ) @@ -3525,7 +3517,7 @@ def test_trends_event_breakdowns_handle_null(self): None, BreakdownFilter( breakdowns=[ - Breakdown(value="$second_bin", histogram_bin_count=10), + Breakdown(property="$second_bin", histogram_bin_count=10), ] ), ) @@ -3541,9 +3533,9 @@ def test_trends_event_breakdowns_can_combine_bool_sting_and_numeric_in_any_order flush_persons_and_events() breakdowns = [ - Breakdown(value="prop", histogram_bin_count=2), - Breakdown(value="$browser"), - Breakdown(value="bool_field"), + Breakdown(property="prop", histogram_bin_count=2), + Breakdown(property="$browser"), + Breakdown(property="bool_field"), ] for breakdown_filter in itertools.combinations(breakdowns, 3): response = self._run_trends_query( @@ -3578,8 +3570,8 @@ def test_trends_event_breakdowns_handle_none_histogram_bin_count(self): None, BreakdownFilter( breakdowns=[ - Breakdown(value="prop", histogram_bin_count=2), - Breakdown(value="$browser", histogram_bin_count=None), + Breakdown(property="prop", histogram_bin_count=2), + Breakdown(property="$browser", histogram_bin_count=None), ] ), ) @@ -3616,7 +3608,7 @@ def test_trends_event_math_session_duration_with_breakdowns(self): [EventsNode(event="$pageview", math=PropertyMathType.MEDIAN, math_property="$session_duration")], None, BreakdownFilter( - breakdowns=[Breakdown(value="$session_duration", type=MultipleBreakdownType.SESSION)], + breakdowns=[Breakdown(property="$session_duration", type=MultipleBreakdownType.SESSION)], ), ) @@ -3651,7 +3643,7 @@ def test_trends_event_math_session_duration_with_breakdowns_and_histogram_bins(s None, BreakdownFilter( breakdowns=[ - Breakdown(value="$session_duration", type=MultipleBreakdownType.SESSION, histogram_bin_count=4) + Breakdown(property="$session_duration", type=MultipleBreakdownType.SESSION, histogram_bin_count=4) ], ), ) @@ -3684,7 +3676,7 @@ def test_trends_event_math_wau_with_breakdowns(self): [EventsNode(event="$pageview", math=BaseMathType.WEEKLY_ACTIVE)], None, BreakdownFilter( - breakdowns=[Breakdown(value="$session_duration", type="session", histogram_bin_count=4)], + breakdowns=[Breakdown(property="$session_duration", type="session", histogram_bin_count=4)], ), ) @@ -3716,7 +3708,7 @@ def test_trends_event_math_mau_with_breakdowns(self): [EventsNode(event="$pageview", math=BaseMathType.MONTHLY_ACTIVE)], None, BreakdownFilter( - breakdowns=[Breakdown(value="$browser", type="event")], + breakdowns=[Breakdown(property="$browser", type="event")], ), ) @@ -3740,7 +3732,7 @@ def test_trends_multiple_breakdowns_hogql(self): IntervalType.DAY, [EventsNode(event="$pageview")], None, - BreakdownFilter(breakdowns=[Breakdown(value="properties.$browser", type=MultipleBreakdownType.HOGQL)]), + BreakdownFilter(breakdowns=[Breakdown(property="properties.$browser", type=MultipleBreakdownType.HOGQL)]), ) breakdown_labels = [result["breakdown_value"] for result in response.results] @@ -3767,8 +3759,8 @@ def test_trends_multiple_breakdowns_hogql_and_numeric_prop(self): None, BreakdownFilter( breakdowns=[ - Breakdown(value="properties.$browser", type=MultipleBreakdownType.HOGQL), - Breakdown(value="prop", histogram_bin_count=2), + Breakdown(property="properties.$browser", type=MultipleBreakdownType.HOGQL), + Breakdown(property="prop", histogram_bin_count=2), ] ), ) @@ -3799,14 +3791,14 @@ def test_trends_event_multiple_breakdowns_combined_types(self): flush_persons_and_events() breakdowns = [ - Breakdown(value="prop", histogram_bin_count=2, type=MultipleBreakdownType.EVENT), - Breakdown(value="$browser", type=MultipleBreakdownType.EVENT), - Breakdown(value="bool_field", type=MultipleBreakdownType.EVENT), - Breakdown(value="properties.$browser", type=MultipleBreakdownType.HOGQL), - Breakdown(value="name", type=MultipleBreakdownType.PERSON), - Breakdown(value="$session_duration", type=MultipleBreakdownType.SESSION), - Breakdown(type="group", group_type_index=1, value="employee_count"), - Breakdown(type="group", group_type_index=0, value="industry"), + Breakdown(property="prop", histogram_bin_count=2, type=MultipleBreakdownType.EVENT), + Breakdown(property="$browser", type=MultipleBreakdownType.EVENT), + Breakdown(property="bool_field", type=MultipleBreakdownType.EVENT), + Breakdown(property="properties.$browser", type=MultipleBreakdownType.HOGQL), + Breakdown(property="name", type=MultipleBreakdownType.PERSON), + Breakdown(property="$session_duration", type=MultipleBreakdownType.SESSION), + Breakdown(type="group", group_type_index=1, property="employee_count"), + Breakdown(type="group", group_type_index=0, property="industry"), ] for breakdown_filter in itertools.permutations(breakdowns, 3): @@ -3832,7 +3824,7 @@ def test_trends_multiple_breakdowns_multiple_hogql(self): IntervalType.DAY, [EventsNode(event="$pageview"), EventsNode(event="$pageleave")], None, - BreakdownFilter(breakdowns=[Breakdown(type=MultipleBreakdownType.HOGQL, value="properties.$browser")]), + BreakdownFilter(breakdowns=[Breakdown(type=MultipleBreakdownType.HOGQL, property="properties.$browser")]), ) breakdown_labels = [result["breakdown_value"] for result in response.results] @@ -3877,9 +3869,9 @@ def test_to_insight_query_applies_multiple_breakdowns(self): TrendsFilter(display=ChartDisplayType.ACTIONS_BAR_VALUE), BreakdownFilter( breakdowns=[ - Breakdown(type=BreakdownType.EVENT, value="$browser"), - Breakdown(type=BreakdownType.EVENT, value="prop", histogram_bin_count=2), - Breakdown(type=BreakdownType.EVENT, value="bool_field"), + Breakdown(type=BreakdownType.EVENT, property="$browser"), + Breakdown(type=BreakdownType.EVENT, property="prop", histogram_bin_count=2), + Breakdown(type=BreakdownType.EVENT, property="bool_field"), ] ), ) @@ -3949,15 +3941,7 @@ def test_to_actors_query_options_orders_options_with_histogram_breakdowns(self): IntervalType.DAY, [EventsNode(event="$pageview")], None, - BreakdownFilter( - breakdowns=[ - Breakdown( - type=BreakdownType.EVENT, - value="prop", - histogram_bin_count=4, - ) - ] - ), + BreakdownFilter(breakdowns=[Breakdown(type=BreakdownType.EVENT, property="prop", histogram_bin_count=4)]), ) response = runner.to_actors_query_options() @@ -3997,7 +3981,7 @@ def test_to_insight_query_applies_breakdown_limit(self): IntervalType.DAY, [EventsNode(event="$pageview")], TrendsFilter(display=ChartDisplayType.ACTIONS_BAR_VALUE), - BreakdownFilter(breakdowns=[Breakdown(value="$browser")], breakdown_limit=2), + BreakdownFilter(breakdowns=[Breakdown(property="$browser")], breakdown_limit=2), ) breakdown_labels = [result["breakdown_value"] for result in response.results] @@ -4042,7 +4026,7 @@ def test_trends_table_uses_breakdown_bins(self): [EventsNode(event="$pageview")], TrendsFilter(display=display), BreakdownFilter( - breakdowns=[Breakdown(value="prop", type=MultipleBreakdownType.EVENT, histogram_bin_count=2)], + breakdowns=[Breakdown(property="prop", type=MultipleBreakdownType.EVENT, histogram_bin_count=2)], breakdown_limit=10, breakdown_hide_other_aggregation=True, ), diff --git a/posthog/hogql_queries/insights/trends/trends_query_runner.py b/posthog/hogql_queries/insights/trends/trends_query_runner.py index 5ebac61537cafc..201cd6fe4a1403 100644 --- a/posthog/hogql_queries/insights/trends/trends_query_runner.py +++ b/posthog/hogql_queries/insights/trends/trends_query_runner.py @@ -272,7 +272,7 @@ def to_actors_query_options(self) -> InsightActorsQueryOptionsResponse: MultipleBreakdownOptions( values=self._get_breakdown_items( values, - breakdown_filter.value, + breakdown_filter.property, breakdown_filter.type, histogram_breakdown=isinstance(breakdown_filter.histogram_bin_count, int), group_type_index=breakdown_filter.group_type_index, @@ -987,7 +987,7 @@ def _format_breakdown_label(self, breakdown_value: Any): if self.query.breakdownFilter is not None and self.query.breakdownFilter.breakdowns is not None: labels = [] for breakdown, label in zip(self.query.breakdownFilter.breakdowns, breakdown_value): - if self._is_breakdown_field_boolean(breakdown.value, breakdown.type, breakdown.group_type_index): + if self._is_breakdown_field_boolean(breakdown.property, breakdown.type, breakdown.group_type_index): labels.append(self._convert_boolean(label)) else: labels.append(label) diff --git a/posthog/hogql_queries/legacy_compatibility/filter_to_query.py b/posthog/hogql_queries/legacy_compatibility/filter_to_query.py index 3ecbcb926e8f54..6bff858aaf130e 100644 --- a/posthog/hogql_queries/legacy_compatibility/filter_to_query.py +++ b/posthog/hogql_queries/legacy_compatibility/filter_to_query.py @@ -321,23 +321,34 @@ def _breakdown_filter(_filter: dict): if breakdownFilter["breakdown_type"] == "events": breakdownFilter["breakdown_type"] = "event" - if _filter.get("breakdowns") is not None and isinstance(_filter["breakdowns"], list): - breakdowns = [] - for breakdown in _filter["breakdowns"]: - if isinstance(breakdown, dict) and ("value" in breakdown or "property" in breakdown): - breakdowns.append( - { - "type": breakdown.get("type", "event"), - "value": breakdown.get("value", breakdown.get("property")), - "normalize_url": breakdown.get("normalize_url", None), - "histogram_bin_count": breakdown.get("histogram_bin_count", None), - "group_type_index": breakdown.get("group_type_index", None), - } + if _filter.get("breakdowns") is not None: + if _insight_type(_filter) == "TRENDS": + # Trends support multiple breakdowns + breakdowns = [] + for breakdown in _filter["breakdowns"]: + if isinstance(breakdown, dict) and "property" in breakdown: + breakdowns.append( + { + "type": breakdown.get("type", "event"), + "property": breakdown.get("property"), + "normalize_url": breakdown.get("normalize_url", None), + "histogram_bin_count": breakdown.get("histogram_bin_count", None), + "group_type_index": breakdown.get("group_type_index", None), + } + ) + + if len(breakdowns) > 0: + # Multiple breakdowns accept up to three breakdowns + breakdownFilter["breakdowns"] = breakdowns[:3] + else: + if isinstance(_filter["breakdowns"], list) and len(_filter["breakdowns"]) == 1: + breakdownFilter["breakdown_type"] = _filter["breakdowns"][0].get("type", None) + breakdownFilter["breakdown"] = _filter["breakdowns"][0].get("property", None) + else: + raise Exception( + "Could not convert multi-breakdown property `breakdowns` - found more than one breakdown" ) - if len(breakdowns) > 0: - breakdownFilter["breakdowns"] = breakdowns[:3] - if breakdownFilter["breakdown"] is not None and breakdownFilter["breakdown_type"] is None: breakdownFilter["breakdown_type"] = "event" diff --git a/posthog/hogql_queries/legacy_compatibility/test/test_filter_to_query.py b/posthog/hogql_queries/legacy_compatibility/test/test_filter_to_query.py index 6fc6beb5361014..310851e70ea9d9 100644 --- a/posthog/hogql_queries/legacy_compatibility/test/test_filter_to_query.py +++ b/posthog/hogql_queries/legacy_compatibility/test/test_filter_to_query.py @@ -1343,7 +1343,7 @@ def test_breakdown_converts_multi(self): assert isinstance(query, TrendsQuery) self.assertEqual( query.breakdownFilter, - BreakdownFilter(breakdowns=[{"type": BreakdownType.EVENT, "value": "$browser"}]), + BreakdownFilter(breakdowns=[{"type": BreakdownType.EVENT, "property": "$browser"}]), ) filter = { @@ -1360,8 +1360,8 @@ def test_breakdown_converts_multi(self): query.breakdownFilter, BreakdownFilter( breakdowns=[ - {"type": BreakdownType.EVENT, "value": "$browser"}, - {"type": BreakdownType.SESSION, "value": "$session_duration"}, + {"type": BreakdownType.EVENT, "property": "$browser"}, + {"type": BreakdownType.SESSION, "property": "$session_duration"}, ] ), ) @@ -1646,10 +1646,10 @@ def test_lifecycle_filter(self): def test_multiple_breakdowns(self): filter = { "breakdowns": [ - {"type": "event", "value": "$url", "normalize_url": True}, - {"type": "group", "value": "$os", "group_type_index": 0}, - {"type": "session", "value": "$session_duration", "histogram_bin_count": 10}, - {"type": "person", "value": "extra_prop"}, + {"type": "event", "property": "$url", "normalize_url": True}, + {"type": "group", "property": "$os", "group_type_index": 0}, + {"type": "session", "property": "$session_duration", "histogram_bin_count": 10}, + {"type": "person", "property": "extra_prop"}, ] } @@ -1660,31 +1660,48 @@ def test_multiple_breakdowns(self): query.breakdownFilter, BreakdownFilter( breakdowns=[ - Breakdown(type=BreakdownType.EVENT, value="$url", normalize_url=True), - Breakdown(type=BreakdownType.GROUP, value="$os", group_type_index=0), - Breakdown(type=BreakdownType.SESSION, value="$session_duration", histogram_bin_count=10), + Breakdown(type=BreakdownType.EVENT, property="$url", normalize_url=True), + Breakdown(type=BreakdownType.GROUP, property="$os", group_type_index=0), + Breakdown(type=BreakdownType.SESSION, property="$session_duration", histogram_bin_count=10), ] ), ) - def test_legacy_multiple_breakdowns(self): + def test_funnels_multiple_breakdowns(self): filter = { + "insight": "FUNNELS", "breakdowns": [ - {"type": "event", "property": "$url"}, {"type": "session", "property": "$session_duration"}, - ] + ], } query = filter_to_query(filter) - assert isinstance(query, TrendsQuery) + assert isinstance(query, FunnelsQuery) self.assertEqual( query.breakdownFilter, BreakdownFilter( - breakdowns=[ - Breakdown(type=BreakdownType.EVENT, value="$url"), - Breakdown(type=BreakdownType.SESSION, value="$session_duration"), - ] + breakdown="$session_duration", + breakdown_type=BreakdownType.SESSION, + ), + ) + + def test_funnels_multiple_breakdowns_no_breakdown_type(self): + filter = { + "insight": "FUNNELS", + "breakdowns": [ + {"property": "prop"}, + ], + } + + query = filter_to_query(filter) + + assert isinstance(query, FunnelsQuery) + self.assertEqual( + query.breakdownFilter, + BreakdownFilter( + breakdown="prop", + breakdown_type=BreakdownType.EVENT, ), ) diff --git a/posthog/schema.py b/posthog/schema.py index b1051a853373ce..cd957832fd8432 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -1351,8 +1351,8 @@ class Breakdown(BaseModel): group_type_index: Optional[int] = None histogram_bin_count: Optional[int] = None normalize_url: Optional[bool] = None + property: str type: Optional[MultipleBreakdownType] = None - value: str class BreakdownFilter(BaseModel): diff --git a/posthog/temporal/data_imports/external_data_job.py b/posthog/temporal/data_imports/external_data_job.py index 4fc6e10200866a..76ca85db9be5f8 100644 --- a/posthog/temporal/data_imports/external_data_job.py +++ b/posthog/temporal/data_imports/external_data_job.py @@ -32,6 +32,14 @@ ExternalDataSource, ) from posthog.temporal.common.logger import bind_temporal_worker_logger +from posthog.warehouse.models.external_data_schema import aupdate_should_sync + + +Non_Retryable_Schema_Errors = [ + "NoSuchTableError", + "401 Client Error: Unauthorized for url: https://api.stripe.com", + "403 Client Error: Forbidden for url: https://api.stripe.com", +] @dataclasses.dataclass @@ -54,6 +62,11 @@ async def update_external_data_job_model(inputs: UpdateExternalDataJobStatusInpu f"External data job failed for external data schema {inputs.schema_id} with error: {inputs.internal_error}" ) + has_non_retryable_error = any(error in inputs.internal_error for error in Non_Retryable_Schema_Errors) + if has_non_retryable_error: + logger.info("Schema has a non-retryable error - turning off syncing") + await aupdate_should_sync(schema_id=inputs.schema_id, team_id=inputs.team_id, should_sync=False) + await sync_to_async(update_external_job_status)( run_id=uuid.UUID(inputs.id), status=inputs.status, @@ -177,7 +190,7 @@ async def run(self, inputs: ExternalDataWorkflowInputs): await workflow.execute_activity( import_data_activity, job_inputs, - heartbeat_timeout=dt.timedelta(minutes=1), + heartbeat_timeout=dt.timedelta(minutes=2), **timeout_params, ) # type: ignore diff --git a/posthog/temporal/data_imports/pipelines/sql_database/__init__.py b/posthog/temporal/data_imports/pipelines/sql_database/__init__.py index 3438db67a941fe..04fb8885701dac 100644 --- a/posthog/temporal/data_imports/pipelines/sql_database/__init__.py +++ b/posthog/temporal/data_imports/pipelines/sql_database/__init__.py @@ -1,20 +1,22 @@ """Source that loads tables form any SQLAlchemy supported database, supports batching requests and incremental loads.""" from datetime import datetime, date -from typing import Any, Optional, Union, List # noqa: UP035 +from typing import Any, Optional, Union, List, cast # noqa: UP035 from collections.abc import Iterable from zoneinfo import ZoneInfo from sqlalchemy import MetaData, Table -from sqlalchemy.engine import Engine +from sqlalchemy.engine import Engine, CursorResult import dlt from dlt.sources import DltResource, DltSource +from dlt.common.schema.typing import TColumnSchema from dlt.sources.credentials import ConnectionStringCredentials from urllib.parse import quote from posthog.warehouse.types import IncrementalFieldType +from sqlalchemy.sql import text from .helpers import ( table_rows, @@ -139,8 +141,36 @@ def sql_database( write_disposition="merge" if incremental else "replace", spec=SqlDatabaseTableConfiguration, table_format="delta", + columns=get_column_hints(engine, schema or "", table.name), )( engine=engine, table=table, incremental=incremental, ) + + +def get_column_hints(engine: Engine, schema_name: str, table_name: str) -> dict[str, TColumnSchema]: + with engine.connect() as conn: + execute_result: CursorResult = conn.execute( + text( + "SELECT column_name, data_type, numeric_precision, numeric_scale FROM information_schema.columns WHERE table_schema = :schema_name AND table_name = :table_name" + ), + {"schema_name": schema_name, "table_name": table_name}, + ) + + cursor_result = cast(CursorResult, execute_result) + results = cursor_result.fetchall() + + columns: dict[str, TColumnSchema] = {} + + for column_name, data_type, numeric_precision, numeric_scale in results: + if data_type != "numeric": + continue + + columns[column_name] = { + "data_type": "decimal", + "precision": numeric_precision or 76, + "scale": numeric_scale or 16, + } + + return columns diff --git a/posthog/temporal/data_imports/pipelines/sql_database/test/test_sql_database.py b/posthog/temporal/data_imports/pipelines/sql_database/test/test_sql_database.py new file mode 100644 index 00000000000000..d604d1e38c35bb --- /dev/null +++ b/posthog/temporal/data_imports/pipelines/sql_database/test/test_sql_database.py @@ -0,0 +1,45 @@ +from unittest.mock import MagicMock + +from posthog.temporal.data_imports.pipelines.sql_database import get_column_hints + + +def _setup(return_value): + mock_engine = MagicMock() + mock_engine_enter = MagicMock() + mock_connection = MagicMock() + mock_result = MagicMock() + + mock_engine.configure_mock(**{"connect.return_value": mock_engine_enter}) + mock_engine_enter.configure_mock(**{"__enter__.return_value": mock_connection}) + mock_connection.configure_mock(**{"execute.return_value": mock_result}) + mock_result.configure_mock(**{"fetchall.return_value": return_value}) + + return mock_engine + + +def test_get_column_hints_numeric_no_results(): + mock_engine = _setup([]) + + assert get_column_hints(mock_engine, "some_schema", "some_table") == {} + + +def test_get_column_hints_numeric_with_scale_and_precision(): + mock_engine = _setup([("column", "numeric", 10, 2)]) + + assert get_column_hints(mock_engine, "some_schema", "some_table") == { + "column": {"data_type": "decimal", "precision": 10, "scale": 2} + } + + +def test_get_column_hints_numeric_with_missing_scale_and_precision(): + mock_engine = _setup([("column", "numeric", None, None)]) + + assert get_column_hints(mock_engine, "some_schema", "some_table") == { + "column": {"data_type": "decimal", "precision": 76, "scale": 16} + } + + +def test_get_column_hints_numeric_with_no_numeric(): + mock_engine = _setup([("column", "bigint", None, None)]) + + assert get_column_hints(mock_engine, "some_schema", "some_table") == {} diff --git a/posthog/temporal/data_imports/workflow_activities/import_data.py b/posthog/temporal/data_imports/workflow_activities/import_data.py index 2cba1697ef44e1..9849339e785c72 100644 --- a/posthog/temporal/data_imports/workflow_activities/import_data.py +++ b/posthog/temporal/data_imports/workflow_activities/import_data.py @@ -4,6 +4,7 @@ from temporalio import activity +from posthog.temporal.common.heartbeat import Heartbeater from posthog.temporal.data_imports.pipelines.helpers import aremove_reset_pipeline, aupdate_job_count from posthog.temporal.data_imports.pipelines.pipeline import DataImportPipeline, PipelineInputs @@ -13,7 +14,6 @@ get_external_data_job, ) from posthog.temporal.common.logger import bind_temporal_worker_logger -import asyncio from structlog.typing import FilteringBoundLogger from posthog.warehouse.models.external_data_schema import ExternalDataSchema, aget_schema_by_id from posthog.warehouse.models.ssh_tunnel import SSHTunnel @@ -250,15 +250,7 @@ async def _run( schema: ExternalDataSchema, reset_pipeline: bool, ): - # Temp background heartbeat for now - async def heartbeat() -> None: - while True: - await asyncio.sleep(10) - activity.heartbeat() - - heartbeat_task = asyncio.create_task(heartbeat()) - - try: + async with Heartbeater(): table_row_counts = await DataImportPipeline( job_inputs, source, logger, reset_pipeline, schema.is_incremental ).run() @@ -266,6 +258,3 @@ async def heartbeat() -> None: await aupdate_job_count(inputs.run_id, inputs.team_id, total_rows_synced) await aremove_reset_pipeline(inputs.source_id) - finally: - heartbeat_task.cancel() - await asyncio.wait([heartbeat_task]) diff --git a/posthog/temporal/tests/external_data/test_external_data_job.py b/posthog/temporal/tests/external_data/test_external_data_job.py index 4734740cb4b47c..aa0a83d9941a6a 100644 --- a/posthog/temporal/tests/external_data/test_external_data_job.py +++ b/posthog/temporal/tests/external_data/test_external_data_job.py @@ -262,6 +262,99 @@ async def test_update_external_job_activity(activity_environment, team, **kwargs assert schema.status == ExternalDataJob.Status.COMPLETED +@pytest.mark.django_db(transaction=True) +@pytest.mark.asyncio +async def test_update_external_job_activity_with_retryable_error(activity_environment, team, **kwargs): + new_source = await sync_to_async(ExternalDataSource.objects.create)( + source_id=uuid.uuid4(), + connection_id=uuid.uuid4(), + destination_id=uuid.uuid4(), + team=team, + status="running", + source_type="Stripe", + ) + + schema = await sync_to_async(ExternalDataSchema.objects.create)( + name=PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING[new_source.source_type][0], + team_id=team.id, + source_id=new_source.pk, + should_sync=True, + ) + + new_job = await sync_to_async(create_external_data_job)( + team_id=team.id, + external_data_source_id=new_source.pk, + workflow_id=activity_environment.info.workflow_id, + workflow_run_id=activity_environment.info.workflow_run_id, + external_data_schema_id=schema.id, + ) + + inputs = UpdateExternalDataJobStatusInputs( + id=str(new_job.id), + run_id=str(new_job.id), + status=ExternalDataJob.Status.COMPLETED, + latest_error=None, + internal_error="Some other retryable error", + schema_id=str(schema.pk), + team_id=team.id, + ) + + await activity_environment.run(update_external_data_job_model, inputs) + await sync_to_async(new_job.refresh_from_db)() + await sync_to_async(schema.refresh_from_db)() + + assert new_job.status == ExternalDataJob.Status.COMPLETED + assert schema.status == ExternalDataJob.Status.COMPLETED + assert schema.should_sync is True + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.asyncio +async def test_update_external_job_activity_with_non_retryable_error(activity_environment, team, **kwargs): + new_source = await sync_to_async(ExternalDataSource.objects.create)( + source_id=uuid.uuid4(), + connection_id=uuid.uuid4(), + destination_id=uuid.uuid4(), + team=team, + status="running", + source_type="Stripe", + ) + + schema = await sync_to_async(ExternalDataSchema.objects.create)( + name=PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING[new_source.source_type][0], + team_id=team.id, + source_id=new_source.pk, + should_sync=True, + ) + + new_job = await sync_to_async(create_external_data_job)( + team_id=team.id, + external_data_source_id=new_source.pk, + workflow_id=activity_environment.info.workflow_id, + workflow_run_id=activity_environment.info.workflow_run_id, + external_data_schema_id=schema.id, + ) + + inputs = UpdateExternalDataJobStatusInputs( + id=str(new_job.id), + run_id=str(new_job.id), + status=ExternalDataJob.Status.COMPLETED, + latest_error=None, + internal_error="NoSuchTableError: TableA", + schema_id=str(schema.pk), + team_id=team.id, + ) + with mock.patch("posthog.warehouse.models.external_data_schema.external_data_workflow_exists", return_value=False): + await activity_environment.run(update_external_data_job_model, inputs) + + await sync_to_async(new_job.refresh_from_db)() + await sync_to_async(schema.refresh_from_db)() + + assert new_job.status == ExternalDataJob.Status.COMPLETED + assert schema.status == ExternalDataJob.Status.COMPLETED + assert schema.should_sync is False + + @pytest.mark.django_db(transaction=True) @pytest.mark.asyncio async def test_run_stripe_job(activity_environment, team, minio_client, **kwargs): diff --git a/posthog/warehouse/data_load/service.py b/posthog/warehouse/data_load/service.py index 2425f186b5fa72..46b3bc5b8de01a 100644 --- a/posthog/warehouse/data_load/service.py +++ b/posthog/warehouse/data_load/service.py @@ -1,5 +1,6 @@ from dataclasses import asdict from datetime import timedelta +from typing import TYPE_CHECKING from temporalio.client import ( Schedule, @@ -28,7 +29,6 @@ unpause_schedule, ) from posthog.temporal.utils import ExternalDataWorkflowInputs -from posthog.warehouse.models import ExternalDataSource import temporalio from temporalio.client import Client as TemporalClient from asgiref.sync import async_to_sync @@ -36,10 +36,12 @@ from django.conf import settings import s3fs -from posthog.warehouse.models.external_data_schema import ExternalDataSchema +if TYPE_CHECKING: + from posthog.warehouse.models import ExternalDataSource + from posthog.warehouse.models.external_data_schema import ExternalDataSchema -def get_sync_schedule(external_data_schema: ExternalDataSchema): +def get_sync_schedule(external_data_schema: "ExternalDataSchema"): inputs = ExternalDataWorkflowInputs( team_id=external_data_schema.team_id, external_data_schema_id=external_data_schema.id, @@ -66,7 +68,9 @@ def get_sync_schedule(external_data_schema: ExternalDataSchema): ) -def get_sync_frequency(external_data_schema: ExternalDataSchema): +def get_sync_frequency(external_data_schema: "ExternalDataSchema"): + from posthog.warehouse.models.external_data_schema import ExternalDataSchema + if external_data_schema.sync_frequency == ExternalDataSchema.SyncFrequency.DAILY: return timedelta(days=1) elif external_data_schema.sync_frequency == ExternalDataSchema.SyncFrequency.WEEKLY: @@ -78,8 +82,8 @@ def get_sync_frequency(external_data_schema: ExternalDataSchema): def sync_external_data_job_workflow( - external_data_schema: ExternalDataSchema, create: bool = False -) -> ExternalDataSchema: + external_data_schema: "ExternalDataSchema", create: bool = False +) -> "ExternalDataSchema": temporal = sync_connect() schedule = get_sync_schedule(external_data_schema) @@ -93,8 +97,8 @@ def sync_external_data_job_workflow( async def a_sync_external_data_job_workflow( - external_data_schema: ExternalDataSchema, create: bool = False -) -> ExternalDataSchema: + external_data_schema: "ExternalDataSchema", create: bool = False +) -> "ExternalDataSchema": temporal = await async_connect() schedule = get_sync_schedule(external_data_schema) @@ -107,17 +111,17 @@ async def a_sync_external_data_job_workflow( return external_data_schema -def trigger_external_data_source_workflow(external_data_source: ExternalDataSource): +def trigger_external_data_source_workflow(external_data_source: "ExternalDataSource"): temporal = sync_connect() trigger_schedule(temporal, schedule_id=str(external_data_source.id)) -def trigger_external_data_workflow(external_data_schema: ExternalDataSchema): +def trigger_external_data_workflow(external_data_schema: "ExternalDataSchema"): temporal = sync_connect() trigger_schedule(temporal, schedule_id=str(external_data_schema.id)) -async def a_trigger_external_data_workflow(external_data_schema: ExternalDataSchema): +async def a_trigger_external_data_workflow(external_data_schema: "ExternalDataSchema"): temporal = await async_connect() await a_trigger_schedule(temporal, schedule_id=str(external_data_schema.id)) @@ -153,7 +157,7 @@ def delete_external_data_schedule(schedule_id: str): raise -async def a_delete_external_data_schedule(external_data_source: ExternalDataSource): +async def a_delete_external_data_schedule(external_data_source: "ExternalDataSource"): temporal = await async_connect() try: await a_delete_schedule(temporal, schedule_id=str(external_data_source.id)) @@ -185,4 +189,6 @@ def delete_data_import_folder(folder_path: str): def is_any_external_data_job_paused(team_id: int) -> bool: + from posthog.warehouse.models import ExternalDataSource + return ExternalDataSource.objects.filter(team_id=team_id, status=ExternalDataSource.Status.PAUSED).exists() diff --git a/posthog/warehouse/models/external_data_schema.py b/posthog/warehouse/models/external_data_schema.py index b2c68fb9e8cab1..fbb65500192d78 100644 --- a/posthog/warehouse/models/external_data_schema.py +++ b/posthog/warehouse/models/external_data_schema.py @@ -6,6 +6,12 @@ from posthog.models.utils import CreatedMetaFields, UUIDModel, sane_repr import uuid import psycopg2 +from posthog.warehouse.data_load.service import ( + external_data_workflow_exists, + pause_external_data_schedule, + sync_external_data_job_workflow, + unpause_external_data_schedule, +) from posthog.warehouse.types import IncrementalFieldType from posthog.warehouse.models.ssh_tunnel import SSHTunnel from posthog.warehouse.util import database_sync_to_async @@ -78,6 +84,26 @@ def aget_schema_by_id(schema_id: str, team_id: int) -> ExternalDataSchema | None return ExternalDataSchema.objects.prefetch_related("source").get(id=schema_id, team_id=team_id) +@database_sync_to_async +def aupdate_should_sync(schema_id: str, team_id: int, should_sync: bool) -> ExternalDataSchema | None: + schema = ExternalDataSchema.objects.get(id=schema_id, team_id=team_id) + schema.should_sync = should_sync + schema.save() + + schedule_exists = external_data_workflow_exists(schema_id) + + if schedule_exists: + if should_sync is False: + pause_external_data_schedule(schema_id) + elif should_sync is True: + unpause_external_data_schedule(schema_id) + else: + if should_sync is True: + sync_external_data_job_workflow(schema, create=True) + + return schema + + @database_sync_to_async def get_active_schemas_for_source_id(source_id: uuid.UUID, team_id: int): return list(ExternalDataSchema.objects.filter(team_id=team_id, source_id=source_id, should_sync=True).all()) diff --git a/requirements.in b/requirements.in index 3d586910f5cd58..9d5e15a4e59a57 100644 --- a/requirements.in +++ b/requirements.in @@ -74,10 +74,10 @@ semantic_version==2.8.5 scikit-learn==1.5.0 slack_sdk==3.17.1 snowflake-connector-python==3.6.0 -snowflake-sqlalchemy==1.5.3 +snowflake-sqlalchemy==1.6.1 social-auth-app-django==5.0.0 social-auth-core==4.3.0 -sqlalchemy==1.4.52 +sqlalchemy==2.0.31 sshtunnel==0.4.0 statshog==1.0.6 structlog==23.2.0 diff --git a/requirements.txt b/requirements.txt index d5df2bf41f3e5d..92b14db6600f4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -570,7 +570,7 @@ snowflake-connector-python==3.6.0 # via # -r requirements.in # snowflake-sqlalchemy -snowflake-sqlalchemy==1.5.3 +snowflake-sqlalchemy==1.6.1 # via -r requirements.in social-auth-app-django==5.0.0 # via -r requirements.in @@ -582,7 +582,7 @@ sortedcontainers==2.4.0 # via # snowflake-connector-python # trio -sqlalchemy==1.4.52 +sqlalchemy==2.0.31 # via # -r requirements.in # snowflake-sqlalchemy @@ -641,6 +641,7 @@ typing-extensions==4.7.1 # pydantic-core # qrcode # snowflake-connector-python + # sqlalchemy # stripe # temporalio tzdata==2023.3