From 98f657584cb445e8821e27a0d9d45175b6cb1e7d Mon Sep 17 00:00:00 2001 From: Ben White Date: Tue, 2 Jan 2024 19:14:19 +0100 Subject: [PATCH 01/15] fix: Padding on bullet lists (#19565) --- frontend/src/scenes/notebooks/Notebook/Notebook.scss | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/scenes/notebooks/Notebook/Notebook.scss b/frontend/src/scenes/notebooks/Notebook/Notebook.scss index d2a228a9829ea..f40140cb209c9 100644 --- a/frontend/src/scenes/notebooks/Notebook/Notebook.scss +++ b/frontend/src/scenes/notebooks/Notebook/Notebook.scss @@ -91,11 +91,11 @@ > ul, > ol { - padding-left: 1rem; + padding-left: 2rem; ul, ol { - padding-left: 1rem; + padding-left: 2rem; } li { From b8b90c184b760f79db27acc795af5e5583bb6b9a Mon Sep 17 00:00:00 2001 From: Bianca Yang Date: Tue, 2 Jan 2024 11:37:49 -0800 Subject: [PATCH 02/15] feat: Feature gate session replay controls using available_product_features (#19401) * feature gate session replay control using available_product_features * separate out feature checks * refine the separate feature checks * Update query snapshots * Update UI snapshots for `chromium` (2) * Update query snapshots * Update UI snapshots for `chromium` (2) * Update UI snapshots for `chromium` (2) * Update UI snapshots for `chromium` (2) * Update UI snapshots for `chromium` (2) * Update UI snapshots for `chromium` (2) * Update UI snapshots for `chromium` (2) --------- Co-authored-by: Bianca Yang Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- frontend/src/scenes/onboarding/Onboarding.tsx | 7 +- frontend/src/scenes/settings/SettingsMap.tsx | 7 + .../project/SessionRecordingSettings.tsx | 343 ++++++++++-------- frontend/src/scenes/settings/settingsLogic.ts | 21 +- frontend/src/scenes/settings/types.ts | 3 + frontend/src/types.ts | 5 +- 6 files changed, 220 insertions(+), 166 deletions(-) diff --git a/frontend/src/scenes/onboarding/Onboarding.tsx b/frontend/src/scenes/onboarding/Onboarding.tsx index 85748159d16da..d0a31a536f730 100644 --- a/frontend/src/scenes/onboarding/Onboarding.tsx +++ b/frontend/src/scenes/onboarding/Onboarding.tsx @@ -4,8 +4,9 @@ import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { useEffect, useState } from 'react' import { SceneExport } from 'scenes/sceneTypes' import { teamLogic } from 'scenes/teamLogic' +import { userLogic } from 'scenes/userLogic' -import { ProductKey } from '~/types' +import { AvailableFeature, ProductKey } from '~/types' import { OnboardingBillingStep } from './OnboardingBillingStep' import { OnboardingInviteTeammates } from './OnboardingInviteTeammates' @@ -109,7 +110,7 @@ const ProductAnalyticsOnboarding = (): JSX.Element => { ) } const SessionReplayOnboarding = (): JSX.Element => { - const { featureFlags } = useValues(featureFlagLogic) + const { hasAvailableFeature } = useValues(userLogic) const configOptions: ProductConfigOption[] = [ { type: 'toggle', @@ -129,7 +130,7 @@ const SessionReplayOnboarding = (): JSX.Element => { }, ] - if (featureFlags[FEATURE_FLAGS.SESSION_RECORDING_SAMPLING] === true) { + if (hasAvailableFeature(AvailableFeature.RECORDING_DURATION_MINIMUM)) { configOptions.push({ type: 'select', title: 'Minimum session duration (seconds)', diff --git a/frontend/src/scenes/settings/SettingsMap.tsx b/frontend/src/scenes/settings/SettingsMap.tsx index 7056d62ee4996..8ed352a5bf88d 100644 --- a/frontend/src/scenes/settings/SettingsMap.tsx +++ b/frontend/src/scenes/settings/SettingsMap.tsx @@ -1,3 +1,5 @@ +import { AvailableFeature } from '~/types' + import { Invites } from './organization/Invites' import { Members } from './organization/Members' import { OrganizationDangerZone } from './organization/OrganizationDangerZone' @@ -158,6 +160,11 @@ export const SettingsMap: SettingSection[] = [ title: 'Ingestion controls', component: , flag: 'SESSION_RECORDING_SAMPLING', + features: [ + AvailableFeature.SESSION_REPLAY_SAMPLING, + AvailableFeature.RECORDING_DURATION_MINIMUM, + AvailableFeature.FEATURE_FLAG_BASED_RECORDING, + ], }, ], }, diff --git a/frontend/src/scenes/settings/project/SessionRecordingSettings.tsx b/frontend/src/scenes/settings/project/SessionRecordingSettings.tsx index 401342663d11a..703b129a51cc8 100644 --- a/frontend/src/scenes/settings/project/SessionRecordingSettings.tsx +++ b/frontend/src/scenes/settings/project/SessionRecordingSettings.tsx @@ -7,8 +7,12 @@ import { FlagSelector } from 'lib/components/FlagSelector' import { FEATURE_FLAGS, SESSION_REPLAY_MINIMUM_DURATION_OPTIONS } from 'lib/constants' import { IconCancel } from 'lib/lemon-ui/icons' import { LemonLabel } from 'lib/lemon-ui/LemonLabel/LemonLabel' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { teamLogic } from 'scenes/teamLogic' import { urls } from 'scenes/urls' +import { userLogic } from 'scenes/userLogic' + +import { AvailableFeature } from '~/types' export function ReplayGeneral(): JSX.Element { const { updateCurrentTeam } = useActions(teamLogic) @@ -176,168 +180,191 @@ export function ReplayAuthorizedDomains(): JSX.Element { export function ReplayCostControl(): JSX.Element { const { updateCurrentTeam } = useActions(teamLogic) const { currentTeam } = useValues(teamLogic) + const { hasAvailableFeature } = useValues(userLogic) + const { featureFlags } = useValues(featureFlagLogic) + const samplingControlFeatureEnabled = hasAvailableFeature(AvailableFeature.SESSION_REPLAY_SAMPLING) + const recordingDurationMinimumFeatureEnabled = hasAvailableFeature(AvailableFeature.RECORDING_DURATION_MINIMUM) + const featureFlagRecordingFeatureEnabled = hasAvailableFeature(AvailableFeature.FEATURE_FLAG_BASED_RECORDING) + const costControlFeaturesEnabled = + featureFlags[FEATURE_FLAGS.SESSION_RECORDING_SAMPLING] || + samplingControlFeatureEnabled || + recordingDurationMinimumFeatureEnabled || + featureFlagRecordingFeatureEnabled - return ( - - <> -

- PostHog offers several tools to let you control the number of recordings you collect and which users - you collect recordings for.{' '} - - Learn more in our docs - -

- - Requires posthog-js version 1.88.2 or greater - -
- Sampling - { - updateCurrentTeam({ session_recording_sample_rate: v }) - }} - dropdownMatchSelectWidth={false} - options={[ - { - label: '100% (no sampling)', - value: '1.00', - }, - { - label: '95%', - value: '0.95', - }, - { - label: '90%', - value: '0.90', - }, - { - label: '85%', - value: '0.85', - }, - { - label: '80%', - value: '0.80', - }, - { - label: '75%', - value: '0.75', - }, - { - label: '70%', - value: '0.70', - }, - { - label: '65%', - value: '0.65', - }, - { - label: '60%', - value: '0.60', - }, - { - label: '55%', - value: '0.55', - }, - { - label: '50%', - value: '0.50', - }, - { - label: '45%', - value: '0.45', - }, - { - label: '40%', - value: '0.40', - }, - { - label: '35%', - value: '0.35', - }, - { - label: '30%', - value: '0.30', - }, - { - label: '25%', - value: '0.25', - }, - { - label: '20%', - value: '0.20', - }, - { - label: '15%', - value: '0.15', - }, - { - label: '10%', - value: '0.10', - }, - { - label: '5%', - value: '0.05', - }, - { - label: '0% (replay disabled)', - value: '0.00', - }, - ]} - value={ - typeof currentTeam?.session_recording_sample_rate === 'string' - ? currentTeam?.session_recording_sample_rate - : '1.00' - } - /> -
-

- Use this setting to restrict the percentage of sessions that will be recorded. This is useful if you - want to reduce the amount of data you collect. 100% means all sessions will be collected. 50% means - roughly half of sessions will be collected. -

-
- Minimum session duration (seconds) - { - updateCurrentTeam({ session_recording_minimum_duration_milliseconds: v }) - }} - options={SESSION_REPLAY_MINIMUM_DURATION_OPTIONS} - value={currentTeam?.session_recording_minimum_duration_milliseconds} - /> -
-

- Setting a minimum session duration will ensure that only sessions that last longer than that value - are collected. This helps you avoid collecting sessions that are too short to be useful. -

-
- Enable recordings using feature flag -
- { - updateCurrentTeam({ session_recording_linked_flag: { id, key } }) + return costControlFeaturesEnabled ? ( + <> +

+ PostHog offers several tools to let you control the number of recordings you collect and which users you + collect recordings for.{' '} + + Learn more in our docs + +

+ + Requires posthog-js version 1.88.2 or greater + + {(featureFlags[FEATURE_FLAGS.SESSION_RECORDING_SAMPLING] || samplingControlFeatureEnabled) && ( + <> +
+ Sampling + { + updateCurrentTeam({ session_recording_sample_rate: v }) + }} + dropdownMatchSelectWidth={false} + options={[ + { + label: '100% (no sampling)', + value: '1.00', + }, + { + label: '95%', + value: '0.95', + }, + { + label: '90%', + value: '0.90', + }, + { + label: '85%', + value: '0.85', + }, + { + label: '80%', + value: '0.80', + }, + { + label: '75%', + value: '0.75', + }, + { + label: '70%', + value: '0.70', + }, + { + label: '65%', + value: '0.65', + }, + { + label: '60%', + value: '0.60', + }, + { + label: '55%', + value: '0.55', + }, + { + label: '50%', + value: '0.50', + }, + { + label: '45%', + value: '0.45', + }, + { + label: '40%', + value: '0.40', + }, + { + label: '35%', + value: '0.35', + }, + { + label: '30%', + value: '0.30', + }, + { + label: '25%', + value: '0.25', + }, + { + label: '20%', + value: '0.20', + }, + { + label: '15%', + value: '0.15', + }, + { + label: '10%', + value: '0.10', + }, + { + label: '5%', + value: '0.05', + }, + { + label: '0% (replay disabled)', + value: '0.00', + }, + ]} + value={ + typeof currentTeam?.session_recording_sample_rate === 'string' + ? currentTeam?.session_recording_sample_rate + : '1.00' + } + /> +
+

+ Use this setting to restrict the percentage of sessions that will be recorded. This is useful if + you want to reduce the amount of data you collect. 100% means all sessions will be collected. + 50% means roughly half of sessions will be collected. +

+ + )} + {(featureFlags[FEATURE_FLAGS.SESSION_RECORDING_SAMPLING] || recordingDurationMinimumFeatureEnabled) && ( + <> +
+ Minimum session duration (seconds) + { + updateCurrentTeam({ session_recording_minimum_duration_milliseconds: v }) }} + options={SESSION_REPLAY_MINIMUM_DURATION_OPTIONS} + value={currentTeam?.session_recording_minimum_duration_milliseconds} /> - {currentTeam?.session_recording_linked_flag && ( - } - size="small" - onClick={() => updateCurrentTeam({ session_recording_linked_flag: null })} - title="Clear selected flag" +
+

+ Setting a minimum session duration will ensure that only sessions that last longer than that + value are collected. This helps you avoid collecting sessions that are too short to be useful. +

+ + )} + {(featureFlags[FEATURE_FLAGS.SESSION_RECORDING_SAMPLING] || featureFlagRecordingFeatureEnabled) && ( + <> +
+ Enable recordings using feature flag +
+ { + updateCurrentTeam({ session_recording_linked_flag: { id, key } }) + }} /> - )} + {currentTeam?.session_recording_linked_flag && ( + } + size="small" + type="secondary" + onClick={() => updateCurrentTeam({ session_recording_linked_flag: null })} + title="Clear selected flag" + /> + )} +
-
-

- Linking a flag means that recordings will only be collected for users who have the flag enabled. - Only supports release toggles (boolean flags). -

- - +

+ Linking a flag means that recordings will only be collected for users who have the flag enabled. + Only supports release toggles (boolean flags). +

+ + )} + + ) : ( + <> ) } diff --git a/frontend/src/scenes/settings/settingsLogic.ts b/frontend/src/scenes/settings/settingsLogic.ts index 8af9bb55f8475..f727e2a2dfe50 100644 --- a/frontend/src/scenes/settings/settingsLogic.ts +++ b/frontend/src/scenes/settings/settingsLogic.ts @@ -3,6 +3,7 @@ import { FEATURE_FLAGS } from 'lib/constants' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { copyToClipboard } from 'lib/utils/copyToClipboard' import { urls } from 'scenes/urls' +import { userLogic } from 'scenes/userLogic' import type { settingsLogicType } from './settingsLogicType' import { SettingsMap } from './SettingsMap' @@ -13,7 +14,7 @@ export const settingsLogic = kea([ key((props) => props.logicKey ?? 'global'), path((key) => ['scenes', 'settings', 'settingsLogic', key]), connect({ - values: [featureFlagLogic, ['featureFlags']], + values: [featureFlagLogic, ['featureFlags'], userLogic, ['hasAvailableFeature']], }), actions({ @@ -65,8 +66,8 @@ export const settingsLogic = kea([ }, ], settings: [ - (s) => [s.selectedLevel, s.selectedSectionId, s.sections, s.featureFlags], - (selectedLevel, selectedSectionId, sections, featureFlags): Setting[] => { + (s) => [s.selectedLevel, s.selectedSectionId, s.sections, s.featureFlags, s.hasAvailableFeature], + (selectedLevel, selectedSectionId, sections, featureFlags, hasAvailableFeature): Setting[] => { let settings: Setting[] = [] if (!selectedSectionId) { @@ -77,7 +78,19 @@ export const settingsLogic = kea([ settings = sections.find((x) => x.id === selectedSectionId)?.settings || [] } - return settings.filter((x) => (x.flag ? featureFlags[FEATURE_FLAGS[x.flag]] : true)) + return settings.filter((x) => { + if (x.flag && x.features) { + return ( + x.features.some((feat) => hasAvailableFeature(feat)) || featureFlags[FEATURE_FLAGS[x.flag]] + ) + } else if (x.features) { + return x.features.some((feat) => hasAvailableFeature(feat)) + } else if (x.flag) { + return featureFlags[FEATURE_FLAGS[x.flag]] + } + + return true + }) }, ], }), diff --git a/frontend/src/scenes/settings/types.ts b/frontend/src/scenes/settings/types.ts index d57e2273fc765..6dad2d9f75193 100644 --- a/frontend/src/scenes/settings/types.ts +++ b/frontend/src/scenes/settings/types.ts @@ -1,5 +1,7 @@ import { EitherMembershipLevel, FEATURE_FLAGS } from 'lib/constants' +import { AvailableFeature } from '~/types' + export type SettingsLogicProps = { logicKey?: string // Optional - if given, renders only the given level @@ -78,6 +80,7 @@ export type Setting = { description?: JSX.Element | string component: JSX.Element flag?: keyof typeof FEATURE_FLAGS + features?: AvailableFeature[] } export type SettingSection = { diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 36483bbdd9eb5..efdb944d97756 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -41,7 +41,7 @@ import { NodeKind } from './queries/schema' export type Optional = Omit & { [K in keyof T]?: T[K] } -// Keep this in sync with backend constants (constants.py) +// Keep this in sync with backend constants/features/{product_name}.yml export enum AvailableFeature { EVENTS = 'events', TRACKED_USERS = 'tracked_users', @@ -91,6 +91,9 @@ export enum AvailableFeature { SURVEYS_STYLING = 'surveys_styling', SURVEYS_TEXT_HTML = 'surveys_text_html', SURVEYS_MULTIPLE_QUESTIONS = 'surveys_multiple_questions', + SESSION_REPLAY_SAMPLING = 'session_replay_sampling', + RECORDING_DURATION_MINIMUM = 'replay_recording_duration_minimum', + FEATURE_FLAG_BASED_RECORDING = 'replay_feature_flag_based_recording', } export enum ProductKey { From 1d6ba3c60ba7378ff94f2dff17bb5dde9c0476c7 Mon Sep 17 00:00:00 2001 From: Eric Duong Date: Tue, 2 Jan 2024 16:02:15 -0500 Subject: [PATCH 03/15] feat(data-warehouse): hubspot integration (#19529) * initial api * urls * refactor source selector frontend * refactor source selector frontend * field config * http working * add hubspot dlt helpers * remove products endpoint and add token refresh * reformat * add limiting * Update UI snapshots for `chromium` (1) * Update UI snapshots for `chromium` (1) * typing and migration * typing * update latest migration * add hubspot logo * Update UI snapshots for `chromium` (1) * add prefix flow --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- frontend/public/hubspot-logo.png | Bin 0 -> 4585 bytes frontend/src/lib/api.ts | 6 +- frontend/src/scenes/appScenes.ts | 1 + .../data-warehouse/external/SourceModal.tsx | 118 ++++++----- .../external/sourceFormLogic.ts | 135 ++++++++++++ .../external/sourceModalLogic.ts | 88 ++++---- .../redirect/DataWarehouseRedirectScene.tsx | 34 +++ .../settings/DataWarehouseSettingsScene.tsx | 2 +- frontend/src/scenes/sceneTypes.ts | 1 + frontend/src/scenes/scenes.ts | 4 + frontend/src/scenes/urls.ts | 1 + frontend/src/types.ts | 15 +- latest_migrations.manifest | 2 +- posthog/api/test/test_preflight.py | 1 + ...80_alter_externaldatasource_source_type.py | 17 ++ posthog/settings/__init__.py | 2 +- .../{airbyte.py => data_warehouse.py} | 3 + .../data_imports/external_data_job.py | 19 ++ .../pipelines/hubspot/__init__.py | 146 +++++++++++++ .../data_imports/pipelines/hubspot/auth.py | 42 ++++ .../data_imports/pipelines/hubspot/helpers.py | 198 ++++++++++++++++++ .../pipelines/hubspot/settings.py | 106 ++++++++++ .../data_imports/pipelines/schemas.py | 8 +- posthog/views.py | 2 + posthog/warehouse/api/external_data_source.py | 70 +++++-- .../api/test/test_external_data_source.py | 10 +- .../warehouse/models/external_data_source.py | 1 + 27 files changed, 915 insertions(+), 117 deletions(-) create mode 100644 frontend/public/hubspot-logo.png create mode 100644 frontend/src/scenes/data-warehouse/external/sourceFormLogic.ts create mode 100644 frontend/src/scenes/data-warehouse/redirect/DataWarehouseRedirectScene.tsx create mode 100644 posthog/migrations/0380_alter_externaldatasource_source_type.py rename posthog/settings/{airbyte.py => data_warehouse.py} (75%) create mode 100644 posthog/temporal/data_imports/pipelines/hubspot/__init__.py create mode 100644 posthog/temporal/data_imports/pipelines/hubspot/auth.py create mode 100644 posthog/temporal/data_imports/pipelines/hubspot/helpers.py create mode 100644 posthog/temporal/data_imports/pipelines/hubspot/settings.py diff --git a/frontend/public/hubspot-logo.png b/frontend/public/hubspot-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..ecb0a3804f05c12b5eb3fd0ff25908572cdc7610 GIT binary patch literal 4585 zcmbVPc|26>|DS1u>{lauj;s}COoQx8vfSb(gveNC)G)?m218^dqL?h9tX*4W&7K&! zkqk+e5*oWCA$wWscj*4Q_kLgBKfbT~$2sTuJkR^{{(R17d7g8kPg$Dsa*1(4AP`=2 zGvm|XvvvF8WC!oX_9dC%14=$^Y6K~HDKQNSdq`#uWC(<>dHaGU9p)1UHCg7yhBnmn zS(ZLQ!gfE$XN1B;l4L)+4nKyC59uI^NpLXXEjH#c#$2RB-{7W(_v4Hrj1}pLccr_# zWYKV@RR(!?@=NnW>Tvs~(GT8O>~nqVWv@Y~)O(-mw6NrIiLp^1IyBF?qcMCFyE_|s z1iPkvWPW&FcV&W^r~qWszfElHib{xqN~2s1qIg6E4#3#2w$lEk7=AAV(R?!AvcdJ$O-hbk|8ZrE~A~ zfyY;~S3}nNSdZLgzcGO=8d|*3PU>w)@yGmAY{_~P8J)S#Nqh;b3N2Qs0o81&xFj#r zfYAZ;T0w9ejCFl6n$0W~t-MIVQ9#T*+oG*;+YkRWK_gepL*3@umktPOm|+Eo9L7C&}4 zIh=TI{A<(YWhwltKw|h8Vt+@V&MWTL>Z!UmMCO3L`DnT1>Ug+|kJ+f6VR=#i!y}@V zQhI~w*B@mgjrAIeo<#QZb#b@u={yV}yI<~{PrTWeVIcky1k=w`J?7_w8CAy-fP${R z;aJrt2lxlpI}Qfzq<*RYhs+|U_Bs2g`7~>TOQu zCq2TfG;AF=Hb3IvzZ9UCnPN9>P%ExjPklD){#`Lv*+EG7gTw*IVvshfAe2t)yI03; zHYY+_2pOs6erXw)d%-m5(V?1POf=ods)qN=yNHo?=+_4wj-_qCR?cQ|=(kY!QwbMa zJbzcjojg`lm}ga4ibj;a3|n$!G0sBwm#hlk>eeB>$9**lCGY(*N@vV4W6jNsKE*#c zGci9C`d#vyz38Fh2@Rm^zMOX$re;;_Wk<^ptNs+5WpZBVldX`NYQA?2;#By}4u-t` z@6mm6-_JKb!;S}zY)&G5fOE1kK$E9BtyEupE~)rjYtp#JnGK%++(c5rIguWg0#{>R zq5pHok7=Xy2L96%8+Ras9=e<|0PWp$=M(PqOV6GAJapDy8d2^DSsJ)a?jGqj^`D~I z5^?U!8GL&X0IlArej%C_##D-F=#h41XL~Dt+lksUq&nWUrgV}j3z6oKRj*K-T=%Xz zku51o;r0XdIQ0SA^5B!LS{Cz?cwslqNBxszqM(8}6grVK(`{Qv{zJ@BFrlaq5vA>0F9=!D2sZN=f>c5*qE)5Djx^Ah+qSC^f|y^zxd6?>)GJzH*1%j=FmD02@AQt_I^R4afbaxM}!=yz0Ntj!Am)9o?hB)3NZ*kl5Z?7G>b+VDR3% zsO_{p#>51-$D7D=v|%uOd4o=oH&zbK*wf)XQa3n+sS**$%1+KI_?-8Z6*AS%P);hg z3p1@NgiC3sJAZGI99h27SAtRR4)N8*cKcemd(s*5-<84R-C(N&!#MW9n|l^;4Pt`ya;7wIAcK?pwNtm4XkBLMFI3TJANz zD_p(FdO7~ym?w(giu-MS=_Pu1eQ;P+`@#$5e&wD4bEnRB7<==zk0C7Eu)M(jwdN#d zIYH`o(S&Q?-)L7uVPnT^7p6zGB+87eG-T6;gG7bF(0ZvaI}7J~Y+aQ(qu93k%UKlc`yUhzWsQcB_xi(n#ij_;`|e?^w4KuUJ<%bC}` zMKu?%lvL(N9nEZRr9H}3n>r+I-Nb)eX^<7H*g|6~T1eb-x9pG)m^x?vfoCIA<#?hFi|Ja5 zoqH}u=mZoW6&f>XkFvYQm%X?$fsV+nFvh#vtZ`9L^?d<#5Q)x@IQnVG!@#MWp_g5j zwArUL*p4+d$>57a0V?=+Ukijbo>$6w_G9E{9&SOA01^YAKchQCz!{< zW0ffy_fAv;| zI5yqGnkiR}sgACa8wIRmnUB9vQQpE1f;NVPlDf8ASKJfi50a&UWk%(5D?! z><*@|n59k&XLS>x3mucr(;Gt9yC={ zIm)NKMxT?Yf`U0vmAEE{mHzywvH~5|Ejrp>gt7TerXqnt00PNljWc+4cU+3b;q!6{T;b@{ZKuG;Wlz$Lq`eVb<>d zS`rCV*K$3-Y|h;louHIxaS`dp(|R|R_fA;dld@?UiP^qolI@wzT4GL2Ok3NN9&ST6 znHJh|c7Gh=Tu8-|Z;z4HWrEGB2wGy+bEYw8elk_938i*ei2q(^e*XCaDV`|v-h{}9 zCZgXh2@yHx>anXStjUUSb`Px0>eNw(gnPyYzg;Ja#1TDbPV;oU)*=*>D0_Wk{d@P} zBhF9SqF}Bcu0p4qS6XXkNT)iFI+RH^8Eznfjn{=d9FmQ+Pxq=yk1Oz)9zp_Tp2xZJ zY8y$p>?!sZ zC$-#tyj5JWK5jS_sy7MzErdXH^r<9QcP|_TaKoL)6Ll5l>z*h8c&x61otg#Gf^-ab z0dE%Mi?a^0v~dsea@WKv=<9LmP_;k;Zyd!Hpn4OCWG$+$!Va$%INqK{D1abetcTWV zW0OB2prosCfkGi^ArJup0V)AURD68TBUCjtH4#V@0)_~PAuvJ=TFt~)b++bSyHOCwydWQ3{; za@#Ec(D{MaGQ^U%#UDnjaTj-l9})q-V~OPIOUBs*l5n~Tr*UK-KVNs;4#T$6pCre8 zajp~`Ru6?lsVXB?l~qwXi2qpHw(*z3aVo_etEcIKc13BrX@V+PRb{j~@~E;a(gUfC z^-x2rp^+X~HS}LZKUDrnW8wy~sG-zQ>S~&*XjKhOwWG+NmUkBaiMR3bbHA|tDtdqM z{x!FQ)j@1Wmw^8flpW#EsQe`U5wah^zoq4WPZjBJRj{3>onqY15V!zl=*XWL0wpa& zKgtCkUp)s`FE1a!!qwBy7Y|tD3H~^|udB{B{{InRnd~_GuN(sF_#cJy(;=!W>2DR literal 0 HcmV?d00001 diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index de94171d3d311..06a914a88a253 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -29,9 +29,9 @@ import { EventType, Experiment, ExportedAssetType, + ExternalDataSourceCreatePayload, ExternalDataSourceSchema, ExternalDataStripeSource, - ExternalDataStripeSourceCreatePayload, FeatureFlagAssociatedRoleType, FeatureFlagType, Group, @@ -1756,9 +1756,7 @@ const api = { async list(): Promise> { return await new ApiRequest().externalDataSources().get() }, - async create( - data: Partial - ): Promise { + async create(data: Partial): Promise { return await new ApiRequest().externalDataSources().create({ data }) }, async delete(sourceId: ExternalDataStripeSource['id']): Promise { diff --git a/frontend/src/scenes/appScenes.ts b/frontend/src/scenes/appScenes.ts index 8c7a8c5ab8c09..021d4b883ae61 100644 --- a/frontend/src/scenes/appScenes.ts +++ b/frontend/src/scenes/appScenes.ts @@ -41,6 +41,7 @@ export const appScenes: Record any> = { [Scene.DataWarehouseExternal]: () => import('./data-warehouse/external/DataWarehouseExternalScene'), [Scene.DataWarehouseSavedQueries]: () => import('./data-warehouse/saved_queries/DataWarehouseSavedQueriesScene'), [Scene.DataWarehouseSettings]: () => import('./data-warehouse/settings/DataWarehouseSettingsScene'), + [Scene.DataWarehouseRedirect]: () => import('./data-warehouse/redirect/DataWarehouseRedirectScene'), [Scene.OrganizationCreateFirst]: () => import('./organization/Create'), [Scene.OrganizationCreationConfirm]: () => import('./organization/ConfirmOrganization/ConfirmOrganization'), [Scene.ProjectHomepage]: () => import('./project-homepage/ProjectHomepage'), diff --git a/frontend/src/scenes/data-warehouse/external/SourceModal.tsx b/frontend/src/scenes/data-warehouse/external/SourceModal.tsx index e09050ca89d18..436fc02002820 100644 --- a/frontend/src/scenes/data-warehouse/external/SourceModal.tsx +++ b/frontend/src/scenes/data-warehouse/external/SourceModal.tsx @@ -1,37 +1,47 @@ -import { LemonButton, LemonDivider, LemonInput, LemonModal, LemonModalProps } from '@posthog/lemon-ui' +import { LemonButton, LemonDivider, LemonInput, LemonModal, LemonModalProps, Link } from '@posthog/lemon-ui' import { useActions, useValues } from 'kea' import { Form } from 'kea-forms' import { Field } from 'lib/forms/Field' +import hubspotLogo from 'public/hubspot-logo.png' import stripeLogo from 'public/stripe-logo.svg' +import { ExternalDataSourceType } from '~/types' + import { DatawarehouseTableForm } from '../new_table/DataWarehouseTableForm' +import { SOURCE_DETAILS, sourceFormLogic } from './sourceFormLogic' import { ConnectorConfigType, sourceModalLogic } from './sourceModalLogic' interface SourceModalProps extends LemonModalProps {} export default function SourceModal(props: SourceModalProps): JSX.Element { - const { tableLoading, isExternalDataSourceSubmitting, selectedConnector, isManualLinkFormVisible, connectors } = + const { tableLoading, selectedConnector, isManualLinkFormVisible, connectors, addToHubspotButtonUrl } = useValues(sourceModalLogic) - const { selectConnector, toggleManualLinkFormVisible, resetExternalDataSource, resetTable } = - useActions(sourceModalLogic) + const { selectConnector, toggleManualLinkFormVisible, onClear } = useActions(sourceModalLogic) const MenuButton = (config: ConnectorConfigType): JSX.Element => { const onClick = (): void => { selectConnector(config) } - return ( - - {`stripe - - ) - } + if (config.name === 'Stripe') { + return ( + + {`stripe + + ) + } + if (config.name === 'Hubspot') { + return ( + + + {`hubspot + Hubspot + + + ) + } - const onClear = (): void => { - selectConnector(null) - toggleManualLinkFormVisible(false) - resetExternalDataSource() - resetTable() + return <> } const onManualLinkClick = (): void => { @@ -40,39 +50,7 @@ export default function SourceModal(props: SourceModalProps): JSX.Element { const formToShow = (): JSX.Element => { if (selectedConnector) { - return ( -
- - - - - - - - - - -
- - Back - - - Link - -
- - ) + return } if (isManualLinkFormVisible) { @@ -131,3 +109,47 @@ export default function SourceModal(props: SourceModalProps): JSX.Element { ) } + +interface SourceFormProps { + sourceType: ExternalDataSourceType +} + +function SourceForm({ sourceType }: SourceFormProps): JSX.Element { + const logic = sourceFormLogic({ sourceType }) + const { isExternalDataSourceSubmitting } = useValues(logic) + const { onBack } = useActions(logic) + + return ( +
+ + + + {SOURCE_DETAILS[sourceType].fields.map((field) => ( + + + + ))} + +
+ + Back + + + Link + +
+ + ) +} diff --git a/frontend/src/scenes/data-warehouse/external/sourceFormLogic.ts b/frontend/src/scenes/data-warehouse/external/sourceFormLogic.ts new file mode 100644 index 0000000000000..67877f9a32205 --- /dev/null +++ b/frontend/src/scenes/data-warehouse/external/sourceFormLogic.ts @@ -0,0 +1,135 @@ +import { lemonToast } from '@posthog/lemon-ui' +import { actions, connect, kea, listeners, path, props } from 'kea' +import { forms } from 'kea-forms' +import { router, urlToAction } from 'kea-router' +import api from 'lib/api' +import { urls } from 'scenes/urls' + +import { ExternalDataSourceCreatePayload, ExternalDataSourceType } from '~/types' + +import type { sourceFormLogicType } from './sourceFormLogicType' +import { getHubspotRedirectUri, sourceModalLogic } from './sourceModalLogic' + +export interface SourceFormProps { + sourceType: ExternalDataSourceType +} + +interface SourceConfig { + name: string + caption: string + fields: FieldConfig[] +} +interface FieldConfig { + name: string + label: string + type: string + required: boolean +} + +export const SOURCE_DETAILS: Record = { + Stripe: { + name: 'Stripe', + caption: 'Enter your Stripe credentials to link your Stripe to PostHog', + fields: [ + { + name: 'account_id', + label: 'Account ID', + type: 'text', + required: true, + }, + { + name: 'client_secret', + label: 'Client Secret', + type: 'text', + required: true, + }, + ], + }, +} + +const getPayloadDefaults = (sourceType: string): Record => { + switch (sourceType) { + case 'Stripe': + return { + account_id: '', + client_secret: '', + } + default: + return {} + } +} + +const getErrorsDefaults = (sourceType: string): ((args: Record) => Record) => { + switch (sourceType) { + case 'Stripe': + return ({ payload }) => ({ + payload: { + account_id: !payload.account_id && 'Please enter an account id.', + client_secret: !payload.client_secret && 'Please enter a client secret.', + }, + }) + default: + return () => ({}) + } +} + +export const sourceFormLogic = kea([ + path(['scenes', 'data-warehouse', 'external', 'sourceFormLogic']), + props({} as SourceFormProps), + connect({ + actions: [sourceModalLogic, ['onClear', 'toggleSourceModal', 'loadSources']], + }), + actions({ + onBack: true, + handleRedirect: (kind: string, searchParams: any) => ({ kind, searchParams }), + }), + listeners(({ actions }) => ({ + onBack: () => { + actions.resetExternalDataSource() + actions.onClear() + }, + submitExternalDataSourceSuccess: () => { + lemonToast.success('New Data Resource Created') + actions.toggleSourceModal(false) + actions.resetExternalDataSource() + actions.loadSources() + router.actions.push(urls.dataWarehouseSettings()) + }, + submitExternalDataSourceFailure: ({ error }) => { + lemonToast.error(error?.message || 'Something went wrong') + }, + handleRedirect: async ({ kind, searchParams }) => { + switch (kind) { + case 'hubspot': { + actions.setExternalDataSourceValue('payload', { + code: searchParams.code, + redirect_uri: getHubspotRedirectUri(), + }) + actions.setExternalDataSourceValue('source_type', 'Hubspot') + return + } + default: + lemonToast.error(`Something went wrong.`) + } + }, + })), + urlToAction(({ actions }) => ({ + '/data-warehouse/:kind/redirect': ({ kind = '' }, searchParams) => { + actions.handleRedirect(kind, searchParams) + }, + })), + forms(({ props }) => ({ + externalDataSource: { + defaults: { + prefix: '', + source_type: props.sourceType, + payload: getPayloadDefaults(props.sourceType), + } as ExternalDataSourceCreatePayload, + errors: getErrorsDefaults(props.sourceType), + submit: async (payload: ExternalDataSourceCreatePayload) => { + const newResource = await api.externalDataSources.create(payload) + return newResource + }, + }, + })), +]) diff --git a/frontend/src/scenes/data-warehouse/external/sourceModalLogic.ts b/frontend/src/scenes/data-warehouse/external/sourceModalLogic.ts index 13e6976b2f988..32ae739a5c4d3 100644 --- a/frontend/src/scenes/data-warehouse/external/sourceModalLogic.ts +++ b/frontend/src/scenes/data-warehouse/external/sourceModalLogic.ts @@ -1,19 +1,16 @@ -import { lemonToast } from '@posthog/lemon-ui' import { actions, connect, kea, listeners, path, reducers, selectors } from 'kea' -import { forms } from 'kea-forms' -import { router } from 'kea-router' -import api from 'lib/api' -import { urls } from 'scenes/urls' +import { preflightLogic } from 'scenes/PreflightCheck/preflightLogic' -import { ExternalDataStripeSourceCreatePayload } from '~/types' +import { ExternalDataSourceType } from '~/types' import { dataWarehouseTableLogic } from '../new_table/dataWarehouseTableLogic' import { dataWarehouseSettingsLogic } from '../settings/dataWarehouseSettingsLogic' import { dataWarehouseSceneLogic } from './dataWarehouseSceneLogic' import type { sourceModalLogicType } from './sourceModalLogicType' +export const getHubspotRedirectUri = (): string => `${window.location.origin}/data-warehouse/hubspot/redirect` export interface ConnectorConfigType { - name: string + name: ExternalDataSourceType fields: string[] caption: string disabledReason: string | null @@ -23,10 +20,16 @@ export interface ConnectorConfigType { export const CONNECTORS: ConnectorConfigType[] = [ { name: 'Stripe', - fields: ['accound_id', 'client_secret'], + fields: ['account_id', 'client_secret'], caption: 'Enter your Stripe credentials to link your Stripe to PostHog', disabledReason: null, }, + { + name: 'Hubspot', + fields: [], + caption: '', + disabledReason: null, + }, ] export const sourceModalLogic = kea([ @@ -34,9 +37,18 @@ export const sourceModalLogic = kea([ actions({ selectConnector: (connector: ConnectorConfigType | null) => ({ connector }), toggleManualLinkFormVisible: (visible: boolean) => ({ visible }), + handleRedirect: (kind: string, searchParams: any) => ({ kind, searchParams }), + onClear: true, }), connect({ - values: [dataWarehouseTableLogic, ['tableLoading'], dataWarehouseSettingsLogic, ['dataWarehouseSources']], + values: [ + dataWarehouseTableLogic, + ['tableLoading'], + dataWarehouseSettingsLogic, + ['dataWarehouseSources'], + preflightLogic, + ['preflight'], + ], actions: [ dataWarehouseSceneLogic, ['toggleSourceModal'], @@ -77,37 +89,39 @@ export const sourceModalLogic = kea([ })) }, ], - }), - forms(() => ({ - externalDataSource: { - defaults: { - account_id: '', - client_secret: '', - prefix: '', - source_type: 'Stripe', - } as ExternalDataStripeSourceCreatePayload, - errors: ({ account_id, client_secret }) => { - return { - account_id: !account_id && 'Please enter an account id.', - client_secret: !client_secret && 'Please enter a client secret.', + addToHubspotButtonUrl: [ + (s) => [s.preflight], + (preflight) => { + return () => { + const clientId = preflight?.data_warehouse_integrations?.hubspot.client_id + + if (!clientId) { + return null + } + + const scopes = [ + 'crm.objects.contacts.read', + 'crm.objects.companies.read', + 'crm.objects.deals.read', + 'tickets', + 'crm.objects.quotes.read', + ] + + const params = new URLSearchParams() + params.set('client_id', clientId) + params.set('redirect_uri', getHubspotRedirectUri()) + params.set('scope', scopes.join(' ')) + + return `https://app.hubspot.com/oauth/authorize?${params.toString()}` } }, - submit: async (payload: ExternalDataStripeSourceCreatePayload) => { - const newResource = await api.externalDataSources.create(payload) - return newResource - }, - }, - })), + ], + }), listeners(({ actions }) => ({ - submitExternalDataSourceSuccess: () => { - lemonToast.success('New Data Resource Created') - actions.toggleSourceModal() - actions.resetExternalDataSource() - actions.loadSources() - router.actions.push(urls.dataWarehouseSettings()) - }, - submitExternalDataSourceFailure: ({ error }) => { - lemonToast.error(error?.message || 'Something went wrong') + onClear: () => { + actions.selectConnector(null) + actions.toggleManualLinkFormVisible(false) + actions.resetTable() }, })), ]) diff --git a/frontend/src/scenes/data-warehouse/redirect/DataWarehouseRedirectScene.tsx b/frontend/src/scenes/data-warehouse/redirect/DataWarehouseRedirectScene.tsx new file mode 100644 index 0000000000000..736132b3b747f --- /dev/null +++ b/frontend/src/scenes/data-warehouse/redirect/DataWarehouseRedirectScene.tsx @@ -0,0 +1,34 @@ +import { LemonButton, LemonInput } from '@posthog/lemon-ui' +import { Form } from 'kea-forms' +import { Field } from 'lib/forms/Field' +import { sourceFormLogic } from 'scenes/data-warehouse/external/sourceFormLogic' +import { SceneExport } from 'scenes/sceneTypes' + +export const scene: SceneExport = { + component: DataWarehouseRedirectScene, + logic: sourceFormLogic, +} + +export function DataWarehouseRedirectScene(): JSX.Element { + return ( +
+

Configure

+

Add a prefix to your tables to avoid conflicts with other data sources

+
+ + + + + Submit + +
+
+ ) +} + +export default DataWarehouseRedirectScene diff --git a/frontend/src/scenes/data-warehouse/settings/DataWarehouseSettingsScene.tsx b/frontend/src/scenes/data-warehouse/settings/DataWarehouseSettingsScene.tsx index c0b13232a6cdd..f2b5db2080e45 100644 --- a/frontend/src/scenes/data-warehouse/settings/DataWarehouseSettingsScene.tsx +++ b/frontend/src/scenes/data-warehouse/settings/DataWarehouseSettingsScene.tsx @@ -106,7 +106,7 @@ export function DataWarehouseSettingsScene(): JSX.Element { }, { title: 'Sync Frequency', - key: 'prefix', + key: 'frequency', render: function RenderFrequency() { return 'Every 24 hours' }, diff --git a/frontend/src/scenes/sceneTypes.ts b/frontend/src/scenes/sceneTypes.ts index 4604a64a697da..82c5ba9e1589c 100644 --- a/frontend/src/scenes/sceneTypes.ts +++ b/frontend/src/scenes/sceneTypes.ts @@ -43,6 +43,7 @@ export enum Scene { DataWarehouseSavedQueries = 'DataWarehouseSavedQueries', DataWarehouseTable = 'DataWarehouseTable', DataWarehouseSettings = 'DataWarehouseSettings', + DataWarehouseRedirect = 'DataWarehouseRedirect', OrganizationCreateFirst = 'OrganizationCreate', ProjectHomepage = 'ProjectHomepage', ProjectCreateFirst = 'ProjectCreate', diff --git a/frontend/src/scenes/scenes.ts b/frontend/src/scenes/scenes.ts index 4ebd425f741f0..596135d6fc68a 100644 --- a/frontend/src/scenes/scenes.ts +++ b/frontend/src/scenes/scenes.ts @@ -206,6 +206,9 @@ export const sceneConfigurations: Record = { name: 'Data warehouse settings', defaultDocsPath: '/docs/data-warehouse', }, + [Scene.DataWarehouseRedirect]: { + name: 'Data warehouse redirect', + }, [Scene.DataWarehouseTable]: { projectBased: true, name: 'Data warehouse table', @@ -490,6 +493,7 @@ export const routes: Record = { [urls.dataWarehouseExternal()]: Scene.DataWarehouseExternal, [urls.dataWarehouseSavedQueries()]: Scene.DataWarehouseSavedQueries, [urls.dataWarehouseSettings()]: Scene.DataWarehouseSettings, + [urls.dataWarehouseRedirect(':kind')]: Scene.DataWarehouseRedirect, [urls.featureFlags()]: Scene.FeatureFlags, [urls.featureFlag(':id')]: Scene.FeatureFlag, [urls.annotations()]: Scene.DataManagement, diff --git a/frontend/src/scenes/urls.ts b/frontend/src/scenes/urls.ts index bd5cbe5939aec..53c1250229d13 100644 --- a/frontend/src/scenes/urls.ts +++ b/frontend/src/scenes/urls.ts @@ -127,6 +127,7 @@ export const urls = { dataWarehouseExternal: (): string => '/data-warehouse/external', dataWarehouseSavedQueries: (): string => '/data-warehouse/views', dataWarehouseSettings: (): string => '/data-warehouse/settings', + dataWarehouseRedirect: (kind: string): string => `/data-warehouse/${kind}/redirect`, annotations: (): string => '/data-management/annotations', annotation: (id: AnnotationType['id'] | ':id'): string => `/data-management/annotations/${id}`, projectApps: (tab?: PluginTab): string => `/project/apps${tab ? `?tab=${tab}` : ''}`, diff --git a/frontend/src/types.ts b/frontend/src/types.ts index efdb944d97756..0f4109f6083b3 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -2503,6 +2503,11 @@ export interface PreflightStatus { available: boolean client_id?: string } + data_warehouse_integrations: { + hubspot: { + client_id?: string + } + } /** Whether PostHog is running in DEBUG mode. */ is_debug?: boolean licensed_users_available?: number | null @@ -3331,13 +3336,13 @@ export interface DataWarehouseViewLink { from_join_key?: string } -export interface ExternalDataStripeSourceCreatePayload { - account_id: string - client_secret: string +export type ExternalDataSourceType = 'Stripe' | 'Hubspot' + +export interface ExternalDataSourceCreatePayload { + source_type: ExternalDataSourceType prefix: string - source_type: string + payload: Record } - export interface ExternalDataStripeSource { id: string source_id: string diff --git a/latest_migrations.manifest b/latest_migrations.manifest index 9deccea94b408..132521b9f10a3 100644 --- a/latest_migrations.manifest +++ b/latest_migrations.manifest @@ -5,7 +5,7 @@ contenttypes: 0002_remove_content_type_name ee: 0015_add_verified_properties otp_static: 0002_throttling otp_totp: 0002_auto_20190420_0723 -posthog: 0379_alter_scheduledchange +posthog: 0380_alter_externaldatasource_source_type sessions: 0001_initial social_django: 0010_uid_db_index two_factor: 0007_auto_20201201_1019 diff --git a/posthog/api/test/test_preflight.py b/posthog/api/test/test_preflight.py index 74d5a8b2e490b..6b63ad28f7542 100644 --- a/posthog/api/test/test_preflight.py +++ b/posthog/api/test/test_preflight.py @@ -29,6 +29,7 @@ def preflight_dict(self, options={}): "db": True, "initiated": True, "cloud": False, + "data_warehouse_integrations": {"hubspot": {"client_id": None}}, "demo": False, "clickhouse": True, "kafka": True, diff --git a/posthog/migrations/0380_alter_externaldatasource_source_type.py b/posthog/migrations/0380_alter_externaldatasource_source_type.py new file mode 100644 index 0000000000000..70ba4013f8013 --- /dev/null +++ b/posthog/migrations/0380_alter_externaldatasource_source_type.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.19 on 2023-12-29 18:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("posthog", "0379_alter_scheduledchange"), + ] + + operations = [ + migrations.AlterField( + model_name="externaldatasource", + name="source_type", + field=models.CharField(choices=[("Stripe", "Stripe"), ("Hubspot", "Hubspot")], max_length=128), + ), + ] diff --git a/posthog/settings/__init__.py b/posthog/settings/__init__.py index 7ee845e134694..3a3225e7deed5 100644 --- a/posthog/settings/__init__.py +++ b/posthog/settings/__init__.py @@ -39,7 +39,7 @@ from posthog.settings.object_storage import * from posthog.settings.temporal import * from posthog.settings.web import * -from posthog.settings.airbyte import * +from posthog.settings.data_warehouse import * from posthog.settings.utils import get_from_env, str_to_bool diff --git a/posthog/settings/airbyte.py b/posthog/settings/data_warehouse.py similarity index 75% rename from posthog/settings/airbyte.py rename to posthog/settings/data_warehouse.py index bcbcf2fefacb5..b53e16e570a13 100644 --- a/posthog/settings/airbyte.py +++ b/posthog/settings/data_warehouse.py @@ -8,3 +8,6 @@ # for DLT BUCKET_URL = os.getenv("BUCKET_URL", None) AIRBYTE_BUCKET_NAME = os.getenv("AIRBYTE_BUCKET_NAME", None) + +HUBSPOT_APP_CLIENT_ID = os.getenv("HUBSPOT_APP_CLIENT_ID", None) +HUBSPOT_APP_CLIENT_SECRET = os.getenv("HUBSPOT_APP_CLIENT_SECRET", None) diff --git a/posthog/temporal/data_imports/external_data_job.py b/posthog/temporal/data_imports/external_data_job.py index cdb218c0cce31..3f9f0096a7e76 100644 --- a/posthog/temporal/data_imports/external_data_job.py +++ b/posthog/temporal/data_imports/external_data_job.py @@ -155,6 +155,25 @@ async def run_external_data_job(inputs: ExternalDataJobInputs) -> None: source = stripe_source( api_key=stripe_secret_key, endpoints=tuple(inputs.schemas), job_id=str(model.id), team_id=inputs.team_id ) + elif model.pipeline.source_type == ExternalDataSource.Type.HUBSPOT: + from posthog.temporal.data_imports.pipelines.hubspot.auth import refresh_access_token + from posthog.temporal.data_imports.pipelines.hubspot import hubspot + + hubspot_access_code = model.pipeline.job_inputs.get("hubspot_secret_key", None) + refresh_token = model.pipeline.job_inputs.get("hubspot_refresh_token", None) + if not refresh_token: + raise ValueError(f"Hubspot refresh token not found for job {model.id}") + + if not hubspot_access_code: + hubspot_access_code = refresh_access_token(refresh_token) + + source = hubspot( + api_key=hubspot_access_code, + refresh_token=refresh_token, + job_id=str(model.id), + team_id=inputs.team_id, + endpoints=tuple(inputs.schemas), + ) else: raise ValueError(f"Source type {model.pipeline.source_type} not supported") diff --git a/posthog/temporal/data_imports/pipelines/hubspot/__init__.py b/posthog/temporal/data_imports/pipelines/hubspot/__init__.py new file mode 100644 index 0000000000000..3275071efe992 --- /dev/null +++ b/posthog/temporal/data_imports/pipelines/hubspot/__init__.py @@ -0,0 +1,146 @@ +""" +This is a module that provides a DLT source to retrieve data from multiple endpoints of the HubSpot API using a specified API key. The retrieved data is returned as a tuple of Dlt resources, one for each endpoint. + +The source retrieves data from the following endpoints: +- CRM Companies +- CRM Contacts +- CRM Deals +- CRM Tickets +- CRM Quotes +- Web Analytics Events + +For each endpoint, a resource and transformer function are defined to retrieve data and transform it to a common format. +The resource functions yield the raw data retrieved from the API, while the transformer functions are used to retrieve +additional information from the Web Analytics Events endpoint. + +The source also supports enabling Web Analytics Events for each endpoint by setting the corresponding enable flag to True. + +Example: +To retrieve data from all endpoints, use the following code: + +python + +>>> resources = hubspot(api_key="hubspot_access_code") +""" + +from typing import Literal, Sequence, Iterator, Iterable + +import dlt +from dlt.common.typing import TDataItems +from dlt.sources import DltResource +from posthog.temporal.data_imports.pipelines.helpers import limit_paginated_generator + +from .helpers import ( + fetch_data, + _get_property_names, + fetch_property_history, +) +from .settings import ( + ALL, + CRM_OBJECT_ENDPOINTS, + DEFAULT_PROPS, + OBJECT_TYPE_PLURAL, + OBJECT_TYPE_SINGULAR, +) + +THubspotObjectType = Literal["company", "contact", "deal", "ticket", "quote"] + + +@dlt.source(name="hubspot") +def hubspot( + api_key: str, + refresh_token: str, + job_id: str, + team_id: int, + endpoints: Sequence[str] = ("companies", "contacts", "deals", "tickets", "quotes"), + include_history: bool = False, +) -> Iterable[DltResource]: + """ + A DLT source that retrieves data from the HubSpot API using the + specified API key. + + This function retrieves data for several HubSpot API endpoints, + including companies, contacts, deals, tickets and web + analytics events. It returns a tuple of Dlt resources, one for + each endpoint. + + Args: + api_key (Optional[str]): + The API key used to authenticate with the HubSpot API. Defaults + to dlt.secrets.value. + include_history (Optional[bool]): + Whether to load history of property changes along with entities. + The history entries are loaded to separate tables. + + Returns: + Sequence[DltResource]: Dlt resources, one for each HubSpot API endpoint. + + Notes: + This function uses the `fetch_data` function to retrieve data from the + HubSpot CRM API. The API key is passed to `fetch_data` as the + `api_key` argument. + """ + + for endpoint in endpoints: + yield dlt.resource( + crm_objects, + name=endpoint, + write_disposition="append", + )( + object_type=OBJECT_TYPE_SINGULAR[endpoint], + api_key=api_key, + refresh_token=refresh_token, + include_history=include_history, + props=DEFAULT_PROPS[endpoint], + include_custom_props=True, + job_id=job_id, + team_id=team_id, + ) + + +@limit_paginated_generator +def crm_objects( + object_type: str, + api_key: str, + refresh_token: str, + include_history: bool, + props: Sequence[str], + include_custom_props: bool = True, +) -> Iterator[TDataItems]: + """Building blocks for CRM resources.""" + if props == ALL: + props = list(_get_property_names(api_key, refresh_token, object_type)) + + if include_custom_props: + all_props = _get_property_names(api_key, refresh_token, object_type) + custom_props = [prop for prop in all_props if not prop.startswith("hs_")] + props = props + custom_props # type: ignore + + props = ",".join(sorted(list(set(props)))) + + if len(props) > 2000: + raise ValueError( + ( + "Your request to Hubspot is too long to process. " + "Maximum allowed query length is 2000 symbols, while " + f"your list of properties `{props[:200]}`... is {len(props)} " + "symbols long. Use the `props` argument of the resource to " + "set the list of properties to extract from the endpoint." + ) + ) + + params = {"properties": props, "limit": 100} + + yield from fetch_data(CRM_OBJECT_ENDPOINTS[object_type], api_key, refresh_token, params=params) + if include_history: + # Get history separately, as requesting both all properties and history together + # is likely to hit hubspot's URL length limit + for history_entries in fetch_property_history( + CRM_OBJECT_ENDPOINTS[object_type], + api_key, + props, + ): + yield dlt.mark.with_table_name( + history_entries, + OBJECT_TYPE_PLURAL[object_type] + "_property_history", + ) diff --git a/posthog/temporal/data_imports/pipelines/hubspot/auth.py b/posthog/temporal/data_imports/pipelines/hubspot/auth.py new file mode 100644 index 0000000000000..490552cfe237d --- /dev/null +++ b/posthog/temporal/data_imports/pipelines/hubspot/auth.py @@ -0,0 +1,42 @@ +import requests +from django.conf import settings +from typing import Tuple + + +def refresh_access_token(refresh_token: str) -> str: + res = requests.post( + "https://api.hubapi.com/oauth/v1/token", + data={ + "grant_type": "refresh_token", + "client_id": settings.HUBSPOT_APP_CLIENT_ID, + "client_secret": settings.HUBSPOT_APP_CLIENT_SECRET, + "refresh_token": refresh_token, + }, + ) + + if res.status_code != 200: + err_message = res.json()["message"] + raise Exception(err_message) + + return res.json()["access_token"] + + +def get_access_token_from_code(code: str, redirect_uri: str) -> Tuple[str, str]: + res = requests.post( + "https://api.hubapi.com/oauth/v1/token", + data={ + "grant_type": "authorization_code", + "client_id": settings.HUBSPOT_APP_CLIENT_ID, + "client_secret": settings.HUBSPOT_APP_CLIENT_SECRET, + "redirect_uri": redirect_uri, + "code": code, + }, + ) + + if res.status_code != 200: + err_message = res.json()["message"] + raise Exception(err_message) + + payload = res.json() + + return payload["access_token"], payload["refresh_token"] diff --git a/posthog/temporal/data_imports/pipelines/hubspot/helpers.py b/posthog/temporal/data_imports/pipelines/hubspot/helpers.py new file mode 100644 index 0000000000000..b724368fe7e40 --- /dev/null +++ b/posthog/temporal/data_imports/pipelines/hubspot/helpers.py @@ -0,0 +1,198 @@ +"""Hubspot source helpers""" + +import urllib.parse +from typing import Iterator, Dict, Any, List, Optional + +from dlt.sources.helpers import requests +import requests as http_requests +from .settings import OBJECT_TYPE_PLURAL +from .auth import refresh_access_token + +BASE_URL = "https://api.hubapi.com/" + + +def get_url(endpoint: str) -> str: + """Get absolute hubspot endpoint URL""" + return urllib.parse.urljoin(BASE_URL, endpoint) + + +def _get_headers(api_key: str) -> Dict[str, str]: + """ + Return a dictionary of HTTP headers to use for API requests, including the specified API key. + + Args: + api_key (str): The API key to use for authentication, as a string. + + Returns: + dict: A dictionary of HTTP headers to include in API requests, with the `Authorization` header + set to the specified API key in the format `Bearer {api_key}`. + + """ + # Construct the dictionary of HTTP headers to use for API requests + return dict(authorization=f"Bearer {api_key}") + + +def extract_property_history(objects: List[Dict[str, Any]]) -> Iterator[Dict[str, Any]]: + for item in objects: + history = item.get("propertiesWithHistory") + if not history: + return + # Yield a flat list of property history entries + for key, changes in history.items(): + if not changes: + continue + for entry in changes: + yield {"object_id": item["id"], "property_name": key, **entry} + + +def fetch_property_history( + endpoint: str, + api_key: str, + props: str, + params: Optional[Dict[str, Any]] = None, +) -> Iterator[List[Dict[str, Any]]]: + """Fetch property history from the given CRM endpoint. + + Args: + endpoint: The endpoint to fetch data from, as a string. + api_key: The API key to use for authentication, as a string. + props: A comma separated list of properties to retrieve the history for + params: Optional dict of query params to include in the request + + Yields: + List of property history entries (dicts) + """ + # Construct the URL and headers for the API request + url = get_url(endpoint) + headers = _get_headers(api_key) + + params = dict(params or {}) + params["propertiesWithHistory"] = props + params["limit"] = 50 + # Make the API request + r = requests.get(url, headers=headers, params=params) + # Parse the API response and yield the properties of each result + + # Parse the response JSON data + _data = r.json() + while _data is not None: + if "results" in _data: + yield list(extract_property_history(_data["results"])) + + # Follow pagination links if they exist + _next = _data.get("paging", {}).get("next", None) + if _next: + next_url = _next["link"] + # Get the next page response + r = requests.get(next_url, headers=headers) + _data = r.json() + else: + _data = None + + +def fetch_data( + endpoint: str, api_key: str, refresh_token: str, params: Optional[Dict[str, Any]] = None +) -> Iterator[List[Dict[str, Any]]]: + """ + Fetch data from HUBSPOT endpoint using a specified API key and yield the properties of each result. + For paginated endpoint this function yields item from all pages. + + Args: + endpoint (str): The endpoint to fetch data from, as a string. + api_key (str): The API key to use for authentication, as a string. + params: Optional dict of query params to include in the request + + Yields: + A List of CRM object dicts + + Raises: + requests.exceptions.HTTPError: If the API returns an HTTP error status code. + + Notes: + This function uses the `requests` library to make a GET request to the specified endpoint, with + the API key included in the headers. If the API returns a non-successful HTTP status code (e.g. + 404 Not Found), a `requests.exceptions.HTTPError` exception will be raised. + + The `endpoint` argument should be a relative URL, which will be appended to the base URL for the + API. The `params` argument is used to pass additional query parameters to the request + + This function also includes a retry decorator that will automatically retry the API call up to + 3 times with a 5-second delay between retries, using an exponential backoff strategy. + """ + # Construct the URL and headers for the API request + url = get_url(endpoint) + headers = _get_headers(api_key) + + # Make the API request + try: + r = requests.get(url, headers=headers, params=params) + except http_requests.exceptions.HTTPError as e: + if e.response.status_code == 401: + # refresh token + api_key = refresh_access_token(refresh_token) + headers = _get_headers(api_key) + r = requests.get(url, headers=headers, params=params) + else: + raise e + # Parse the API response and yield the properties of each result + # Parse the response JSON data + + _data = r.json() + # Yield the properties of each result in the API response + while _data is not None: + if "results" in _data: + _objects: List[Dict[str, Any]] = [] + for _result in _data["results"]: + _obj = _result.get("properties", _result) + if "id" not in _obj and "id" in _result: + # Move id from properties to top level + _obj["id"] = _result["id"] + if "associations" in _result: + for association in _result["associations"]: + __values = [ + { + "value": _obj["hs_object_id"], + f"{association}_id": __r["id"], + } + for __r in _result["associations"][association]["results"] + ] + + # remove duplicates from list of dicts + __values = [dict(t) for t in {tuple(d.items()) for d in __values}] + + _obj[association] = __values + _objects.append(_obj) + + yield _objects + + # Follow pagination links if they exist + _next = _data.get("paging", {}).get("next", None) + if _next: + next_url = _next["link"] + # Get the next page response + r = requests.get(next_url, headers=headers) + _data = r.json() + else: + _data = None + + +def _get_property_names(api_key: str, refresh_token: str, object_type: str) -> List[str]: + """ + Retrieve property names for a given entity from the HubSpot API. + + Args: + entity: The entity name for which to retrieve property names. + + Returns: + A list of property names. + + Raises: + Exception: If an error occurs during the API request. + """ + properties = [] + endpoint = f"/crm/v3/properties/{OBJECT_TYPE_PLURAL[object_type]}" + + for page in fetch_data(endpoint, api_key, refresh_token): + properties.extend([prop["name"] for prop in page]) + + return properties diff --git a/posthog/temporal/data_imports/pipelines/hubspot/settings.py b/posthog/temporal/data_imports/pipelines/hubspot/settings.py new file mode 100644 index 0000000000000..10af4c47b5a31 --- /dev/null +++ b/posthog/temporal/data_imports/pipelines/hubspot/settings.py @@ -0,0 +1,106 @@ +"""Hubspot source settings and constants""" + +from dlt.common import pendulum + +STARTDATE = pendulum.datetime(year=2000, month=1, day=1) + +CONTACT = "contact" +COMPANY = "company" +DEAL = "deal" +TICKET = "ticket" +QUOTE = "quote" + +CRM_CONTACTS_ENDPOINT = "/crm/v3/objects/contacts?associations=deals,tickets,quotes" +CRM_COMPANIES_ENDPOINT = "/crm/v3/objects/companies?associations=contacts,deals,tickets,quotes" +CRM_DEALS_ENDPOINT = "/crm/v3/objects/deals" +CRM_TICKETS_ENDPOINT = "/crm/v3/objects/tickets" +CRM_QUOTES_ENDPOINT = "/crm/v3/objects/quotes" + +CRM_OBJECT_ENDPOINTS = { + CONTACT: CRM_CONTACTS_ENDPOINT, + COMPANY: CRM_COMPANIES_ENDPOINT, + DEAL: CRM_DEALS_ENDPOINT, + TICKET: CRM_TICKETS_ENDPOINT, + QUOTE: CRM_QUOTES_ENDPOINT, +} + +WEB_ANALYTICS_EVENTS_ENDPOINT = "/events/v3/events?objectType={objectType}&objectId={objectId}&occurredAfter={occurredAfter}&occurredBefore={occurredBefore}&sort=-occurredAt" + +OBJECT_TYPE_SINGULAR = { + "companies": COMPANY, + "contacts": CONTACT, + "deals": DEAL, + "tickets": TICKET, + "quotes": QUOTE, +} + +OBJECT_TYPE_PLURAL = {v: k for k, v in OBJECT_TYPE_SINGULAR.items()} + + +ENDPOINTS = ( + OBJECT_TYPE_PLURAL[CONTACT], + OBJECT_TYPE_PLURAL[DEAL], + OBJECT_TYPE_PLURAL[COMPANY], + OBJECT_TYPE_PLURAL[TICKET], + OBJECT_TYPE_PLURAL[QUOTE], +) + +DEFAULT_DEAL_PROPS = [ + "amount", + "closedate", + "createdate", + "dealname", + "dealstage", + "hs_lastmodifieddate", + "hs_object_id", + "pipeline", +] + +DEFAULT_COMPANY_PROPS = [ + "createdate", + "domain", + "hs_lastmodifieddate", + "hs_object_id", + "name", +] + +DEFAULT_CONTACT_PROPS = [ + "createdate", + "email", + "firstname", + "hs_object_id", + "lastmodifieddate", + "lastname", +] + +DEFAULT_TICKET_PROPS = [ + "createdate", + "content", + "hs_lastmodifieddate", + "hs_object_id", + "hs_pipeline", + "hs_pipeline_stage", + "hs_ticket_category", + "hs_ticket_priority", + "subject", +] + +DEFAULT_QUOTE_PROPS = [ + "hs_createdate", + "hs_expiration_date", + "hs_lastmodifieddate", + "hs_object_id", + "hs_public_url_key", + "hs_status", + "hs_title", +] + +DEFAULT_PROPS = { + OBJECT_TYPE_PLURAL[CONTACT]: DEFAULT_CONTACT_PROPS, + OBJECT_TYPE_PLURAL[COMPANY]: DEFAULT_COMPANY_PROPS, + OBJECT_TYPE_PLURAL[DEAL]: DEFAULT_DEAL_PROPS, + OBJECT_TYPE_PLURAL[TICKET]: DEFAULT_TICKET_PROPS, + OBJECT_TYPE_PLURAL[QUOTE]: DEFAULT_QUOTE_PROPS, +} + +ALL = ("ALL",) diff --git a/posthog/temporal/data_imports/pipelines/schemas.py b/posthog/temporal/data_imports/pipelines/schemas.py index a62db7d664e40..eaaa431d7aef9 100644 --- a/posthog/temporal/data_imports/pipelines/schemas.py +++ b/posthog/temporal/data_imports/pipelines/schemas.py @@ -1,4 +1,8 @@ from posthog.warehouse.models import ExternalDataSource -from posthog.temporal.data_imports.pipelines.stripe.settings import ENDPOINTS +from posthog.temporal.data_imports.pipelines.stripe.settings import ENDPOINTS as STRIPE_ENDPOINTS +from posthog.temporal.data_imports.pipelines.hubspot.settings import ENDPOINTS as HUBSPOT_ENDPOINTS -PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING = {ExternalDataSource.Type.STRIPE: ENDPOINTS} +PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING = { + ExternalDataSource.Type.STRIPE: STRIPE_ENDPOINTS, + ExternalDataSource.Type.HUBSPOT: HUBSPOT_ENDPOINTS, +} diff --git a/posthog/views.py b/posthog/views.py index 4750cf170cc27..1f757f9833734 100644 --- a/posthog/views.py +++ b/posthog/views.py @@ -92,6 +92,7 @@ def security_txt(request): @never_cache def preflight_check(request: HttpRequest) -> JsonResponse: slack_client_id = SlackIntegration.slack_config().get("SLACK_APP_CLIENT_ID") + hubspot_client_id = settings.HUBSPOT_APP_CLIENT_ID response = { "django": True, @@ -113,6 +114,7 @@ def preflight_check(request: HttpRequest) -> JsonResponse: "available": bool(slack_client_id), "client_id": slack_client_id or None, }, + "data_warehouse_integrations": {"hubspot": {"client_id": hubspot_client_id}}, "object_storage": is_cloud() or is_object_storage_available(), } diff --git a/posthog/warehouse/api/external_data_source.py b/posthog/warehouse/api/external_data_source.py index 48f8babed4a5a..843821e2f2749 100644 --- a/posthog/warehouse/api/external_data_source.py +++ b/posthog/warehouse/api/external_data_source.py @@ -25,6 +25,9 @@ from posthog.temporal.data_imports.pipelines.schemas import ( PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING, ) +from posthog.temporal.data_imports.pipelines.hubspot.auth import ( + get_access_token_from_code, +) import temporalio logger = structlog.get_logger(__name__) @@ -107,7 +110,6 @@ def get_queryset(self): ) def create(self, request: Request, *args: Any, **kwargs: Any) -> Response: - client_secret = request.data["client_secret"] prefix = request.data.get("prefix", None) source_type = request.data["source_type"] @@ -127,18 +129,12 @@ def create(self, request: Request, *args: Any, **kwargs: Any) -> Response: ) # TODO: remove dummy vars - new_source_model = ExternalDataSource.objects.create( - source_id=str(uuid.uuid4()), - connection_id=str(uuid.uuid4()), - destination_id=str(uuid.uuid4()), - team=self.team, - status="Running", - source_type=source_type, - job_inputs={ - "stripe_secret_key": client_secret, - }, - prefix=prefix, - ) + if source_type == ExternalDataSource.Type.STRIPE: + new_source_model = self._handle_stripe_source(request, *args, **kwargs) + elif source_type == ExternalDataSource.Type.HUBSPOT: + new_source_model = self._handle_hubspot_source(request, *args, **kwargs) + else: + raise NotImplementedError(f"Source type {source_type} not implemented") schemas = PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING[source_type] for schema in schemas: @@ -156,6 +152,54 @@ def create(self, request: Request, *args: Any, **kwargs: Any) -> Response: return Response(status=status.HTTP_201_CREATED, data={"id": new_source_model.pk}) + def _handle_stripe_source(self, request: Request, *args: Any, **kwargs: Any) -> ExternalDataSource: + payload = request.data["payload"] + client_secret = payload.get("client_secret") + prefix = request.data.get("prefix", None) + source_type = request.data["source_type"] + + # TODO: remove dummy vars + new_source_model = ExternalDataSource.objects.create( + source_id=str(uuid.uuid4()), + connection_id=str(uuid.uuid4()), + destination_id=str(uuid.uuid4()), + team=self.team, + status="Running", + source_type=source_type, + job_inputs={ + "stripe_secret_key": client_secret, + }, + prefix=prefix, + ) + + return new_source_model + + def _handle_hubspot_source(self, request: Request, *args: Any, **kwargs: Any) -> ExternalDataSource: + payload = request.data["payload"] + code = payload.get("code") + redirect_uri = payload.get("redirect_uri") + prefix = request.data.get("prefix", None) + source_type = request.data["source_type"] + + access_token, refresh_token = get_access_token_from_code(code, redirect_uri=redirect_uri) + + # TODO: remove dummy vars + new_source_model = ExternalDataSource.objects.create( + source_id=str(uuid.uuid4()), + connection_id=str(uuid.uuid4()), + destination_id=str(uuid.uuid4()), + team=self.team, + status="Running", + source_type=source_type, + job_inputs={ + "hubspot_secret_key": access_token, + "hubspot_refresh_token": refresh_token, + }, + prefix=prefix, + ) + + return new_source_model + def prefix_required(self, source_type: str) -> bool: source_type_exists = ExternalDataSource.objects.filter(team_id=self.team.pk, source_type=source_type).exists() return source_type_exists diff --git a/posthog/warehouse/api/test/test_external_data_source.py b/posthog/warehouse/api/test/test_external_data_source.py index 2ad741b453a29..955c032c0373e 100644 --- a/posthog/warehouse/api/test/test_external_data_source.py +++ b/posthog/warehouse/api/test/test_external_data_source.py @@ -30,7 +30,7 @@ def _create_external_data_schema(self, source_id) -> ExternalDataSchema: def test_create_external_data_source(self): response = self.client.post( f"/api/projects/{self.team.id}/external_data_sources/", - data={"source_type": "Stripe", "client_secret": "sk_test_123"}, + data={"source_type": "Stripe", "payload": {"client_secret": "sk_test_123"}}, ) payload = response.json() @@ -46,7 +46,7 @@ def test_prefix_external_data_source(self): response = self.client.post( f"/api/projects/{self.team.id}/external_data_sources/", - data={"source_type": "Stripe", "client_secret": "sk_test_123"}, + data={"source_type": "Stripe", "payload": {"client_secret": "sk_test_123"}}, ) self.assertEqual(response.status_code, 201) @@ -54,7 +54,7 @@ def test_prefix_external_data_source(self): response = self.client.post( f"/api/projects/{self.team.id}/external_data_sources/", - data={"source_type": "Stripe", "client_secret": "sk_test_123"}, + data={"source_type": "Stripe", "payload": {"client_secret": "sk_test_123"}}, ) self.assertEqual(response.status_code, 400) @@ -63,7 +63,7 @@ def test_prefix_external_data_source(self): # Create with prefix response = self.client.post( f"/api/projects/{self.team.id}/external_data_sources/", - data={"source_type": "Stripe", "client_secret": "sk_test_123", "prefix": "test_"}, + data={"source_type": "Stripe", "payload": {"client_secret": "sk_test_123"}, "prefix": "test_"}, ) self.assertEqual(response.status_code, 201) @@ -71,7 +71,7 @@ def test_prefix_external_data_source(self): # Try to create same type with same prefix again response = self.client.post( f"/api/projects/{self.team.id}/external_data_sources/", - data={"source_type": "Stripe", "client_secret": "sk_test_123", "prefix": "test_"}, + data={"source_type": "Stripe", "payload": {"client_secret": "sk_test_123"}, "prefix": "test_"}, ) self.assertEqual(response.status_code, 400) diff --git a/posthog/warehouse/models/external_data_source.py b/posthog/warehouse/models/external_data_source.py index 287a4a3f2cd99..5d8f736a77b94 100644 --- a/posthog/warehouse/models/external_data_source.py +++ b/posthog/warehouse/models/external_data_source.py @@ -8,6 +8,7 @@ class ExternalDataSource(CreatedMetaFields, UUIDModel): class Type(models.TextChoices): STRIPE = "Stripe", "Stripe" + HUBSPOT = "Hubspot", "Hubspot" class Status(models.TextChoices): RUNNING = "Running", "Running" From cc0c937bb3c40442fece7e4ed4cb1578088a5bf5 Mon Sep 17 00:00:00 2001 From: Eric Duong Date: Tue, 2 Jan 2024 16:02:34 -0500 Subject: [PATCH 04/15] Revert "feat(data-warehouse): hubspot integration" (#19569) Revert "feat(data-warehouse): hubspot integration (#19529)" This reverts commit 1d6ba3c60ba7378ff94f2dff17bb5dde9c0476c7. --- frontend/public/hubspot-logo.png | Bin 4585 -> 0 bytes frontend/src/lib/api.ts | 6 +- frontend/src/scenes/appScenes.ts | 1 - .../data-warehouse/external/SourceModal.tsx | 118 +++++------ .../external/sourceFormLogic.ts | 135 ------------ .../external/sourceModalLogic.ts | 88 ++++---- .../redirect/DataWarehouseRedirectScene.tsx | 34 --- .../settings/DataWarehouseSettingsScene.tsx | 2 +- frontend/src/scenes/sceneTypes.ts | 1 - frontend/src/scenes/scenes.ts | 4 - frontend/src/scenes/urls.ts | 1 - frontend/src/types.ts | 15 +- latest_migrations.manifest | 2 +- posthog/api/test/test_preflight.py | 1 - ...80_alter_externaldatasource_source_type.py | 17 -- posthog/settings/__init__.py | 2 +- .../{data_warehouse.py => airbyte.py} | 3 - .../data_imports/external_data_job.py | 19 -- .../pipelines/hubspot/__init__.py | 146 ------------- .../data_imports/pipelines/hubspot/auth.py | 42 ---- .../data_imports/pipelines/hubspot/helpers.py | 198 ------------------ .../pipelines/hubspot/settings.py | 106 ---------- .../data_imports/pipelines/schemas.py | 8 +- posthog/views.py | 2 - posthog/warehouse/api/external_data_source.py | 70 ++----- .../api/test/test_external_data_source.py | 10 +- .../warehouse/models/external_data_source.py | 1 - 27 files changed, 117 insertions(+), 915 deletions(-) delete mode 100644 frontend/public/hubspot-logo.png delete mode 100644 frontend/src/scenes/data-warehouse/external/sourceFormLogic.ts delete mode 100644 frontend/src/scenes/data-warehouse/redirect/DataWarehouseRedirectScene.tsx delete mode 100644 posthog/migrations/0380_alter_externaldatasource_source_type.py rename posthog/settings/{data_warehouse.py => airbyte.py} (75%) delete mode 100644 posthog/temporal/data_imports/pipelines/hubspot/__init__.py delete mode 100644 posthog/temporal/data_imports/pipelines/hubspot/auth.py delete mode 100644 posthog/temporal/data_imports/pipelines/hubspot/helpers.py delete mode 100644 posthog/temporal/data_imports/pipelines/hubspot/settings.py diff --git a/frontend/public/hubspot-logo.png b/frontend/public/hubspot-logo.png deleted file mode 100644 index ecb0a3804f05c12b5eb3fd0ff25908572cdc7610..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4585 zcmbVPc|26>|DS1u>{lauj;s}COoQx8vfSb(gveNC)G)?m218^dqL?h9tX*4W&7K&! zkqk+e5*oWCA$wWscj*4Q_kLgBKfbT~$2sTuJkR^{{(R17d7g8kPg$Dsa*1(4AP`=2 zGvm|XvvvF8WC!oX_9dC%14=$^Y6K~HDKQNSdq`#uWC(<>dHaGU9p)1UHCg7yhBnmn zS(ZLQ!gfE$XN1B;l4L)+4nKyC59uI^NpLXXEjH#c#$2RB-{7W(_v4Hrj1}pLccr_# zWYKV@RR(!?@=NnW>Tvs~(GT8O>~nqVWv@Y~)O(-mw6NrIiLp^1IyBF?qcMCFyE_|s z1iPkvWPW&FcV&W^r~qWszfElHib{xqN~2s1qIg6E4#3#2w$lEk7=AAV(R?!AvcdJ$O-hbk|8ZrE~A~ zfyY;~S3}nNSdZLgzcGO=8d|*3PU>w)@yGmAY{_~P8J)S#Nqh;b3N2Qs0o81&xFj#r zfYAZ;T0w9ejCFl6n$0W~t-MIVQ9#T*+oG*;+YkRWK_gepL*3@umktPOm|+Eo9L7C&}4 zIh=TI{A<(YWhwltKw|h8Vt+@V&MWTL>Z!UmMCO3L`DnT1>Ug+|kJ+f6VR=#i!y}@V zQhI~w*B@mgjrAIeo<#QZb#b@u={yV}yI<~{PrTWeVIcky1k=w`J?7_w8CAy-fP${R z;aJrt2lxlpI}Qfzq<*RYhs+|U_Bs2g`7~>TOQu zCq2TfG;AF=Hb3IvzZ9UCnPN9>P%ExjPklD){#`Lv*+EG7gTw*IVvshfAe2t)yI03; zHYY+_2pOs6erXw)d%-m5(V?1POf=ods)qN=yNHo?=+_4wj-_qCR?cQ|=(kY!QwbMa zJbzcjojg`lm}ga4ibj;a3|n$!G0sBwm#hlk>eeB>$9**lCGY(*N@vV4W6jNsKE*#c zGci9C`d#vyz38Fh2@Rm^zMOX$re;;_Wk<^ptNs+5WpZBVldX`NYQA?2;#By}4u-t` z@6mm6-_JKb!;S}zY)&G5fOE1kK$E9BtyEupE~)rjYtp#JnGK%++(c5rIguWg0#{>R zq5pHok7=Xy2L96%8+Ras9=e<|0PWp$=M(PqOV6GAJapDy8d2^DSsJ)a?jGqj^`D~I z5^?U!8GL&X0IlArej%C_##D-F=#h41XL~Dt+lksUq&nWUrgV}j3z6oKRj*K-T=%Xz zku51o;r0XdIQ0SA^5B!LS{Cz?cwslqNBxszqM(8}6grVK(`{Qv{zJ@BFrlaq5vA>0F9=!D2sZN=f>c5*qE)5Djx^Ah+qSC^f|y^zxd6?>)GJzH*1%j=FmD02@AQt_I^R4afbaxM}!=yz0Ntj!Am)9o?hB)3NZ*kl5Z?7G>b+VDR3% zsO_{p#>51-$D7D=v|%uOd4o=oH&zbK*wf)XQa3n+sS**$%1+KI_?-8Z6*AS%P);hg z3p1@NgiC3sJAZGI99h27SAtRR4)N8*cKcemd(s*5-<84R-C(N&!#MW9n|l^;4Pt`ya;7wIAcK?pwNtm4XkBLMFI3TJANz zD_p(FdO7~ym?w(giu-MS=_Pu1eQ;P+`@#$5e&wD4bEnRB7<==zk0C7Eu)M(jwdN#d zIYH`o(S&Q?-)L7uVPnT^7p6zGB+87eG-T6;gG7bF(0ZvaI}7J~Y+aQ(qu93k%UKlc`yUhzWsQcB_xi(n#ij_;`|e?^w4KuUJ<%bC}` zMKu?%lvL(N9nEZRr9H}3n>r+I-Nb)eX^<7H*g|6~T1eb-x9pG)m^x?vfoCIA<#?hFi|Ja5 zoqH}u=mZoW6&f>XkFvYQm%X?$fsV+nFvh#vtZ`9L^?d<#5Q)x@IQnVG!@#MWp_g5j zwArUL*p4+d$>57a0V?=+Ukijbo>$6w_G9E{9&SOA01^YAKchQCz!{< zW0ffy_fAv;| zI5yqGnkiR}sgACa8wIRmnUB9vQQpE1f;NVPlDf8ASKJfi50a&UWk%(5D?! z><*@|n59k&XLS>x3mucr(;Gt9yC={ zIm)NKMxT?Yf`U0vmAEE{mHzywvH~5|Ejrp>gt7TerXqnt00PNljWc+4cU+3b;q!6{T;b@{ZKuG;Wlz$Lq`eVb<>d zS`rCV*K$3-Y|h;louHIxaS`dp(|R|R_fA;dld@?UiP^qolI@wzT4GL2Ok3NN9&ST6 znHJh|c7Gh=Tu8-|Z;z4HWrEGB2wGy+bEYw8elk_938i*ei2q(^e*XCaDV`|v-h{}9 zCZgXh2@yHx>anXStjUUSb`Px0>eNw(gnPyYzg;Ja#1TDbPV;oU)*=*>D0_Wk{d@P} zBhF9SqF}Bcu0p4qS6XXkNT)iFI+RH^8Eznfjn{=d9FmQ+Pxq=yk1Oz)9zp_Tp2xZJ zY8y$p>?!sZ zC$-#tyj5JWK5jS_sy7MzErdXH^r<9QcP|_TaKoL)6Ll5l>z*h8c&x61otg#Gf^-ab z0dE%Mi?a^0v~dsea@WKv=<9LmP_;k;Zyd!Hpn4OCWG$+$!Va$%INqK{D1abetcTWV zW0OB2prosCfkGi^ArJup0V)AURD68TBUCjtH4#V@0)_~PAuvJ=TFt~)b++bSyHOCwydWQ3{; za@#Ec(D{MaGQ^U%#UDnjaTj-l9})q-V~OPIOUBs*l5n~Tr*UK-KVNs;4#T$6pCre8 zajp~`Ru6?lsVXB?l~qwXi2qpHw(*z3aVo_etEcIKc13BrX@V+PRb{j~@~E;a(gUfC z^-x2rp^+X~HS}LZKUDrnW8wy~sG-zQ>S~&*XjKhOwWG+NmUkBaiMR3bbHA|tDtdqM z{x!FQ)j@1Wmw^8flpW#EsQe`U5wah^zoq4WPZjBJRj{3>onqY15V!zl=*XWL0wpa& zKgtCkUp)s`FE1a!!qwBy7Y|tD3H~^|udB{B{{InRnd~_GuN(sF_#cJy(;=!W>2DR diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 06a914a88a253..de94171d3d311 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -29,9 +29,9 @@ import { EventType, Experiment, ExportedAssetType, - ExternalDataSourceCreatePayload, ExternalDataSourceSchema, ExternalDataStripeSource, + ExternalDataStripeSourceCreatePayload, FeatureFlagAssociatedRoleType, FeatureFlagType, Group, @@ -1756,7 +1756,9 @@ const api = { async list(): Promise> { return await new ApiRequest().externalDataSources().get() }, - async create(data: Partial): Promise { + async create( + data: Partial + ): Promise { return await new ApiRequest().externalDataSources().create({ data }) }, async delete(sourceId: ExternalDataStripeSource['id']): Promise { diff --git a/frontend/src/scenes/appScenes.ts b/frontend/src/scenes/appScenes.ts index 021d4b883ae61..8c7a8c5ab8c09 100644 --- a/frontend/src/scenes/appScenes.ts +++ b/frontend/src/scenes/appScenes.ts @@ -41,7 +41,6 @@ export const appScenes: Record any> = { [Scene.DataWarehouseExternal]: () => import('./data-warehouse/external/DataWarehouseExternalScene'), [Scene.DataWarehouseSavedQueries]: () => import('./data-warehouse/saved_queries/DataWarehouseSavedQueriesScene'), [Scene.DataWarehouseSettings]: () => import('./data-warehouse/settings/DataWarehouseSettingsScene'), - [Scene.DataWarehouseRedirect]: () => import('./data-warehouse/redirect/DataWarehouseRedirectScene'), [Scene.OrganizationCreateFirst]: () => import('./organization/Create'), [Scene.OrganizationCreationConfirm]: () => import('./organization/ConfirmOrganization/ConfirmOrganization'), [Scene.ProjectHomepage]: () => import('./project-homepage/ProjectHomepage'), diff --git a/frontend/src/scenes/data-warehouse/external/SourceModal.tsx b/frontend/src/scenes/data-warehouse/external/SourceModal.tsx index 436fc02002820..e09050ca89d18 100644 --- a/frontend/src/scenes/data-warehouse/external/SourceModal.tsx +++ b/frontend/src/scenes/data-warehouse/external/SourceModal.tsx @@ -1,47 +1,37 @@ -import { LemonButton, LemonDivider, LemonInput, LemonModal, LemonModalProps, Link } from '@posthog/lemon-ui' +import { LemonButton, LemonDivider, LemonInput, LemonModal, LemonModalProps } from '@posthog/lemon-ui' import { useActions, useValues } from 'kea' import { Form } from 'kea-forms' import { Field } from 'lib/forms/Field' -import hubspotLogo from 'public/hubspot-logo.png' import stripeLogo from 'public/stripe-logo.svg' -import { ExternalDataSourceType } from '~/types' - import { DatawarehouseTableForm } from '../new_table/DataWarehouseTableForm' -import { SOURCE_DETAILS, sourceFormLogic } from './sourceFormLogic' import { ConnectorConfigType, sourceModalLogic } from './sourceModalLogic' interface SourceModalProps extends LemonModalProps {} export default function SourceModal(props: SourceModalProps): JSX.Element { - const { tableLoading, selectedConnector, isManualLinkFormVisible, connectors, addToHubspotButtonUrl } = + const { tableLoading, isExternalDataSourceSubmitting, selectedConnector, isManualLinkFormVisible, connectors } = useValues(sourceModalLogic) - const { selectConnector, toggleManualLinkFormVisible, onClear } = useActions(sourceModalLogic) + const { selectConnector, toggleManualLinkFormVisible, resetExternalDataSource, resetTable } = + useActions(sourceModalLogic) const MenuButton = (config: ConnectorConfigType): JSX.Element => { const onClick = (): void => { selectConnector(config) } - if (config.name === 'Stripe') { - return ( - - {`stripe - - ) - } - if (config.name === 'Hubspot') { - return ( - - - {`hubspot - Hubspot - - - ) - } + return ( + + {`stripe + + ) + } - return <> + const onClear = (): void => { + selectConnector(null) + toggleManualLinkFormVisible(false) + resetExternalDataSource() + resetTable() } const onManualLinkClick = (): void => { @@ -50,7 +40,39 @@ export default function SourceModal(props: SourceModalProps): JSX.Element { const formToShow = (): JSX.Element => { if (selectedConnector) { - return + return ( +
+ + + + + + + + + + +
+ + Back + + + Link + +
+ + ) } if (isManualLinkFormVisible) { @@ -109,47 +131,3 @@ export default function SourceModal(props: SourceModalProps): JSX.Element { ) } - -interface SourceFormProps { - sourceType: ExternalDataSourceType -} - -function SourceForm({ sourceType }: SourceFormProps): JSX.Element { - const logic = sourceFormLogic({ sourceType }) - const { isExternalDataSourceSubmitting } = useValues(logic) - const { onBack } = useActions(logic) - - return ( -
- - - - {SOURCE_DETAILS[sourceType].fields.map((field) => ( - - - - ))} - -
- - Back - - - Link - -
- - ) -} diff --git a/frontend/src/scenes/data-warehouse/external/sourceFormLogic.ts b/frontend/src/scenes/data-warehouse/external/sourceFormLogic.ts deleted file mode 100644 index 67877f9a32205..0000000000000 --- a/frontend/src/scenes/data-warehouse/external/sourceFormLogic.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { lemonToast } from '@posthog/lemon-ui' -import { actions, connect, kea, listeners, path, props } from 'kea' -import { forms } from 'kea-forms' -import { router, urlToAction } from 'kea-router' -import api from 'lib/api' -import { urls } from 'scenes/urls' - -import { ExternalDataSourceCreatePayload, ExternalDataSourceType } from '~/types' - -import type { sourceFormLogicType } from './sourceFormLogicType' -import { getHubspotRedirectUri, sourceModalLogic } from './sourceModalLogic' - -export interface SourceFormProps { - sourceType: ExternalDataSourceType -} - -interface SourceConfig { - name: string - caption: string - fields: FieldConfig[] -} -interface FieldConfig { - name: string - label: string - type: string - required: boolean -} - -export const SOURCE_DETAILS: Record = { - Stripe: { - name: 'Stripe', - caption: 'Enter your Stripe credentials to link your Stripe to PostHog', - fields: [ - { - name: 'account_id', - label: 'Account ID', - type: 'text', - required: true, - }, - { - name: 'client_secret', - label: 'Client Secret', - type: 'text', - required: true, - }, - ], - }, -} - -const getPayloadDefaults = (sourceType: string): Record => { - switch (sourceType) { - case 'Stripe': - return { - account_id: '', - client_secret: '', - } - default: - return {} - } -} - -const getErrorsDefaults = (sourceType: string): ((args: Record) => Record) => { - switch (sourceType) { - case 'Stripe': - return ({ payload }) => ({ - payload: { - account_id: !payload.account_id && 'Please enter an account id.', - client_secret: !payload.client_secret && 'Please enter a client secret.', - }, - }) - default: - return () => ({}) - } -} - -export const sourceFormLogic = kea([ - path(['scenes', 'data-warehouse', 'external', 'sourceFormLogic']), - props({} as SourceFormProps), - connect({ - actions: [sourceModalLogic, ['onClear', 'toggleSourceModal', 'loadSources']], - }), - actions({ - onBack: true, - handleRedirect: (kind: string, searchParams: any) => ({ kind, searchParams }), - }), - listeners(({ actions }) => ({ - onBack: () => { - actions.resetExternalDataSource() - actions.onClear() - }, - submitExternalDataSourceSuccess: () => { - lemonToast.success('New Data Resource Created') - actions.toggleSourceModal(false) - actions.resetExternalDataSource() - actions.loadSources() - router.actions.push(urls.dataWarehouseSettings()) - }, - submitExternalDataSourceFailure: ({ error }) => { - lemonToast.error(error?.message || 'Something went wrong') - }, - handleRedirect: async ({ kind, searchParams }) => { - switch (kind) { - case 'hubspot': { - actions.setExternalDataSourceValue('payload', { - code: searchParams.code, - redirect_uri: getHubspotRedirectUri(), - }) - actions.setExternalDataSourceValue('source_type', 'Hubspot') - return - } - default: - lemonToast.error(`Something went wrong.`) - } - }, - })), - urlToAction(({ actions }) => ({ - '/data-warehouse/:kind/redirect': ({ kind = '' }, searchParams) => { - actions.handleRedirect(kind, searchParams) - }, - })), - forms(({ props }) => ({ - externalDataSource: { - defaults: { - prefix: '', - source_type: props.sourceType, - payload: getPayloadDefaults(props.sourceType), - } as ExternalDataSourceCreatePayload, - errors: getErrorsDefaults(props.sourceType), - submit: async (payload: ExternalDataSourceCreatePayload) => { - const newResource = await api.externalDataSources.create(payload) - return newResource - }, - }, - })), -]) diff --git a/frontend/src/scenes/data-warehouse/external/sourceModalLogic.ts b/frontend/src/scenes/data-warehouse/external/sourceModalLogic.ts index 32ae739a5c4d3..13e6976b2f988 100644 --- a/frontend/src/scenes/data-warehouse/external/sourceModalLogic.ts +++ b/frontend/src/scenes/data-warehouse/external/sourceModalLogic.ts @@ -1,16 +1,19 @@ +import { lemonToast } from '@posthog/lemon-ui' import { actions, connect, kea, listeners, path, reducers, selectors } from 'kea' -import { preflightLogic } from 'scenes/PreflightCheck/preflightLogic' +import { forms } from 'kea-forms' +import { router } from 'kea-router' +import api from 'lib/api' +import { urls } from 'scenes/urls' -import { ExternalDataSourceType } from '~/types' +import { ExternalDataStripeSourceCreatePayload } from '~/types' import { dataWarehouseTableLogic } from '../new_table/dataWarehouseTableLogic' import { dataWarehouseSettingsLogic } from '../settings/dataWarehouseSettingsLogic' import { dataWarehouseSceneLogic } from './dataWarehouseSceneLogic' import type { sourceModalLogicType } from './sourceModalLogicType' -export const getHubspotRedirectUri = (): string => `${window.location.origin}/data-warehouse/hubspot/redirect` export interface ConnectorConfigType { - name: ExternalDataSourceType + name: string fields: string[] caption: string disabledReason: string | null @@ -20,16 +23,10 @@ export interface ConnectorConfigType { export const CONNECTORS: ConnectorConfigType[] = [ { name: 'Stripe', - fields: ['account_id', 'client_secret'], + fields: ['accound_id', 'client_secret'], caption: 'Enter your Stripe credentials to link your Stripe to PostHog', disabledReason: null, }, - { - name: 'Hubspot', - fields: [], - caption: '', - disabledReason: null, - }, ] export const sourceModalLogic = kea([ @@ -37,18 +34,9 @@ export const sourceModalLogic = kea([ actions({ selectConnector: (connector: ConnectorConfigType | null) => ({ connector }), toggleManualLinkFormVisible: (visible: boolean) => ({ visible }), - handleRedirect: (kind: string, searchParams: any) => ({ kind, searchParams }), - onClear: true, }), connect({ - values: [ - dataWarehouseTableLogic, - ['tableLoading'], - dataWarehouseSettingsLogic, - ['dataWarehouseSources'], - preflightLogic, - ['preflight'], - ], + values: [dataWarehouseTableLogic, ['tableLoading'], dataWarehouseSettingsLogic, ['dataWarehouseSources']], actions: [ dataWarehouseSceneLogic, ['toggleSourceModal'], @@ -89,39 +77,37 @@ export const sourceModalLogic = kea([ })) }, ], - addToHubspotButtonUrl: [ - (s) => [s.preflight], - (preflight) => { - return () => { - const clientId = preflight?.data_warehouse_integrations?.hubspot.client_id - - if (!clientId) { - return null - } - - const scopes = [ - 'crm.objects.contacts.read', - 'crm.objects.companies.read', - 'crm.objects.deals.read', - 'tickets', - 'crm.objects.quotes.read', - ] - - const params = new URLSearchParams() - params.set('client_id', clientId) - params.set('redirect_uri', getHubspotRedirectUri()) - params.set('scope', scopes.join(' ')) - - return `https://app.hubspot.com/oauth/authorize?${params.toString()}` + }), + forms(() => ({ + externalDataSource: { + defaults: { + account_id: '', + client_secret: '', + prefix: '', + source_type: 'Stripe', + } as ExternalDataStripeSourceCreatePayload, + errors: ({ account_id, client_secret }) => { + return { + account_id: !account_id && 'Please enter an account id.', + client_secret: !client_secret && 'Please enter a client secret.', } }, - ], - }), + submit: async (payload: ExternalDataStripeSourceCreatePayload) => { + const newResource = await api.externalDataSources.create(payload) + return newResource + }, + }, + })), listeners(({ actions }) => ({ - onClear: () => { - actions.selectConnector(null) - actions.toggleManualLinkFormVisible(false) - actions.resetTable() + submitExternalDataSourceSuccess: () => { + lemonToast.success('New Data Resource Created') + actions.toggleSourceModal() + actions.resetExternalDataSource() + actions.loadSources() + router.actions.push(urls.dataWarehouseSettings()) + }, + submitExternalDataSourceFailure: ({ error }) => { + lemonToast.error(error?.message || 'Something went wrong') }, })), ]) diff --git a/frontend/src/scenes/data-warehouse/redirect/DataWarehouseRedirectScene.tsx b/frontend/src/scenes/data-warehouse/redirect/DataWarehouseRedirectScene.tsx deleted file mode 100644 index 736132b3b747f..0000000000000 --- a/frontend/src/scenes/data-warehouse/redirect/DataWarehouseRedirectScene.tsx +++ /dev/null @@ -1,34 +0,0 @@ -import { LemonButton, LemonInput } from '@posthog/lemon-ui' -import { Form } from 'kea-forms' -import { Field } from 'lib/forms/Field' -import { sourceFormLogic } from 'scenes/data-warehouse/external/sourceFormLogic' -import { SceneExport } from 'scenes/sceneTypes' - -export const scene: SceneExport = { - component: DataWarehouseRedirectScene, - logic: sourceFormLogic, -} - -export function DataWarehouseRedirectScene(): JSX.Element { - return ( -
-

Configure

-

Add a prefix to your tables to avoid conflicts with other data sources

-
- - - - - Submit - -
-
- ) -} - -export default DataWarehouseRedirectScene diff --git a/frontend/src/scenes/data-warehouse/settings/DataWarehouseSettingsScene.tsx b/frontend/src/scenes/data-warehouse/settings/DataWarehouseSettingsScene.tsx index f2b5db2080e45..c0b13232a6cdd 100644 --- a/frontend/src/scenes/data-warehouse/settings/DataWarehouseSettingsScene.tsx +++ b/frontend/src/scenes/data-warehouse/settings/DataWarehouseSettingsScene.tsx @@ -106,7 +106,7 @@ export function DataWarehouseSettingsScene(): JSX.Element { }, { title: 'Sync Frequency', - key: 'frequency', + key: 'prefix', render: function RenderFrequency() { return 'Every 24 hours' }, diff --git a/frontend/src/scenes/sceneTypes.ts b/frontend/src/scenes/sceneTypes.ts index 82c5ba9e1589c..4604a64a697da 100644 --- a/frontend/src/scenes/sceneTypes.ts +++ b/frontend/src/scenes/sceneTypes.ts @@ -43,7 +43,6 @@ export enum Scene { DataWarehouseSavedQueries = 'DataWarehouseSavedQueries', DataWarehouseTable = 'DataWarehouseTable', DataWarehouseSettings = 'DataWarehouseSettings', - DataWarehouseRedirect = 'DataWarehouseRedirect', OrganizationCreateFirst = 'OrganizationCreate', ProjectHomepage = 'ProjectHomepage', ProjectCreateFirst = 'ProjectCreate', diff --git a/frontend/src/scenes/scenes.ts b/frontend/src/scenes/scenes.ts index 596135d6fc68a..4ebd425f741f0 100644 --- a/frontend/src/scenes/scenes.ts +++ b/frontend/src/scenes/scenes.ts @@ -206,9 +206,6 @@ export const sceneConfigurations: Record = { name: 'Data warehouse settings', defaultDocsPath: '/docs/data-warehouse', }, - [Scene.DataWarehouseRedirect]: { - name: 'Data warehouse redirect', - }, [Scene.DataWarehouseTable]: { projectBased: true, name: 'Data warehouse table', @@ -493,7 +490,6 @@ export const routes: Record = { [urls.dataWarehouseExternal()]: Scene.DataWarehouseExternal, [urls.dataWarehouseSavedQueries()]: Scene.DataWarehouseSavedQueries, [urls.dataWarehouseSettings()]: Scene.DataWarehouseSettings, - [urls.dataWarehouseRedirect(':kind')]: Scene.DataWarehouseRedirect, [urls.featureFlags()]: Scene.FeatureFlags, [urls.featureFlag(':id')]: Scene.FeatureFlag, [urls.annotations()]: Scene.DataManagement, diff --git a/frontend/src/scenes/urls.ts b/frontend/src/scenes/urls.ts index 53c1250229d13..bd5cbe5939aec 100644 --- a/frontend/src/scenes/urls.ts +++ b/frontend/src/scenes/urls.ts @@ -127,7 +127,6 @@ export const urls = { dataWarehouseExternal: (): string => '/data-warehouse/external', dataWarehouseSavedQueries: (): string => '/data-warehouse/views', dataWarehouseSettings: (): string => '/data-warehouse/settings', - dataWarehouseRedirect: (kind: string): string => `/data-warehouse/${kind}/redirect`, annotations: (): string => '/data-management/annotations', annotation: (id: AnnotationType['id'] | ':id'): string => `/data-management/annotations/${id}`, projectApps: (tab?: PluginTab): string => `/project/apps${tab ? `?tab=${tab}` : ''}`, diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 0f4109f6083b3..efdb944d97756 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -2503,11 +2503,6 @@ export interface PreflightStatus { available: boolean client_id?: string } - data_warehouse_integrations: { - hubspot: { - client_id?: string - } - } /** Whether PostHog is running in DEBUG mode. */ is_debug?: boolean licensed_users_available?: number | null @@ -3336,13 +3331,13 @@ export interface DataWarehouseViewLink { from_join_key?: string } -export type ExternalDataSourceType = 'Stripe' | 'Hubspot' - -export interface ExternalDataSourceCreatePayload { - source_type: ExternalDataSourceType +export interface ExternalDataStripeSourceCreatePayload { + account_id: string + client_secret: string prefix: string - payload: Record + source_type: string } + export interface ExternalDataStripeSource { id: string source_id: string diff --git a/latest_migrations.manifest b/latest_migrations.manifest index 132521b9f10a3..9deccea94b408 100644 --- a/latest_migrations.manifest +++ b/latest_migrations.manifest @@ -5,7 +5,7 @@ contenttypes: 0002_remove_content_type_name ee: 0015_add_verified_properties otp_static: 0002_throttling otp_totp: 0002_auto_20190420_0723 -posthog: 0380_alter_externaldatasource_source_type +posthog: 0379_alter_scheduledchange sessions: 0001_initial social_django: 0010_uid_db_index two_factor: 0007_auto_20201201_1019 diff --git a/posthog/api/test/test_preflight.py b/posthog/api/test/test_preflight.py index 6b63ad28f7542..74d5a8b2e490b 100644 --- a/posthog/api/test/test_preflight.py +++ b/posthog/api/test/test_preflight.py @@ -29,7 +29,6 @@ def preflight_dict(self, options={}): "db": True, "initiated": True, "cloud": False, - "data_warehouse_integrations": {"hubspot": {"client_id": None}}, "demo": False, "clickhouse": True, "kafka": True, diff --git a/posthog/migrations/0380_alter_externaldatasource_source_type.py b/posthog/migrations/0380_alter_externaldatasource_source_type.py deleted file mode 100644 index 70ba4013f8013..0000000000000 --- a/posthog/migrations/0380_alter_externaldatasource_source_type.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 3.2.19 on 2023-12-29 18:53 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("posthog", "0379_alter_scheduledchange"), - ] - - operations = [ - migrations.AlterField( - model_name="externaldatasource", - name="source_type", - field=models.CharField(choices=[("Stripe", "Stripe"), ("Hubspot", "Hubspot")], max_length=128), - ), - ] diff --git a/posthog/settings/__init__.py b/posthog/settings/__init__.py index 3a3225e7deed5..7ee845e134694 100644 --- a/posthog/settings/__init__.py +++ b/posthog/settings/__init__.py @@ -39,7 +39,7 @@ from posthog.settings.object_storage import * from posthog.settings.temporal import * from posthog.settings.web import * -from posthog.settings.data_warehouse import * +from posthog.settings.airbyte import * from posthog.settings.utils import get_from_env, str_to_bool diff --git a/posthog/settings/data_warehouse.py b/posthog/settings/airbyte.py similarity index 75% rename from posthog/settings/data_warehouse.py rename to posthog/settings/airbyte.py index b53e16e570a13..bcbcf2fefacb5 100644 --- a/posthog/settings/data_warehouse.py +++ b/posthog/settings/airbyte.py @@ -8,6 +8,3 @@ # for DLT BUCKET_URL = os.getenv("BUCKET_URL", None) AIRBYTE_BUCKET_NAME = os.getenv("AIRBYTE_BUCKET_NAME", None) - -HUBSPOT_APP_CLIENT_ID = os.getenv("HUBSPOT_APP_CLIENT_ID", None) -HUBSPOT_APP_CLIENT_SECRET = os.getenv("HUBSPOT_APP_CLIENT_SECRET", None) diff --git a/posthog/temporal/data_imports/external_data_job.py b/posthog/temporal/data_imports/external_data_job.py index 3f9f0096a7e76..cdb218c0cce31 100644 --- a/posthog/temporal/data_imports/external_data_job.py +++ b/posthog/temporal/data_imports/external_data_job.py @@ -155,25 +155,6 @@ async def run_external_data_job(inputs: ExternalDataJobInputs) -> None: source = stripe_source( api_key=stripe_secret_key, endpoints=tuple(inputs.schemas), job_id=str(model.id), team_id=inputs.team_id ) - elif model.pipeline.source_type == ExternalDataSource.Type.HUBSPOT: - from posthog.temporal.data_imports.pipelines.hubspot.auth import refresh_access_token - from posthog.temporal.data_imports.pipelines.hubspot import hubspot - - hubspot_access_code = model.pipeline.job_inputs.get("hubspot_secret_key", None) - refresh_token = model.pipeline.job_inputs.get("hubspot_refresh_token", None) - if not refresh_token: - raise ValueError(f"Hubspot refresh token not found for job {model.id}") - - if not hubspot_access_code: - hubspot_access_code = refresh_access_token(refresh_token) - - source = hubspot( - api_key=hubspot_access_code, - refresh_token=refresh_token, - job_id=str(model.id), - team_id=inputs.team_id, - endpoints=tuple(inputs.schemas), - ) else: raise ValueError(f"Source type {model.pipeline.source_type} not supported") diff --git a/posthog/temporal/data_imports/pipelines/hubspot/__init__.py b/posthog/temporal/data_imports/pipelines/hubspot/__init__.py deleted file mode 100644 index 3275071efe992..0000000000000 --- a/posthog/temporal/data_imports/pipelines/hubspot/__init__.py +++ /dev/null @@ -1,146 +0,0 @@ -""" -This is a module that provides a DLT source to retrieve data from multiple endpoints of the HubSpot API using a specified API key. The retrieved data is returned as a tuple of Dlt resources, one for each endpoint. - -The source retrieves data from the following endpoints: -- CRM Companies -- CRM Contacts -- CRM Deals -- CRM Tickets -- CRM Quotes -- Web Analytics Events - -For each endpoint, a resource and transformer function are defined to retrieve data and transform it to a common format. -The resource functions yield the raw data retrieved from the API, while the transformer functions are used to retrieve -additional information from the Web Analytics Events endpoint. - -The source also supports enabling Web Analytics Events for each endpoint by setting the corresponding enable flag to True. - -Example: -To retrieve data from all endpoints, use the following code: - -python - ->>> resources = hubspot(api_key="hubspot_access_code") -""" - -from typing import Literal, Sequence, Iterator, Iterable - -import dlt -from dlt.common.typing import TDataItems -from dlt.sources import DltResource -from posthog.temporal.data_imports.pipelines.helpers import limit_paginated_generator - -from .helpers import ( - fetch_data, - _get_property_names, - fetch_property_history, -) -from .settings import ( - ALL, - CRM_OBJECT_ENDPOINTS, - DEFAULT_PROPS, - OBJECT_TYPE_PLURAL, - OBJECT_TYPE_SINGULAR, -) - -THubspotObjectType = Literal["company", "contact", "deal", "ticket", "quote"] - - -@dlt.source(name="hubspot") -def hubspot( - api_key: str, - refresh_token: str, - job_id: str, - team_id: int, - endpoints: Sequence[str] = ("companies", "contacts", "deals", "tickets", "quotes"), - include_history: bool = False, -) -> Iterable[DltResource]: - """ - A DLT source that retrieves data from the HubSpot API using the - specified API key. - - This function retrieves data for several HubSpot API endpoints, - including companies, contacts, deals, tickets and web - analytics events. It returns a tuple of Dlt resources, one for - each endpoint. - - Args: - api_key (Optional[str]): - The API key used to authenticate with the HubSpot API. Defaults - to dlt.secrets.value. - include_history (Optional[bool]): - Whether to load history of property changes along with entities. - The history entries are loaded to separate tables. - - Returns: - Sequence[DltResource]: Dlt resources, one for each HubSpot API endpoint. - - Notes: - This function uses the `fetch_data` function to retrieve data from the - HubSpot CRM API. The API key is passed to `fetch_data` as the - `api_key` argument. - """ - - for endpoint in endpoints: - yield dlt.resource( - crm_objects, - name=endpoint, - write_disposition="append", - )( - object_type=OBJECT_TYPE_SINGULAR[endpoint], - api_key=api_key, - refresh_token=refresh_token, - include_history=include_history, - props=DEFAULT_PROPS[endpoint], - include_custom_props=True, - job_id=job_id, - team_id=team_id, - ) - - -@limit_paginated_generator -def crm_objects( - object_type: str, - api_key: str, - refresh_token: str, - include_history: bool, - props: Sequence[str], - include_custom_props: bool = True, -) -> Iterator[TDataItems]: - """Building blocks for CRM resources.""" - if props == ALL: - props = list(_get_property_names(api_key, refresh_token, object_type)) - - if include_custom_props: - all_props = _get_property_names(api_key, refresh_token, object_type) - custom_props = [prop for prop in all_props if not prop.startswith("hs_")] - props = props + custom_props # type: ignore - - props = ",".join(sorted(list(set(props)))) - - if len(props) > 2000: - raise ValueError( - ( - "Your request to Hubspot is too long to process. " - "Maximum allowed query length is 2000 symbols, while " - f"your list of properties `{props[:200]}`... is {len(props)} " - "symbols long. Use the `props` argument of the resource to " - "set the list of properties to extract from the endpoint." - ) - ) - - params = {"properties": props, "limit": 100} - - yield from fetch_data(CRM_OBJECT_ENDPOINTS[object_type], api_key, refresh_token, params=params) - if include_history: - # Get history separately, as requesting both all properties and history together - # is likely to hit hubspot's URL length limit - for history_entries in fetch_property_history( - CRM_OBJECT_ENDPOINTS[object_type], - api_key, - props, - ): - yield dlt.mark.with_table_name( - history_entries, - OBJECT_TYPE_PLURAL[object_type] + "_property_history", - ) diff --git a/posthog/temporal/data_imports/pipelines/hubspot/auth.py b/posthog/temporal/data_imports/pipelines/hubspot/auth.py deleted file mode 100644 index 490552cfe237d..0000000000000 --- a/posthog/temporal/data_imports/pipelines/hubspot/auth.py +++ /dev/null @@ -1,42 +0,0 @@ -import requests -from django.conf import settings -from typing import Tuple - - -def refresh_access_token(refresh_token: str) -> str: - res = requests.post( - "https://api.hubapi.com/oauth/v1/token", - data={ - "grant_type": "refresh_token", - "client_id": settings.HUBSPOT_APP_CLIENT_ID, - "client_secret": settings.HUBSPOT_APP_CLIENT_SECRET, - "refresh_token": refresh_token, - }, - ) - - if res.status_code != 200: - err_message = res.json()["message"] - raise Exception(err_message) - - return res.json()["access_token"] - - -def get_access_token_from_code(code: str, redirect_uri: str) -> Tuple[str, str]: - res = requests.post( - "https://api.hubapi.com/oauth/v1/token", - data={ - "grant_type": "authorization_code", - "client_id": settings.HUBSPOT_APP_CLIENT_ID, - "client_secret": settings.HUBSPOT_APP_CLIENT_SECRET, - "redirect_uri": redirect_uri, - "code": code, - }, - ) - - if res.status_code != 200: - err_message = res.json()["message"] - raise Exception(err_message) - - payload = res.json() - - return payload["access_token"], payload["refresh_token"] diff --git a/posthog/temporal/data_imports/pipelines/hubspot/helpers.py b/posthog/temporal/data_imports/pipelines/hubspot/helpers.py deleted file mode 100644 index b724368fe7e40..0000000000000 --- a/posthog/temporal/data_imports/pipelines/hubspot/helpers.py +++ /dev/null @@ -1,198 +0,0 @@ -"""Hubspot source helpers""" - -import urllib.parse -from typing import Iterator, Dict, Any, List, Optional - -from dlt.sources.helpers import requests -import requests as http_requests -from .settings import OBJECT_TYPE_PLURAL -from .auth import refresh_access_token - -BASE_URL = "https://api.hubapi.com/" - - -def get_url(endpoint: str) -> str: - """Get absolute hubspot endpoint URL""" - return urllib.parse.urljoin(BASE_URL, endpoint) - - -def _get_headers(api_key: str) -> Dict[str, str]: - """ - Return a dictionary of HTTP headers to use for API requests, including the specified API key. - - Args: - api_key (str): The API key to use for authentication, as a string. - - Returns: - dict: A dictionary of HTTP headers to include in API requests, with the `Authorization` header - set to the specified API key in the format `Bearer {api_key}`. - - """ - # Construct the dictionary of HTTP headers to use for API requests - return dict(authorization=f"Bearer {api_key}") - - -def extract_property_history(objects: List[Dict[str, Any]]) -> Iterator[Dict[str, Any]]: - for item in objects: - history = item.get("propertiesWithHistory") - if not history: - return - # Yield a flat list of property history entries - for key, changes in history.items(): - if not changes: - continue - for entry in changes: - yield {"object_id": item["id"], "property_name": key, **entry} - - -def fetch_property_history( - endpoint: str, - api_key: str, - props: str, - params: Optional[Dict[str, Any]] = None, -) -> Iterator[List[Dict[str, Any]]]: - """Fetch property history from the given CRM endpoint. - - Args: - endpoint: The endpoint to fetch data from, as a string. - api_key: The API key to use for authentication, as a string. - props: A comma separated list of properties to retrieve the history for - params: Optional dict of query params to include in the request - - Yields: - List of property history entries (dicts) - """ - # Construct the URL and headers for the API request - url = get_url(endpoint) - headers = _get_headers(api_key) - - params = dict(params or {}) - params["propertiesWithHistory"] = props - params["limit"] = 50 - # Make the API request - r = requests.get(url, headers=headers, params=params) - # Parse the API response and yield the properties of each result - - # Parse the response JSON data - _data = r.json() - while _data is not None: - if "results" in _data: - yield list(extract_property_history(_data["results"])) - - # Follow pagination links if they exist - _next = _data.get("paging", {}).get("next", None) - if _next: - next_url = _next["link"] - # Get the next page response - r = requests.get(next_url, headers=headers) - _data = r.json() - else: - _data = None - - -def fetch_data( - endpoint: str, api_key: str, refresh_token: str, params: Optional[Dict[str, Any]] = None -) -> Iterator[List[Dict[str, Any]]]: - """ - Fetch data from HUBSPOT endpoint using a specified API key and yield the properties of each result. - For paginated endpoint this function yields item from all pages. - - Args: - endpoint (str): The endpoint to fetch data from, as a string. - api_key (str): The API key to use for authentication, as a string. - params: Optional dict of query params to include in the request - - Yields: - A List of CRM object dicts - - Raises: - requests.exceptions.HTTPError: If the API returns an HTTP error status code. - - Notes: - This function uses the `requests` library to make a GET request to the specified endpoint, with - the API key included in the headers. If the API returns a non-successful HTTP status code (e.g. - 404 Not Found), a `requests.exceptions.HTTPError` exception will be raised. - - The `endpoint` argument should be a relative URL, which will be appended to the base URL for the - API. The `params` argument is used to pass additional query parameters to the request - - This function also includes a retry decorator that will automatically retry the API call up to - 3 times with a 5-second delay between retries, using an exponential backoff strategy. - """ - # Construct the URL and headers for the API request - url = get_url(endpoint) - headers = _get_headers(api_key) - - # Make the API request - try: - r = requests.get(url, headers=headers, params=params) - except http_requests.exceptions.HTTPError as e: - if e.response.status_code == 401: - # refresh token - api_key = refresh_access_token(refresh_token) - headers = _get_headers(api_key) - r = requests.get(url, headers=headers, params=params) - else: - raise e - # Parse the API response and yield the properties of each result - # Parse the response JSON data - - _data = r.json() - # Yield the properties of each result in the API response - while _data is not None: - if "results" in _data: - _objects: List[Dict[str, Any]] = [] - for _result in _data["results"]: - _obj = _result.get("properties", _result) - if "id" not in _obj and "id" in _result: - # Move id from properties to top level - _obj["id"] = _result["id"] - if "associations" in _result: - for association in _result["associations"]: - __values = [ - { - "value": _obj["hs_object_id"], - f"{association}_id": __r["id"], - } - for __r in _result["associations"][association]["results"] - ] - - # remove duplicates from list of dicts - __values = [dict(t) for t in {tuple(d.items()) for d in __values}] - - _obj[association] = __values - _objects.append(_obj) - - yield _objects - - # Follow pagination links if they exist - _next = _data.get("paging", {}).get("next", None) - if _next: - next_url = _next["link"] - # Get the next page response - r = requests.get(next_url, headers=headers) - _data = r.json() - else: - _data = None - - -def _get_property_names(api_key: str, refresh_token: str, object_type: str) -> List[str]: - """ - Retrieve property names for a given entity from the HubSpot API. - - Args: - entity: The entity name for which to retrieve property names. - - Returns: - A list of property names. - - Raises: - Exception: If an error occurs during the API request. - """ - properties = [] - endpoint = f"/crm/v3/properties/{OBJECT_TYPE_PLURAL[object_type]}" - - for page in fetch_data(endpoint, api_key, refresh_token): - properties.extend([prop["name"] for prop in page]) - - return properties diff --git a/posthog/temporal/data_imports/pipelines/hubspot/settings.py b/posthog/temporal/data_imports/pipelines/hubspot/settings.py deleted file mode 100644 index 10af4c47b5a31..0000000000000 --- a/posthog/temporal/data_imports/pipelines/hubspot/settings.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Hubspot source settings and constants""" - -from dlt.common import pendulum - -STARTDATE = pendulum.datetime(year=2000, month=1, day=1) - -CONTACT = "contact" -COMPANY = "company" -DEAL = "deal" -TICKET = "ticket" -QUOTE = "quote" - -CRM_CONTACTS_ENDPOINT = "/crm/v3/objects/contacts?associations=deals,tickets,quotes" -CRM_COMPANIES_ENDPOINT = "/crm/v3/objects/companies?associations=contacts,deals,tickets,quotes" -CRM_DEALS_ENDPOINT = "/crm/v3/objects/deals" -CRM_TICKETS_ENDPOINT = "/crm/v3/objects/tickets" -CRM_QUOTES_ENDPOINT = "/crm/v3/objects/quotes" - -CRM_OBJECT_ENDPOINTS = { - CONTACT: CRM_CONTACTS_ENDPOINT, - COMPANY: CRM_COMPANIES_ENDPOINT, - DEAL: CRM_DEALS_ENDPOINT, - TICKET: CRM_TICKETS_ENDPOINT, - QUOTE: CRM_QUOTES_ENDPOINT, -} - -WEB_ANALYTICS_EVENTS_ENDPOINT = "/events/v3/events?objectType={objectType}&objectId={objectId}&occurredAfter={occurredAfter}&occurredBefore={occurredBefore}&sort=-occurredAt" - -OBJECT_TYPE_SINGULAR = { - "companies": COMPANY, - "contacts": CONTACT, - "deals": DEAL, - "tickets": TICKET, - "quotes": QUOTE, -} - -OBJECT_TYPE_PLURAL = {v: k for k, v in OBJECT_TYPE_SINGULAR.items()} - - -ENDPOINTS = ( - OBJECT_TYPE_PLURAL[CONTACT], - OBJECT_TYPE_PLURAL[DEAL], - OBJECT_TYPE_PLURAL[COMPANY], - OBJECT_TYPE_PLURAL[TICKET], - OBJECT_TYPE_PLURAL[QUOTE], -) - -DEFAULT_DEAL_PROPS = [ - "amount", - "closedate", - "createdate", - "dealname", - "dealstage", - "hs_lastmodifieddate", - "hs_object_id", - "pipeline", -] - -DEFAULT_COMPANY_PROPS = [ - "createdate", - "domain", - "hs_lastmodifieddate", - "hs_object_id", - "name", -] - -DEFAULT_CONTACT_PROPS = [ - "createdate", - "email", - "firstname", - "hs_object_id", - "lastmodifieddate", - "lastname", -] - -DEFAULT_TICKET_PROPS = [ - "createdate", - "content", - "hs_lastmodifieddate", - "hs_object_id", - "hs_pipeline", - "hs_pipeline_stage", - "hs_ticket_category", - "hs_ticket_priority", - "subject", -] - -DEFAULT_QUOTE_PROPS = [ - "hs_createdate", - "hs_expiration_date", - "hs_lastmodifieddate", - "hs_object_id", - "hs_public_url_key", - "hs_status", - "hs_title", -] - -DEFAULT_PROPS = { - OBJECT_TYPE_PLURAL[CONTACT]: DEFAULT_CONTACT_PROPS, - OBJECT_TYPE_PLURAL[COMPANY]: DEFAULT_COMPANY_PROPS, - OBJECT_TYPE_PLURAL[DEAL]: DEFAULT_DEAL_PROPS, - OBJECT_TYPE_PLURAL[TICKET]: DEFAULT_TICKET_PROPS, - OBJECT_TYPE_PLURAL[QUOTE]: DEFAULT_QUOTE_PROPS, -} - -ALL = ("ALL",) diff --git a/posthog/temporal/data_imports/pipelines/schemas.py b/posthog/temporal/data_imports/pipelines/schemas.py index eaaa431d7aef9..a62db7d664e40 100644 --- a/posthog/temporal/data_imports/pipelines/schemas.py +++ b/posthog/temporal/data_imports/pipelines/schemas.py @@ -1,8 +1,4 @@ from posthog.warehouse.models import ExternalDataSource -from posthog.temporal.data_imports.pipelines.stripe.settings import ENDPOINTS as STRIPE_ENDPOINTS -from posthog.temporal.data_imports.pipelines.hubspot.settings import ENDPOINTS as HUBSPOT_ENDPOINTS +from posthog.temporal.data_imports.pipelines.stripe.settings import ENDPOINTS -PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING = { - ExternalDataSource.Type.STRIPE: STRIPE_ENDPOINTS, - ExternalDataSource.Type.HUBSPOT: HUBSPOT_ENDPOINTS, -} +PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING = {ExternalDataSource.Type.STRIPE: ENDPOINTS} diff --git a/posthog/views.py b/posthog/views.py index 1f757f9833734..4750cf170cc27 100644 --- a/posthog/views.py +++ b/posthog/views.py @@ -92,7 +92,6 @@ def security_txt(request): @never_cache def preflight_check(request: HttpRequest) -> JsonResponse: slack_client_id = SlackIntegration.slack_config().get("SLACK_APP_CLIENT_ID") - hubspot_client_id = settings.HUBSPOT_APP_CLIENT_ID response = { "django": True, @@ -114,7 +113,6 @@ def preflight_check(request: HttpRequest) -> JsonResponse: "available": bool(slack_client_id), "client_id": slack_client_id or None, }, - "data_warehouse_integrations": {"hubspot": {"client_id": hubspot_client_id}}, "object_storage": is_cloud() or is_object_storage_available(), } diff --git a/posthog/warehouse/api/external_data_source.py b/posthog/warehouse/api/external_data_source.py index 843821e2f2749..48f8babed4a5a 100644 --- a/posthog/warehouse/api/external_data_source.py +++ b/posthog/warehouse/api/external_data_source.py @@ -25,9 +25,6 @@ from posthog.temporal.data_imports.pipelines.schemas import ( PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING, ) -from posthog.temporal.data_imports.pipelines.hubspot.auth import ( - get_access_token_from_code, -) import temporalio logger = structlog.get_logger(__name__) @@ -110,6 +107,7 @@ def get_queryset(self): ) def create(self, request: Request, *args: Any, **kwargs: Any) -> Response: + client_secret = request.data["client_secret"] prefix = request.data.get("prefix", None) source_type = request.data["source_type"] @@ -129,12 +127,18 @@ def create(self, request: Request, *args: Any, **kwargs: Any) -> Response: ) # TODO: remove dummy vars - if source_type == ExternalDataSource.Type.STRIPE: - new_source_model = self._handle_stripe_source(request, *args, **kwargs) - elif source_type == ExternalDataSource.Type.HUBSPOT: - new_source_model = self._handle_hubspot_source(request, *args, **kwargs) - else: - raise NotImplementedError(f"Source type {source_type} not implemented") + new_source_model = ExternalDataSource.objects.create( + source_id=str(uuid.uuid4()), + connection_id=str(uuid.uuid4()), + destination_id=str(uuid.uuid4()), + team=self.team, + status="Running", + source_type=source_type, + job_inputs={ + "stripe_secret_key": client_secret, + }, + prefix=prefix, + ) schemas = PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING[source_type] for schema in schemas: @@ -152,54 +156,6 @@ def create(self, request: Request, *args: Any, **kwargs: Any) -> Response: return Response(status=status.HTTP_201_CREATED, data={"id": new_source_model.pk}) - def _handle_stripe_source(self, request: Request, *args: Any, **kwargs: Any) -> ExternalDataSource: - payload = request.data["payload"] - client_secret = payload.get("client_secret") - prefix = request.data.get("prefix", None) - source_type = request.data["source_type"] - - # TODO: remove dummy vars - new_source_model = ExternalDataSource.objects.create( - source_id=str(uuid.uuid4()), - connection_id=str(uuid.uuid4()), - destination_id=str(uuid.uuid4()), - team=self.team, - status="Running", - source_type=source_type, - job_inputs={ - "stripe_secret_key": client_secret, - }, - prefix=prefix, - ) - - return new_source_model - - def _handle_hubspot_source(self, request: Request, *args: Any, **kwargs: Any) -> ExternalDataSource: - payload = request.data["payload"] - code = payload.get("code") - redirect_uri = payload.get("redirect_uri") - prefix = request.data.get("prefix", None) - source_type = request.data["source_type"] - - access_token, refresh_token = get_access_token_from_code(code, redirect_uri=redirect_uri) - - # TODO: remove dummy vars - new_source_model = ExternalDataSource.objects.create( - source_id=str(uuid.uuid4()), - connection_id=str(uuid.uuid4()), - destination_id=str(uuid.uuid4()), - team=self.team, - status="Running", - source_type=source_type, - job_inputs={ - "hubspot_secret_key": access_token, - "hubspot_refresh_token": refresh_token, - }, - prefix=prefix, - ) - - return new_source_model - def prefix_required(self, source_type: str) -> bool: source_type_exists = ExternalDataSource.objects.filter(team_id=self.team.pk, source_type=source_type).exists() return source_type_exists diff --git a/posthog/warehouse/api/test/test_external_data_source.py b/posthog/warehouse/api/test/test_external_data_source.py index 955c032c0373e..2ad741b453a29 100644 --- a/posthog/warehouse/api/test/test_external_data_source.py +++ b/posthog/warehouse/api/test/test_external_data_source.py @@ -30,7 +30,7 @@ def _create_external_data_schema(self, source_id) -> ExternalDataSchema: def test_create_external_data_source(self): response = self.client.post( f"/api/projects/{self.team.id}/external_data_sources/", - data={"source_type": "Stripe", "payload": {"client_secret": "sk_test_123"}}, + data={"source_type": "Stripe", "client_secret": "sk_test_123"}, ) payload = response.json() @@ -46,7 +46,7 @@ def test_prefix_external_data_source(self): response = self.client.post( f"/api/projects/{self.team.id}/external_data_sources/", - data={"source_type": "Stripe", "payload": {"client_secret": "sk_test_123"}}, + data={"source_type": "Stripe", "client_secret": "sk_test_123"}, ) self.assertEqual(response.status_code, 201) @@ -54,7 +54,7 @@ def test_prefix_external_data_source(self): response = self.client.post( f"/api/projects/{self.team.id}/external_data_sources/", - data={"source_type": "Stripe", "payload": {"client_secret": "sk_test_123"}}, + data={"source_type": "Stripe", "client_secret": "sk_test_123"}, ) self.assertEqual(response.status_code, 400) @@ -63,7 +63,7 @@ def test_prefix_external_data_source(self): # Create with prefix response = self.client.post( f"/api/projects/{self.team.id}/external_data_sources/", - data={"source_type": "Stripe", "payload": {"client_secret": "sk_test_123"}, "prefix": "test_"}, + data={"source_type": "Stripe", "client_secret": "sk_test_123", "prefix": "test_"}, ) self.assertEqual(response.status_code, 201) @@ -71,7 +71,7 @@ def test_prefix_external_data_source(self): # Try to create same type with same prefix again response = self.client.post( f"/api/projects/{self.team.id}/external_data_sources/", - data={"source_type": "Stripe", "payload": {"client_secret": "sk_test_123"}, "prefix": "test_"}, + data={"source_type": "Stripe", "client_secret": "sk_test_123", "prefix": "test_"}, ) self.assertEqual(response.status_code, 400) diff --git a/posthog/warehouse/models/external_data_source.py b/posthog/warehouse/models/external_data_source.py index 5d8f736a77b94..287a4a3f2cd99 100644 --- a/posthog/warehouse/models/external_data_source.py +++ b/posthog/warehouse/models/external_data_source.py @@ -8,7 +8,6 @@ class ExternalDataSource(CreatedMetaFields, UUIDModel): class Type(models.TextChoices): STRIPE = "Stripe", "Stripe" - HUBSPOT = "Hubspot", "Hubspot" class Status(models.TextChoices): RUNNING = "Running", "Running" From cb97d1b88ecec8393fa64673df3b132c085b66d1 Mon Sep 17 00:00:00 2001 From: Eric Duong Date: Tue, 2 Jan 2024 16:22:11 -0500 Subject: [PATCH 05/15] chore(data-warehouse): cleanup unused celery code and extend time (#19568) cleanup --- posthog/celery.py | 33 ---- posthog/tasks/test/test_warehouse.py | 160 ----------------- posthog/tasks/warehouse.py | 163 +----------------- .../data_imports/external_data_job.py | 2 +- 4 files changed, 2 insertions(+), 356 deletions(-) diff --git a/posthog/celery.py b/posthog/celery.py index 5256cee72bfad..69d961edf8ffd 100644 --- a/posthog/celery.py +++ b/posthog/celery.py @@ -342,12 +342,6 @@ def setup_periodic_tasks(sender: Celery, **kwargs): name="delete expired exported assets", ) - sender.add_periodic_task( - crontab(minute="*/10"), - sync_datawarehouse_sources.s(), - name="sync datawarehouse sources that have settled in s3 bucket", - ) - sender.add_periodic_task( crontab(minute="23", hour="*"), check_data_import_row_limits.s(), @@ -938,23 +932,6 @@ def calculate_decide_usage() -> None: ph_client.shutdown() -@app.task(ignore_result=True) -def calculate_external_data_rows_synced() -> None: - from django.db.models import Q - - from posthog.models import Team - from posthog.tasks.warehouse import ( - capture_workspace_rows_synced_by_team, - check_external_data_source_billing_limit_by_team, - ) - - for team in Team.objects.select_related("organization").exclude( - Q(organization__for_internal_metrics=True) | Q(is_demo=True) | Q(external_data_workspace_id__isnull=True) - ): - capture_workspace_rows_synced_by_team.delay(team.pk) - check_external_data_source_billing_limit_by_team.delay(team.pk) - - @app.task(ignore_result=True) def find_flags_with_enriched_analytics(): from datetime import datetime, timedelta @@ -1109,16 +1086,6 @@ def ee_persist_finished_recordings(): persist_finished_recordings() -@app.task(ignore_result=True) -def sync_datawarehouse_sources(): - try: - from posthog.tasks.warehouse import sync_resources - except ImportError: - pass - else: - sync_resources() - - @app.task(ignore_result=True) def check_data_import_row_limits(): try: diff --git a/posthog/tasks/test/test_warehouse.py b/posthog/tasks/test/test_warehouse.py index 01b5ac561f5dd..9581e5af0284c 100644 --- a/posthog/tasks/test/test_warehouse.py +++ b/posthog/tasks/test/test_warehouse.py @@ -1,172 +1,12 @@ from posthog.test.base import APIBaseTest -import datetime from unittest.mock import patch, MagicMock from posthog.tasks.warehouse import ( - _traverse_jobs_by_field, - capture_workspace_rows_synced_by_team, - check_external_data_source_billing_limit_by_team, check_synced_row_limits_of_team, ) from posthog.warehouse.models import ExternalDataSource, ExternalDataJob -from freezegun import freeze_time class TestWarehouse(APIBaseTest): - @patch("posthog.tasks.warehouse.send_request") - @freeze_time("2023-11-07") - def test_traverse_jobs_by_field(self, send_request_mock: MagicMock) -> None: - send_request_mock.return_value = { - "data": [ - { - "jobId": 5827835, - "status": "succeeded", - "jobType": "sync", - "startTime": "2023-11-07T16:50:49Z", - "connectionId": "fake", - "lastUpdatedAt": "2023-11-07T16:52:54Z", - "duration": "PT2M5S", - "rowsSynced": 93353, - }, - { - "jobId": 5783573, - "status": "succeeded", - "jobType": "sync", - "startTime": "2023-11-05T18:32:41Z", - "connectionId": "fake-2", - "lastUpdatedAt": "2023-11-05T18:35:11Z", - "duration": "PT2M30S", - "rowsSynced": 97747, - }, - ] - } - mock_capture = MagicMock() - response = _traverse_jobs_by_field(mock_capture, self.team, "fake-url", "rowsSynced") - - self.assertEqual( - response, - [ - {"count": 93353, "startTime": "2023-11-07T16:50:49Z"}, - {"count": 97747, "startTime": "2023-11-05T18:32:41Z"}, - ], - ) - - self.assertEqual(mock_capture.capture.call_count, 2) - mock_capture.capture.assert_called_with( - self.team.pk, - "external data sync job", - { - "count": 97747, - "workspace_id": self.team.external_data_workspace_id, - "team_id": self.team.pk, - "team_uuid": self.team.uuid, - "startTime": "2023-11-05T18:32:41Z", - "job_id": "5783573", - }, - ) - - @patch("posthog.tasks.warehouse._traverse_jobs_by_field") - @patch("posthog.tasks.warehouse.get_ph_client") - @freeze_time("2023-11-07") - def test_capture_workspace_rows_synced_by_team( - self, mock_capture: MagicMock, traverse_jobs_mock: MagicMock - ) -> None: - traverse_jobs_mock.return_value = [ - {"count": 97747, "startTime": "2023-11-05T18:32:41Z"}, - {"count": 93353, "startTime": "2023-11-07T16:50:49Z"}, - ] - - capture_workspace_rows_synced_by_team(self.team.pk) - - self.team.refresh_from_db() - self.assertEqual( - self.team.external_data_workspace_last_synced_at, - datetime.datetime(2023, 11, 7, 16, 50, 49, tzinfo=datetime.timezone.utc), - ) - - @patch("posthog.tasks.warehouse._traverse_jobs_by_field") - @patch("posthog.tasks.warehouse.get_ph_client") - @freeze_time("2023-11-07") - def test_capture_workspace_rows_synced_by_team_month_cutoff( - self, mock_capture: MagicMock, traverse_jobs_mock: MagicMock - ) -> None: - # external_data_workspace_last_synced_at unset - traverse_jobs_mock.return_value = [ - {"count": 93353, "startTime": "2023-11-07T16:50:49Z"}, - ] - - capture_workspace_rows_synced_by_team(self.team.pk) - - self.team.refresh_from_db() - self.assertEqual( - self.team.external_data_workspace_last_synced_at, - datetime.datetime(2023, 11, 7, 16, 50, 49, tzinfo=datetime.timezone.utc), - ) - - @patch("posthog.tasks.warehouse._traverse_jobs_by_field") - @patch("posthog.tasks.warehouse.get_ph_client") - @freeze_time("2023-11-07") - def test_capture_workspace_rows_synced_by_team_month_cutoff_field_set( - self, mock_capture: MagicMock, traverse_jobs_mock: MagicMock - ) -> None: - self.team.external_data_workspace_last_synced_at = datetime.datetime( - 2023, 10, 29, 18, 32, 41, tzinfo=datetime.timezone.utc - ) - self.team.save() - traverse_jobs_mock.return_value = [ - {"count": 97747, "startTime": "2023-10-30T18:32:41Z"}, - {"count": 93353, "startTime": "2023-11-07T16:50:49Z"}, - ] - - capture_workspace_rows_synced_by_team(self.team.pk) - - self.team.refresh_from_db() - self.assertEqual( - self.team.external_data_workspace_last_synced_at, - datetime.datetime(2023, 11, 7, 16, 50, 49, tzinfo=datetime.timezone.utc), - ) - - @patch("posthog.warehouse.external_data_source.connection.send_request") - @patch("ee.billing.quota_limiting.list_limited_team_attributes") - def test_external_data_source_billing_limit_deactivate( - self, usage_limit_mock: MagicMock, send_request_mock: MagicMock - ) -> None: - usage_limit_mock.return_value = [self.team.pk] - - external_source = ExternalDataSource.objects.create( - source_id="test_id", - connection_id="fake connectino_id", - destination_id="fake destination_id", - team=self.team, - status="running", - source_type="Stripe", - ) - - check_external_data_source_billing_limit_by_team(self.team.pk) - - external_source.refresh_from_db() - self.assertEqual(external_source.status, "inactive") - - @patch("posthog.warehouse.external_data_source.connection.send_request") - @patch("ee.billing.quota_limiting.list_limited_team_attributes") - def test_external_data_source_billing_limit_activate( - self, usage_limit_mock: MagicMock, send_request_mock: MagicMock - ) -> None: - usage_limit_mock.return_value = [] - - external_source = ExternalDataSource.objects.create( - source_id="test_id", - connection_id="fake connectino_id", - destination_id="fake destination_id", - team=self.team, - status="inactive", - source_type="Stripe", - ) - - check_external_data_source_billing_limit_by_team(self.team.pk) - - external_source.refresh_from_db() - self.assertEqual(external_source.status, "running") - @patch("posthog.tasks.warehouse.MONTHLY_LIMIT", 100) @patch("posthog.tasks.warehouse.cancel_external_data_workflow") @patch("posthog.tasks.warehouse.pause_external_data_schedule") diff --git a/posthog/tasks/warehouse.py b/posthog/tasks/warehouse.py index 5ab889fcd54a1..de48d10a28bdc 100644 --- a/posthog/tasks/warehouse.py +++ b/posthog/tasks/warehouse.py @@ -1,176 +1,15 @@ -from django.conf import settings import datetime -from posthog.models import Team -from posthog.warehouse.external_data_source.client import send_request from posthog.warehouse.data_load.service import ( cancel_external_data_workflow, pause_external_data_schedule, unpause_external_data_schedule, ) -from posthog.warehouse.models import DataWarehouseCredential, DataWarehouseTable, ExternalDataSource, ExternalDataJob -from posthog.warehouse.external_data_source.connection import retrieve_sync -from urllib.parse import urlencode -from posthog.ph_client import get_ph_client -from typing import Any, Dict, List, TYPE_CHECKING +from posthog.warehouse.models import ExternalDataSource, ExternalDataJob from posthog.celery import app import structlog logger = structlog.get_logger(__name__) -AIRBYTE_JOBS_URL = "https://api.airbyte.com/v1/jobs" -DEFAULT_DATE_TIME = datetime.datetime(2023, 11, 7, tzinfo=datetime.timezone.utc) - -if TYPE_CHECKING: - from posthoganalytics import Posthog - - -def sync_resources() -> None: - resources = ExternalDataSource.objects.filter(are_tables_created=False, status__in=["running", "error"]) - - for resource in resources: - sync_resource.delay(resource.pk) - - -@app.task(ignore_result=True) -def sync_resource(resource_id: str) -> None: - resource = ExternalDataSource.objects.get(pk=resource_id) - - try: - job = retrieve_sync(resource.connection_id) - except Exception as e: - logger.exception("Data Warehouse: Sync Resource failed with an unexpected exception.", exc_info=e) - resource.status = "error" - resource.save() - return - - if job is None: - logger.error(f"Data Warehouse: No jobs found for connection: {resource.connection_id}") - resource.status = "error" - resource.save() - return - - if job["status"] == "succeeded": - resource = ExternalDataSource.objects.get(pk=resource_id) - credential, _ = DataWarehouseCredential.objects.get_or_create( - team_id=resource.team.pk, - access_key=settings.AIRBYTE_BUCKET_KEY, - access_secret=settings.AIRBYTE_BUCKET_SECRET, - ) - - data = { - "credential": credential, - "name": "stripe_customers", - "format": "Parquet", - "url_pattern": f"https://{settings.AIRBYTE_BUCKET_DOMAIN}/airbyte/{resource.team.pk}/customers/*.parquet", - "team_id": resource.team.pk, - } - - table = DataWarehouseTable(**data) - try: - table.columns = table.get_columns() - except Exception as e: - logger.exception( - f"Data Warehouse: Sync Resource failed with an unexpected exception for connection: {resource.connection_id}", - exc_info=e, - ) - else: - table.save() - - resource.are_tables_created = True - resource.status = job["status"] - resource.save() - - else: - resource.status = job["status"] - resource.save() - - -DEFAULT_USAGE_LIMIT = 1000000 -ROWS_PER_DOLLAR = 66666 # 1 million rows per $15 - - -@app.task(ignore_result=True, max_retries=2) -def check_external_data_source_billing_limit_by_team(team_id: int) -> None: - from posthog.warehouse.external_data_source.connection import deactivate_connection_by_id, activate_connection_by_id - from ee.billing.quota_limiting import list_limited_team_attributes, QuotaResource - - limited_teams_rows_synced = list_limited_team_attributes(QuotaResource.ROWS_SYNCED) - - team = Team.objects.get(pk=team_id) - all_active_connections = ExternalDataSource.objects.filter(team=team, status__in=["running", "succeeded"]) - all_inactive_connections = ExternalDataSource.objects.filter(team=team, status="inactive") - - # TODO: consider more boundaries - if team_id in limited_teams_rows_synced: - for connection in all_active_connections: - deactivate_connection_by_id(connection.connection_id) - connection.status = "inactive" - connection.save() - else: - for connection in all_inactive_connections: - activate_connection_by_id(connection.connection_id) - connection.status = "running" - connection.save() - - -@app.task(ignore_result=True, max_retries=2) -def capture_workspace_rows_synced_by_team(team_id: int) -> None: - ph_client = get_ph_client() - team = Team.objects.get(pk=team_id) - now = datetime.datetime.now(datetime.timezone.utc) - begin = team.external_data_workspace_last_synced_at or DEFAULT_DATE_TIME - - params = { - "workspaceIds": team.external_data_workspace_id, - "limit": 100, - "offset": 0, - "status": "succeeded", - "orderBy": "createdAt|ASC", - "updatedAtStart": begin.strftime("%Y-%m-%dT%H:%M:%SZ"), - "updatedAtEnd": now.strftime("%Y-%m-%dT%H:%M:%SZ"), - } - result_totals = _traverse_jobs_by_field(ph_client, team, AIRBYTE_JOBS_URL + "?" + urlencode(params), "rowsSynced") - - # TODO: check assumption that ordering is possible with API - team.external_data_workspace_last_synced_at = result_totals[-1]["startTime"] if result_totals else now - team.save() - - ph_client.shutdown() - - -def _traverse_jobs_by_field( - ph_client: "Posthog", team: Team, url: str, field: str, acc: List[Dict[str, Any]] = [] -) -> List[Dict[str, Any]]: - response = send_request(url, method="GET") - response_data = response.get("data", []) - response_next = response.get("next", None) - - for job in response_data: - acc.append( - { - "count": job[field], - "startTime": job["startTime"], - } - ) - ph_client.capture( - team.pk, - "external data sync job", - { - "count": job[field], - "workspace_id": team.external_data_workspace_id, - "team_id": team.pk, - "team_uuid": team.uuid, - "startTime": job["startTime"], - "job_id": str(job["jobId"]), - }, - ) - - if response_next: - return _traverse_jobs_by_field(ph_client, team, response_next, field, acc) - - return acc - - MONTHLY_LIMIT = 1_000_000 diff --git a/posthog/temporal/data_imports/external_data_job.py b/posthog/temporal/data_imports/external_data_job.py index cdb218c0cce31..97c05ee2c4b16 100644 --- a/posthog/temporal/data_imports/external_data_job.py +++ b/posthog/temporal/data_imports/external_data_job.py @@ -218,7 +218,7 @@ async def run(self, inputs: ExternalDataWorkflowInputs): await workflow.execute_activity( run_external_data_job, job_inputs, - start_to_close_timeout=dt.timedelta(minutes=90), + start_to_close_timeout=dt.timedelta(hours=4), retry_policy=RetryPolicy(maximum_attempts=5), heartbeat_timeout=dt.timedelta(minutes=1), ) From 9f779771b06933d5155c9172422ecaae9bbbcfec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Jan 2024 09:12:03 +0000 Subject: [PATCH 06/15] chore(deps): bump peter-evans/find-comment from 1 to 2 (#19559) Bumps [peter-evans/find-comment](https://github.com/peter-evans/find-comment) from 1 to 2. - [Release notes](https://github.com/peter-evans/find-comment/releases) - [Commits](https://github.com/peter-evans/find-comment/compare/v1...v2) --- updated-dependencies: - dependency-name: peter-evans/find-comment dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 465ed1e734836..5be167503e287 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -136,7 +136,7 @@ jobs: - name: Find Comment if: ${{ github.event_name == 'pull_request' }} - uses: peter-evans/find-comment@v1 + uses: peter-evans/find-comment@v2 id: fc with: issue-number: ${{ github.event.number }} From 388d88db7c8b19d6a6f022626b1c448dbd4ce626 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Wed, 3 Jan 2024 10:31:12 +0000 Subject: [PATCH 07/15] feat: incremental updates for mobile transformer (#19567) first pass at adding incremental mutations created some fake mobile recordings locally and these behaved as expected this adds: removals these can be passed through - they're the same as rrweb adds the added wireframes can be converted just like in full snapshots (I think) updates these are implemented as removes and then adds this might not work on recordings with very many mutations but it'll be at least enough for us to get some folk testing it and we should have few enough mutations that it'll be ok --- .../__snapshots__/transform.test.ts.snap | 1517 ++++++++++++++--- ee/frontend/mobile-replay/mobile.types.ts | 34 +- .../schema/mobile/rr-mobile-schema.json | 77 + ee/frontend/mobile-replay/transform.test.ts | 69 + ee/frontend/mobile-replay/transformers.ts | 114 +- 5 files changed, 1522 insertions(+), 289 deletions(-) diff --git a/ee/frontend/mobile-replay/__snapshots__/transform.test.ts.snap b/ee/frontend/mobile-replay/__snapshots__/transform.test.ts.snap index d8d1d103bcaca..07dc030451f9c 100644 --- a/ee/frontend/mobile-replay/__snapshots__/transform.test.ts.snap +++ b/ee/frontend/mobile-replay/__snapshots__/transform.test.ts.snap @@ -667,7 +667,7 @@ exports[`replay/transform transform inputs input - $inputType - hello 1`] = ` { "attributes": {}, "childNodes": [], - "id": 224, + "id": 322, "tagName": "div", "type": 2, }, @@ -734,7 +734,7 @@ exports[`replay/transform transform inputs input - button - click me 1`] = ` }, "childNodes": [ { - "id": 216, + "id": 314, "textContent": "click me", "type": 3, }, @@ -744,7 +744,7 @@ exports[`replay/transform transform inputs input - button - click me 1`] = ` "type": 2, }, ], - "id": 215, + "id": 313, "tagName": "div", "type": 2, }, @@ -821,17 +821,17 @@ exports[`replay/transform transform inputs input - checkbox - $value 1`] = ` "type": 2, }, { - "id": 184, + "id": 282, "textContent": "first", "type": 3, }, ], - "id": 185, + "id": 283, "tagName": "label", "type": 2, }, ], - "id": 183, + "id": 281, "tagName": "div", "type": 2, }, @@ -907,17 +907,17 @@ exports[`replay/transform transform inputs input - checkbox - $value 2`] = ` "type": 2, }, { - "id": 187, + "id": 285, "textContent": "second", "type": 3, }, ], - "id": 188, + "id": 286, "tagName": "label", "type": 2, }, ], - "id": 186, + "id": 284, "tagName": "div", "type": 2, }, @@ -995,17 +995,17 @@ exports[`replay/transform transform inputs input - checkbox - $value 3`] = ` "type": 2, }, { - "id": 190, + "id": 288, "textContent": "third", "type": 3, }, ], - "id": 191, + "id": 289, "tagName": "label", "type": 2, }, ], - "id": 189, + "id": 287, "tagName": "div", "type": 2, }, @@ -1077,7 +1077,7 @@ exports[`replay/transform transform inputs input - checkbox - $value 4`] = ` "type": 2, }, ], - "id": 192, + "id": 290, "tagName": "div", "type": 2, }, @@ -1149,7 +1149,7 @@ exports[`replay/transform transform inputs input - email - $value 1`] = ` "type": 2, }, ], - "id": 176, + "id": 274, "tagName": "div", "type": 2, }, @@ -1221,7 +1221,7 @@ exports[`replay/transform transform inputs input - number - $value 1`] = ` "type": 2, }, ], - "id": 177, + "id": 275, "tagName": "div", "type": 2, }, @@ -1293,7 +1293,7 @@ exports[`replay/transform transform inputs input - password - $value 1`] = ` "type": 2, }, ], - "id": 175, + "id": 273, "tagName": "div", "type": 2, }, @@ -1369,17 +1369,17 @@ exports[`replay/transform transform inputs input - progress - $value 1`] = ` }, "childNodes": [ { - "id": 227, + "id": 325, "textContent": "@keyframes spin { 0% { transform: rotate(0deg); } 100% { transform: rotate(360deg); } }", "type": 3, }, ], - "id": 226, + "id": 324, "tagName": "style", "type": 2, }, ], - "id": 228, + "id": 326, "tagName": "div", "type": 2, }, @@ -1389,7 +1389,7 @@ exports[`replay/transform transform inputs input - progress - $value 1`] = ` "type": 2, }, ], - "id": 225, + "id": 323, "tagName": "div", "type": 2, }, @@ -1462,7 +1462,7 @@ exports[`replay/transform transform inputs input - progress - $value 2`] = ` "type": 2, }, ], - "id": 229, + "id": 327, "tagName": "div", "type": 2, }, @@ -1535,7 +1535,7 @@ exports[`replay/transform transform inputs input - progress - 0.75 1`] = ` "type": 2, }, ], - "id": 230, + "id": 328, "tagName": "div", "type": 2, }, @@ -1608,7 +1608,7 @@ exports[`replay/transform transform inputs input - progress - 0.75 2`] = ` "type": 2, }, ], - "id": 231, + "id": 329, "tagName": "div", "type": 2, }, @@ -1680,7 +1680,7 @@ exports[`replay/transform transform inputs input - search - $value 1`] = ` "type": 2, }, ], - "id": 178, + "id": 276, "tagName": "div", "type": 2, }, @@ -1753,12 +1753,12 @@ exports[`replay/transform transform inputs input - select - hello 1`] = ` }, "childNodes": [ { - "id": 221, + "id": 319, "textContent": "hello", "type": 3, }, ], - "id": 220, + "id": 318, "tagName": "option", "type": 2, }, @@ -1766,12 +1766,12 @@ exports[`replay/transform transform inputs input - select - hello 1`] = ` "attributes": {}, "childNodes": [ { - "id": 223, + "id": 321, "textContent": "world", "type": 3, }, ], - "id": 222, + "id": 320, "tagName": "option", "type": 2, }, @@ -1781,7 +1781,7 @@ exports[`replay/transform transform inputs input - select - hello 1`] = ` "type": 2, }, ], - "id": 219, + "id": 317, "tagName": "div", "type": 2, }, @@ -1854,7 +1854,7 @@ exports[`replay/transform transform inputs input - tel - $value 1`] = ` "type": 2, }, ], - "id": 179, + "id": 277, "tagName": "div", "type": 2, }, @@ -1926,7 +1926,7 @@ exports[`replay/transform transform inputs input - text - $value 1`] = ` "type": 2, }, ], - "id": 174, + "id": 272, "tagName": "div", "type": 2, }, @@ -1998,7 +1998,7 @@ exports[`replay/transform transform inputs input - text - hello 1`] = ` "type": 2, }, ], - "id": 173, + "id": 271, "tagName": "div", "type": 2, }, @@ -2070,7 +2070,7 @@ exports[`replay/transform transform inputs input - textArea - $value 1`] = ` "type": 2, }, ], - "id": 218, + "id": 316, "tagName": "div", "type": 2, }, @@ -2142,7 +2142,7 @@ exports[`replay/transform transform inputs input - textArea - hello 1`] = ` "type": 2, }, ], - "id": 217, + "id": 315, "tagName": "div", "type": 2, }, @@ -2208,7 +2208,7 @@ exports[`replay/transform transform inputs input - toggle - $value 1`] = ` }, "childNodes": [ { - "id": 197, + "id": 295, "textContent": "first", "type": 3, }, @@ -2228,7 +2228,7 @@ exports[`replay/transform transform inputs input - toggle - $value 1`] = ` "style": "position:absolute;top:33%;left:5%;display:inline-block;width:75%;height:33%;background-color:#1d4aff;opacity: 0.2;border-radius:7.5%;", }, "childNodes": [], - "id": 195, + "id": 293, "tagName": "div", "type": 2, }, @@ -2238,12 +2238,12 @@ exports[`replay/transform transform inputs input - toggle - $value 1`] = ` "style": "position:absolute;top:1.5%;right:5%;display:flex;align-items:center;justify-content:center;width:40%;height:75%;cursor:inherit;background-color:#1d4aff;border:2px solid #1d4aff;border-radius:50%;", }, "childNodes": [], - "id": 196, + "id": 294, "tagName": "div", "type": 2, }, ], - "id": 194, + "id": 292, "tagName": "div", "type": 2, }, @@ -2253,12 +2253,12 @@ exports[`replay/transform transform inputs input - toggle - $value 1`] = ` "type": 2, }, ], - "id": 198, + "id": 296, "tagName": "label", "type": 2, }, ], - "id": 193, + "id": 291, "tagName": "div", "type": 2, }, @@ -2324,7 +2324,7 @@ exports[`replay/transform transform inputs input - toggle - $value 2`] = ` }, "childNodes": [ { - "id": 203, + "id": 301, "textContent": "second", "type": 3, }, @@ -2344,7 +2344,7 @@ exports[`replay/transform transform inputs input - toggle - $value 2`] = ` "style": "position:absolute;top:33%;left:5%;display:inline-block;width:75%;height:33%;background-color:#f3f4ef;opacity: 0.2;border-radius:7.5%;", }, "childNodes": [], - "id": 201, + "id": 299, "tagName": "div", "type": 2, }, @@ -2354,12 +2354,12 @@ exports[`replay/transform transform inputs input - toggle - $value 2`] = ` "style": "position:absolute;top:1.5%;left:5%;display:flex;align-items:center;justify-content:center;width:40%;height:75%;cursor:inherit;background-color:#f3f4ef;border:2px solid #f3f4ef;border-radius:50%;", }, "childNodes": [], - "id": 202, + "id": 300, "tagName": "div", "type": 2, }, ], - "id": 200, + "id": 298, "tagName": "div", "type": 2, }, @@ -2369,12 +2369,12 @@ exports[`replay/transform transform inputs input - toggle - $value 2`] = ` "type": 2, }, ], - "id": 204, + "id": 302, "tagName": "label", "type": 2, }, ], - "id": 199, + "id": 297, "tagName": "div", "type": 2, }, @@ -2440,7 +2440,7 @@ exports[`replay/transform transform inputs input - toggle - $value 3`] = ` }, "childNodes": [ { - "id": 209, + "id": 307, "textContent": "third", "type": 3, }, @@ -2460,7 +2460,7 @@ exports[`replay/transform transform inputs input - toggle - $value 3`] = ` "style": "position:absolute;top:33%;left:5%;display:inline-block;width:75%;height:33%;background-color:#1d4aff;opacity: 0.2;border-radius:7.5%;", }, "childNodes": [], - "id": 207, + "id": 305, "tagName": "div", "type": 2, }, @@ -2470,12 +2470,12 @@ exports[`replay/transform transform inputs input - toggle - $value 3`] = ` "style": "position:absolute;top:1.5%;right:5%;display:flex;align-items:center;justify-content:center;width:40%;height:75%;cursor:inherit;background-color:#1d4aff;border:2px solid #1d4aff;border-radius:50%;", }, "childNodes": [], - "id": 208, + "id": 306, "tagName": "div", "type": 2, }, ], - "id": 206, + "id": 304, "tagName": "div", "type": 2, }, @@ -2485,12 +2485,12 @@ exports[`replay/transform transform inputs input - toggle - $value 3`] = ` "type": 2, }, ], - "id": 210, + "id": 308, "tagName": "label", "type": 2, }, ], - "id": 205, + "id": 303, "tagName": "div", "type": 2, }, @@ -2566,7 +2566,7 @@ exports[`replay/transform transform inputs input - toggle - $value 4`] = ` "style": "position:absolute;top:33%;left:5%;display:inline-block;width:75%;height:33%;background-color:#1d4aff;opacity: 0.2;border-radius:7.5%;", }, "childNodes": [], - "id": 213, + "id": 311, "tagName": "div", "type": 2, }, @@ -2576,12 +2576,12 @@ exports[`replay/transform transform inputs input - toggle - $value 4`] = ` "style": "position:absolute;top:1.5%;right:5%;display:flex;align-items:center;justify-content:center;width:40%;height:75%;cursor:inherit;background-color:#1d4aff;border:2px solid #1d4aff;border-radius:50%;", }, "childNodes": [], - "id": 214, + "id": 312, "tagName": "div", "type": 2, }, ], - "id": 212, + "id": 310, "tagName": "div", "type": 2, }, @@ -2591,7 +2591,7 @@ exports[`replay/transform transform inputs input - toggle - $value 4`] = ` "type": 2, }, ], - "id": 211, + "id": 309, "tagName": "div", "type": 2, }, @@ -2663,7 +2663,7 @@ exports[`replay/transform transform inputs input - url - https://example.io 1`] "type": 2, }, ], - "id": 180, + "id": 278, "tagName": "div", "type": 2, }, @@ -2687,7 +2687,7 @@ exports[`replay/transform transform inputs input - url - https://example.io 1`] } `; -exports[`replay/transform transform inputs open keyboard custom event 1`] = ` +exports[`replay/transform transform inputs isolated add mutation 1`] = ` { "data": { "adds": [ @@ -2695,236 +2695,1223 @@ exports[`replay/transform transform inputs open keyboard custom event 1`] = ` "nextId": null, "node": { "attributes": { - "style": "color: #35373e;background-color: #f3f4ef;width: 100vw;height: 150px;bottom: 0;position: fixed;align-items: center;justify-content: center;display: flex;", - }, - "childNodes": [ - { - "id": 170, - "textContent": "keyboard", - "type": 3, - }, - ], - "id": 6, - "tagName": "div", - "type": 2, - }, - "parentId": 5, - }, - { - "nextId": null, - "node": { - "id": 171, - "textContent": "keyboard", - "type": 3, - }, - "parentId": 6, - }, - ], - "attributes": [], - "removes": [], - "source": 0, - "texts": [], - }, - "timestamp": 1, - "type": 3, -} -`; - -exports[`replay/transform transform inputs placeholder - $inputType - $value 1`] = ` -{ - "data": { - "initialOffset": { - "left": 0, - "top": 0, - }, - "node": { - "childNodes": [ - { - "id": 2, - "name": "html", - "publicId": "", - "systemId": "", - "type": 1, - }, - { - "attributes": { - "style": "height: 100vh; width: 100vw;", + "style": "width: 100px;height: 30px;position: fixed;left: 0px;top: 0px;", }, "childNodes": [ - { - "attributes": {}, - "childNodes": [], - "id": 4, - "tagName": "head", - "type": 2, - }, { "attributes": { - "style": "height: 100vh; width: 100vw;", + "style": "position: relative; display: flex; flex-direction: row; padding: 2px 4px;", }, "childNodes": [ { - "attributes": {}, + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 174, + "textContent": "filled star", + "type": 3, + }, + ], + "id": 173, + "isSVG": true, + "tagName": "title", + "type": 2, + }, { "attributes": { - "style": "color: #35373e;background-color: #f3f4ef;width: 100px;height: 30px;position: fixed;left: 0px;top: 0px;align-items: center;justify-content: center;display: flex;", + "d": "M12,17.27L18.18,21L16.54,13.97L22,9.24L14.81,8.62L12,2L9.19,8.62L2,9.24L7.45,13.97L5.82,21L12,17.27Z", }, + "childNodes": [], + "id": 175, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 172, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, "childNodes": [ { - "id": 233, - "textContent": "hello", + "id": 178, + "textContent": "filled star", "type": 3, }, ], - "id": 12365, - "tagName": "div", + "id": 177, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,17.27L18.18,21L16.54,13.97L22,9.24L14.81,8.62L12,2L9.19,8.62L2,9.24L7.45,13.97L5.82,21L12,17.27Z", + }, + "childNodes": [], + "id": 179, + "isSVG": true, + "tagName": "path", "type": 2, }, ], - "id": 232, - "tagName": "div", + "id": 176, + "isSVG": true, + "tagName": "svg", "type": 2, }, - ], - "id": 5, - "tagName": "body", - "type": 2, - }, - ], - "id": 3, - "tagName": "html", - "type": 2, - }, - ], - "id": 1, - "type": 0, - }, - }, - "timestamp": 1, - "type": 2, -} -`; - -exports[`replay/transform transform inputs progress rating 1`] = ` -{ - "data": { - "initialOffset": { - "left": 0, - "top": 0, - }, - "node": { - "childNodes": [ - { - "id": 2, - "name": "html", - "publicId": "", - "systemId": "", - "type": 1, - }, - { - "attributes": { - "style": "height: 100vh; width: 100vw;", - }, - "childNodes": [ - { - "attributes": {}, - "childNodes": [], - "id": 4, - "tagName": "head", - "type": 2, - }, - { - "attributes": { - "style": "height: 100vh; width: 100vw;", - }, - "childNodes": [ { - "attributes": {}, + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 182, + "textContent": "filled star", + "type": 3, + }, + ], + "id": 181, + "isSVG": true, + "tagName": "title", + "type": 2, + }, { "attributes": { - "style": "width: 100px;height: 30px;position: fixed;left: 0px;top: 0px;", + "d": "M12,17.27L18.18,21L16.54,13.97L22,9.24L14.81,8.62L12,2L9.19,8.62L2,9.24L7.45,13.97L5.82,21L12,17.27Z", }, + "childNodes": [], + "id": 183, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 180, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, "childNodes": [ { - "attributes": { - "style": "position: relative; display: flex; flex-direction: row; padding: 2px 4px;", - }, - "childNodes": [ - { - "attributes": { - "fill": "currentColor", - "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", - "viewBox": "0 0 24 24", - }, - "childNodes": [ - { - "attributes": {}, - "childNodes": [ - { - "id": 123, - "textContent": "filled star", - "type": 3, - }, - ], - "id": 122, - "isSVG": true, - "tagName": "title", - "type": 2, - }, - { - "attributes": { - "d": "M12,17.27L18.18,21L16.54,13.97L22,9.24L14.81,8.62L12,2L9.19,8.62L2,9.24L7.45,13.97L5.82,21L12,17.27Z", - }, - "childNodes": [], - "id": 124, - "isSVG": true, - "tagName": "path", - "type": 2, - }, - ], - "id": 121, - "isSVG": true, - "tagName": "svg", - "type": 2, - }, - { - "attributes": { - "fill": "currentColor", - "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", - "viewBox": "0 0 24 24", - }, - "childNodes": [ - { - "attributes": {}, - "childNodes": [ - { - "id": 127, - "textContent": "filled star", - "type": 3, - }, - ], - "id": 126, - "isSVG": true, - "tagName": "title", - "type": 2, - }, - { - "attributes": { - "d": "M12,17.27L18.18,21L16.54,13.97L22,9.24L14.81,8.62L12,2L9.19,8.62L2,9.24L7.45,13.97L5.82,21L12,17.27Z", - }, - "childNodes": [], - "id": 128, - "isSVG": true, - "tagName": "path", - "type": 2, - }, - ], - "id": 125, - "isSVG": true, - "tagName": "svg", - "type": 2, + "id": 186, + "textContent": "filled star", + "type": 3, + }, + ], + "id": 185, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,17.27L18.18,21L16.54,13.97L22,9.24L14.81,8.62L12,2L9.19,8.62L2,9.24L7.45,13.97L5.82,21L12,17.27Z", + }, + "childNodes": [], + "id": 187, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 184, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 190, + "textContent": "filled star", + "type": 3, + }, + ], + "id": 189, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,17.27L18.18,21L16.54,13.97L22,9.24L14.81,8.62L12,2L9.19,8.62L2,9.24L7.45,13.97L5.82,21L12,17.27Z", + }, + "childNodes": [], + "id": 191, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 188, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 194, + "textContent": "filled star", + "type": 3, + }, + ], + "id": 193, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,17.27L18.18,21L16.54,13.97L22,9.24L14.81,8.62L12,2L9.19,8.62L2,9.24L7.45,13.97L5.82,21L12,17.27Z", + }, + "childNodes": [], + "id": 195, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 192, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 198, + "textContent": "half-filled star", + "type": 3, + }, + ], + "id": 197, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,15.4V6.1L13.71,10.13L18.09,10.5L14.77,13.39L15.76,17.67M22,9.24L14.81,8.63L12,2L9.19,8.63L2,9.24L7.45,13.97L5.82,21L12,17.27L18.18,21L16.54,13.97L22,9.24Z", + }, + "childNodes": [], + "id": 199, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 196, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 202, + "textContent": "empty star", + "type": 3, + }, + ], + "id": 201, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,15.39L8.24,17.66L9.23,13.38L5.91,10.5L10.29,10.13L12,6.09L13.71,10.13L18.09,10.5L14.77,13.38L15.76,17.66M22,9.24L14.81,8.63L12,2L9.19,8.63L2,9.24L7.45,13.97L5.82,21L12,17.27L18.18,21L16.54,13.97L22,9.24Z", + }, + "childNodes": [], + "id": 203, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 200, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 206, + "textContent": "empty star", + "type": 3, + }, + ], + "id": 205, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,15.39L8.24,17.66L9.23,13.38L5.91,10.5L10.29,10.13L12,6.09L13.71,10.13L18.09,10.5L14.77,13.38L15.76,17.66M22,9.24L14.81,8.63L12,2L9.19,8.63L2,9.24L7.45,13.97L5.82,21L12,17.27L18.18,21L16.54,13.97L22,9.24Z", + }, + "childNodes": [], + "id": 207, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 204, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 210, + "textContent": "empty star", + "type": 3, + }, + ], + "id": 209, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,15.39L8.24,17.66L9.23,13.38L5.91,10.5L10.29,10.13L12,6.09L13.71,10.13L18.09,10.5L14.77,13.38L15.76,17.66M22,9.24L14.81,8.63L12,2L9.19,8.63L2,9.24L7.45,13.97L5.82,21L12,17.27L18.18,21L16.54,13.97L22,9.24Z", + }, + "childNodes": [], + "id": 211, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 208, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 214, + "textContent": "empty star", + "type": 3, + }, + ], + "id": 213, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,15.39L8.24,17.66L9.23,13.38L5.91,10.5L10.29,10.13L12,6.09L13.71,10.13L18.09,10.5L14.77,13.38L15.76,17.66M22,9.24L14.81,8.63L12,2L9.19,8.63L2,9.24L7.45,13.97L5.82,21L12,17.27L18.18,21L16.54,13.97L22,9.24Z", + }, + "childNodes": [], + "id": 215, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 212, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 218, + "textContent": "empty star", + "type": 3, + }, + ], + "id": 217, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,15.39L8.24,17.66L9.23,13.38L5.91,10.5L10.29,10.13L12,6.09L13.71,10.13L18.09,10.5L14.77,13.38L15.76,17.66M22,9.24L14.81,8.63L12,2L9.19,8.63L2,9.24L7.45,13.97L5.82,21L12,17.27L18.18,21L16.54,13.97L22,9.24Z", + }, + "childNodes": [], + "id": 219, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 216, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + ], + "id": 220, + "tagName": "div", + "type": 2, + }, + ], + "id": 12365, + "tagName": "div", + "type": 2, + }, + "parentId": 54321, + }, + ], + "attributes": [], + "removes": [], + "source": 0, + "texts": [], + }, + "timestamp": 1, + "type": 3, +} +`; + +exports[`replay/transform transform inputs isolated remove mutation 1`] = ` +{ + "data": { + "removes": [ + { + "id": 12345, + "parentId": 54321, + }, + ], + "source": 0, + }, + "timestamp": 1, + "type": 3, +} +`; + +exports[`replay/transform transform inputs isolated update mutation 1`] = ` +{ + "data": { + "adds": [ + { + "nextId": null, + "node": { + "attributes": { + "style": "width: 100px;height: 30px;position: fixed;left: 0px;top: 0px;", + }, + "childNodes": [ + { + "attributes": { + "style": "position: relative; display: flex; flex-direction: row; padding: 2px 4px;", + }, + "childNodes": [ + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 223, + "textContent": "filled star", + "type": 3, + }, + ], + "id": 222, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,17.27L18.18,21L16.54,13.97L22,9.24L14.81,8.62L12,2L9.19,8.62L2,9.24L7.45,13.97L5.82,21L12,17.27Z", + }, + "childNodes": [], + "id": 224, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 221, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 227, + "textContent": "filled star", + "type": 3, + }, + ], + "id": 226, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,17.27L18.18,21L16.54,13.97L22,9.24L14.81,8.62L12,2L9.19,8.62L2,9.24L7.45,13.97L5.82,21L12,17.27Z", + }, + "childNodes": [], + "id": 228, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 225, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 231, + "textContent": "filled star", + "type": 3, + }, + ], + "id": 230, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,17.27L18.18,21L16.54,13.97L22,9.24L14.81,8.62L12,2L9.19,8.62L2,9.24L7.45,13.97L5.82,21L12,17.27Z", + }, + "childNodes": [], + "id": 232, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 229, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 235, + "textContent": "filled star", + "type": 3, + }, + ], + "id": 234, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,17.27L18.18,21L16.54,13.97L22,9.24L14.81,8.62L12,2L9.19,8.62L2,9.24L7.45,13.97L5.82,21L12,17.27Z", + }, + "childNodes": [], + "id": 236, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 233, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 239, + "textContent": "filled star", + "type": 3, + }, + ], + "id": 238, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,17.27L18.18,21L16.54,13.97L22,9.24L14.81,8.62L12,2L9.19,8.62L2,9.24L7.45,13.97L5.82,21L12,17.27Z", + }, + "childNodes": [], + "id": 240, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 237, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 243, + "textContent": "filled star", + "type": 3, + }, + ], + "id": 242, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,17.27L18.18,21L16.54,13.97L22,9.24L14.81,8.62L12,2L9.19,8.62L2,9.24L7.45,13.97L5.82,21L12,17.27Z", + }, + "childNodes": [], + "id": 244, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 241, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 247, + "textContent": "half-filled star", + "type": 3, + }, + ], + "id": 246, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,15.4V6.1L13.71,10.13L18.09,10.5L14.77,13.39L15.76,17.67M22,9.24L14.81,8.63L12,2L9.19,8.63L2,9.24L7.45,13.97L5.82,21L12,17.27L18.18,21L16.54,13.97L22,9.24Z", + }, + "childNodes": [], + "id": 248, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 245, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 251, + "textContent": "empty star", + "type": 3, + }, + ], + "id": 250, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,15.39L8.24,17.66L9.23,13.38L5.91,10.5L10.29,10.13L12,6.09L13.71,10.13L18.09,10.5L14.77,13.38L15.76,17.66M22,9.24L14.81,8.63L12,2L9.19,8.63L2,9.24L7.45,13.97L5.82,21L12,17.27L18.18,21L16.54,13.97L22,9.24Z", + }, + "childNodes": [], + "id": 252, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 249, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 255, + "textContent": "empty star", + "type": 3, + }, + ], + "id": 254, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,15.39L8.24,17.66L9.23,13.38L5.91,10.5L10.29,10.13L12,6.09L13.71,10.13L18.09,10.5L14.77,13.38L15.76,17.66M22,9.24L14.81,8.63L12,2L9.19,8.63L2,9.24L7.45,13.97L5.82,21L12,17.27L18.18,21L16.54,13.97L22,9.24Z", + }, + "childNodes": [], + "id": 256, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 253, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 259, + "textContent": "empty star", + "type": 3, + }, + ], + "id": 258, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,15.39L8.24,17.66L9.23,13.38L5.91,10.5L10.29,10.13L12,6.09L13.71,10.13L18.09,10.5L14.77,13.38L15.76,17.66M22,9.24L14.81,8.63L12,2L9.19,8.63L2,9.24L7.45,13.97L5.82,21L12,17.27L18.18,21L16.54,13.97L22,9.24Z", + }, + "childNodes": [], + "id": 260, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 257, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 263, + "textContent": "empty star", + "type": 3, + }, + ], + "id": 262, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,15.39L8.24,17.66L9.23,13.38L5.91,10.5L10.29,10.13L12,6.09L13.71,10.13L18.09,10.5L14.77,13.38L15.76,17.66M22,9.24L14.81,8.63L12,2L9.19,8.63L2,9.24L7.45,13.97L5.82,21L12,17.27L18.18,21L16.54,13.97L22,9.24Z", + }, + "childNodes": [], + "id": 264, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 261, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 267, + "textContent": "empty star", + "type": 3, + }, + ], + "id": 266, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,15.39L8.24,17.66L9.23,13.38L5.91,10.5L10.29,10.13L12,6.09L13.71,10.13L18.09,10.5L14.77,13.38L15.76,17.66M22,9.24L14.81,8.63L12,2L9.19,8.63L2,9.24L7.45,13.97L5.82,21L12,17.27L18.18,21L16.54,13.97L22,9.24Z", + }, + "childNodes": [], + "id": 268, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 265, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + ], + "id": 269, + "tagName": "div", + "type": 2, + }, + ], + "id": 12365, + "tagName": "div", + "type": 2, + }, + "parentId": 54321, + }, + ], + "attributes": [], + "removes": [ + { + "id": 12365, + "parentId": 54321, + }, + ], + "source": 0, + "texts": [], + }, + "timestamp": 1, + "type": 3, +} +`; + +exports[`replay/transform transform inputs open keyboard custom event 1`] = ` +{ + "data": { + "adds": [ + { + "nextId": null, + "node": { + "attributes": { + "style": "color: #35373e;background-color: #f3f4ef;width: 100vw;height: 150px;bottom: 0;position: fixed;align-items: center;justify-content: center;display: flex;", + }, + "childNodes": [ + { + "id": 170, + "textContent": "keyboard", + "type": 3, + }, + ], + "id": 6, + "tagName": "div", + "type": 2, + }, + "parentId": 5, + }, + { + "nextId": null, + "node": { + "id": 171, + "textContent": "keyboard", + "type": 3, + }, + "parentId": 6, + }, + ], + "attributes": [], + "removes": [], + "source": 0, + "texts": [], + }, + "timestamp": 1, + "type": 3, +} +`; + +exports[`replay/transform transform inputs placeholder - $inputType - $value 1`] = ` +{ + "data": { + "initialOffset": { + "left": 0, + "top": 0, + }, + "node": { + "childNodes": [ + { + "id": 2, + "name": "html", + "publicId": "", + "systemId": "", + "type": 1, + }, + { + "attributes": { + "style": "height: 100vh; width: 100vw;", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [], + "id": 4, + "tagName": "head", + "type": 2, + }, + { + "attributes": { + "style": "height: 100vh; width: 100vw;", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "attributes": { + "style": "color: #35373e;background-color: #f3f4ef;width: 100px;height: 30px;position: fixed;left: 0px;top: 0px;align-items: center;justify-content: center;display: flex;", + }, + "childNodes": [ + { + "id": 331, + "textContent": "hello", + "type": 3, + }, + ], + "id": 12365, + "tagName": "div", + "type": 2, + }, + ], + "id": 330, + "tagName": "div", + "type": 2, + }, + ], + "id": 5, + "tagName": "body", + "type": 2, + }, + ], + "id": 3, + "tagName": "html", + "type": 2, + }, + ], + "id": 1, + "type": 0, + }, + }, + "timestamp": 1, + "type": 2, +} +`; + +exports[`replay/transform transform inputs progress rating 1`] = ` +{ + "data": { + "initialOffset": { + "left": 0, + "top": 0, + }, + "node": { + "childNodes": [ + { + "id": 2, + "name": "html", + "publicId": "", + "systemId": "", + "type": 1, + }, + { + "attributes": { + "style": "height: 100vh; width: 100vw;", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [], + "id": 4, + "tagName": "head", + "type": 2, + }, + { + "attributes": { + "style": "height: 100vh; width: 100vw;", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "attributes": { + "style": "width: 100px;height: 30px;position: fixed;left: 0px;top: 0px;", + }, + "childNodes": [ + { + "attributes": { + "style": "position: relative; display: flex; flex-direction: row; padding: 2px 4px;", + }, + "childNodes": [ + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 123, + "textContent": "filled star", + "type": 3, + }, + ], + "id": 122, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,17.27L18.18,21L16.54,13.97L22,9.24L14.81,8.62L12,2L9.19,8.62L2,9.24L7.45,13.97L5.82,21L12,17.27Z", + }, + "childNodes": [], + "id": 124, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 121, + "isSVG": true, + "tagName": "svg", + "type": 2, + }, + { + "attributes": { + "fill": "currentColor", + "style": "height: 100%;overflow-clip-margin: content-box;overflow:hidden", + "viewBox": "0 0 24 24", + }, + "childNodes": [ + { + "attributes": {}, + "childNodes": [ + { + "id": 127, + "textContent": "filled star", + "type": 3, + }, + ], + "id": 126, + "isSVG": true, + "tagName": "title", + "type": 2, + }, + { + "attributes": { + "d": "M12,17.27L18.18,21L16.54,13.97L22,9.24L14.81,8.62L12,2L9.19,8.62L2,9.24L7.45,13.97L5.82,21L12,17.27Z", + }, + "childNodes": [], + "id": 128, + "isSVG": true, + "tagName": "path", + "type": 2, + }, + ], + "id": 125, + "isSVG": true, + "tagName": "svg", + "type": 2, }, { "attributes": { @@ -3367,7 +4354,7 @@ exports[`replay/transform transform inputs radio group - $inputType - $value 1`] { "attributes": {}, "childNodes": [], - "id": 182, + "id": 280, "tagName": "div", "type": 2, }, @@ -3437,7 +4424,7 @@ exports[`replay/transform transform inputs radio_group - $inputType - $value 1`] "type": 2, }, ], - "id": 181, + "id": 279, "tagName": "div", "type": 2, }, @@ -3507,7 +4494,7 @@ exports[`replay/transform transform inputs radio_group 1`] = ` "type": 2, }, ], - "id": 172, + "id": 270, "tagName": "div", "type": 2, }, @@ -3573,7 +4560,7 @@ exports[`replay/transform transform inputs web_view - $inputType - $value 1`] = }, "childNodes": [ { - "id": 235, + "id": 333, "textContent": "web_view", "type": 3, }, @@ -3583,7 +4570,7 @@ exports[`replay/transform transform inputs web_view - $inputType - $value 1`] = "type": 2, }, ], - "id": 234, + "id": 332, "tagName": "div", "type": 2, }, diff --git a/ee/frontend/mobile-replay/mobile.types.ts b/ee/frontend/mobile-replay/mobile.types.ts index 992860d730918..bdf080b40e0ee 100644 --- a/ee/frontend/mobile-replay/mobile.types.ts +++ b/ee/frontend/mobile-replay/mobile.types.ts @@ -1,5 +1,5 @@ // copied from rrweb-snapshot, not included in rrweb types -import { customEvent, EventType } from '@rrweb/types' +import { customEvent, EventType, IncrementalSource } from '@rrweb/types' export enum NodeType { Document = 0, @@ -278,9 +278,37 @@ export type fullSnapshotEvent = { } } -export type incrementalSnapshotEvent = { +export type incrementalSnapshotEvent = + | { + type: EventType.IncrementalSnapshot + data: any // keeps a loose incremental type so that we can accept any rrweb incremental snapshot event type + } + | MobileIncrementalSnapshotEvent + +export type MobileNodeMutation = { + parentId: number + wireframe: wireframe +} + +export type MobileAddedNodeMutationData = { + source: IncrementalSource.Mutation + adds: MobileNodeMutation[] +} + +export type MobileUpdatedNodeMutationData = { + source: IncrementalSource.Mutation + /** + * @description An update is implemented as a remove and then an add, so the updates array contains the ID of the removed node and the wireframe for the added node + */ + updates: MobileNodeMutation[] +} + +export type MobileIncrementalSnapshotEvent = { type: EventType.IncrementalSnapshot - data: any // TODO: this will change as we implement incremental snapshots + /** + * @description This sits alongside the RRWeb incremental snapshot event type, mobile replay can send any of the RRWeb incremental snapshot event types, which will be passed unchanged to the player - for example to send touch events. removed node mutations are passed unchanged to the player. + */ + data: MobileAddedNodeMutationData | MobileUpdatedNodeMutationData } export type metaEvent = { diff --git a/ee/frontend/mobile-replay/schema/mobile/rr-mobile-schema.json b/ee/frontend/mobile-replay/schema/mobile/rr-mobile-schema.json index bfbdd9fd4155d..3f5e4265dcf87 100644 --- a/ee/frontend/mobile-replay/schema/mobile/rr-mobile-schema.json +++ b/ee/frontend/mobile-replay/schema/mobile/rr-mobile-schema.json @@ -120,6 +120,33 @@ "required": ["data", "timestamp", "type"], "type": "object" }, + { + "additionalProperties": false, + "properties": { + "data": { + "anyOf": [ + { + "$ref": "#/definitions/MobileAddedNodeMutationData" + }, + { + "$ref": "#/definitions/MobileUpdatedNodeMutationData" + } + ], + "description": "This sits alongside the RRWeb incremental snapshot event type, mobile replay can send any of the RRWeb incremental snapshot event types, which will be passed unchanged to the player - for example to send touch events. removed node mutations are passed unchanged to the player." + }, + "delay": { + "type": "number" + }, + "timestamp": { + "type": "number" + }, + "type": { + "$ref": "#/definitions/EventType.IncrementalSnapshot" + } + }, + "required": ["data", "timestamp", "type"], + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -207,6 +234,39 @@ "const": 4, "type": "number" }, + "IncrementalSource.Mutation": { + "const": 0, + "type": "number" + }, + "MobileAddedNodeMutationData": { + "additionalProperties": false, + "properties": { + "adds": { + "items": { + "$ref": "#/definitions/MobileNodeMutation" + }, + "type": "array" + }, + "source": { + "$ref": "#/definitions/IncrementalSource.Mutation" + } + }, + "required": ["source", "adds"], + "type": "object" + }, + "MobileNodeMutation": { + "additionalProperties": false, + "properties": { + "parentId": { + "type": "number" + }, + "wireframe": { + "$ref": "#/definitions/wireframe" + } + }, + "required": ["parentId", "wireframe"], + "type": "object" + }, "MobileNodeType": { "enum": ["text", "image", "rectangle", "placeholder", "web_view", "input", "div", "radio_group"], "type": "string" @@ -255,6 +315,23 @@ }, "type": "object" }, + "MobileUpdatedNodeMutationData": { + "additionalProperties": false, + "properties": { + "source": { + "$ref": "#/definitions/IncrementalSource.Mutation" + }, + "updates": { + "description": "An update is implemented as a remove and then an add, so the updates array contains the ID of the removed node and the wireframe for the added node", + "items": { + "$ref": "#/definitions/MobileNodeMutation" + }, + "type": "array" + } + }, + "required": ["source", "updates"], + "type": "object" + }, "wireframe": { "anyOf": [ { diff --git a/ee/frontend/mobile-replay/transform.test.ts b/ee/frontend/mobile-replay/transform.test.ts index ccd5fb097c517..c2ac4d3fd3330 100644 --- a/ee/frontend/mobile-replay/transform.test.ts +++ b/ee/frontend/mobile-replay/transform.test.ts @@ -486,6 +486,75 @@ describe('replay/transform', () => { ).toMatchSnapshot() }) + test('isolated add mutation', () => { + expect( + posthogEEModule.mobileReplay?.transformEventToWeb({ + timestamp: 1, + type: EventType.IncrementalSnapshot, + data: { + source: 0, + adds: [ + { + parentId: 54321, + wireframe: { + id: 12365, + width: 100, + height: 30, + type: 'input', + inputType: 'progress', + style: { bar: 'rating' }, + max: '12', + value: '6.5', + }, + }, + ], + }, + }) + ).toMatchSnapshot() + }) + + test('isolated remove mutation', () => { + expect( + posthogEEModule.mobileReplay?.transformEventToWeb({ + timestamp: 1, + type: EventType.IncrementalSnapshot, + data: { + source: 0, + removes: [{ parentId: 54321, id: 12345 }], + }, + }) + ).toMatchSnapshot() + }) + + test('isolated update mutation', () => { + expect( + posthogEEModule.mobileReplay?.transformEventToWeb({ + timestamp: 1, + type: EventType.IncrementalSnapshot, + data: { + source: 0, + texts: [], + attributes: [], + updates: [ + { + parentId: 54321, + wireframe: { + id: 12365, + width: 100, + height: 30, + type: 'input', + inputType: 'progress', + style: { bar: 'rating' }, + max: '12', + value: '6.5', + }, + }, + ], + }, + }) + ).toMatchSnapshot() + }) + test('closed keyboard custom event', () => { expect( posthogEEModule.mobileReplay?.transformEventToWeb({ diff --git a/ee/frontend/mobile-replay/transformers.ts b/ee/frontend/mobile-replay/transformers.ts index 18ab4333541b9..d68df59db2b5e 100644 --- a/ee/frontend/mobile-replay/transformers.ts +++ b/ee/frontend/mobile-replay/transformers.ts @@ -7,16 +7,19 @@ import { IncrementalSource, metaEvent, mutationData, + removedNodeMutation, } from '@rrweb/types' import { captureMessage } from '@sentry/react' +import { isObject } from 'lib/utils' import { attributes, elementNode, fullSnapshotEvent as MobileFullSnapshotEvent, - incrementalSnapshotEvent as MobileIncrementalSnapshotEvent, keyboardEvent, metaEvent as MobileMetaEvent, + MobileIncrementalSnapshotEvent, + MobileNodeMutation, MobileNodeType, NodeType, serializedNodeWithId, @@ -84,15 +87,7 @@ const BODY_ID = 5 const KEYBOARD_ID = 6 function isKeyboardEvent(x: unknown): x is keyboardEvent { - return ( - typeof x === 'object' && - x !== null && - 'data' in x && - typeof x.data === 'object' && - x.data !== null && - 'tag' in x.data && - x.data.tag === 'keyboard' - ) + return isObject(x) && 'data' in x && isObject(x.data) && 'tag' in x.data && x.data.tag === 'keyboard' } export const makeCustomEvent = ( @@ -775,19 +770,19 @@ function chooseConverter( return converterMapping[converterType] } +function convertWireframe(wireframe: wireframe): serializedNodeWithId | null { + const children = convertWireframesFor(wireframe.childWireframes) + const converter = chooseConverter(wireframe) + return converter?.(wireframe, children) || null +} + function convertWireframesFor(wireframes: wireframe[] | undefined): serializedNodeWithId[] { if (!wireframes) { return [] } return wireframes.reduce((acc, wireframe) => { - const children = convertWireframesFor(wireframe.childWireframes) - const converter = chooseConverter(wireframe) - if (!converter) { - console.error(`No converter for wireframe type ${wireframe.type}`) - return acc - } - const convertedEl = converter(wireframe, children) + const convertedEl = convertWireframe(wireframe) if (convertedEl !== null) { acc.push(convertedEl) } @@ -795,15 +790,56 @@ function convertWireframesFor(wireframes: wireframe[] | undefined): serializedNo }, [] as serializedNodeWithId[]) } +function isMobileIncrementalSnapshotEvent(x: unknown): x is MobileIncrementalSnapshotEvent { + const isIncrementalSnapshot = isObject(x) && 'type' in x && x.type === EventType.IncrementalSnapshot + if (!isIncrementalSnapshot) { + return false + } + const hasData = isObject(x) && 'data' in x + const data = hasData ? x.data : null + + const hasMutationSource = isObject(data) && 'source' in data && data.source === IncrementalSource.Mutation + + const adds = isObject(data) && 'adds' in data && Array.isArray(data.adds) ? data.adds : null + const updates = isObject(data) && 'updates' in data && Array.isArray(data.updates) ? data.updates : null + + const hasUpdatedWireframe = !!updates && updates.length > 0 && isObject(updates[0]) && 'wireframe' in updates[0] + const hasAddedWireframe = !!adds && adds.length > 0 && isObject(adds[0]) && 'wireframe' in adds[0] + + return hasMutationSource && (hasAddedWireframe || hasUpdatedWireframe) +} + +function makeIncrementalAdd(add: MobileNodeMutation): addedNodeMutation | null { + const converted = convertWireframe(add.wireframe) + return converted + ? { + parentId: add.parentId, + nextId: null, + node: converted, + } + : null +} + +function makeIncrementalRemove(update: MobileNodeMutation): removedNodeMutation { + return { + parentId: update.parentId, + id: update.wireframe.id, + } +} + /** - * We've not implemented mutations, until then this is almost an index function. + * We want to ensure that any events don't use id = 0. + * They must always represent a valid ID from the dom, so we swap in the body id when the id = 0. * - * But, we want to ensure that any mouse/touch events don't use id = 0. - * They must always represent a valid ID from the dom, so we swap in the body id. + * For "removes", we don't need to do anything, the id of the element to be removed remains valid. We won't try and remove other elements that we added during transformation in order to show that element. + * + * "adds" are converted from wireframes to nodes and converted to incrementalSnapshotEvent.adds + * + * "updates" are converted to a remove and an add. * */ export const makeIncrementalEvent = ( - mobileEvent: MobileIncrementalSnapshotEvent & { + mobileEvent: (MobileIncrementalSnapshotEvent | incrementalSnapshotEvent) & { timestamp: number delay?: number } @@ -818,6 +854,42 @@ export const makeIncrementalEvent = ( if ('id' in converted.data && converted.data.id === 0) { converted.data.id = BODY_ID } + + if (isMobileIncrementalSnapshotEvent(mobileEvent)) { + const adds: addedNodeMutation[] = [] + const removes: removedNodeMutation[] = [] + if ('adds' in mobileEvent.data && Array.isArray(mobileEvent.data.adds)) { + mobileEvent.data.adds.forEach((add) => { + const converted = makeIncrementalAdd(add) + if (converted) { + // TODO when implementing keyboard placeholder we had to flatten the mutations not nest them + adds.push(converted) + } + }) + } + if ('updates' in mobileEvent.data && Array.isArray(mobileEvent.data.updates)) { + mobileEvent.data.updates.forEach((update) => { + const removal = makeIncrementalRemove(update) + if (removal) { + removes.push(removal) + } + const addition = makeIncrementalAdd(update) + if (addition) { + adds.push(addition) + } + }) + } + + converted.data = { + source: IncrementalSource.Mutation, + attributes: [], + texts: [], + adds, + // TODO: this assumes that removes are processed before adds 🤞 + removes, + } + } + return converted } From 55bdadd13d7b4f01185bbe4c9cdd945a5e020a3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Oberm=C3=BCller?= Date: Wed, 3 Jan 2024 12:01:04 +0100 Subject: [PATCH 08/15] fix(trends): fix breakdown legend (#19533) --- .../InsightLegend/InsightLegendRow.tsx | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/InsightLegend/InsightLegendRow.tsx b/frontend/src/lib/components/InsightLegend/InsightLegendRow.tsx index cf20e9d60d5e5..cfdb3f7067bd1 100644 --- a/frontend/src/lib/components/InsightLegend/InsightLegendRow.tsx +++ b/frontend/src/lib/components/InsightLegend/InsightLegendRow.tsx @@ -1,11 +1,16 @@ +import { useValues } from 'kea' import { getSeriesColor } from 'lib/colors' import { InsightLabel } from 'lib/components/InsightLabel' import { LemonCheckbox } from 'lib/lemon-ui/LemonCheckbox' import { useEffect, useRef } from 'react' import { formatAggregationAxisValue } from 'scenes/insights/aggregationAxisFormat' +import { isTrendsFilter } from 'scenes/insights/sharedUtils' +import { formatBreakdownLabel } from 'scenes/insights/utils' import { formatCompareLabel } from 'scenes/insights/views/InsightsTable/columns/SeriesColumn' import { IndexedTrendResult } from 'scenes/trends/types' +import { cohortsModel } from '~/models/cohortsModel' +import { propertyDefinitionsModel } from '~/models/propertyDefinitionsModel' import { TrendsFilter } from '~/queries/schema' import { ChartDisplayType } from '~/types' @@ -32,6 +37,9 @@ export function InsightLegendRow({ trendsFilter, highlighted, }: InsightLegendRowProps): JSX.Element { + const { cohorts } = useValues(cohortsModel) + const { formatPropertyValueForDisplay } = useValues(propertyDefinitionsModel) + const highlightStyle: Record = highlighted ? { style: { backgroundColor: getSeriesColor(item.seriesIndex, false, true) }, @@ -45,6 +53,15 @@ export function InsightLegendRow({ } }, [highlighted]) + const formattedBreakdownValue = formatBreakdownLabel( + cohorts, + formatPropertyValueForDisplay, + item.breakdown_value, + item.filter?.breakdown, + item.filter?.breakdown_type, + item.filter && isTrendsFilter(item.filter) && item.filter?.breakdown_histogram_bin_count !== undefined + ) + return (
@@ -61,7 +78,7 @@ export function InsightLegendRow({ action={item.action} fallbackName={item.breakdown_value === '' ? 'None' : item.label} hasMultipleSeries={hasMultipleSeries} - breakdownValue={item.breakdown_value === '' ? 'None' : item.breakdown_value?.toString()} + breakdownValue={formattedBreakdownValue} compareValue={compare ? formatCompareLabel(item) : undefined} pillMidEllipsis={item?.filter?.breakdown === '$current_url'} // TODO: define set of breakdown values that would benefit from mid ellipsis truncation hideIcon From c8e0857f3cf1d2998aaf0c41e18f30db26d2ba8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Oberm=C3=BCller?= Date: Wed, 3 Jan 2024 12:02:18 +0100 Subject: [PATCH 09/15] fix(trends): fix breakdowns persons label (#19534) --- .../persons-modal/persons-modal-utils.tsx | 63 +++++++++++-------- .../trends/viz/ActionsHorizontalBar.tsx | 2 +- .../scenes/trends/viz/ActionsLineGraph.tsx | 11 +++- frontend/src/scenes/trends/viz/ActionsPie.tsx | 2 +- 4 files changed, 50 insertions(+), 28 deletions(-) diff --git a/frontend/src/scenes/trends/persons-modal/persons-modal-utils.tsx b/frontend/src/scenes/trends/persons-modal/persons-modal-utils.tsx index 3f44300480fe5..3a82c70c9e6b6 100644 --- a/frontend/src/scenes/trends/persons-modal/persons-modal-utils.tsx +++ b/frontend/src/scenes/trends/persons-modal/persons-modal-utils.tsx @@ -5,10 +5,13 @@ import { PropertyKeyInfo } from 'lib/components/PropertyKeyInfo' import { dayjs } from 'lib/dayjs' import { capitalizeFirstLetter, pluralize, toParams } from 'lib/utils' import md5 from 'md5' -import { isFunnelsFilter, isPathsFilter } from 'scenes/insights/sharedUtils' +import { isFunnelsFilter, isPathsFilter, isTrendsFilter } from 'scenes/insights/sharedUtils' +import { formatBreakdownLabel } from 'scenes/insights/utils' import { cleanFilters } from 'scenes/insights/utils/cleanFilters' +import { FormatPropertyValueForDisplayFunction } from '~/models/propertyDefinitionsModel' import { + CohortType, FunnelsFilterType, FunnelVizType, GraphDataset, @@ -60,7 +63,9 @@ export const pathsTitle = (props: { isDropOff: boolean; label: string }): React. export const urlsForDatasets = ( crossDataset: GraphDataset[] | undefined, - index: number + index: number, + cohorts: CohortType[], + formatPropertyValueForDisplay: FormatPropertyValueForDisplayFunction ): { value: string; label: JSX.Element }[] => { const showCountedByTag = !!crossDataset?.find(({ action }) => action?.math && action.math !== 'total') const hasMultipleSeries = !!crossDataset?.find(({ action }) => action?.order) @@ -88,29 +93,37 @@ export const urlsForDatasets = ( return ( crossDataset - ?.map((dataset) => ({ - value: dataset.persons_urls?.[index].url || dataset.personsValues?.[index]?.url || '', - label: ( - - ), - })) + ?.map((dataset) => { + const formattedBreakdownValue = dataset.status + ? capitalizeFirstLetter(dataset.status) + : formatBreakdownLabel( + cohorts, + formatPropertyValueForDisplay, + dataset.breakdown_value, + dataset.filter?.breakdown, + dataset.filter?.breakdown_type, + dataset.filter && + isTrendsFilter(dataset.filter) && + dataset.filter?.breakdown_histogram_bin_count !== undefined + ) + return { + value: dataset.persons_urls?.[index].url || dataset.personsValues?.[index]?.url || '', + label: ( + + ), + } + }) .filter((x) => x.value) || [] ) } diff --git a/frontend/src/scenes/trends/viz/ActionsHorizontalBar.tsx b/frontend/src/scenes/trends/viz/ActionsHorizontalBar.tsx index a1478f30142ce..41f26f7dfafa3 100644 --- a/frontend/src/scenes/trends/viz/ActionsHorizontalBar.tsx +++ b/frontend/src/scenes/trends/viz/ActionsHorizontalBar.tsx @@ -90,7 +90,7 @@ export function ActionsHorizontalBar({ showPersonsModal = true }: ChartParams): const dataset = points.referencePoint.dataset const label = dataset.labels?.[point.index] - const urls = urlsForDatasets(crossDataset, index) + const urls = urlsForDatasets(crossDataset, index, cohorts, formatPropertyValueForDisplay) const selectedUrl = urls[index]?.value if (selectedUrl) { diff --git a/frontend/src/scenes/trends/viz/ActionsLineGraph.tsx b/frontend/src/scenes/trends/viz/ActionsLineGraph.tsx index a80f2cdd279b1..38c1476bcf348 100644 --- a/frontend/src/scenes/trends/viz/ActionsLineGraph.tsx +++ b/frontend/src/scenes/trends/viz/ActionsLineGraph.tsx @@ -7,6 +7,8 @@ import { capitalizeFirstLetter, isMultiSeriesFormula } from 'lib/utils' import { insightDataLogic } from 'scenes/insights/insightDataLogic' import { insightLogic } from 'scenes/insights/insightLogic' +import { cohortsModel } from '~/models/cohortsModel' +import { propertyDefinitionsModel } from '~/models/propertyDefinitionsModel' import { NodeKind } from '~/queries/schema' import { isInsightVizNode, isLifecycleQuery } from '~/queries/utils' import { ChartDisplayType, ChartParams, GraphType } from '~/types' @@ -24,6 +26,8 @@ export function ActionsLineGraph({ }: ChartParams): JSX.Element | null { const { insightProps, hiddenLegendKeys } = useValues(insightLogic) const { featureFlags } = useValues(featureFlagLogic) + const { cohorts } = useValues(cohortsModel) + const { formatPropertyValueForDisplay } = useValues(propertyDefinitionsModel) const { query } = useValues(insightDataLogic(insightProps)) const { indexedResults, @@ -126,7 +130,12 @@ export function ActionsLineGraph({ }, }) } else { - const datasetUrls = urlsForDatasets(crossDataset, index) + const datasetUrls = urlsForDatasets( + crossDataset, + index, + cohorts, + formatPropertyValueForDisplay + ) if (datasetUrls?.length) { openPersonsModal({ urls: datasetUrls, diff --git a/frontend/src/scenes/trends/viz/ActionsPie.tsx b/frontend/src/scenes/trends/viz/ActionsPie.tsx index cb2bcecb1fc43..b9a6aebe28294 100644 --- a/frontend/src/scenes/trends/viz/ActionsPie.tsx +++ b/frontend/src/scenes/trends/viz/ActionsPie.tsx @@ -92,7 +92,7 @@ export function ActionsPie({ const dataset = points.referencePoint.dataset const label = dataset.labels?.[index] - const urls = urlsForDatasets(crossDataset, index) + const urls = urlsForDatasets(crossDataset, index, cohorts, formatPropertyValueForDisplay) const selectedUrl = urls[index]?.value if (selectedUrl) { From 4ebd823453a0f04f417c88b4d285760be18b76d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Jan 2024 12:13:16 +0100 Subject: [PATCH 10/15] chore(deps): bump chromaui/action from 1 to 10 (#19560) --- .github/workflows/storybook-chromatic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/storybook-chromatic.yml b/.github/workflows/storybook-chromatic.yml index 4faed2ea8e084..43b15131affa0 100644 --- a/.github/workflows/storybook-chromatic.yml +++ b/.github/workflows/storybook-chromatic.yml @@ -42,7 +42,7 @@ jobs: run: pnpm i -D chromatic - name: Publish to Chromatic - uses: chromaui/action@v1 + uses: chromaui/action@v10 id: publish with: token: ${{ secrets.GITHUB_TOKEN }} From 24b1cb6c84e583c92c76e97bfe9fa64704090dbe Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Wed, 3 Jan 2024 11:22:14 +0000 Subject: [PATCH 11/15] fix: assume typeless series nodes are of type events node (#19550) --- .../nodes/InsightQuery/utils/filtersToQueryNode.test.ts | 8 ++++++++ .../nodes/InsightQuery/utils/filtersToQueryNode.ts | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/frontend/src/queries/nodes/InsightQuery/utils/filtersToQueryNode.test.ts b/frontend/src/queries/nodes/InsightQuery/utils/filtersToQueryNode.test.ts index c86915aab1d89..4a774f755a3d2 100644 --- a/frontend/src/queries/nodes/InsightQuery/utils/filtersToQueryNode.test.ts +++ b/frontend/src/queries/nodes/InsightQuery/utils/filtersToQueryNode.test.ts @@ -72,6 +72,14 @@ describe('actionsAndEventsToSeries', () => { expect(result[1].name).toEqual('item1') expect(result[2].name).toEqual('item2') }) + + it('assumes typeless series is an event series', () => { + const events: ActionFilter[] = [{ id: '$pageview', order: 0, name: 'item1' } as any] + + const result = actionsAndEventsToSeries({ events }) + + expect(result[0].kind === NodeKind.EventsNode) + }) }) describe('cleanHiddenLegendIndexes', () => { diff --git a/frontend/src/queries/nodes/InsightQuery/utils/filtersToQueryNode.ts b/frontend/src/queries/nodes/InsightQuery/utils/filtersToQueryNode.ts index 31628318c2e7c..75f16b6421e64 100644 --- a/frontend/src/queries/nodes/InsightQuery/utils/filtersToQueryNode.ts +++ b/frontend/src/queries/nodes/InsightQuery/utils/filtersToQueryNode.ts @@ -73,7 +73,7 @@ export const actionsAndEventsToSeries = ({ id: f.id, ...shared, } - } else if (f.type === 'events') { + } else { return { kind: NodeKind.EventsNode, event: f.id, From 1b4733d55acc9a0b38f88c9f8e44616f829ef54e Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Wed, 3 Jan 2024 11:27:19 +0000 Subject: [PATCH 12/15] fix(bi): fixed some of the query duplications (#19573) Fixed some of the query duplications --- .../Components/SeriesTab.tsx | 4 +- .../DataVisualization/DataVisualization.tsx | 95 ++++++++++--------- frontend/src/queries/query.ts | 2 +- 3 files changed, 52 insertions(+), 49 deletions(-) diff --git a/frontend/src/queries/nodes/DataVisualization/Components/SeriesTab.tsx b/frontend/src/queries/nodes/DataVisualization/Components/SeriesTab.tsx index 0d86d7ee5c235..2d1c921bd2869 100644 --- a/frontend/src/queries/nodes/DataVisualization/Components/SeriesTab.tsx +++ b/frontend/src/queries/nodes/DataVisualization/Components/SeriesTab.tsx @@ -2,12 +2,10 @@ import { LemonButton, LemonLabel, LemonSelect } from '@posthog/lemon-ui' import { useActions, useValues } from 'kea' import { IconDelete, IconPlusMini } from 'lib/lemon-ui/icons' -import { dataNodeLogic } from '../../DataNode/dataNodeLogic' import { dataVisualizationLogic } from '../dataVisualizationLogic' export const SeriesTab = (): JSX.Element => { - const { columns, xData, yData } = useValues(dataVisualizationLogic) - const { responseLoading } = useValues(dataNodeLogic) + const { columns, xData, yData, responseLoading } = useValues(dataVisualizationLogic) const { updateXSeries, updateYSeries, addYSeries, deleteYSeries } = useActions(dataVisualizationLogic) const options = columns.map(({ name, label }) => ({ diff --git a/frontend/src/queries/nodes/DataVisualization/DataVisualization.tsx b/frontend/src/queries/nodes/DataVisualization/DataVisualization.tsx index 0d64ded3608ac..e95b58f2a15a4 100644 --- a/frontend/src/queries/nodes/DataVisualization/DataVisualization.tsx +++ b/frontend/src/queries/nodes/DataVisualization/DataVisualization.tsx @@ -46,7 +46,6 @@ export function DataTableVisualization(props: DataTableVisualizationProps): JSX. setQuery: props.setQuery, cachedResults: props.cachedResults, } - const builtDataVisualizationLogic = dataVisualizationLogic(dataVisualizationLogicProps) const dataNodeLogicProps: DataNodeLogicProps = { query: props.query.source, @@ -54,8 +53,20 @@ export function DataTableVisualization(props: DataTableVisualizationProps): JSX. cachedResults: props.cachedResults, } + return ( + + + + + + + + ) +} + +function InternalDataTableVisualization(props: DataTableVisualizationProps): JSX.Element { const { query, visualizationType, showEditingUI, showResultControls, sourceFeatures, response, responseLoading } = - useValues(builtDataVisualizationLogic) + useValues(dataVisualizationLogic) const setQuerySource = useCallback( (source: HogQLQuery) => props.setQuery?.({ ...props.query, source }), @@ -72,7 +83,7 @@ export function DataTableVisualization(props: DataTableVisualizationProps): JSX. } else if (visualizationType === ChartDisplayType.ActionsTable) { component = ( - - -
-
- {showEditingUI && ( - <> - - {sourceFeatures.has(QueryFeature.dateRangePicker) && ( -
- { - if (query.kind === NodeKind.HogQLQuery) { - setQuerySource(query) - } - }} - /> -
- )} - - )} - {showResultControls && ( - <> - -
-
- - -
-
- -
-
- - )} - {component} +
+
+ {showEditingUI && ( + <> + + {sourceFeatures.has(QueryFeature.dateRangePicker) && ( +
+ { + if (query.kind === NodeKind.HogQLQuery) { + setQuerySource(query) + } + }} + /> +
+ )} + + )} + {showResultControls && ( + <> + +
+
+ + +
+
+ +
-
- - - + + )} + {component} +
+
) } diff --git a/frontend/src/queries/query.ts b/frontend/src/queries/query.ts index 348a57059a2c3..b8a147b570e8d 100644 --- a/frontend/src/queries/query.ts +++ b/frontend/src/queries/query.ts @@ -107,7 +107,7 @@ async function executeQuery( queryId?: string ): Promise> { const queryAsyncEnabled = Boolean(featureFlagLogic.findMounted()?.values.featureFlags?.[FEATURE_FLAGS.QUERY_ASYNC]) - const excludedKinds = ['HogQLMetadata', 'EventsQuery', 'DataVisualizationNode'] + const excludedKinds = ['HogQLMetadata', 'EventsQuery'] const queryAsync = queryAsyncEnabled && !excludedKinds.includes(queryNode.kind) const response = await api.query(queryNode, methodOptions, queryId, refresh, queryAsync) From e28d34c6e90772f8c5a00dd5d2b49821466e7074 Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto <5731772+marandaneto@users.noreply.github.com> Date: Wed, 3 Jan 2024 13:19:09 +0100 Subject: [PATCH 13/15] chore: Add Flutter feature flags snippets (#19563) --- README.md | 2 +- cypress/e2e/featureFlags.cy.ts | 2 +- .../feature-flags/FeatureFlagCodeOptions.tsx | 26 ++++++++++++------- .../feature-flags/FeatureFlagSnippets.tsx | 11 ++++++++ .../FeatureFlagsSDKInstructions.tsx | 4 ++- .../onboarding/sdks/feature-flags/flutter.tsx | 13 ++++++++++ .../onboarding/sdks/feature-flags/index.tsx | 1 + .../sdks/sdk-install-instructions/flutter.tsx | 2 +- .../settings/project/ProjectSettings.tsx | 4 +-- .../project/SessionRecordingSettings.tsx | 2 +- .../settings/project/WebhookIntegration.tsx | 4 +-- .../src/scenes/surveys/SurveySettings.tsx | 2 +- 12 files changed, 54 insertions(+), 19 deletions(-) create mode 100644 frontend/src/scenes/onboarding/sdks/feature-flags/flutter.tsx diff --git a/README.md b/README.md index 4629af2ac101e..7e2f8d5ac44b8 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ PostHog brings all the tools and data you need to build better products. ### Data and infrastructure tools - **Import and export your data:** Import from and export to the services that matter to you with [the PostHog CDP](https://posthog.com/docs/cdp) -- **Ready-made libraries:** We’ve built libraries for [JavaScript](https://posthog.com/docs/integrations/js-integration), [Python](https://posthog.com/docs/integrations/python-integration), [Ruby](https://posthog.com/docs/integrations/ruby-integration), [Node](https://posthog.com/docs/integrations/node-integration), [Go](https://posthog.com/docs/integrations/go-integration), [Android](https://posthog.com/docs/integrations/android-integration), [iOS](https://posthog.com/docs/integrations/ios-integration), [PHP](https://posthog.com/docs/integrations/php-integration), [Flutter](https://posthog.com/docs/integrations/flutter-integration), [React Native](https://posthog.com/docs/integrations/react-native-integration), [Elixir](https://posthog.com/docs/integrations/elixir-integration), [Nim](https://github.com/Yardanico/posthog-nim), and an [API](https://posthog.com/docs/integrations/api) for anything else +- **Ready-made libraries:** We’ve built libraries for [JavaScript](https://posthog.com/docs/libraries/js), [Python](https://posthog.com/docs/libraries/python), [Ruby](https://posthog.com/docs/libraries/ruby), [Node](https://posthog.com/docs/libraries/node), [Go](https://posthog.com/docs/libraries/go), [Android](https://posthog.com/docs/libraries/android), [iOS](https://posthog.com/docs/libraries/ios), [PHP](https://posthog.com/docs/libraries/php), [Flutter](https://posthog.com/docs/libraries/flutter), [React Native](https://posthog.com/docs/libraries/react-native), [Elixir](https://posthog.com/docs/libraries/elixir), [Nim](https://github.com/Yardanico/posthog-nim), and an [API](https://posthog.com/docs/api) for anything else - **Plays nicely with data warehouses:** import events or user data from your warehouse by writing a simple transformation plugin, and export data with pre-built apps - such as [BigQuery](https://posthog.com/apps/bigquery-export), [Redshift](https://posthog.com/apps/redshift-export), [Snowflake](https://posthog.com/apps/snowflake-export), and [S3](https://posthog.com/apps/s3-expo) [Read a full list of PostHog features](https://posthog.com/product). diff --git a/cypress/e2e/featureFlags.cy.ts b/cypress/e2e/featureFlags.cy.ts index 9f1c395493e05..2d6b30aaabb04 100644 --- a/cypress/e2e/featureFlags.cy.ts +++ b/cypress/e2e/featureFlags.cy.ts @@ -28,7 +28,7 @@ describe('Feature Flags', () => { cy.get('[data-attr=feature-flag-doc-link]').should( 'have.attr', 'href', - 'https://posthog.com/docs/integrations/php-integration?utm_medium=in-product&utm_campaign=feature-flag#feature-flags' + 'https://posthog.com/docs/libraries/php?utm_medium=in-product&utm_campaign=feature-flag#feature-flags' ) // select "add filter" and "property" diff --git a/frontend/src/scenes/feature-flags/FeatureFlagCodeOptions.tsx b/frontend/src/scenes/feature-flags/FeatureFlagCodeOptions.tsx index 2d5d2c618d6ae..ebfe728f181df 100644 --- a/frontend/src/scenes/feature-flags/FeatureFlagCodeOptions.tsx +++ b/frontend/src/scenes/feature-flags/FeatureFlagCodeOptions.tsx @@ -4,6 +4,7 @@ import { AndroidSnippet, APISnippet, FeatureFlagSnippet, + FlutterSnippet, GolangSnippet, iOSSnippet, JSBootstrappingSnippet, @@ -39,28 +40,28 @@ export enum LibraryType { export const OPTIONS: InstructionOption[] = [ { value: 'JavaScript', - documentationLink: `${DOC_BASE_URL}integrations/js-integration${UTM_TAGS}`, + documentationLink: `${DOC_BASE_URL}libraries/js${UTM_TAGS}`, Snippet: JSSnippet, type: LibraryType.Client, key: SDKKey.JS_WEB, }, { value: 'Android', - documentationLink: `${DOC_BASE_URL}integrate/client/android${UTM_TAGS}`, + documentationLink: `${DOC_BASE_URL}libraries/android${UTM_TAGS}`, Snippet: AndroidSnippet, type: LibraryType.Client, key: SDKKey.ANDROID, }, { value: 'iOS', - documentationLink: `${DOC_BASE_URL}integrate/client/ios${UTM_TAGS}`, + documentationLink: `${DOC_BASE_URL}libraries/ios${UTM_TAGS}`, Snippet: iOSSnippet, type: LibraryType.Client, key: SDKKey.IOS, }, { value: 'React Native', - documentationLink: `${DOC_BASE_URL}integrate/client/react-native${UTM_TAGS}`, + documentationLink: `${DOC_BASE_URL}libraries/react-native${UTM_TAGS}`, Snippet: ReactNativeSnippet, type: LibraryType.Client, key: SDKKey.REACT_NATIVE, @@ -74,21 +75,21 @@ export const OPTIONS: InstructionOption[] = [ }, { value: 'Node.js', - documentationLink: `${DOC_BASE_URL}integrations/node-integration${UTM_TAGS}`, + documentationLink: `${DOC_BASE_URL}libraries/node${UTM_TAGS}`, Snippet: NodeJSSnippet, type: LibraryType.Server, key: SDKKey.NODE_JS, }, { value: 'Python', - documentationLink: `${DOC_BASE_URL}integrations/python-integration${UTM_TAGS}`, + documentationLink: `${DOC_BASE_URL}libraries/python${UTM_TAGS}`, Snippet: PythonSnippet, type: LibraryType.Server, key: SDKKey.PYTHON, }, { value: 'Ruby', - documentationLink: `${DOC_BASE_URL}integrations/ruby-integration${UTM_TAGS}`, + documentationLink: `${DOC_BASE_URL}libraries/ruby${UTM_TAGS}`, Snippet: RubySnippet, type: LibraryType.Server, key: SDKKey.RUBY, @@ -102,18 +103,25 @@ export const OPTIONS: InstructionOption[] = [ }, { value: 'PHP', - documentationLink: `${DOC_BASE_URL}integrations/php-integration${UTM_TAGS}`, + documentationLink: `${DOC_BASE_URL}libraries/php${UTM_TAGS}`, Snippet: PHPSnippet, type: LibraryType.Server, key: SDKKey.PHP, }, { value: 'Go', - documentationLink: `${DOC_BASE_URL}integrations/go-integration${UTM_TAGS}`, + documentationLink: `${DOC_BASE_URL}libraries/go${UTM_TAGS}`, Snippet: GolangSnippet, type: LibraryType.Server, key: SDKKey.GO, }, + { + value: 'Flutter', + documentationLink: `${DOC_BASE_URL}libraries/flutter${UTM_TAGS}`, + Snippet: FlutterSnippet, + type: LibraryType.Client, + key: SDKKey.FLUTTER, + }, ] export const LOCAL_EVALUATION_LIBRARIES: string[] = [SDKKey.NODE_JS, SDKKey.PYTHON, SDKKey.RUBY, SDKKey.PHP, SDKKey.GO] diff --git a/frontend/src/scenes/feature-flags/FeatureFlagSnippets.tsx b/frontend/src/scenes/feature-flags/FeatureFlagSnippets.tsx index 7bee754183d1a..54f8d6c1ff728 100644 --- a/frontend/src/scenes/feature-flags/FeatureFlagSnippets.tsx +++ b/frontend/src/scenes/feature-flags/FeatureFlagSnippets.tsx @@ -336,6 +336,17 @@ export function AndroidSnippet({ flagKey, multivariant, payload }: FeatureFlagSn ) } +export function FlutterSnippet({ flagKey }: FeatureFlagSnippet): JSX.Element { + return ( + + {`if (await Posthog().isFeatureEnabled('${flagKey}') ?? false) { + // do something +} + `} + + ) +} + export function iOSSnippet({ flagKey, multivariant, payload }: FeatureFlagSnippet): JSX.Element { const clientSuffix = 'posthog.' diff --git a/frontend/src/scenes/onboarding/sdks/feature-flags/FeatureFlagsSDKInstructions.tsx b/frontend/src/scenes/onboarding/sdks/feature-flags/FeatureFlagsSDKInstructions.tsx index fee11b72005d0..db8b896c25a58 100644 --- a/frontend/src/scenes/onboarding/sdks/feature-flags/FeatureFlagsSDKInstructions.tsx +++ b/frontend/src/scenes/onboarding/sdks/feature-flags/FeatureFlagsSDKInstructions.tsx @@ -3,6 +3,7 @@ import { SDKInstructionsMap, SDKKey } from '~/types' import { FeatureFlagsAndroidInstructions, FeatureFlagsAPIInstructions, + FeatureFlagsFlutterInstructions, FeatureFlagsGoInstructions, FeatureFlagsIOSInstructions, FeatureFlagsJSWebInstructions, @@ -22,11 +23,12 @@ export const FeatureFlagsSDKInstructions: SDKInstructionsMap = { [SDKKey.IOS]: FeatureFlagsIOSInstructions, [SDKKey.REACT_NATIVE]: FeatureFlagsRNInstructions, [SDKKey.ANDROID]: FeatureFlagsAndroidInstructions, + [SDKKey.FLUTTER]: FeatureFlagsFlutterInstructions, [SDKKey.NODE_JS]: FeatureFlagsNodeInstructions, [SDKKey.PYTHON]: FeatureFlagsPythonInstructions, [SDKKey.RUBY]: FeatureFlagsRubyInstructions, [SDKKey.PHP]: FeatureFlagsPHPInstructions, [SDKKey.GO]: FeatureFlagsGoInstructions, [SDKKey.API]: FeatureFlagsAPIInstructions, - // add flutter, rust, gatsby, nuxt, vue, svelte, and others here + // add rust, gatsby, nuxt, vue, svelte, and others here } diff --git a/frontend/src/scenes/onboarding/sdks/feature-flags/flutter.tsx b/frontend/src/scenes/onboarding/sdks/feature-flags/flutter.tsx new file mode 100644 index 0000000000000..ac4e3750e6d6e --- /dev/null +++ b/frontend/src/scenes/onboarding/sdks/feature-flags/flutter.tsx @@ -0,0 +1,13 @@ +import { SDKKey } from '~/types' + +import { SDKInstallFlutterInstructions } from '../sdk-install-instructions' +import { FlagImplementationSnippet } from './flagImplementationSnippet' + +export function FeatureFlagsFlutterInstructions(): JSX.Element { + return ( + <> + + + + ) +} diff --git a/frontend/src/scenes/onboarding/sdks/feature-flags/index.tsx b/frontend/src/scenes/onboarding/sdks/feature-flags/index.tsx index 69d5234c62f1e..c7a6d79949424 100644 --- a/frontend/src/scenes/onboarding/sdks/feature-flags/index.tsx +++ b/frontend/src/scenes/onboarding/sdks/feature-flags/index.tsx @@ -1,5 +1,6 @@ export * from './android' export * from './api' +export * from './flutter' export * from './go' export * from './ios' export * from './js-web' diff --git a/frontend/src/scenes/onboarding/sdks/sdk-install-instructions/flutter.tsx b/frontend/src/scenes/onboarding/sdks/sdk-install-instructions/flutter.tsx index b2284e00d6596..496eec8c7d822 100644 --- a/frontend/src/scenes/onboarding/sdks/sdk-install-instructions/flutter.tsx +++ b/frontend/src/scenes/onboarding/sdks/sdk-install-instructions/flutter.tsx @@ -31,7 +31,7 @@ function FlutterIOSSetupSnippet(): JSX.Element { currentTeam?.api_token + '\n\tcom.posthog.posthog.POSTHOG_HOST\n\t' + url + - '\n\tcom.posthog.posthog.TRACK_APPLICATION_LIFECYCLE_EVENTS\n\t\n\t[...]\n'} + '\n\tcom.posthog.posthog.CAPTURE_APPLICATION_LIFECYCLE_EVENTS\n\t\n\t[...]\n'} ) } diff --git a/frontend/src/scenes/settings/project/ProjectSettings.tsx b/frontend/src/scenes/settings/project/ProjectSettings.tsx index 77ace6cdb4675..3a50c902d8ac0 100644 --- a/frontend/src/scenes/settings/project/ProjectSettings.tsx +++ b/frontend/src/scenes/settings/project/ProjectSettings.tsx @@ -55,7 +55,7 @@ export function WebSnippet(): JSX.Element {

For more guidance, including on identifying users,{' '} - see PostHog Docs. + see PostHog Docs.

{currentTeamLoading && !currentTeam ? (
@@ -97,7 +97,7 @@ export function ProjectVariables(): JSX.Element {

You can use this write-only key in any one of{' '} - our libraries. + our libraries.

PostHog snippet or the latest version of{' '} posthog-js diff --git a/frontend/src/scenes/settings/project/WebhookIntegration.tsx b/frontend/src/scenes/settings/project/WebhookIntegration.tsx index 060826fac741b..a3e30b57fcedb 100644 --- a/frontend/src/scenes/settings/project/WebhookIntegration.tsx +++ b/frontend/src/scenes/settings/project/WebhookIntegration.tsx @@ -43,8 +43,8 @@ export function WebhookIntegration(): JSX.Element {
Guidance on integrating with webhooks available in our docs,{' '} for Slack and{' '} - for Microsoft Teams. Discord is - also supported. + for Microsoft Teams. Discord is also + supported.

diff --git a/frontend/src/scenes/surveys/SurveySettings.tsx b/frontend/src/scenes/surveys/SurveySettings.tsx index ac5fd8c458be4..83703bcfce52c 100644 --- a/frontend/src/scenes/surveys/SurveySettings.tsx +++ b/frontend/src/scenes/surveys/SurveySettings.tsx @@ -33,7 +33,7 @@ export function SurveySettings({ inModal = false }: SurveySettingsProps): JSX.El Please note your website needs to have the{' '} PostHog snippet or at least version 1.81.1 of{' '} posthog-js From 50c21c9b5befef51f21f93e183c3357ad5e783b0 Mon Sep 17 00:00:00 2001 From: Tiina Turban Date: Wed, 3 Jan 2024 14:37:48 +0100 Subject: [PATCH 14/15] chore: Make plugin-server ignore deleted plugin configs (#18444) --- plugin-server/src/utils/db/sql.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugin-server/src/utils/db/sql.ts b/plugin-server/src/utils/db/sql.ts index 202f7f4ece5eb..c4e0f378fa766 100644 --- a/plugin-server/src/utils/db/sql.ts +++ b/plugin-server/src/utils/db/sql.ts @@ -21,7 +21,9 @@ function pluginConfigsInForceQuery(specificField?: keyof PluginConfig): string { LEFT JOIN posthog_organization ON posthog_organization.id = posthog_team.organization_id LEFT JOIN posthog_plugin ON posthog_plugin.id = posthog_pluginconfig.plugin_id WHERE ( - posthog_pluginconfig.enabled='t' AND posthog_organization.plugins_access_level > 0 + posthog_pluginconfig.enabled='t' + AND (posthog_pluginconfig.deleted is NULL OR posthog_pluginconfig.deleted!='t') + AND posthog_organization.plugins_access_level > 0 )` } From ed1611a2b95deafde05f1cc0c890fe715ff57d8d Mon Sep 17 00:00:00 2001 From: Tiina Turban Date: Wed, 3 Jan 2024 15:26:22 +0100 Subject: [PATCH 15/15] feat: populate plugin capabilities on install and edit (#19188) --- plugin-server/src/capabilities.ts | 4 +- plugin-server/src/main/pluginsServer.ts | 6 +++ plugin-server/src/types.ts | 2 +- plugin-server/src/utils/db/sql.ts | 31 ++++++++------ plugin-server/src/worker/tasks.ts | 4 ++ plugin-server/src/worker/vm/capabilities.ts | 10 ++--- plugin-server/src/worker/vm/lazy.ts | 40 +++++++++++++++++-- .../tests/worker/capabilities.test.ts | 2 +- posthog/api/plugin.py | 6 +++ posthog/models/plugin.py | 6 +++ 10 files changed, 87 insertions(+), 24 deletions(-) diff --git a/plugin-server/src/capabilities.ts b/plugin-server/src/capabilities.ts index 7a30b46438b36..d5cbe7f05cf5a 100644 --- a/plugin-server/src/capabilities.ts +++ b/plugin-server/src/capabilities.ts @@ -20,7 +20,7 @@ export function getPluginServerCapabilities(config: PluginsServerConfig): Plugin processAsyncWebhooksHandlers: true, sessionRecordingBlobIngestion: true, personOverrides: config.POE_DEFERRED_WRITES_ENABLED, - transpileFrontendApps: true, + appManagementSingleton: true, preflightSchedules: true, ...sharedCapabilities, } @@ -73,7 +73,7 @@ export function getPluginServerCapabilities(config: PluginsServerConfig): Plugin case PluginServerMode.scheduler: return { pluginScheduledTasks: true, - transpileFrontendApps: true, // TODO: move this away from pod startup, into a graphile job + appManagementSingleton: true, ...sharedCapabilities, } case PluginServerMode.person_overrides: diff --git a/plugin-server/src/main/pluginsServer.ts b/plugin-server/src/main/pluginsServer.ts index 5b94900e807f9..edf8a2f787833 100644 --- a/plugin-server/src/main/pluginsServer.ts +++ b/plugin-server/src/main/pluginsServer.ts @@ -397,6 +397,12 @@ export async function startPluginsServer( 'reset-available-features-cache': async (message) => { await piscina?.broadcastTask({ task: 'resetAvailableFeaturesCache', args: JSON.parse(message) }) }, + 'populate-plugin-capabilities': async (message) => { + // We need this to be done in only once + if (hub?.capabilities.appManagementSingleton && piscina) { + await piscina?.broadcastTask({ task: 'populatePluginCapabilities', args: JSON.parse(message) }) + } + }, }) await pubSub.start() diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts index 0d15899c84aa2..a7f45b18aeb21 100644 --- a/plugin-server/src/types.ts +++ b/plugin-server/src/types.ts @@ -296,7 +296,7 @@ export interface PluginServerCapabilities { processAsyncWebhooksHandlers?: boolean sessionRecordingBlobIngestion?: boolean personOverrides?: boolean - transpileFrontendApps?: boolean // TODO: move this away from pod startup, into a graphile job + appManagementSingleton?: boolean preflightSchedules?: boolean // Used for instance health checks on hobby deploy, not useful on cloud http?: boolean mmdb?: boolean diff --git a/plugin-server/src/utils/db/sql.ts b/plugin-server/src/utils/db/sql.ts index c4e0f378fa766..9d3643ce5a079 100644 --- a/plugin-server/src/utils/db/sql.ts +++ b/plugin-server/src/utils/db/sql.ts @@ -27,15 +27,7 @@ function pluginConfigsInForceQuery(specificField?: keyof PluginConfig): string { )` } -export async function getPluginRows(hub: Hub): Promise { - const { rows }: { rows: Plugin[] } = await hub.db.postgres.query( - PostgresUse.COMMON_READ, - // `posthog_plugin` columns have to be listed individually, as we want to exclude a few columns - // and Postgres syntax unfortunately doesn't have a column exclusion feature. The excluded columns are: - // - archive - this is a potentially large blob, only extracted in Django as a plugin server optimization - // - latest_tag - not used in this service - // - latest_tag_checked_at - not used in this service - `SELECT +const PLUGIN_SELECT = `SELECT posthog_plugin.id, posthog_plugin.name, posthog_plugin.url, @@ -63,7 +55,22 @@ export async function getPluginRows(hub: Hub): Promise { LEFT JOIN posthog_pluginsourcefile psf__frontend_tsx ON (psf__frontend_tsx.plugin_id = posthog_plugin.id AND psf__frontend_tsx.filename = 'frontend.tsx') LEFT JOIN posthog_pluginsourcefile psf__site_ts - ON (psf__site_ts.plugin_id = posthog_plugin.id AND psf__site_ts.filename = 'site.ts') + ON (psf__site_ts.plugin_id = posthog_plugin.id AND psf__site_ts.filename = 'site.ts')` + +export async function getPlugin(hub: Hub, pluginId: number): Promise { + const result = await hub.db.postgres.query( + PostgresUse.COMMON_READ, + `${PLUGIN_SELECT} WHERE posthog_plugin.id = $1`, + [pluginId], + 'getPlugin' + ) + return result.rows[0] +} + +export async function getPluginRows(hub: Hub): Promise { + const { rows }: { rows: Plugin[] } = await hub.db.postgres.query( + PostgresUse.COMMON_READ, + `${PLUGIN_SELECT} WHERE posthog_plugin.id IN (${pluginConfigsInForceQuery('plugin_id')} GROUP BY posthog_pluginconfig.plugin_id)`, undefined, @@ -96,13 +103,13 @@ export async function getPluginConfigRows(hub: Hub): Promise { export async function setPluginCapabilities( hub: Hub, - pluginConfig: PluginConfig, + pluginId: number, capabilities: PluginCapabilities ): Promise { await hub.db.postgres.query( PostgresUse.COMMON_WRITE, 'UPDATE posthog_plugin SET capabilities = ($1) WHERE id = $2', - [capabilities, pluginConfig.plugin_id], + [capabilities, pluginId], 'setPluginCapabilities' ) } diff --git a/plugin-server/src/worker/tasks.ts b/plugin-server/src/worker/tasks.ts index 87b216d07ac57..71243d7b2a18d 100644 --- a/plugin-server/src/worker/tasks.ts +++ b/plugin-server/src/worker/tasks.ts @@ -8,6 +8,7 @@ import { loadSchedule } from './plugins/loadSchedule' import { runPluginTask, runProcessEvent } from './plugins/run' import { setupPlugins } from './plugins/setup' import { teardownPlugins } from './plugins/teardown' +import { populatePluginCapabilities } from './vm/lazy' type TaskRunner = (hub: Hub, args: any) => Promise | any @@ -88,6 +89,9 @@ export const workerTasks: Record = { resetAvailableFeaturesCache: (hub, args: { organization_id: string }) => { hub.organizationManager.resetAvailableFeatureCache(args.organization_id) }, + populatePluginCapabilities: async (hub, args: { plugin_id: string }) => { + await populatePluginCapabilities(hub, Number(args.plugin_id)) + }, // Exported only for tests _testsRunProcessEvent: async (hub, args: { event: PluginEvent }) => { return runProcessEvent(hub, args.event) diff --git a/plugin-server/src/worker/vm/capabilities.ts b/plugin-server/src/worker/vm/capabilities.ts index bca6cc36f1e4a..5c4fa2e90386e 100644 --- a/plugin-server/src/worker/vm/capabilities.ts +++ b/plugin-server/src/worker/vm/capabilities.ts @@ -1,14 +1,14 @@ -import { PluginCapabilities, PluginConfigVMResponse, VMMethods } from '../../types' +import { PluginCapabilities, PluginTask, PluginTaskType, VMMethods } from '../../types' import { PluginServerCapabilities } from './../../types' const PROCESS_EVENT_CAPABILITIES = new Set(['ingestion', 'ingestionOverflow', 'ingestionHistorical']) -export function getVMPluginCapabilities(vm: PluginConfigVMResponse): PluginCapabilities { +export function getVMPluginCapabilities( + methods: VMMethods, + tasks: Record> +): PluginCapabilities { const capabilities: Required = { scheduled_tasks: [], jobs: [], methods: [] } - const tasks = vm?.tasks - const methods = vm?.methods - if (methods) { for (const [key, value] of Object.entries(methods)) { if (value as VMMethods[keyof VMMethods] | undefined) { diff --git a/plugin-server/src/worker/vm/lazy.ts b/plugin-server/src/worker/vm/lazy.ts index 52286b13ff937..9c1964a792269 100644 --- a/plugin-server/src/worker/vm/lazy.ts +++ b/plugin-server/src/worker/vm/lazy.ts @@ -14,7 +14,7 @@ import { VMMethods, } from '../../types' import { processError } from '../../utils/db/error' -import { disablePlugin, setPluginCapabilities } from '../../utils/db/sql' +import { disablePlugin, getPlugin, setPluginCapabilities } from '../../utils/db/sql' import { instrument } from '../../utils/metrics' import { getNextRetryMs } from '../../utils/retries' import { status } from '../../utils/status' @@ -307,12 +307,46 @@ export class LazyPluginVM { } private async updatePluginCapabilitiesIfNeeded(vm: PluginConfigVMResponse): Promise { - const capabilities = getVMPluginCapabilities(vm) + const capabilities = getVMPluginCapabilities(vm.methods, vm.tasks) const prevCapabilities = this.pluginConfig.plugin!.capabilities if (!equal(prevCapabilities, capabilities)) { - await setPluginCapabilities(this.hub, this.pluginConfig, capabilities) + await setPluginCapabilities(this.hub, this.pluginConfig.plugin_id, capabilities) this.pluginConfig.plugin!.capabilities = capabilities } } } + +export async function populatePluginCapabilities(hub: Hub, pluginId: number): Promise { + status.info('🔌', `Populating plugin capabilities for plugin ID ${pluginId}...`) + const plugin = await getPlugin(hub, pluginId) + if (!plugin) { + status.error('🔌', `Plugin with ID ${pluginId} not found for populating capabilities.`) + return + } + if (!plugin.source__index_ts) { + status.error('🔌', `Plugin with ID ${pluginId} has no index.ts file for populating capabilities.`) + return + } + + const { methods, tasks } = createPluginConfigVM( + hub, + { + id: 0, + plugin: plugin, + plugin_id: plugin.id, + team_id: 0, + enabled: false, + order: 0, + created_at: '0', + config: {}, + }, + plugin.source__index_ts || '' + ) + const capabilities = getVMPluginCapabilities(methods, tasks) + + const prevCapabilities = plugin.capabilities + if (!equal(prevCapabilities, capabilities)) { + await setPluginCapabilities(hub, pluginId, capabilities) + } +} diff --git a/plugin-server/tests/worker/capabilities.test.ts b/plugin-server/tests/worker/capabilities.test.ts index 6dadbef2e88da..8376f49de584b 100644 --- a/plugin-server/tests/worker/capabilities.test.ts +++ b/plugin-server/tests/worker/capabilities.test.ts @@ -28,7 +28,7 @@ describe('capabilities', () => { describe('getVMPluginCapabilities()', () => { function getCapabilities(indexJs: string): PluginCapabilities { const vm = createPluginConfigVM(hub, pluginConfig39, indexJs) - return getVMPluginCapabilities(vm) + return getVMPluginCapabilities(vm.methods, vm.tasks) } it('handles processEvent', () => { diff --git a/posthog/api/plugin.py b/posthog/api/plugin.py index 2cda937296883..c6cf5e28a5775 100644 --- a/posthog/api/plugin.py +++ b/posthog/api/plugin.py @@ -48,6 +48,7 @@ from posthog.plugins import can_configure_plugins, can_install_plugins, parse_url from posthog.plugins.access import can_globally_manage_plugins from posthog.queries.app_metrics.app_metrics import TeamPluginsDeliveryRateQuery +from posthog.redis import get_client from posthog.utils import format_query_params_absolute_url # Keep this in sync with: frontend/scenes/plugins/utils.ts @@ -439,6 +440,11 @@ def update_source(self, request: request.Request, **kwargs): if performed_changes: plugin.updated_at = now() plugin.save() + # Trigger capabilities update in plugin server, in case the app source changed the methods etc + get_client().publish( + "populate-plugin-capabilities", + json.dumps({"plugin_id": str(plugin.id)}), + ) return Response(response) @action(methods=["POST"], detail=True) diff --git a/posthog/models/plugin.py b/posthog/models/plugin.py index 34afc1918b1d8..bdd1a5f8f496e 100644 --- a/posthog/models/plugin.py +++ b/posthog/models/plugin.py @@ -1,4 +1,5 @@ import datetime +import json import os from dataclasses import dataclass from enum import Enum @@ -29,6 +30,7 @@ load_json_file, parse_url, ) +from posthog.redis import get_client from .utils import UUIDModel, sane_repr @@ -129,6 +131,10 @@ def install(self, **kwargs) -> "Plugin": plugin = Plugin.objects.create(**kwargs) if plugin_json: PluginSourceFile.objects.sync_from_plugin_archive(plugin, plugin_json) + get_client().publish( + "populate-plugin-capabilities", + json.dumps({"plugin_id": str(plugin.id)}), + ) return plugin