Merge branch 'master' into dw-assume-schema-timestamp

.github/workflows/ci-backend-depot.yml

@@ -5,15 +5,7 @@
 name: Backend CI (depot)
 
 on:
-    push:
-        branches:
-            - master
     pull_request:
-    workflow_dispatch:
-        inputs:
-            clickhouseServerVersion:
-                description: ClickHouse server version. Leave blank for default
-                type: string
 
 concurrency:
     group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -41,7 +33,7 @@ jobs:
     changes:
         runs-on: depot-ubuntu-latest-4
         timeout-minutes: 5
-        if: github.repository == 'PostHog/posthog'
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'test-depot') }}
         name: Determine need to run backend checks
         # Set job outputs to values from filter step
         outputs:

.github/workflows/ci-e2e-depot.yml

@@ -16,7 +16,7 @@ jobs:
     changes:
         runs-on: depot-ubuntu-latest-4
         timeout-minutes: 5
-        if: github.repository == 'PostHog/posthog'
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'test-depot') }}
         name: Determine need to run E2E checks
         # Set job outputs to values from filter step
         outputs:

ee/session_recordings/ai/error_clustering.py

@@ -6,6 +6,7 @@
 import pandas as pd
 import numpy as np
 from posthog.session_recordings.models.session_recording_event import SessionRecordingViewed
+from datetime import date
 
 CLUSTER_REPLAY_ERRORS_TIMING = Histogram(
     "posthog_session_recordings_cluster_replay_errors",
@@ -30,7 +31,7 @@ def error_clustering(team: Team, user: User):
     if not results:
         return []
 
-    df = pd.DataFrame(results, columns=["session_id", "input", "embeddings"])
+    df = pd.DataFrame(results, columns=["session_id", "error", "embeddings", "timestamp"])
 
     df["cluster"] = cluster_embeddings(df["embeddings"].tolist())
 
@@ -42,7 +43,7 @@ def error_clustering(team: Team, user: User):
 def fetch_error_embeddings(team_id: int):
     query = """
             SELECT
-                session_id, input, embeddings
+                session_id, input, embeddings, generation_timestamp
             FROM
                 session_replay_embeddings
             WHERE
@@ -76,13 +77,21 @@ def construct_response(df: pd.DataFrame, team: Team, user: User):
     clusters = []
     for cluster, rows in df.groupby("cluster"):
         session_ids = rows["session_id"].unique()
-        sample = rows.sample(n=1)[["session_id", "input"]].rename(columns={"input": "error"}).to_dict("records")[0]
+        sample = rows.sample(n=1)[["session_id", "error"]].to_dict("records")[0]
+
+        date_series = (
+            df.groupby([df["timestamp"].dt.date])
+            .size()
+            .reindex(pd.date_range(end=date.today(), periods=7), fill_value=0)
+        )
+        sparkline = dict(zip(date_series.index.astype(str), date_series))
         clusters.append(
             {
                 "cluster": cluster,
                 "sample": sample.get("error"),
                 "session_ids": np.random.choice(session_ids, size=DBSCAN_MIN_SAMPLES - 1),
                 "occurrences": rows.size,
+                "sparkline": sparkline,
                 "unique_sessions": len(session_ids),
                 "viewed": len(np.intersect1d(session_ids, viewed_session_ids, assume_unique=True)),
             }

frontend/src/lib/components/DefinitionPopover/DefinitionPopoverContents.tsx

@@ -111,8 +111,16 @@ function DefinitionView({ group }: { group: TaxonomicFilterGroup }): JSX.Element
     } = useValues(definitionPopoverLogic)
 
     const { setLocalDefinition } = useActions(definitionPopoverLogic)
+    const { selectedItemMeta } = useValues(taxonomicFilterLogic)
     const { selectItem } = useActions(taxonomicFilterLogic)
 
+    // Use effect here to make definition view stateful. TaxonomicFilterLogic won't mount within definitionPopoverLogic
+    useEffect(() => {
+        if (selectedItemMeta && definition.name == selectedItemMeta.id) {
+            setLocalDefinition(selectedItemMeta)
+        }
+    }, [definition])
+
     if (!definition) {
         return <></>
     }
@@ -280,6 +288,7 @@ function DefinitionView({ group }: { group: TaxonomicFilterGroup }): JSX.Element
             value: column.key,
         }))
         const itemValue = localDefinition ? group?.getValue?.(localDefinition) : null
+
         return (
             <form className="definition-popover-data-warehouse-schema-form">
                 <div className="flex flex-col justify-between gap-4">

frontend/src/lib/components/TaxonomicFilter/InfiniteList.tsx

@@ -173,7 +173,6 @@ export function InfiniteList({ popupAnchorElement }: InfiniteListProps): JSX.Ele
     const { mouseInteractionsEnabled, activeTab, searchQuery, value, groupType, eventNames } =
         useValues(taxonomicFilterLogic)
     const { selectItem } = useActions(taxonomicFilterLogic)
-
     const {
         isLoading,
         results,

frontend/src/lib/components/TaxonomicFilter/TaxonomicFilter.tsx

@@ -21,6 +21,7 @@ export function TaxonomicFilter({
     taxonomicFilterLogicKey: taxonomicFilterLogicKeyInput,
     groupType,
     value,
+    filter,
     onChange,
     onClose,
     taxonomicGroupTypes,
@@ -48,6 +49,7 @@ export function TaxonomicFilter({
         taxonomicFilterLogicKey,
         groupType,
         value,
+        filter,
         onChange,
         taxonomicGroupTypes,
         optionsFromProp,

frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.tsx

@@ -141,6 +141,7 @@ export const taxonomicFilterLogic = kea<taxonomicFilterLogicType>([
         ],
     })),
     selectors({
+        selectedItemMeta: [() => [(_, props) => props.filter], (filter) => filter],
         taxonomicFilterLogicKey: [
             (_, p) => [p.taxonomicFilterLogicKey],
             (taxonomicFilterLogicKey) => taxonomicFilterLogicKey,
@@ -218,7 +219,7 @@ export const taxonomicFilterLogic = kea<taxonomicFilterLogicType>([
                         logic: dataWarehouseSceneLogic,
                         value: 'externalTables',
                         getName: (table: DataWarehouseTableType) => table.name,
-                        getValue: (table: DataWarehouseTableType) => table.id,
+                        getValue: (table: DataWarehouseTableType) => table.name,
                         getPopoverHeader: () => 'Data Warehouse Table',
                         getIcon: () => <IconServer />,
                     },

frontend/src/lib/components/TaxonomicFilter/types.ts

@@ -1,6 +1,7 @@
 import Fuse from 'fuse.js'
 import { BuiltLogic, LogicWrapper } from 'kea'
 import { DataWarehouseTableType } from 'scenes/data-warehouse/types'
+import { LocalFilter } from 'scenes/insights/filters/ActionFilter/entityFilterLogic'
 
 import { AnyDataNode, DatabaseSchemaQueryResponseField } from '~/queries/schema'
 import {
@@ -22,6 +23,7 @@ export interface TaxonomicFilterProps {
     value?: TaxonomicFilterValue
     onChange?: (group: TaxonomicFilterGroup, value: TaxonomicFilterValue, item: any) => void
     onClose?: () => void
+    filter?: LocalFilter
     taxonomicGroupTypes: TaxonomicFilterGroupType[]
     taxonomicFilterLogicKey?: string
     optionsFromProp?: Partial<Record<TaxonomicFilterGroupType, SimpleOption[]>>

frontend/src/lib/components/TaxonomicPopover/TaxonomicPopover.tsx

@@ -4,6 +4,7 @@ import { TaxonomicFilterGroupType, TaxonomicFilterValue } from 'lib/components/T
 import { LemonButton, LemonButtonProps } from 'lib/lemon-ui/LemonButton'
 import { LemonDropdown } from 'lib/lemon-ui/LemonDropdown'
 import { useEffect, useState } from 'react'
+import { LocalFilter } from 'scenes/insights/filters/ActionFilter/entityFilterLogic'
 
 import { AnyDataNode, DatabaseSchemaQueryResponseField } from '~/queries/schema'
 
@@ -13,6 +14,7 @@ export interface TaxonomicPopoverProps<ValueType extends TaxonomicFilterValue =
     value?: ValueType
     onChange: (value: ValueType, groupType: TaxonomicFilterGroupType, item: any) => void
 
+    filter?: LocalFilter
     groupTypes?: TaxonomicFilterGroupType[]
     renderValue?: (value: ValueType) => JSX.Element | null
     eventNames?: string[]
@@ -41,6 +43,7 @@ export function TaxonomicStringPopover(props: TaxonomicPopoverProps<string>): JS
 export function TaxonomicPopover<ValueType extends TaxonomicFilterValue = TaxonomicFilterValue>({
     groupType,
     value,
+    filter,
     onChange,
     renderValue,
     groupTypes,
@@ -81,6 +84,7 @@ export function TaxonomicPopover<ValueType extends TaxonomicFilterValue = Taxono
                 <TaxonomicFilter
                     groupType={groupType}
                     value={value}
+                    filter={filter}
                     onChange={({ type }, payload, item) => {
                         onChange?.(payload as ValueType, type, item)
                         setVisible(false)

frontend/src/queries/query.ts

@@ -220,7 +220,13 @@ export async function query<N extends DataNode = DataNode>(
                 (hogQLInsightsFunnelsFlagEnabled && isFunnelsQuery(queryNode))
             ) {
                 if (hogQLInsightsLiveCompareEnabled) {
-                    const legacyFunction = legacyUrl ? fetchLegacyUrl : fetchLegacyInsights
+                    const legacyFunction = (): any => {
+                        try {
+                            return legacyUrl ? fetchLegacyUrl : fetchLegacyInsights
+                        } catch (e) {
+                            console.error('Error fetching legacy insights', e)
+                        }
+                    }
                     let legacyResponse: any
                     ;[response, legacyResponse] = await Promise.all([
                         executeQuery(queryNode, methodOptions, refresh, queryId),

frontend/src/scenes/batch_exports/BatchExportEditForm.tsx

@@ -242,6 +242,15 @@ export function BatchExportsEditFields({
                                     ]}
                                 />
                             </LemonField>
+
+                            <LemonField name="file_format" label="Format" className="flex-1">
+                                <LemonSelect
+                                    options={[
+                                        { value: 'JSONLines', label: 'JSON lines' },
+                                        { value: 'Parquet', label: 'Apache Parquet' },
+                                    ]}
+                                />
+                            </LemonField>
                         </div>
 
                         <div className="flex gap-4">
@@ -375,17 +384,21 @@ export function BatchExportsEditFields({
                         </LemonField>
 
                         <LemonField name="has_self_signed_cert">
-                            <LemonCheckbox
-                                bordered
-                                label={
-                                    <span className="flex items-center gap-2">
-                                        Does your Postgres instance have a self-signed SSL certificate?
-                                        <Tooltip title="In most cases, Heroku and RDS users should check this.">
-                                            <IconInfo className=" text-lg text-muted-alt" />
-                                        </Tooltip>
-                                    </span>
-                                }
-                            />
+                            {({ value, onChange }) => (
+                                <LemonCheckbox
+                                    bordered
+                                    label={
+                                        <span className="flex items-center gap-2">
+                                            Does your Postgres instance have a self-signed SSL certificate?
+                                            <Tooltip title="In most cases, Heroku and RDS users should check this.">
+                                                <IconInfo className=" text-lg text-muted-alt" />
+                                            </Tooltip>
+                                        </span>
+                                    }
+                                    checked={!!value}
+                                    onChange={onChange}
+                                />
+                            )}
                         </LemonField>
 
                         <LemonField name="exclude_events" label="Events to exclude" className="flex-1">

frontend/src/scenes/batch_exports/BatchExports.stories.tsx

@@ -42,6 +42,7 @@ export default {
                             include_events: [],
                             encryption: null,
                             kms_key_id: null,
+                            file_format: 'JSONLines',
                         },
                     },
                     start_at: null,

frontend/src/scenes/batch_exports/batchExportEditLogic.ts

@@ -90,6 +90,7 @@ export const batchExportFormFields = (
                   aws_secret_access_key: isNew ? (!config.aws_secret_access_key ? 'This field is required' : '') : '',
                   compression: '',
                   encryption: '',
+                  file_format: isNew ? (!config.file_format ? 'This field is required' : '') : '',
                   kms_key_id: !config.kms_key_id && config.encryption == 'aws:kms' ? 'This field is required' : '',
                   exclude_events: '',
                   include_events: '',

frontend/src/scenes/data-warehouse/ViewLinkModal.tsx

@@ -1,6 +1,6 @@
 import './ViewLinkModal.scss'
 
-import { IconTrash } from '@posthog/icons'
+import { IconCollapse, IconExpand, IconTrash } from '@posthog/icons'
 import {
     LemonButton,
     LemonDivider,
@@ -35,7 +35,7 @@ export function ViewLinkModal(): JSX.Element {
             }
             isOpen={isJoinTableModalOpen}
             onClose={toggleJoinTableModal}
-            width={600}
+            width={700}
         >
             <ViewLinkForm />
         </LemonModal>
@@ -57,6 +57,7 @@ export function ViewLinkForm(): JSX.Element {
         selectedJoiningKey,
         sourceIsUsingHogQLExpression,
         joiningIsUsingHogQLExpression,
+        isViewLinkSubmitting,
     } = useValues(viewLinkLogic)
     const {
         selectJoiningTable,
@@ -66,12 +67,13 @@ export function ViewLinkForm(): JSX.Element {
         selectSourceKey,
         selectJoiningKey,
     } = useActions(viewLinkLogic)
+    const [advancedSettingsExpanded, setAdvancedSettingsExpanded] = useState(false)
 
     return (
         <Form logic={viewLinkLogic} formKey="viewLink" enableFormOnSubmit>
             <div className="flex flex-col w-full justify-between items-center">
                 <div className="flex flex-row w-full justify-between">
-                    <div className={isNewJoin ? 'w-50' : 'flex flex-col'}>
+                    <div className={isNewJoin ? 'w-60' : 'flex flex-col'}>
                         <span className="l4">Source Table</span>
                         {isNewJoin ? (
                             <Field name="source_table_name">
@@ -86,7 +88,7 @@ export function ViewLinkForm(): JSX.Element {
                             selectedSourceTableName ?? ''
                         )}
                     </div>
-                    <div className="w-50">
+                    <div className="w-60">
                         <span className="l4">Joining Table</span>
                         <Field name="joining_table_name">
                             <LemonSelect
@@ -98,8 +100,8 @@ export function ViewLinkForm(): JSX.Element {
                         </Field>
                     </div>
                 </div>
-                <div className="mt-3 flex flex-row justify-between items-center w-full">
-                    <div className="w-50">
+                <div className="mt-4 flex flex-row justify-between items-center w-full">
+                    <div className="w-60">
                         <span className="l4">Source Table Key</span>
                         <Field name="source_table_key">
                             <>
@@ -124,7 +126,7 @@ export function ViewLinkForm(): JSX.Element {
                     <div className="mt-5">
                         <IconSwapHoriz />
                     </div>
-                    <div className="w-50">
+                    <div className="w-60">
                         <span className="l4">Joining Table Key</span>
                         <Field name="joining_table_key">
                             <>
@@ -148,8 +150,22 @@ export function ViewLinkForm(): JSX.Element {
                     </div>
                 </div>
                 {sqlCodeSnippet && (
-                    <>
+                    <div className="w-full mt-2">
                         <LemonDivider className="mt-4 mb-4" />
+                        <LemonButton
+                            fullWidth
+                            onClick={() => setAdvancedSettingsExpanded(!advancedSettingsExpanded)}
+                            sideIcon={advancedSettingsExpanded ? <IconCollapse /> : <IconExpand />}
+                        >
+                            <div>
+                                <h3 className="l4 mt-2">Advanced settings</h3>
+                                <div className="text-muted mb-2 font-medium">Customize how the fields are accessed</div>
+                            </div>
+                        </LemonButton>
+                    </div>
+                )}
+                {sqlCodeSnippet && advancedSettingsExpanded && (
+                    <>
                         <div className="mt-3 flex flex-row justify-between items-center w-full">
                             <div className="w-full">
                                 <span className="l4">Field Name</span>
@@ -185,7 +201,7 @@ export function ViewLinkForm(): JSX.Element {
                 <LemonButton className="mr-3" type="secondary" onClick={toggleJoinTableModal}>
                     Close
                 </LemonButton>
-                <LemonButton type="primary" htmlType="submit">
+                <LemonButton type="primary" htmlType="submit" loading={isViewLinkSubmitting}>
                     Save
                 </LemonButton>
             </div>

frontend/src/scenes/insights/filters/ActionFilter/ActionFilterRow/ActionFilterRow.tsx

@@ -228,6 +228,7 @@ export function ActionFilterRow({
             fullWidth
             groupType={filter.type as TaxonomicFilterGroupType}
             value={getValue(value, filter)}
+            filter={filter}
             onChange={(changedValue, taxonomicGroupType, item) => {
                 const groupType = taxonomicFilterGroupTypeToEntityType(taxonomicGroupType)
                 if (groupType === EntityTypes.DATA_WAREHOUSE) {