feat(hog): fix elements matching

frontend/src/scenes/pipeline/hogfunctions/hogFunctionConfigurationLogic.tsx

@@ -36,6 +36,7 @@ import {
     PipelineTab,
     PropertyFilterType,
     PropertyGroupFilter,
+    PropertyGroupFilterValue,
 } from '~/types'
 
 import { EmailTemplate } from './email-templater/emailTemplaterLogic'
@@ -448,10 +449,14 @@ export const hogFunctionConfigurationLogic = kea<hogFunctionConfigurationLogicTy
         sparklineQuery: [
             (s) => [s.configuration],
             (configuration): TrendsQuery => {
-                const properties: PropertyGroupFilter = {
+                const seriesProperties: PropertyGroupFilterValue = {
                     type: FilterLogicalOperator.Or,
                     values: [],
                 }
+                const properties: PropertyGroupFilter = {
+                    type: FilterLogicalOperator.And,
+                    values: [seriesProperties],
+                }
                 for (const event of configuration.filters?.events ?? []) {
                     const eventProperties: AnyPropertyFilter[] = [...(event.properties ?? [])]
                     if (event.id) {
@@ -466,7 +471,7 @@ export const hogFunctionConfigurationLogic = kea<hogFunctionConfigurationLogicTy
                             key: 'true',
                         })
                     }
-                    properties.values.push({
+                    seriesProperties.values.push({
                         type: FilterLogicalOperator.And,
                         values: eventProperties,
                     })
@@ -479,11 +484,21 @@ export const hogFunctionConfigurationLogic = kea<hogFunctionConfigurationLogicTy
                             key: hogql`matchesAction(${parseInt(action.id)})`,
                         })
                     }
-                    properties.values.push({
+                    seriesProperties.values.push({
                         type: FilterLogicalOperator.And,
                         values: actionProperties,
                     })
                 }
+                if ((configuration.filters?.properties?.length ?? 0) > 0) {
+                    const globalProperties: PropertyGroupFilterValue = {
+                        type: FilterLogicalOperator.And,
+                        values: [],
+                    }
+                    for (const property of configuration.filters?.properties ?? []) {
+                        globalProperties.values.push(property as AnyPropertyFilter)
+                    }
+                    properties.values.push(globalProperties)
+                }
 
                 return {
                     kind: NodeKind.TrendsQuery,

plugin-server/src/cdp/types.ts

@@ -46,20 +46,6 @@ export interface HogFunctionFilters {
     bytecode?: HogBytecode
 }
 
-// We have a "parsed" clickhous event type to make it easier to work with calls from kafka as well as those from the frontend
-export interface ParsedClickhouseEvent {
-    uuid: string
-    event: string
-    team_id: number
-    distinct_id: string
-    person_id?: string
-    timestamp: string
-    created_at: string
-    properties: Record<string, any>
-    person_created_at?: string
-    person_properties: Record<string, any>
-}
-
 export type GroupType = {
     id: string // the "key" of the group
     type: string
@@ -109,6 +95,10 @@ export type HogFunctionFilterGlobals = {
     event: string
     timestamp: string
     elements_chain: string
+    elements_chain_href: string
+    elements_chain_texts: string[]
+    elements_chain_ids: string[]
+    elements_chain_elements: string[]
     properties: Record<string, any>
 
     person?: {

plugin-server/src/cdp/utils.ts

@@ -1,6 +1,7 @@
 // NOTE: PostIngestionEvent is our context event - it should never be sent directly to an output, but rather transformed into a lightweight schema
 
 import { DateTime } from 'luxon'
+import RE2 from 're2'
 import { gunzip, gzip } from 'zlib'
 
 import { RawClickHouseEvent, Team, TimestampFormat } from '../types'
@@ -12,7 +13,6 @@ import {
     HogFunctionInvocationGlobals,
     HogFunctionInvocationResult,
     HogFunctionLogEntrySerialized,
-    ParsedClickhouseEvent,
 } from './types'
 
 export const PERSON_DEFAULT_DISPLAY_NAME_PROPERTIES = [
@@ -36,26 +36,6 @@ const getPersonDisplayName = (team: Team, distinctId: string, properties: Record
     return (customIdentifier || distinctId)?.trim()
 }
 
-export function convertToParsedClickhouseEvent(event: RawClickHouseEvent): ParsedClickhouseEvent {
-    const properties = event.properties ? JSON.parse(event.properties) : {}
-    if (event.elements_chain) {
-        properties['$elements_chain'] = event.elements_chain
-    }
-
-    return {
-        uuid: event.uuid,
-        event: event.event,
-        team_id: event.team_id,
-        distinct_id: event.distinct_id,
-        person_id: event.person_id,
-        timestamp: clickHouseTimestampToISO(event.timestamp),
-        created_at: clickHouseTimestampToISO(event.created_at),
-        properties: properties,
-        person_created_at: event.person_created_at ? clickHouseTimestampToISO(event.person_created_at) : undefined,
-        person_properties: event.person_properties ? JSON.parse(event.person_properties) : {},
-    }
-}
-
 // that we can keep to as a contract
 export function convertToHogFunctionInvocationGlobals(
     event: RawClickHouseEvent,
@@ -105,6 +85,46 @@ export function convertToHogFunctionInvocationGlobals(
     return context
 }
 
+function getElementsChainHref(elementsChain: string): string {
+    // Adapted from SQL: extract(elements_chain, '(?::|\")href="(.*?)"'),
+    const hrefRegex = new RE2(/(?::|")href="(.*?)"/)
+    const hrefMatch = hrefRegex.exec(elementsChain)
+    return hrefMatch ? hrefMatch[1] : ''
+}
+
+function getElementsChainTexts(elementsChain: string): string[] {
+    // Adapted from SQL: arrayDistinct(extractAll(elements_chain, '(?::|\")text="(.*?)"')),
+    const textRegex = new RE2(/(?::|")text="(.*?)"/g)
+    const textMatches = new Set<string>()
+    let textMatch
+    while ((textMatch = textRegex.exec(elementsChain)) !== null) {
+        textMatches.add(textMatch[1])
+    }
+    return Array.from(textMatches)
+}
+
+function getElementsChainIds(elementsChain: string): string[] {
+    // Adapted from SQL: arrayDistinct(extractAll(elements_chain, '(?::|\")id="(.*?)"')),
+    const idRegex = new RE2(/(?::|")id="(.*?)"/g)
+    const idMatches = new Set<string>()
+    let idMatch
+    while ((idMatch = idRegex.exec(elementsChain)) !== null) {
+        idMatches.add(idMatch[1])
+    }
+    return Array.from(idMatches)
+}
+
+function getElementsChainElements(elementsChain: string): string[] {
+    // Adapted from SQL: arrayDistinct(extractAll(elements_chain, '(?:^|;)(a|button|form|input|select|textarea|label)(?:\\.|$|:)'))
+    const elementRegex = new RE2(/(?:^|;)(a|button|form|input|select|textarea|label)(?:\.|$|:)/g)
+    const elementMatches = new Set<string>()
+    let elementMatch
+    while ((elementMatch = elementRegex.exec(elementsChain)) !== null) {
+        elementMatches.add(elementMatch[1])
+    }
+    return Array.from(elementMatches)
+}
+
 export function convertToHogFunctionFilterGlobal(globals: HogFunctionInvocationGlobals): HogFunctionFilterGlobals {
     const groups: Record<string, any> = {}
 
@@ -114,14 +134,36 @@ export function convertToHogFunctionFilterGlobal(globals: HogFunctionInvocationG
         }
     }
 
-    return {
+    const elementsChain = globals.event.properties['$elements_chain']
+    const response = {
         event: globals.event.name,
-        elements_chain: globals.event.properties['$elements_chain'],
+        elements_chain: elementsChain,
+        elements_chain_href: '',
+        elements_chain_texts: [],
+        elements_chain_ids: [],
+        elements_chain_elements: [],
         timestamp: globals.event.timestamp,
         properties: globals.event.properties,
         person: globals.person ? { properties: globals.person.properties } : undefined,
         ...groups,
     }
+    if (elementsChain) {
+        Object.defineProperties(response, {
+            elements_chain_href: {
+                get: () => getElementsChainHref(elementsChain),
+            },
+            elements_chain_texts: {
+                get: () => getElementsChainTexts(elementsChain),
+            },
+            elements_chain_ids: {
+                get: () => getElementsChainIds(elementsChain),
+            },
+            elements_chain_elements: {
+                get: () => getElementsChainElements(elementsChain),
+            },
+        })
+    }
+    return response
 }
 
 export const convertToCaptureEvent = (event: HogFunctionCapturedEvent, team: Team): any => {

plugin-server/tests/cdp/examples.ts

@@ -417,6 +417,56 @@ export const HOG_FILTERS_EXAMPLES: Record<string, Pick<HogFunctionType, 'filters
             ],
         },
     },
+    elements_text_filter: {
+        filters: {
+            actions: [],
+            bytecode: [
+                '_H',
+                1,
+                32,
+                '$autocapture',
+                32,
+                'event',
+                1,
+                1,
+                11,
+                52,
+                'lambda',
+                1,
+                0,
+                6,
+                32,
+                '%reload%',
+                36,
+                0,
+                18,
+                38,
+                53,
+                0,
+                32,
+                'elements_chain_texts',
+                1,
+                1,
+                2,
+                'arrayExists',
+                2,
+                3,
+                2,
+                4,
+                1,
+            ],
+            events: [
+                {
+                    id: '$autocapture',
+                    name: '$autocapture',
+                    order: 0,
+                    properties: [{ key: 'text', operator: 'icontains', type: 'element', value: 'reload' }],
+                    type: 'events',
+                },
+            ],
+            filter_test_accounts: false,
+        },
+    },
 }
 
 export const HOG_MASK_EXAMPLES: Record<string, Pick<HogFunctionType, 'masking'>> = {

plugin-server/tests/cdp/hog-executor.test.ts

@@ -263,6 +263,56 @@ describe('Hog Executor', () => {
             expect(resultsShouldMatch.matchingFunctions).toHaveLength(1)
             expect(resultsShouldMatch.nonMatchingFunctions).toHaveLength(0)
         })
+
+        it('can use elements_chain materialized fields', () => {
+            const fn = createHogFunction({
+                ...HOG_EXAMPLES.simple_fetch,
+                ...HOG_INPUTS_EXAMPLES.simple_fetch,
+                ...HOG_FILTERS_EXAMPLES.elements_text_filter,
+            })
+
+            mockFunctionManager.getTeamHogFunctions.mockReturnValue([fn])
+            const elementsChain = (buttonText: string) =>
+                `span.LemonButton__content:attr__class="LemonButton__content"nth-child="2"nth-of-type="2"text="${buttonText}";span.LemonButton__chrome:attr__class="LemonButton__chrome"nth-child="1"nth-of-type="1";button.LemonButton.LemonButton--has-icon.LemonButton--secondary.LemonButton--status-default:attr__class="LemonButton LemonButton--secondary LemonButton--status-default LemonButton--has-icon"attr__type="button"nth-child="1"nth-of-type="1"text="${buttonText}";div.flex.gap-4.items-center:attr__class="flex gap-4 items-center"nth-child="1"nth-of-type="1";div.flex.flex-wrap.gap-4.justify-between:attr__class="flex gap-4 justify-between flex-wrap"nth-child="3"nth-of-type="3";div.flex.flex-1.flex-col.gap-4.h-full.relative.w-full:attr__class="relative w-full flex flex-col gap-4 flex-1 h-full"nth-child="1"nth-of-type="1";div.LemonTabs__content:attr__class="LemonTabs__content"nth-child="2"nth-of-type="1";div.LemonTabs.LemonTabs--medium:attr__class="LemonTabs LemonTabs--medium"attr__style="--lemon-tabs-slider-width: 48px; --lemon-tabs-slider-offset: 0px;"nth-child="1"nth-of-type="1";div.Navigation3000__scene:attr__class="Navigation3000__scene"nth-child="2"nth-of-type="2";main:nth-child="2"nth-of-type="1";div.Navigation3000:attr__class="Navigation3000"nth-child="1"nth-of-type="1";div:attr__id="root"attr_id="root"nth-child="3"nth-of-type="1";body.overflow-hidden:attr__class="overflow-hidden"attr__theme="light"nth-child="2"nth-of-type="1"`
+
+            const hogGlobals1 = createHogExecutionGlobals({
+                groups: {},
+                event: {
+                    uuid: 'uuid',
+                    name: '$autocapture',
+                    distinct_id: 'distinct_id',
+                    url: 'http://localhost:8000/events/1',
+                    properties: {
+                        $lib_version: '1.2.3',
+                        $elements_chain: elementsChain('Not our text'),
+                    },
+                    timestamp: new Date().toISOString(),
+                },
+            })
+
+            const resultsShouldntMatch = executor.findMatchingFunctions(hogGlobals1)
+            expect(resultsShouldntMatch.matchingFunctions).toHaveLength(0)
+            expect(resultsShouldntMatch.nonMatchingFunctions).toHaveLength(1)
+
+            const hogGlobals2 = createHogExecutionGlobals({
+                groups: {},
+                event: {
+                    uuid: 'uuid',
+                    name: '$autocapture',
+                    distinct_id: 'distinct_id',
+                    url: 'http://localhost:8000/events/1',
+                    properties: {
+                        $lib_version: '1.2.3',
+                        $elements_chain: elementsChain('Reload'),
+                    },
+                    timestamp: new Date().toISOString(),
+                },
+            })
+
+            const resultsShouldMatch = executor.findMatchingFunctions(hogGlobals2)
+            expect(resultsShouldMatch.matchingFunctions).toHaveLength(1)
+            expect(resultsShouldMatch.nonMatchingFunctions).toHaveLength(0)
+        })
     })
 
     describe('async function responses', () => {

posthog/hogql/property.py

@@ -550,23 +550,26 @@ def action_to_expr(action: Action) -> ast.Expr:
             if step.text is not None:
                 value = step.text
                 if step.text_matching == "regex":
-                    match = ast.CompareOperationOp.Regex
+                    exprs.append(
+                        parse_expr(
+                            "arrayExists(x -> x =~ {value}, elements_chain_texts)",
+                            {"value": ast.Constant(value=value)},
+                        )
+                    )
                 elif step.text_matching == "contains":
-                    match = ast.CompareOperationOp.ILike
-                    value = f"%{value}%"
+                    exprs.append(
+                        parse_expr(
+                            "arrayExists(x -> x ilike {value}, elements_chain_texts)",
+                            {"value": ast.Constant(value=f"%{value}%")},
+                        )
+                    )
                 else:
-                    match = ast.CompareOperationOp.Eq
-
-                exprs.append(
-                    parse_expr(
-                        "arrayExists(x -> {match}, elements_chain_texts)",
-                        {
-                            "match": ast.CompareOperation(
-                                op=match, left=ast.Field(chain=["x"]), right=ast.Constant(value=value)
-                            )
-                        },
+                    exprs.append(
+                        parse_expr(
+                            "arrayExists(x -> x = {value}, elements_chain_texts)",
+                            {"value": ast.Constant(value=value)},
+                        )
                     )
-                )
         if step.url:
             if step.url_matching == "exact":
                 expr = parse_expr(

posthog/hogql/test/test_property.py

@@ -649,6 +649,24 @@ def test_action_to_expr(self):
             self._parse_expr("event = '$autocapture' and arrayExists(x -> x =~ 'blabla', elements_chain_texts)"),
         )
 
+        action7 = Action.objects.create(
+            team=self.team,
+            steps_json=[{"event": "$autocapture", "text": "blabla", "text_matching": "contains"}],
+        )
+        self.assertEqual(
+            clear_locations(action_to_expr(action7)),
+            self._parse_expr("event = '$autocapture' and arrayExists(x -> x ilike '%blabla%', elements_chain_texts)"),
+        )
+
+        action8 = Action.objects.create(
+            team=self.team,
+            steps_json=[{"event": "$autocapture", "text": "blabla", "text_matching": "exact"}],
+        )
+        self.assertEqual(
+            clear_locations(action_to_expr(action8)),
+            self._parse_expr("event = '$autocapture' and arrayExists(x -> x = 'blabla', elements_chain_texts)"),
+        )
+
     def test_cohort_filter_static(self):
         cohort = Cohort.objects.create(
             team=self.team,