Skip to content

Commit

Permalink
Merge branch 'master' into speed-up-person-queries
Browse files Browse the repository at this point in the history
  • Loading branch information
timgl committed Oct 17, 2024
2 parents d57b6c4 + f79f573 commit 9f2d558
Show file tree
Hide file tree
Showing 52 changed files with 1,434 additions and 1,141 deletions.
14 changes: 7 additions & 7 deletions docker/clickhouse/user_defined_function.xml
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@
<function>
<type>executable_pool</type>
<name>aggregate_funnel_trends</name>
<return_type>Array(Tuple(UInt64, Int8, Nullable(String)))</return_type>
<return_type>Array(Tuple(UInt64, Int8, Nullable(String), UUID))</return_type>
<return_name>result</return_name>
<argument>
<type>UInt8</type>
Expand Down Expand Up @@ -169,7 +169,7 @@
<name>prop_vals</name>
</argument>
<argument>
<type>Array(Tuple(Nullable(Float64), UInt64, Nullable(String), Array(Int8)))</type>
<type>Array(Tuple(Nullable(Float64), UInt64, UUID, Nullable(String), Array(Int8)))</type>
<name>value</name>
</argument>
<format>JSONEachRow</format>
Expand All @@ -181,7 +181,7 @@
<type>executable_pool</type>
<name>aggregate_funnel_array_trends</name>
<!-- Return type for trends is a start interval time, a success flag (1 or -1), and a breakdown value -->
<return_type>Array(Tuple(UInt64, Int8, Array(String)))</return_type>
<return_type>Array(Tuple(UInt64, Int8, Array(String), UUID))</return_type>
<return_name>result</return_name>
<argument>
<type>UInt8</type>
Expand All @@ -208,7 +208,7 @@
<name>prop_vals</name>
</argument>
<argument>
<type>Array(Tuple(Nullable(Float64), UInt64, Array(String), Array(Int8)))</type>
<type>Array(Tuple(Nullable(Float64), UInt64, UUID, Array(String), Array(Int8)))</type>
<name>value</name>
</argument>
<format>JSONEachRow</format>
Expand All @@ -220,7 +220,7 @@
<type>executable_pool</type>
<name>aggregate_funnel_cohort_trends</name>
<!-- Return type for trends is a start interval time, a success flag (1 or -1), and a breakdown value -->
<return_type>Array(Tuple(UInt64, Int8, UInt64))</return_type>
<return_type>Array(Tuple(UInt64, Int8, UInt64, UUID))</return_type>
<return_name>result</return_name>
<argument>
<type>UInt8</type>
Expand All @@ -247,7 +247,7 @@
<name>prop_vals</name>
</argument>
<argument>
<type>Array(Tuple(Nullable(Float64), UInt64, UInt64, Array(Int8)))</type>
<type>Array(Tuple(Nullable(Float64), UInt64, UUID, UInt64, Array(Int8)))</type>
<name>value</name>
</argument>
<format>JSONEachRow</format>
Expand Down Expand Up @@ -285,7 +285,7 @@
<name>prop_vals</name>
</argument>
<argument>
<type>Array(Tuple(Nullable(Float64), UInt64, Array(String), Array(Int8)))</type>
<type>Array(Tuple(Nullable(Float64), UInt64, UUID, Array(String), Array(Int8)))</type>
<name>value</name>
</argument>
<format>JSONEachRow</format>
Expand Down
54 changes: 24 additions & 30 deletions ee/clickhouse/views/groups.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,38 +177,32 @@ def property_definitions(self, request: request.Request, **kw):

return response.Response(group_type_index_to_properties)

@extend_schema(
parameters=[
OpenApiParameter(
"group_type_index",
OpenApiTypes.INT,
description="Specify the group type to find property values of",
required=True,
),
OpenApiParameter(
"key",
OpenApiTypes.STR,
description="Specify the property key to find values for",
required=True,
),
]
)
@action(methods=["GET"], detail=False)
def property_values(self, request: request.Request, **kw):
rows = sync_execute(
f"""
SELECT {trim_quotes_expr("tupleElement(keysAndValues, 2)")} as value
value_filter = request.GET.get("value")

query = f"""
SELECT {trim_quotes_expr("tupleElement(keysAndValues, 2)")} as value, count(*) as count
FROM groups
ARRAY JOIN JSONExtractKeysAndValuesRaw(group_properties) as keysAndValues
WHERE team_id = %(team_id)s AND group_type_index = %(group_type_index)s AND tupleElement(keysAndValues, 1) = %(key)s
GROUP BY tupleElement(keysAndValues, 2)
ORDER BY value ASC
""",
{
"team_id": self.team.pk,
"group_type_index": request.GET["group_type_index"],
"key": request.GET["key"],
},
)
WHERE team_id = %(team_id)s
AND group_type_index = %(group_type_index)s
AND tupleElement(keysAndValues, 1) = %(key)s
{f"AND {trim_quotes_expr('tupleElement(keysAndValues, 2)')} ILIKE %(value_filter)s" if value_filter else ""}
GROUP BY value
ORDER BY count DESC, value ASC
LIMIT 20
"""

params = {
"team_id": self.team.pk,
"group_type_index": request.GET["group_type_index"],
"key": request.GET["key"],
}

if value_filter:
params["value_filter"] = f"%{value_filter}%"

rows = sync_execute(query, params)

return response.Response([{"name": name[0]} for name in rows])
return response.Response([{"name": name, "count": count} for name, count in rows])
56 changes: 55 additions & 1 deletion ee/clickhouse/views/test/test_clickhouse_groups.py
Original file line number Diff line number Diff line change
Expand Up @@ -309,17 +309,71 @@ def test_property_values(self):
group_key="org:6",
properties={"industry": "technology"},
)
create_group(
team_id=self.team.pk,
group_type_index=0,
group_key="org:7",
properties={"industry": "finance-technology"},
)
create_group(
team_id=self.team.pk,
group_type_index=1,
group_key="org:1",
properties={"industry": "finance"},
)

# Test without query parameter
response_data = self.client.get(
f"/api/projects/{self.team.id}/groups/property_values/?key=industry&group_type_index=0"
).json()
self.assertEqual(len(response_data), 3)
self.assertEqual(
response_data,
[
{"name": "finance", "count": 1},
{"name": "finance-technology", "count": 1},
{"name": "technology", "count": 1},
],
)

# Test with query parameter
response_data = self.client.get(
f"/api/projects/{self.team.id}/groups/property_values/?key=industry&group_type_index=0&value=fin"
).json()
self.assertEqual(len(response_data), 2)
self.assertEqual(response_data, [{"name": "finance", "count": 1}, {"name": "finance-technology", "count": 1}])

# Test with query parameter - case insensitive
response_data = self.client.get(
f"/api/projects/{self.team.id}/groups/property_values/?key=industry&group_type_index=0&value=TECH"
).json()
self.assertEqual(len(response_data), 2)
self.assertEqual(response_data, [{"name": "finance"}, {"name": "technology"}])
self.assertEqual(
response_data, [{"name": "finance-technology", "count": 1}, {"name": "technology", "count": 1}]
)

# Test with query parameter - no matches
response_data = self.client.get(
f"/api/projects/{self.team.id}/groups/property_values/?key=industry&group_type_index=0&value=healthcare"
).json()
self.assertEqual(len(response_data), 0)
self.assertEqual(response_data, [])

# Test with query parameter - exact match
response_data = self.client.get(
f"/api/projects/{self.team.id}/groups/property_values/?key=industry&group_type_index=0&value=technology"
).json()
self.assertEqual(len(response_data), 2)
self.assertEqual(
response_data, [{"name": "finance-technology", "count": 1}, {"name": "technology", "count": 1}]
)

# Test with different group_type_index
response_data = self.client.get(
f"/api/projects/{self.team.id}/groups/property_values/?key=industry&group_type_index=1&value=fin"
).json()
self.assertEqual(len(response_data), 1)
self.assertEqual(response_data, [{"name": "finance", "count": 1}])

def test_empty_property_values(self):
create_group(
Expand Down
Binary file added frontend/public/services/airtable.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { IconBalance } from '@posthog/icons'
import { LemonBanner, LemonButton, LemonDivider, LemonInput, LemonTable, Link, Spinner } from '@posthog/lemon-ui'
import { useActions, useValues } from 'kea'
import { router } from 'kea-router'
Expand Down Expand Up @@ -99,8 +100,11 @@ export const SidePanelExperimentFeatureFlag = (): JSX.Element => {
title: (
<div className="flex items-center justify-between space-x-2">
<span>Rollout Percentage</span>
<LemonButton type="secondary" size="xsmall" onClick={distributeVariantsEqually}>
Redistribute
<LemonButton
onClick={distributeVariantsEqually}
tooltip="Redistribute variant rollout percentages equally"
>
<IconBalance />
</LemonButton>
</div>
),
Expand All @@ -121,6 +125,7 @@ export const SidePanelExperimentFeatureFlag = (): JSX.Element => {
}}
min={0}
max={100}
suffix={<span>%</span>}
/>
),
},
Expand Down
11 changes: 11 additions & 0 deletions frontend/src/layout/navigation/ProjectSwitcher.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,17 @@ function OtherProjectButton({ team }: { team: TeamBasicType; onClickInside?: ()
// and after switching is on a different page than before.
let route = removeProjectIdIfPresent(location.pathname)
route = removeFlagIdIfPresent(route)

// List of routes that should redirect to project home
// instead of keeping the current path.
const redirectToHomeRoutes = ['/products', '/onboarding']

const shouldRedirectToHome = redirectToHomeRoutes.some((redirectRoute) => route.includes(redirectRoute))

if (shouldRedirectToHome) {
return urls.project(team.id) // Go to project home
}

return urls.project(team.id, route)
}, [location.pathname])

Expand Down
2 changes: 1 addition & 1 deletion frontend/src/lib/lemon-ui/icons/categories.ts
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ export const TEAMS_AND_COMPANIES = {
'IconRewindPlay',
'IconVideoCamera',
],
'Feature Success': ['IconFlask', 'IconTestTube', 'IconMultivariateTesting', 'IconSplitTesting'],
'Feature Success': ['IconFlask', 'IconTestTube', 'IconMultivariateTesting', 'IconSplitTesting', 'IconBalance'],
Pipeline: ['IconWebhooks', 'IconDecisionTree'],
'Product OS': ['IconNotebook', 'IconHogQL', 'IconDashboard', 'IconSupport'],
Logos: ['IconLogomark', 'IconGithub'],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ const getIncrementalSyncSupported = (
if (!schema.incremental_available) {
return {
disabled: true,
disabledReason: "Incremental append replication isn't supported on this table",
disabledReason: "Incremental replication isn't supported on this table",
}
}

Expand Down Expand Up @@ -109,23 +109,18 @@ export const SyncMethodForm = ({
label: (
<div className="mb-6 font-normal">
<div className="items-center flex leading-[normal] overflow-hidden mb-2.5">
<h2 className="mb-0 mr-2">Incremental append replication</h2>
<h2 className="mb-0 mr-2">Incremental replication</h2>
{!incrementalSyncSupported.disabled && (
<LemonTag type="success">Recommended</LemonTag>
)}
</div>
<p>
When using incremental append replication, we'll store the max value of the below
field on each sync and only sync rows with greater or equal value on the next run.
When using incremental replication, we'll store the max value of the below field on
each sync and only sync rows with greater or equal value on the next run.
</p>
<p>
You should pick a field that increments for each row, such as a{' '}
<code>created_at</code> timestamp.
</p>
<p>
This method will append all new rows to your existing table - this means duplicate
data can exist if the incremental field updates for updated rows (such as when using
an <code>updated_at</code> field)
You should pick a field that increments or updates each time the row is updated,
such as a <code>updated_at</code> timestamp.
</p>
<LemonSelect
value={incrementalFieldValue}
Expand Down Expand Up @@ -167,8 +162,7 @@ export const SyncMethodForm = ({
/>
{showRefreshMessage && (
<p className="text-danger">
Note: Changing the sync type or incremental append replication field will trigger a full table
refresh
Note: Changing the sync type or incremental replication field will trigger a full table refresh
</p>
)}
<div className="flex flex-row justify-end w-full">
Expand All @@ -185,7 +179,7 @@ export const SyncMethodForm = ({
(n) => n.field === incrementalFieldValue
)
if (!fieldSelected) {
lemonToast.error('Selected field for incremental append replication not found')
lemonToast.error('Selected field for incremental replication not found')
return
}

Expand Down
12 changes: 9 additions & 3 deletions frontend/src/scenes/feature-flags/FeatureFlag.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import './FeatureFlag.scss'

import { IconCollapse, IconExpand, IconPlus, IconTrash } from '@posthog/icons'
import { IconBalance, IconCollapse, IconExpand, IconPlus, IconTrash } from '@posthog/icons'
import { LemonDialog, LemonSegmentedButton, LemonSkeleton, LemonSwitch } from '@posthog/lemon-ui'
import { useActions, useValues } from 'kea'
import { Form, Group } from 'kea-forms'
Expand Down Expand Up @@ -950,9 +950,14 @@ function FeatureFlagRollout({ readOnly }: { readOnly?: boolean }): JSX.Element {
</span>
</div>
</div>
<div className="col-span-4 flex items-center gap-1">
<div className="col-span-3 flex justify-between items-center gap-1">
<span>Rollout</span>
<LemonButton onClick={distributeVariantsEqually}>(Redistribute)</LemonButton>
<LemonButton
onClick={distributeVariantsEqually}
tooltip="Normalize variant rollout percentages"
>
<IconBalance />
</LemonButton>
</div>
</div>
{variants.map((variant, index) => (
Expand Down Expand Up @@ -1023,6 +1028,7 @@ function FeatureFlagRollout({ readOnly }: { readOnly?: boolean }): JSX.Element {
}
}
}}
suffix={<span>%</span>}
/>
{filterGroups.filter((group) => group.variant === variant.key)
.length > 0 && (
Expand Down
1 change: 1 addition & 0 deletions frontend/src/scenes/funnels/FunnelLineGraph.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ export function FunnelLineGraph({
kind: NodeKind.FunnelsActorsQuery,
source: querySource,
funnelTrendsDropOff: false,
includeRecordings: true,
funnelTrendsEntrancePeriodStart: dayjs(day).format('YYYY-MM-DD HH:mm:ss'),
}
openPersonsModal({
Expand Down
Loading

0 comments on commit 9f2d558

Please sign in to comment.