Skip to content

Commit

Permalink
feat(web-analytics): Add limit and hasMore to stats_table (#20120)
Browse files Browse the repository at this point in the history 
* Add limit and hasMore to stats_table

* Add test

* Use paginator helper class

* Fix

* Add asType

* Update posthog/hogql_queries/web_analytics/stats_table.py

Co-authored-by: Julian Bez <[email protected]>

* Add limit and offset

* Formatting

* Set limit in modal

* Add more asType

---------

Co-authored-by: Julian Bez <[email protected]>
  • Loading branch information
@robbie-c @webjunkie
robbie-c and webjunkie authored Feb 6, 2024
1 parent 1803fe6 commit 8d6ebe7
Show file tree
Hide file tree
Showing 6 changed files with 122 additions and 19 deletions.
21 changes: 21 additions & 0 deletions frontend/src/queries/schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3487,6 +3487,9 @@
"items": {},
"type": "array"
},
"hasMore": {
"type": "boolean"
},
"hogql": {
"type": "string"
},
Expand All @@ -3496,9 +3499,15 @@
"last_refresh": {
"type": "string"
},
"limit": {
"type": "integer"
},
"next_allowed_client_refresh": {
"type": "string"
},
"offset": {
"type": "integer"
},
"results": {
"items": {},
"type": "array"
Expand Down Expand Up @@ -4939,6 +4948,9 @@
"const": "WebStatsTableQuery",
"type": "string"
},
"limit": {
"type": "integer"
},
"properties": {
"$ref": "#/definitions/WebAnalyticsPropertyFilters"
},
Expand Down Expand Up @@ -4968,6 +4980,9 @@
"items": {},
"type": "array"
},
"hasMore": {
"type": "boolean"
},
"hogql": {
"type": "string"
},
Expand All @@ -4977,9 +4992,15 @@
"last_refresh": {
"type": "string"
},
"limit": {
"type": "integer"
},
"next_allowed_client_refresh": {
"type": "string"
},
"offset": {
"type": "integer"
},
"results": {
"items": {},
"type": "array"
Expand Down
7 changes: 7 additions & 0 deletions frontend/src/queries/schema.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -935,13 +935,20 @@ export interface WebStatsTableQuery extends WebAnalyticsQueryBase {
response?: WebStatsTableQueryResponse
includeScrollDepth?: boolean // automatically sets includeBounceRate to true
includeBounceRate?: boolean
/** @asType integer */
limit?: number
}
export interface WebStatsTableQueryResponse extends QueryResponse {
results: unknown[]
types?: unknown[]
columns?: unknown[]
hogql?: string
samplingRate?: SamplingRate
hasMore?: boolean
/** @asType integer */
limit?: number
/** @asType integer */
offset?: number
}

export type InsightQueryNode =
Expand Down
15 changes: 14 additions & 1 deletion frontend/src/scenes/web-analytics/webAnalyticsLogic.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -552,6 +552,7 @@ export const webAnalyticsLogic = kea<webAnalyticsLogicType>([
includeScrollDepth: statusCheck?.isSendingPageLeavesScroll,
includeBounceRate: true,
sampling,
limit: 10,
},
embedded: false,
},
Expand All @@ -572,6 +573,7 @@ export const webAnalyticsLogic = kea<webAnalyticsLogicType>([
dateRange,
includeScrollDepth: statusCheck?.isSendingPageLeavesScroll,
sampling,
limit: 10,
},
embedded: false,
},
Expand Down Expand Up @@ -602,6 +604,7 @@ export const webAnalyticsLogic = kea<webAnalyticsLogicType>([
breakdownBy: WebStatsBreakdown.InitialReferringDomain,
dateRange,
sampling,
limit: 10,
},
},
insightProps: createInsightProps(TileId.SOURCES, SourceTab.REFERRING_DOMAIN),
Expand All @@ -620,6 +623,7 @@ export const webAnalyticsLogic = kea<webAnalyticsLogicType>([
breakdownBy: WebStatsBreakdown.InitialChannelType,
dateRange,
sampling,
limit: 10,
},
},
insightProps: createInsightProps(TileId.SOURCES, SourceTab.CHANNEL),
Expand All @@ -638,6 +642,7 @@ export const webAnalyticsLogic = kea<webAnalyticsLogicType>([
breakdownBy: WebStatsBreakdown.InitialUTMSource,
dateRange,
sampling,
limit: 10,
},
},
insightProps: createInsightProps(TileId.SOURCES, SourceTab.UTM_SOURCE),
Expand All @@ -656,6 +661,7 @@ export const webAnalyticsLogic = kea<webAnalyticsLogicType>([
breakdownBy: WebStatsBreakdown.InitialUTMMedium,
dateRange,
sampling,
limit: 10,
},
},
insightProps: createInsightProps(TileId.SOURCES, SourceTab.UTM_MEDIUM),
Expand All @@ -674,6 +680,7 @@ export const webAnalyticsLogic = kea<webAnalyticsLogicType>([
breakdownBy: WebStatsBreakdown.InitialUTMCampaign,
dateRange,
sampling,
limit: 10,
},
},
insightProps: createInsightProps(TileId.SOURCES, SourceTab.UTM_CAMPAIGN),
Expand All @@ -692,6 +699,7 @@ export const webAnalyticsLogic = kea<webAnalyticsLogicType>([
breakdownBy: WebStatsBreakdown.InitialUTMContent,
dateRange,
sampling,
limit: 10,
},
},
insightProps: createInsightProps(TileId.SOURCES, SourceTab.UTM_CONTENT),
Expand All @@ -710,6 +718,7 @@ export const webAnalyticsLogic = kea<webAnalyticsLogicType>([
breakdownBy: WebStatsBreakdown.InitialUTMTerm,
dateRange,
sampling,
limit: 10,
},
},
insightProps: createInsightProps(TileId.SOURCES, SourceTab.UTM_TERM),
Expand Down Expand Up @@ -794,6 +803,7 @@ export const webAnalyticsLogic = kea<webAnalyticsLogicType>([
breakdownBy: WebStatsBreakdown.OS,
dateRange,
sampling,
limit: 10,
},
embedded: false,
},
Expand Down Expand Up @@ -857,6 +867,7 @@ export const webAnalyticsLogic = kea<webAnalyticsLogicType>([
breakdownBy: WebStatsBreakdown.Country,
dateRange,
sampling,
limit: 10,
},
},
insightProps: createInsightProps(TileId.GEOGRAPHY, GeographyTab.COUNTRIES),
Expand All @@ -875,6 +886,7 @@ export const webAnalyticsLogic = kea<webAnalyticsLogicType>([
breakdownBy: WebStatsBreakdown.Region,
dateRange,
sampling,
limit: 10,
},
},
insightProps: createInsightProps(TileId.GEOGRAPHY, GeographyTab.REGIONS),
Expand All @@ -893,6 +905,7 @@ export const webAnalyticsLogic = kea<webAnalyticsLogicType>([
breakdownBy: WebStatsBreakdown.City,
dateRange,
sampling,
limit: 10,
},
},
insightProps: createInsightProps(TileId.GEOGRAPHY, GeographyTab.CITIES),
Expand Down Expand Up @@ -956,7 +969,7 @@ export const webAnalyticsLogic = kea<webAnalyticsLogicType>([
...query,
source: {
...query.source,
// limit: 50, // wait for https://github.com/PostHog/posthog/pull/20120
limit: 50,
},
}
} else {
Expand Down
40 changes: 24 additions & 16 deletions posthog/hogql_queries/web_analytics/stats_table.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from posthog.hogql import ast
from posthog.hogql.constants import LimitContext
from posthog.hogql.parser import parse_select, parse_expr
from posthog.hogql.query import execute_hogql_query
from posthog.hogql_queries.insights.paginators import HogQLHasMorePaginator
from posthog.hogql_queries.web_analytics.ctes import (
COUNTS_CTE,
BOUNCE_RATE_CTE,
Expand All @@ -19,6 +20,13 @@
class WebStatsTableQueryRunner(WebAnalyticsQueryRunner):
query: WebStatsTableQuery
query_type = WebStatsTableQuery
paginator: HogQLHasMorePaginator

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.paginator = HogQLHasMorePaginator.from_limit_context(
limit_context=LimitContext.QUERY, limit=self.query.limit if self.query.limit else None
)

def _bounce_rate_subquery(self):
with self.timings.measure("bounce_rate_query"):
Expand Down Expand Up @@ -57,13 +65,12 @@ def _scroll_depth_subquery(self):
},
)

def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery:
def to_query(self) -> ast.SelectQuery:
# special case for channel, as some hogql features to use the general code are still being worked on
if self.query.breakdownBy == WebStatsBreakdown.InitialChannelType:
return self.to_channel_query()

if self.query.includeScrollDepth:
return parse_select(
query = self.to_channel_query()
elif self.query.includeScrollDepth:
query = parse_select(
"""
SELECT
counts.breakdown_value as "context.columns.breakdown_value",
Expand All @@ -87,7 +94,6 @@ def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery:
ORDER BY
"context.columns.views" DESC,
"context.columns.breakdown_value" DESC
LIMIT 10
""",
timings=self.timings,
placeholders={
Expand All @@ -100,7 +106,7 @@ def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery:
)
elif self.query.includeBounceRate:
with self.timings.measure("stats_table_query"):
return parse_select(
query = parse_select(
"""
SELECT
counts.breakdown_value as "context.columns.breakdown_value",
Expand All @@ -118,7 +124,6 @@ def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery:
ORDER BY
"context.columns.views" DESC,
"context.columns.breakdown_value" DESC
LIMIT 10
""",
timings=self.timings,
placeholders={
Expand All @@ -129,7 +134,7 @@ def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery:
)
else:
with self.timings.measure("stats_table_query"):
return parse_select(
query = parse_select(
"""
SELECT
counts.breakdown_value as "context.columns.breakdown_value",
Expand All @@ -142,24 +147,27 @@ def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery:
ORDER BY
"context.columns.views" DESC,
"context.columns.breakdown_value" DESC
LIMIT 10
""",
timings=self.timings,
placeholders={
"counts_query": self._counts_subquery(),
"where_breakdown": self.where_breakdown(),
},
)
assert isinstance(query, ast.SelectQuery)
return query

def calculate(self):
response = execute_hogql_query(
response = self.paginator.execute_hogql_query(
query_type="top_sources_query",
query=self.to_query(),
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
)
assert response.results is not None
results = self.paginator.results

assert results is not None

def to_data(col_val, col_idx):
if col_idx == 0: # breakdown_value
Expand All @@ -173,14 +181,15 @@ def to_data(col_val, col_idx):
else:
return col_val

results = [[to_data(c, i) for (i, c) in enumerate(r)] for r in response.results]
results_mapped = [[to_data(c, i) for (i, c) in enumerate(r)] for r in results]

return WebStatsTableQueryResponse(
columns=response.columns,
results=results,
results=results_mapped,
timings=response.timings,
types=response.types,
hogql=response.hogql,
**self.paginator.response_params(),
)

def counts_breakdown(self):
Expand Down Expand Up @@ -344,7 +353,6 @@ def to_channel_query(self):
ORDER BY
"context.columns.views" DESC,
"context.columns.breakdown_value" DESC
LIMIT 10
""",
timings=self.timings,
backend="cpp",
Expand Down
34 changes: 32 additions & 2 deletions posthog/hogql_queries/web_analytics/test/test_web_stats_table.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,12 @@ def _create_events(self, data, event="$pageview"):
)
return person_result

def _run_web_stats_table_query(self, date_from, date_to, breakdown_by=WebStatsBreakdown.Page):
def _run_web_stats_table_query(self, date_from, date_to, breakdown_by=WebStatsBreakdown.Page, limit=None):
query = WebStatsTableQuery(
dateRange=DateRange(date_from=date_from, date_to=date_to), properties=[], breakdownBy=breakdown_by
dateRange=DateRange(date_from=date_from, date_to=date_to),
properties=[],
breakdownBy=breakdown_by,
limit=limit,
)
runner = WebStatsTableQueryRunner(team=self.team, query=query)
return runner.calculate()
Expand Down Expand Up @@ -111,3 +114,30 @@ def test_breakdown_channel_type_doesnt_throw(self):
1,
len(results),
)

def test_limit(self):
self._create_events(
[
("p1", [("2023-12-02", "s1", "/"), ("2023-12-03", "s1", "/login")]),
("p2", [("2023-12-10", "s2", "/")]),
]
)

response_1 = self._run_web_stats_table_query("all", "2023-12-15", limit=1)
self.assertEqual(
[
["/", 2, 2],
],
response_1.results,
)
self.assertEqual(True, response_1.hasMore)

response_2 = self._run_web_stats_table_query("all", "2023-12-15", limit=2)
self.assertEqual(
[
["/", 2, 2],
["/login", 1, 1],
],
response_2.results,
)
self.assertEqual(False, response_2.hasMore)
Loading

0 comments on commit 8d6ebe7

Please sign in to comment.