Skip to content

Commit

Permalink
feat(queries): Make query runner add more tags to both query log and …
Browse files Browse the repository at this point in the history 
…Sentry (#24254)
  • Loading branch information
@webjunkie
webjunkie authored Aug 8, 2024
1 parent 8233b4f commit 4aedc89
Show file tree
Hide file tree
Showing 6 changed files with 126 additions and 44 deletions.
Binary file modified frontend/__snapshots__/exporter-exporter--user-paths-insight--dark.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified frontend/__snapshots__/exporter-exporter--user-paths-insight--light.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
115 changes: 92 additions & 23 deletions frontend/src/lib/components/CommandPalette/DebugCHQueries.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import { LemonBanner } from 'lib/lemon-ui/LemonBanner'
import { LemonButton } from 'lib/lemon-ui/LemonButton'
import { LemonDialog } from 'lib/lemon-ui/LemonDialog'
import { LemonTable } from 'lib/lemon-ui/LemonTable'
import { LemonTag } from 'lib/lemon-ui/LemonTag'
import { Link } from 'lib/lemon-ui/Link'
import { humanizeBytes } from 'lib/utils'
import { copyToClipboard } from 'lib/utils/copyToClipboard'
import { useState } from 'react'
Expand All @@ -30,7 +32,6 @@ export interface Query {
timestamp: string
query: string
query_id: string
queryJson: string
exception: string
/**
* 1 means running, 2 means finished, 3 means errored before execution, 4 means errored during execution.
Expand All @@ -39,6 +40,10 @@ export interface Query {
status: 1 | 2 | 3 | 4
execution_time: number
path: string
logComment: {
query: any
[key: string]: any
}
}

const debugCHQueriesLogic = kea<debugCHQueriesLogicType>([
Expand All @@ -59,7 +64,7 @@ const debugCHQueriesLogic = kea<debugCHQueriesLogicType>([
[] as Query[],
{
loadQueries: async () => {
return await api.get('api/debug_ch_queries/')
return (await api.get('api/debug_ch_queries/')).queries
},
},
],
Expand Down Expand Up @@ -128,9 +133,20 @@ function DebugCHQueries(): JSX.Element {
title: 'Timestamp',
render: function Timestamp(_, item) {
return (
<span className="font-mono whitespace-pre">
{dayjs.tz(item.timestamp, 'UTC').tz().format().replace('T', '\n')}
</span>
<>
<div className="font-mono whitespace-pre mb-2">
{dayjs.tz(item.timestamp, 'UTC').tz().format().replace('T', '\n')}
</div>
<div>
{item.status === 1 ? (
'In progress…'
) : (
<>
Took {Math.round((item.execution_time + Number.EPSILON) * 100) / 100} ms
</>
)}
</div>
</>
)
},
width: 160,
Expand All @@ -141,12 +157,71 @@ function DebugCHQueries(): JSX.Element {
return (
<div className="max-w-200 py-1 space-y-2">
<div>
<span className="font-bold tracking-wide">ID:</span>{' '}
<span className="font-mono">{item.query_id}</span>
<LemonTag className="inline-block">
<span className="font-bold tracking-wide">ID:</span>{' '}
<span className="font-mono">{item.query_id}</span>
</LemonTag>{' '}
{item.logComment.cache_key ? (
<LemonTag className="inline-block">
<span className="font-bold tracking-wide">Cache key:</span>{' '}
<span className="font-mono">{item.logComment.cache_key}</span>{' '}
<Link
to={`https://sentry.io/issues/?query=is%3Aunresolved+cache_key%3A${item.logComment.cache_key}&referrer=issue-list&statsPeriod=7d`}
className="inline-block"
target="_blank"
targetBlankIcon
/>
</LemonTag>
) : null}{' '}
{item.logComment.insight_id ? (
<LemonTag className="inline-block">
<span className="font-bold tracking-wide">Insight ID:</span>{' '}
<span className="font-mono">{item.logComment.insight_id}</span>{' '}
<Link
to={`https://sentry.io/issues/?query=is%3Aunresolved+insight_id%3A${item.logComment.insight_id}&referrer=issue-list&statsPeriod=7d`}
className="inline-block"
target="_blank"
targetBlankIcon
/>
</LemonTag>
) : null}{' '}
{item.logComment.dashboard_id ? (
<LemonTag className="inline-block">
<span className="font-bold tracking-wide">Dashboard ID:</span>{' '}
<span className="font-mono">{item.logComment.dashboard_id}</span>{' '}
<Link
to={`https://sentry.io/issues/?query=is%3Aunresolved+dashboard_id%3A${item.logComment.dashboard_id}&referrer=issue-list&statsPeriod=7d`}
className="inline-block"
target="_blank"
targetBlankIcon
/>
</LemonTag>
) : null}{' '}
{item.logComment.user_id ? (
<LemonTag className="inline-block">
<span className="font-bold tracking-wide">User ID:</span>{' '}
<span className="font-mono">{item.logComment.user_id}</span>{' '}
<Link
to={`https://sentry.io/issues/?query=is%3Aunresolved+user%3A%22id%3A${item.logComment.user_id}%22&referrer=issue-list&statsPeriod=7d`}
className="inline-block"
target="_blank"
targetBlankIcon
/>
</LemonTag>
) : null}
</div>
{item.exception && (
<LemonBanner type="error" className="text-xs font-mono">
{item.exception}
<div>{item.exception}</div>
<LemonButton
type="secondary"
size="xsmall"
to={`https://sentry.io/issues/?query=is%3Aunresolved+issue.priority%3A%5Bhigh%2C+medium%5D+trace%3A${item.logComment.sentry_trace}&statsPeriod=1d`}
targetBlank
className="mt-4 mb-1"
>
View in Sentry
</LemonButton>
</LemonBanner>
)}
<CodeSnippet
Expand All @@ -157,40 +232,33 @@ function DebugCHQueries(): JSX.Element {
>
{item.query}
</CodeSnippet>
{item.queryJson ? (
{item.logComment.query ? (
<LemonButton
type="primary"
size="small"
fullWidth
center
icon={<IconCodeInsert />}
to={urls.debugQuery(item.queryJson)}
to={urls.debugQuery(item.logComment.query)}
targetBlank
sideAction={{
icon: <IconCopy />,
onClick: () => void copyToClipboard(item.queryJson, 'query JSON'),
onClick: () =>
void copyToClipboard(
JSON.stringify(item.logComment.query),
'query JSON'
),
tooltip: 'Copy query JSON to clipboard',
}}
className="my-0"
>
Debug {JSON.parse(item.queryJson).kind || 'query'} in new tab
Debug {item.logComment.query.kind || 'query'} in new tab
</LemonButton>
) : null}
</div>
)
},
},

{
title: 'Duration',
render: function Duration(_, item) {
if (item.status === 1) {
return 'In progress…'
}
return <>{Math.round((item.execution_time + Number.EPSILON) * 100) / 100} ms</>
},
align: 'right',
},
{
title: 'Profiling stats',
render: function ProfilingStats(_, item) {
Expand Down Expand Up @@ -287,6 +355,7 @@ function DebugCHQueries(): JSX.Element {
loading={queriesLoading}
loadingSkeletonRows={5}
pagination={undefined}
rowClassName="align-top"
/>
</>
)
Expand Down
36 changes: 19 additions & 17 deletions posthog/api/debug_ch_queries.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import json
import re
from typing import Optional

Expand Down Expand Up @@ -33,17 +34,16 @@ def list(self, request):
SELECT
query_id,
argMax(query, type) AS query,
argMax(query_json, type) AS query_json,
argMax(query_start_time, type) AS query_start_time,
argMax(exception, type) AS exception,
argMax(query_duration_ms, type) AS query_duration_ms,
argMax(ProfileEvents, type) as profile_events,
argMax(log_comment, type) AS log_comment,
max(type) AS status
FROM (
SELECT
query_id, query, query_start_time, exception, query_duration_ms, toInt8(type) AS type,
JSONExtractRaw(log_comment, 'query') as query_json,
ProfileEvents
ProfileEvents, log_comment
FROM clusterAllReplicas(%(cluster)s, system, query_log)
WHERE
query LIKE %(query)s AND
Expand All @@ -63,18 +63,20 @@ def list(self, request):
},
)
return Response(
[
{
"query_id": resp[0],
"query": resp[1],
"queryJson": resp[2],
"timestamp": resp[3],
"exception": resp[4],
"execution_time": resp[5],
"profile_events": resp[6],
"status": resp[7],
"path": self._get_path(resp[1]),
}
for resp in response
]
{
"queries": [
{
"query_id": resp[0],
"query": resp[1],
"timestamp": resp[2],
"exception": resp[3],
"execution_time": resp[4],
"profile_events": resp[5],
"logComment": json.loads(resp[6]),
"status": resp[7],
"path": self._get_path(resp[1]),
}
for resp in response
]
}
)
7 changes: 4 additions & 3 deletions posthog/api/query.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,13 @@
from django.http import JsonResponse
from drf_spectacular.utils import OpenApiResponse
from pydantic import BaseModel
from posthog.hogql_queries.query_runner import ExecutionMode, execution_mode_from_refresh
from rest_framework import status
from rest_framework import viewsets
from rest_framework.decorators import action
from rest_framework.exceptions import ValidationError, NotAuthenticated
from rest_framework.request import Request
from rest_framework.response import Response
from sentry_sdk import capture_exception
from rest_framework import status
from sentry_sdk import capture_exception, set_tag

from posthog.api.documentation import extend_schema
from posthog.api.mixins import PydanticModelMixin
Expand All @@ -25,6 +24,7 @@
from posthog.errors import ExposedCHQueryError
from posthog.hogql.ai import PromptUnclear, write_sql_from_prompt
from posthog.hogql.errors import ExposedHogQLError
from posthog.hogql_queries.query_runner import ExecutionMode, execution_mode_from_refresh
from posthog.models.user import User
from posthog.rate_limit import (
AIBurstRateThrottle,
Expand Down Expand Up @@ -159,3 +159,4 @@ def _tag_client_query_id(self, query_id: str | None):
return

tag_queries(client_query_id=query_id)
set_tag("client_query_id", query_id)
12 changes: 11 additions & 1 deletion posthog/hogql_queries/query_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import structlog
from prometheus_client import Counter
from pydantic import BaseModel, ConfigDict
from sentry_sdk import capture_exception, push_scope
from sentry_sdk import capture_exception, push_scope, set_tag, get_traceparent

from posthog.caching.utils import is_stale, ThresholdMode, cache_target_age, last_refresh_from_cached_result
from posthog.clickhouse.client.execute_async import enqueue_process_query_task, get_query_status, QueryNotFoundError
Expand Down Expand Up @@ -538,7 +538,17 @@ def run(
dashboard_id: Optional[int] = None,
) -> CR | CacheMissResponse | QueryStatusResponse:
cache_key = self.get_cache_key()

tag_queries(cache_key=cache_key)
tag_queries(sentry_trace=get_traceparent())
set_tag("cache_key", cache_key)
if insight_id:
tag_queries(insight_id=insight_id)
set_tag("insight_id", str(insight_id))
if dashboard_id:
tag_queries(dashboard_id=dashboard_id)
set_tag("dashboard_id", str(dashboard_id))

self.query_id = query_id or self.query_id
CachedResponse: type[CR] = self.cached_response_type
cache_manager = QueryCacheManager(
Expand Down

0 comments on commit 4aedc89

Please sign in to comment.