Skip to content

Commit

Permalink
Merge branch 'master' into dw-virtual-data-warehouse-table-breakdowns
Browse files Browse the repository at this point in the history
  • Loading branch information
EDsCODE authored Mar 19, 2024
2 parents c309f16 + 16fc714 commit d6c1d16
Show file tree
Hide file tree
Showing 15 changed files with 328 additions and 73 deletions.
36 changes: 35 additions & 1 deletion .github/workflows/ci-backend-depot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ jobs:
needs: changes
timeout-minutes: 30

name: Django tests – ${{ matrix.segment }} (persons-on-events ${{ matrix.person-on-events && 'on' || 'off' }}), Py ${{ matrix.python-version }}, ${{ matrix.clickhouse-server-image }} (${{matrix.group}}/${{ matrix.concurrency }})
name: Django tests – ${{ matrix.segment }} (persons-on-events ${{ matrix.person-on-events && 'on' || 'off' }}), Py ${{ matrix.python-version }}, ${{ matrix.clickhouse-server-image }} (${{matrix.group}}/${{ matrix.concurrency }}) (depot)
runs-on: depot-ubuntu-latest

strategy:
Expand Down Expand Up @@ -371,3 +371,37 @@ jobs:
- name: Run async migrations tests
run: |
pytest -m "async_migrations"
calculate-running-time:
name: Calculate running time
needs: [django, async-migrations]
runs-on: ubuntu-latest
if: needs.changes.outputs.backend == 'true'
steps:
- name: Calculate running time
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token
run_id=${GITHUB_RUN_ID}
repo=${GITHUB_REPOSITORY}
run_info=$(gh api repos/${repo}/actions/runs/${run_id})
echo run_info: ${run_info}
# name is the name of the workflow file
# run_started_at is the start time of the workflow
# we want to get the number of seconds between the start time and now
name=$(echo ${run_info} | jq -r '.name')
run_url=$(echo ${run_info} | jq -r '.url')
run_started_at=$(echo ${run_info} | jq -r '.run_started_at')
run_attempt=$(echo ${run_info} | jq -r '.run_attempt')
start_seconds=$(date -d "${run_started_at}" +%s)
now_seconds=$(date +%s)
duration=$((now_seconds-start_seconds))
echo running_time_duration_seconds=${duration} >> $GITHUB_ENV
echo running_time_run_url=${run_url} >> $GITHUB_ENV
echo running_time_run_attempt=${run_attempt} >> $GITHUB_ENV
echo running_time_run_id=${run_id} >> $GITHUB_ENV
echo running_time_run_started_at=${run_started_at} >> $GITHUB_ENV
- name: Capture running time to PostHog
uses: PostHog/[email protected]
with:
posthog-token: ${{secrets.POSTHOG_API_TOKEN}}
event: 'posthog-ci-running-time'
properties: '{"duration_seconds": ${{ env.running_time_duration_seconds }}, "run_url": "${{ env.running_time_run_url }}", "run_attempt": "${{ env.running_time_run_attempt }}", "run_id": "${{ env.running_time_run_id }}", "run_started_at": "${{ env.running_time_run_started_at }}"}'
35 changes: 35 additions & 0 deletions .github/workflows/ci-backend.yml
Original file line number Diff line number Diff line change
Expand Up @@ -371,3 +371,38 @@ jobs:
- name: Run async migrations tests
run: |
pytest -m "async_migrations"
calculate-running-time:
name: Calculate running time
needs: [django, async-migrations]
runs-on: ubuntu-latest
if: needs.changes.outputs.backend == 'true'
steps:
- name: Calculate running time
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token
run_id=${GITHUB_RUN_ID}
repo=${GITHUB_REPOSITORY}
run_info=$(gh api repos/${repo}/actions/runs/${run_id})
echo run_info: ${run_info}
# name is the name of the workflow file
# run_started_at is the start time of the workflow
# we want to get the number of seconds between the start time and now
name=$(echo ${run_info} | jq -r '.name')
run_url=$(echo ${run_info} | jq -r '.url')
run_started_at=$(echo ${run_info} | jq -r '.run_started_at')
run_attempt=$(echo ${run_info} | jq -r '.run_attempt')
start_seconds=$(date -d "${run_started_at}" +%s)
now_seconds=$(date +%s)
duration=$((now_seconds-start_seconds))
echo running_time_duration_seconds=${duration} >> $GITHUB_ENV
echo running_time_run_url=${run_url} >> $GITHUB_ENV
echo running_time_run_attempt=${run_attempt} >> $GITHUB_ENV
echo running_time_run_id=${run_id} >> $GITHUB_ENV
echo running_time_run_started_at=${run_started_at} >> $GITHUB_ENV
- name: Capture running time to PostHog
uses: PostHog/[email protected]
with:
posthog-token: ${{secrets.POSTHOG_API_TOKEN}}
event: 'posthog-ci-running-time'
properties: '{"duration_seconds": ${{ env.running_time_duration_seconds }}, "run_url": "${{ env.running_time_run_url }}", "run_attempt": "${{ env.running_time_run_attempt }}", "run_id": "${{ env.running_time_run_id }}", "run_started_at": "${{ env.running_time_run_started_at }}"}'
2 changes: 1 addition & 1 deletion .github/workflows/ci-e2e-depot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ jobs:
actions-id-token-request-url: ${{ env.ACTIONS_ID_TOKEN_REQUEST_URL }}

cypress:
name: Cypress E2E tests (${{ strategy.job-index }})
name: Cypress E2E tests (${{ strategy.job-index }}) (depot)
runs-on: depot-ubuntu-latest
timeout-minutes: 60
needs: [chunks, changes, container]
Expand Down
40 changes: 27 additions & 13 deletions ee/session_recordings/ai/error_clustering.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
from prometheus_client import Histogram
from django.conf import settings
from posthog.clickhouse.client import sync_execute
from posthog.models.team import Team
from posthog.models import Team, User
from sklearn.cluster import DBSCAN
import pandas as pd
import numpy as np
from posthog.session_recordings.models.session_recording_event import SessionRecordingViewed

CLUSTER_REPLAY_ERRORS_TIMING = Histogram(
"posthog_session_recordings_cluster_replay_errors",
Expand All @@ -22,7 +24,7 @@
DBSCAN_MIN_SAMPLES = settings.REPLAY_EMBEDDINGS_CLUSTERING_DBSCAN_MIN_SAMPLES


def error_clustering(team: Team):
def error_clustering(team: Team, user: User):
results = fetch_error_embeddings(team.pk)

if not results:
Expand All @@ -34,7 +36,7 @@ def error_clustering(team: Team):

CLUSTER_REPLAY_ERRORS_CLUSTER_COUNT.labels(team_id=team.pk).observe(df["cluster"].nunique())

return construct_response(df)
return construct_response(df, team, user)


def fetch_error_embeddings(team_id: int):
Expand Down Expand Up @@ -64,13 +66,25 @@ def cluster_embeddings(embeddings):
return dbscan.labels_


def construct_response(df):
return [
{
"cluster": cluster,
"samples": rows.head(n=DBSCAN_MIN_SAMPLES)[["session_id", "input"]].to_dict("records"),
"occurrences": rows.size,
"unique_sessions": rows["session_id"].count(),
}
for cluster, rows in df.groupby("cluster")
]
def construct_response(df: pd.DataFrame, team: Team, user: User):
viewed_session_ids = list(
SessionRecordingViewed.objects.filter(team=team, user=user, session_id__in=df["session_id"].unique())
.values_list("session_id", flat=True)
.distinct()
)

clusters = []
for cluster, rows in df.groupby("cluster"):
session_ids = rows["session_id"].unique()
sample = rows.sample(n=1)[["session_id", "input"]].rename(columns={"input": "error"}).to_dict("records")
clusters.append(
{
"cluster": cluster,
"sample": sample,
"session_ids": session_ids,
"occurrences": rows.size,
"unique_sessions": len(session_ids),
"viewed": len(np.intersect1d(session_ids, viewed_session_ids, assume_unique=True)),
}
)
return clusters
1 change: 1 addition & 0 deletions frontend/src/lib/constants.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,7 @@ export const FEATURE_FLAGS = {
REPLAY_ERROR_CLUSTERING: 'session-replay-error-clustering', // owner: #team-replay
AUDIT_LOGS_ACCESS: 'audit-logs-access', // owner: #team-growth
SUBSCRIBE_FROM_PAYGATE: 'subscribe-from-paygate', // owner: #team-growth
REVERSE_PROXY_ONBOARDING: 'reverse-proxy-onboarding', // owner: @zlwaterfield
} as const
export type FeatureFlagKey = (typeof FEATURE_FLAGS)[keyof typeof FEATURE_FLAGS]

Expand Down
5 changes: 5 additions & 0 deletions frontend/src/lib/taxonomy.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -672,6 +672,11 @@ export const CORE_FILTER_DEFINITIONS_BY_GROUP = {
description: 'What library was used to send the event.',
examples: ['web', 'posthog-ios'],
},
$lib_custom_api_host: {
label: 'Library Custom API Host',
description: 'The custom API host used to send the event.',
examples: ['https://ph.example.com'],
},
$lib_version: {
label: 'Library Version',
description: 'Version of the library used to send the event. Used in combination with Library.',
Expand Down
8 changes: 7 additions & 1 deletion frontend/src/scenes/onboarding/Onboarding.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { onboardingLogic, OnboardingStepKey } from './onboardingLogic'
import { OnboardingProductConfiguration } from './OnboardingProductConfiguration'
import { ProductConfigOption } from './onboardingProductConfigurationLogic'
import { OnboardingProductIntroduction } from './OnboardingProductIntroduction'
import { OnboardingReverseProxy } from './OnboardingReverseProxy'
import { FeatureFlagsSDKInstructions } from './sdks/feature-flags/FeatureFlagsSDKInstructions'
import { ProductAnalyticsSDKInstructions } from './sdks/product-analytics/ProductAnalyticsSDKInstructions'
import { SDKs } from './sdks/SDKs'
Expand All @@ -28,7 +29,8 @@ export const scene: SceneExport = {
* Wrapper for custom onboarding content. This automatically includes billing, other products, and invite steps.
*/
const OnboardingWrapper = ({ children }: { children: React.ReactNode }): JSX.Element => {
const { currentOnboardingStep, shouldShowBillingStep, product, includeIntro } = useValues(onboardingLogic)
const { currentOnboardingStep, shouldShowBillingStep, shouldShowReverseProxyStep, product, includeIntro } =
useValues(onboardingLogic)
const { setAllOnboardingSteps } = useActions(onboardingLogic)
const [allSteps, setAllSteps] = useState<JSX.Element[]>([])

Expand Down Expand Up @@ -58,6 +60,10 @@ const OnboardingWrapper = ({ children }: { children: React.ReactNode }): JSX.Ele
const IntroStep = <OnboardingProductIntroduction stepKey={OnboardingStepKey.PRODUCT_INTRO} />
steps = [IntroStep, ...steps]
}
if (shouldShowReverseProxyStep) {
const ReverseProxyStep = <OnboardingReverseProxy stepKey={OnboardingStepKey.REVERSE_PROXY} />
steps = [...steps, ReverseProxyStep]
}
if (shouldShowBillingStep) {
const BillingStep = <OnboardingBillingStep product={product} stepKey={OnboardingStepKey.PLANS} />
steps = [...steps, BillingStep]
Expand Down
87 changes: 87 additions & 0 deletions frontend/src/scenes/onboarding/OnboardingReverseProxy.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
import { LemonDivider, Link } from '@posthog/lemon-ui'

import { InviteMembersButton } from '~/layout/navigation/TopBar/AccountPopover'

import { OnboardingStepKey } from './onboardingLogic'
import { OnboardingStep } from './OnboardingStep'

const proxyDocs = [
{
title: 'AWS CloudFront',
link: 'https://posthog.com/docs/advanced/proxy/cloudfront',
},
{
title: 'Caddy',
link: 'https://posthog.com/docs/advanced/proxy/caddy',
},
{
title: 'Cloudflare',
link: 'https://posthog.com/docs/advanced/proxy/cloudflare',
},
{
title: 'Kubernetes Ingress Controller',
link: 'https://posthog.com/docs/advanced/proxy/kubernetes-ingress-controller',
},
{
title: 'Netlify',
link: 'https://posthog.com/docs/advanced/proxy/netlify',
},
{
title: 'Next.js rewrites',
link: 'https://posthog.com/docs/advanced/proxy/nextjs',
},
{
title: 'Next.js middleware',
link: 'https://posthog.com/docs/advanced/proxy/nextjs-middleware',
},
{
title: 'Vercel',
link: 'https://posthog.com/docs/advanced/proxy/vercel',
},
{
title: 'Nuxt',
link: 'https://posthog.com/docs/advanced/proxy/nuxt',
},
]

export const OnboardingReverseProxy = ({ stepKey }: { stepKey: OnboardingStepKey }): JSX.Element => {
return (
<OnboardingStep
title="Reverse proxy (optional)"
stepKey={stepKey}
continueText="I've already done this"
showSkip
>
<div className="mb-6 mt-6">
<p>A reverse proxy allows you to send events to PostHog Cloud using your own domain.</p>
<p>
This means that events are sent from your own domain and are less likely to be intercepted by
tracking blockers. You'll be able to capture more usage data without having to self-host PostHog.
</p>
<p>
Setting up a reverse proxy means setting up a service to redirect requests from a subdomain you
choose (like <span className="font-mono break-keep">e.yourdomain.com</span>) to PostHog. It is best
practice to use a subdomain that does not include posthog, analytics, tracking, or other similar
words.
</p>
<h3>Documentation</h3>
<p>Here are some popular reverse proxy options:</p>
<ul className="list-disc list-inside ml-2">
{proxyDocs.map(({ title, link }) => (
<li key={title}>
<Link to={link} target="_blank">
{title}
</Link>
</li>
))}
</ul>
<LemonDivider className="my-6" />
<h3>Need help with this step?</h3>
<p>Invite a team member to help you get set up.</p>
<div className="mt-3 w-40">
<InviteMembersButton type="secondary" />
</div>
</div>
</OnboardingStep>
)
}
4 changes: 3 additions & 1 deletion frontend/src/scenes/onboarding/OnboardingStep.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export const OnboardingStep = ({
showHelpButton = false,
onSkip,
continueAction,
continueText,
continueOverride,
hideHeader,
}: {
Expand All @@ -27,6 +28,7 @@ export const OnboardingStep = ({
showHelpButton?: boolean
onSkip?: () => void
continueAction?: () => void
continueText?: string
continueOverride?: JSX.Element
hideHeader?: boolean
}): JSX.Element => {
Expand Down Expand Up @@ -114,7 +116,7 @@ export const OnboardingStep = ({
}}
sideIcon={hasNextStep ? <IconArrowRight /> : null}
>
{!hasNextStep ? 'Finish' : 'Next'}
{continueText ? continueText : !hasNextStep ? 'Finish' : 'Next'}
</LemonButton>
)}
</div>
Expand Down
11 changes: 11 additions & 0 deletions frontend/src/scenes/onboarding/onboardingLogic.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ export enum OnboardingStepKey {
PLANS = 'plans',
VERIFY = 'verify',
PRODUCT_CONFIGURATION = 'configure',
REVERSE_PROXY = 'proxy',
INVITE_TEAMMATES = 'invite_teammates',
}

Expand Down Expand Up @@ -208,6 +209,16 @@ export const onboardingLogic = kea<onboardingLogicType>([
return !product?.subscribed || !hasAllAddons || subscribedDuringOnboarding
},
],
shouldShowReverseProxyStep: [
(s) => [s.product, s.featureFlags],
(product: BillingProductV2Type | null, featureFlags: FeatureFlagsSet) => {
const productsWithReverseProxy = []
if (featureFlags[FEATURE_FLAGS.REVERSE_PROXY_ONBOARDING] === 'test') {
productsWithReverseProxy.push(ProductKey.FEATURE_FLAGS)
}
return productsWithReverseProxy.includes(product?.type as ProductKey)
},
],
isStepKeyInvalid: [
(s) => [s.stepKey, s.allOnboardingSteps, s.currentOnboardingStep],
(stepKey: string, allOnboardingSteps: AllOnboardingSteps, currentOnboardingStep: React.ReactNode | null) =>
Expand Down
Loading

0 comments on commit d6c1d16

Please sign in to comment.