From 3b7288f7c71cf0778f42e5c7922040a175a862a1 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Wed, 13 Mar 2024 11:42:57 +0000 Subject: [PATCH 1/3] fix: replay ingestion was ignoring pageview urls (#20883) --- .../session-recording/process-event.ts | 19 +++- .../session-recording/process-event.test.ts | 87 ++++++++++++++++++- 2 files changed, 101 insertions(+), 5 deletions(-) diff --git a/plugin-server/src/main/ingestion-queues/session-recording/process-event.ts b/plugin-server/src/main/ingestion-queues/session-recording/process-event.ts index 9795bbcbeef2e..a729fb23fcff6 100644 --- a/plugin-server/src/main/ingestion-queues/session-recording/process-event.ts +++ b/plugin-server/src/main/ingestion-queues/session-recording/process-event.ts @@ -236,6 +236,18 @@ function isAnyMouseActivity(event: RRWebEvent) { ) } +/** + * meta event has type = 4 and event.data.href + * and custom events have type = 5 and _might_ have event.data.payload.href + * + * we don't really care what type of event they are just whether they have a href + */ +function hrefFrom(event: RRWebEvent): string | undefined { + const metaHref = event.data?.href?.trim() + const customHref = event.data?.payload?.href?.trim() + return metaHref || customHref || undefined +} + export const createSessionReplayEvent = ( uuid: string, team_id: number, @@ -275,9 +287,12 @@ export const createSessionReplayEvent = ( keypressCount += 1 } } - if (url === null && !!event.data?.href?.trim().length) { - url = event.data.href + + const eventUrl: string | undefined = hrefFrom(event) + if (url === null && eventUrl) { + url = eventUrl } + if (event.type === RRWebEventType.Plugin && event.data?.plugin === 'rrweb/console@1') { const level = safeLevel(event.data.payload?.level) if (level === 'info') { diff --git a/plugin-server/tests/main/ingestion-queues/session-recording/process-event.test.ts b/plugin-server/tests/main/ingestion-queues/session-recording/process-event.test.ts index 8bca6952603a1..d74d3a2de9e23 100644 --- a/plugin-server/tests/main/ingestion-queues/session-recording/process-event.test.ts +++ b/plugin-server/tests/main/ingestion-queues/session-recording/process-event.test.ts @@ -181,7 +181,44 @@ describe('session recording process event', () => { }, }, { - testDescription: 'first url detection', + testDescription: 'url can be detected in meta event', + snapshotData: { + events_summary: [ + { + timestamp: 1682449093693, + type: 3, + data: {}, + windowId: '1', + }, + { + timestamp: 1682449093469, + type: 4, + data: { + href: 'http://127.0.0.1:8000/the/url', + }, + windowId: '1', + }, + ], + }, + expected: { + click_count: 0, + keypress_count: 0, + mouse_activity_count: 0, + first_url: 'http://127.0.0.1:8000/the/url', + first_timestamp: '2023-04-25 18:58:13.469', + last_timestamp: '2023-04-25 18:58:13.693', + active_milliseconds: 0, // no data.source, so no activity + console_log_count: 0, + console_warn_count: 0, + console_error_count: 0, + size: 163, + event_count: 2, + message_count: 1, + snapshot_source: 'web', + }, + }, + { + testDescription: 'first url detection takes the first url whether meta url or payload url', snapshotData: { events_summary: [ { @@ -189,7 +226,6 @@ describe('session recording process event', () => { type: 5, data: { payload: { - // doesn't match because href is nested in payload href: 'http://127.0.0.1:8000/home', }, }, @@ -209,7 +245,7 @@ describe('session recording process event', () => { click_count: 0, keypress_count: 0, mouse_activity_count: 0, - first_url: 'http://127.0.0.1:8000/second/url', + first_url: 'http://127.0.0.1:8000/home', first_timestamp: '2023-04-25 18:58:13.469', last_timestamp: '2023-04-25 18:58:13.693', active_milliseconds: 0, // no data.source, so no activity @@ -222,6 +258,51 @@ describe('session recording process event', () => { snapshot_source: 'web', }, }, + { + testDescription: 'first url detection can use payload url', + snapshotData: { + events_summary: [ + { + timestamp: 1682449093469, + type: 5, + data: { + payload: { + // we don't read just any URL + 'the-page-url': 'http://127.0.0.1:8000/second/url', + }, + }, + windowId: '1', + }, + { + timestamp: 1682449093693, + type: 5, + data: { + payload: { + // matches href nested in payload + href: 'http://127.0.0.1:8000/my-spa', + }, + }, + windowId: '1', + }, + ], + }, + expected: { + click_count: 0, + keypress_count: 0, + mouse_activity_count: 0, + first_url: 'http://127.0.0.1:8000/my-spa', + first_timestamp: '2023-04-25 18:58:13.469', + last_timestamp: '2023-04-25 18:58:13.693', + active_milliseconds: 0, // no data.source, so no activity + console_log_count: 0, + console_warn_count: 0, + console_error_count: 0, + size: 235, + event_count: 2, + message_count: 1, + snapshot_source: 'web', + }, + }, { testDescription: 'negative timestamps are not included when picking timestamps', snapshotData: { From 7bfb0cff15fd7211e75641f126f0b797be041a8c Mon Sep 17 00:00:00 2001 From: Ben White Date: Wed, 13 Mar 2024 13:29:19 +0100 Subject: [PATCH 2/3] fix: Docker build not reporting errors (#20892) --- .../sceneDashboardChoiceModalLogic.ts | 2 +- .../insights/EmptyStates/EmptyStates.tsx | 2 +- .../filters/DurationTypeSelect.tsx | 2 +- frontend/src/toolbar/index.tsx | 2 +- frontend/src/types.ts | 2 +- frontend/utils.mjs | 28 +++++++++++++------ production.Dockerfile | 10 +++---- 7 files changed, 29 insertions(+), 19 deletions(-) diff --git a/frontend/src/lib/components/SceneDashboardChoice/sceneDashboardChoiceModalLogic.ts b/frontend/src/lib/components/SceneDashboardChoice/sceneDashboardChoiceModalLogic.ts index 171a80749d209..944e4ef450f25 100644 --- a/frontend/src/lib/components/SceneDashboardChoice/sceneDashboardChoiceModalLogic.ts +++ b/frontend/src/lib/components/SceneDashboardChoice/sceneDashboardChoiceModalLogic.ts @@ -1,7 +1,7 @@ import Fuse from 'fuse.js' import { actions, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea' import { eventUsageLogic } from 'lib/utils/eventUsageLogic' -import { posthog } from 'posthog-js' +import posthog from 'posthog-js' import { Scene } from 'scenes/sceneTypes' import { teamLogic } from 'scenes/teamLogic' import { userLogic } from 'scenes/userLogic' diff --git a/frontend/src/scenes/insights/EmptyStates/EmptyStates.tsx b/frontend/src/scenes/insights/EmptyStates/EmptyStates.tsx index 71150131d7b66..6693835f6564c 100644 --- a/frontend/src/scenes/insights/EmptyStates/EmptyStates.tsx +++ b/frontend/src/scenes/insights/EmptyStates/EmptyStates.tsx @@ -11,7 +11,7 @@ import { supportLogic } from 'lib/components/Support/supportLogic' import { IconErrorOutline, IconOpenInNew } from 'lib/lemon-ui/icons' import { Link } from 'lib/lemon-ui/Link' import { Tooltip } from 'lib/lemon-ui/Tooltip' -import { posthog } from 'posthog-js' +import posthog from 'posthog-js' import { funnelDataLogic } from 'scenes/funnels/funnelDataLogic' import { entityFilterLogic } from 'scenes/insights/filters/ActionFilter/entityFilterLogic' import { insightLogic } from 'scenes/insights/insightLogic' diff --git a/frontend/src/scenes/session-recordings/filters/DurationTypeSelect.tsx b/frontend/src/scenes/session-recordings/filters/DurationTypeSelect.tsx index e6585bf4306d4..09c4517b1c4d4 100644 --- a/frontend/src/scenes/session-recordings/filters/DurationTypeSelect.tsx +++ b/frontend/src/scenes/session-recordings/filters/DurationTypeSelect.tsx @@ -1,5 +1,5 @@ import { LemonSelect } from '@posthog/lemon-ui' -import { posthog } from 'posthog-js' +import posthog from 'posthog-js' import { DurationType } from '~/types' diff --git a/frontend/src/toolbar/index.tsx b/frontend/src/toolbar/index.tsx index 66f36bd4f45ad..a5bdb7923fa1c 100644 --- a/frontend/src/toolbar/index.tsx +++ b/frontend/src/toolbar/index.tsx @@ -1,7 +1,7 @@ import '~/styles' import './styles.scss' -import { PostHog } from 'posthog-js' +import type { PostHog } from 'posthog-js' import { createRoot } from 'react-dom/client' import { initKea } from '~/initKea' diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 662f83942c1b6..511595e56a896 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -20,7 +20,7 @@ import { } from 'lib/constants' import { Dayjs, dayjs } from 'lib/dayjs' import { PopoverProps } from 'lib/lemon-ui/Popover/Popover' -import { PostHog } from 'posthog-js' +import type { PostHog } from 'posthog-js' import { Layout } from 'react-grid-layout' import { LogLevel } from 'rrweb' import { BehavioralFilterKey, BehavioralFilterType } from 'scenes/cohorts/CohortFilters/types' diff --git a/frontend/utils.mjs b/frontend/utils.mjs index 16ad8b984c51e..e4e9bbd5d9c70 100644 --- a/frontend/utils.mjs +++ b/frontend/utils.mjs @@ -194,15 +194,21 @@ function getChunks(result) { } export async function buildInParallel(configs, { onBuildStart, onBuildComplete } = {}) { - await Promise.all( - configs.map((config) => - buildOrWatch({ - ...config, - onBuildStart, - onBuildComplete, - }) + try { + await Promise.all( + configs.map((config) => + buildOrWatch({ + ...config, + onBuildStart, + onBuildComplete, + }) + ) ) - ) + } catch (e) { + if (!isDev) { + process.exit(1) + } + } if (!isDev) { process.exit(0) @@ -338,7 +344,11 @@ export async function buildOrWatch(config) { ...buildResult.metafile, } } catch (e) { - log({ success: false, name, time }) + if (isDev) { + log({ success: false, name, time }) + } else { + throw e + } } } diff --git a/production.Dockerfile b/production.Dockerfile index 9b71e97f34b69..6cac69754aea4 100644 --- a/production.Dockerfile +++ b/production.Dockerfile @@ -23,7 +23,7 @@ # FROM node:18.12.1-bullseye-slim AS frontend-build WORKDIR /code -SHELL ["/bin/bash", "-o", "pipefail", "-c"] +SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] COPY package.json pnpm-lock.yaml ./ COPY patches/ patches/ @@ -44,7 +44,7 @@ RUN pnpm build # FROM node:18.12.1-bullseye-slim AS plugin-server-build WORKDIR /code/plugin-server -SHELL ["/bin/bash", "-o", "pipefail", "-c"] +SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] # Compile and install Node.js dependencies. COPY ./plugin-server/package.json ./plugin-server/pnpm-lock.yaml ./plugin-server/tsconfig.json ./ @@ -85,7 +85,7 @@ RUN corepack enable && \ # FROM python:3.10.10-slim-bullseye AS posthog-build WORKDIR /code -SHELL ["/bin/bash", "-o", "pipefail", "-c"] +SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] # Compile and install Python dependencies. # We install those dependencies on a custom folder that we will @@ -120,7 +120,7 @@ RUN SKIP_SERVICE_VERSION_REQUIREMENTS=1 SECRET_KEY='unsafe secret key for collec # FROM debian:bullseye-slim AS fetch-geoip-db WORKDIR /code -SHELL ["/bin/bash", "-o", "pipefail", "-c"] +SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] # Fetch the GeoLite2-City database that will be used for IP geolocation within Django. RUN apt-get update && \ @@ -237,7 +237,7 @@ CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] # FROM unit-131-python-310 WORKDIR /code -SHELL ["/bin/bash", "-o", "pipefail", "-c"] +SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] ENV PYTHONUNBUFFERED 1 # Install OS runtime dependencies. From f0f230ea4db228dfbb01cb441616b5d7719b23e6 Mon Sep 17 00:00:00 2001 From: danielxnj <115043334+danielxnj@users.noreply.github.com> Date: Wed, 13 Mar 2024 05:29:29 -0700 Subject: [PATCH 3/3] chore: bump base image to fix vulns (#20885) Co-authored-by: xneyder --- production.Dockerfile | 46 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/production.Dockerfile b/production.Dockerfile index 6cac69754aea4..6d43498a5350b 100644 --- a/production.Dockerfile +++ b/production.Dockerfile @@ -21,7 +21,7 @@ # # --------------------------------------------------------- # -FROM node:18.12.1-bullseye-slim AS frontend-build +FROM node:18.19.1-bullseye-slim AS frontend-build WORKDIR /code SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] @@ -42,7 +42,7 @@ RUN pnpm build # # --------------------------------------------------------- # -FROM node:18.12.1-bullseye-slim AS plugin-server-build +FROM node:18.19.1-bullseye-slim AS plugin-server-build WORKDIR /code/plugin-server SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] @@ -166,19 +166,19 @@ RUN set -ex \ && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS)" \ && CONFIGURE_ARGS_MODULES="--prefix=/usr \ - --statedir=/var/lib/unit \ - --control=unix:/var/run/control.unit.sock \ - --runstatedir=/var/run \ - --pid=/var/run/unit.pid \ - --logdir=/var/log \ - --log=/var/log/unit.log \ - --tmpdir=/var/tmp \ - --user=unit \ - --group=unit \ - --openssl \ - --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + --statedir=/var/lib/unit \ + --control=unix:/var/run/control.unit.sock \ + --runstatedir=/var/run \ + --pid=/var/run/unit.pid \ + --logdir=/var/log \ + --log=/var/log/unit.log \ + --tmpdir=/var/tmp \ + --user=unit \ + --group=unit \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ && CONFIGURE_ARGS="$CONFIGURE_ARGS_MODULES \ - --njs" \ + --njs" \ && make -j $NCPU -C pkg/contrib .njs \ && export PKG_CONFIG_PATH=$(pwd)/pkg/contrib/njs/build \ && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ @@ -200,8 +200,8 @@ RUN set -ex \ && cd \ && rm -rf /usr/src/unit \ && for f in /usr/sbin/unitd /usr/lib/unit/modules/*.unit.so; do \ - ldd $f | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $n; done | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq >> /requirements.apt; \ - done \ + ldd $f | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $n; done | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq >> /requirements.apt; \ + done \ && apt-mark showmanual | xargs apt-mark auto > /dev/null \ && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ && /bin/true \ @@ -209,13 +209,13 @@ RUN set -ex \ && mkdir -p /docker-entrypoint.d/ \ && groupadd --gid 998 unit \ && useradd \ - --uid 998 \ - --gid unit \ - --no-create-home \ - --home /nonexistent \ - --comment "unit user" \ - --shell /bin/false \ - unit \ + --uid 998 \ + --gid unit \ + --no-create-home \ + --home /nonexistent \ + --comment "unit user" \ + --shell /bin/false \ + unit \ && apt-get update \ && apt-get --no-install-recommends --no-install-suggests -y install curl $(cat /requirements.apt) \ && apt-get purge -y --auto-remove build-essential \